diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
new file mode 100644
index 0000000000000..d1c404e6fe35a
--- /dev/null
+++ b/.github/workflows/build_and_test.yml
@@ -0,0 +1,372 @@
+name: Build and test
+
+on:
+  push:
+    branches:
+    - branch-3.0
+  pull_request:
+    branches:
+    - branch-3.0
+
+jobs:
+  # Build: build Spark and run the tests for specified modules.
+  build:
+    name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }} (JDK ${{ matrix.java }}, ${{ matrix.hadoop }}, ${{ matrix.hive }})"
+    # Ubuntu 20.04 is the latest LTS. The next LTS is 22.04.
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - 8
+        hadoop:
+          - hadoop2.7
+        hive:
+          - hive2.3
+        # TODO(SPARK-32246): We don't test 'streaming-kinesis-asl' for now.
+        # Kinesis tests depends on external Amazon kinesis service.
+        # Note that the modules below are from sparktestsupport/modules.py.
+        modules:
+          - >-
+            core, unsafe, kvstore, avro,
+            network-common, network-shuffle, repl, launcher,
+            examples, sketch, graphx
+          - >-
+            catalyst, hive-thriftserver
+          - >-
+            streaming, sql-kafka-0-10, streaming-kafka-0-10,
+            mllib-local, mllib,
+            yarn, mesos, kubernetes, hadoop-cloud, spark-ganglia-lgpl
+        # Here, we split Hive and SQL tests into some of slow ones and the rest of them.
+        included-tags: [""]
+        # Some tests are disabled in GitHun Actions. Ideally, we should remove this tag
+        # and run all tests.
+        excluded-tags: ["org.apache.spark.tags.GitHubActionsUnstableTest"]
+        comment: [""]
+        include:
+          # Hive tests
+          - modules: hive
+            java: 8
+            hadoop: hadoop2.7
+            hive: hive2.3
+            included-tags: org.apache.spark.tags.SlowHiveTest
+            comment: "- slow tests"
+          - modules: hive
+            java: 8
+            hadoop: hadoop2.7
+            hive: hive2.3
+            excluded-tags: org.apache.spark.tags.SlowHiveTest,org.apache.spark.tags.GitHubActionsUnstableTest
+            comment: "- other tests"
+          # SQL tests
+          - modules: sql
+            java: 8
+            hadoop: hadoop2.7
+            hive: hive2.3
+            included-tags: org.apache.spark.tags.ExtendedSQLTest
+            comment: "- slow tests"
+          - modules: sql
+            java: 8
+            hadoop: hadoop2.7
+            hive: hive2.3
+            excluded-tags: org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.GitHubActionsUnstableTest
+            comment: "- other tests"
+    env:
+      MODULES_TO_TEST: ${{ matrix.modules }}
+      EXCLUDED_TAGS: ${{ matrix.excluded-tags }}
+      INCLUDED_TAGS: ${{ matrix.included-tags }}
+      HADOOP_PROFILE: ${{ matrix.hadoop }}
+      HIVE_PROFILE: ${{ matrix.hive }}
+      # GitHub Actions' default miniconda to use in pip packaging test.
+      CONDA_PREFIX: /usr/share/miniconda
+      GITHUB_PREV_SHA: ${{ github.event.before }}
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+      # In order to fetch changed files
+      with:
+        fetch-depth: 0
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT, Maven and Zinc
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/zinc-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Ivy local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.ivy2/cache
+        key: ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-
+    - name: Install Java ${{ matrix.java }}
+      uses: actions/setup-java@v1
+      with:
+        java-version: ${{ matrix.java }}
+    - name: Install Python 3.8
+      uses: actions/setup-python@v2
+      # We should install one Python that is higher then 3+ for SQL and Yarn because:
+      # - SQL component also has Python related tests, for example, IntegratedUDFTestUtils.
+      # - Yarn has a Python specific test too, for example, YarnClusterSuite.
+      if: contains(matrix.modules, 'yarn') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
+      with:
+        python-version: 3.8
+        architecture: x64
+    - name: Install Python packages (Python 3.8)
+      if: (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
+      run: |
+        python3.8 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
+        python3.8 -m pip list
+    # Run the tests.
+    - name: Run tests
+      run: |
+        # Hive and SQL tests become flaky when running in parallel as it's too intensive.
+        if [[ "$MODULES_TO_TEST" == "hive" ]] || [[ "$MODULES_TO_TEST" == "sql" ]]; then export SERIAL_SBT_TESTS=1; fi
+        ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS"
+    - name: Upload test results to report
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-results-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
+        path: "**/target/test-reports/*.xml"
+    - name: Upload unit tests log files
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
+        path: "**/target/unit-tests.log"
+
+  pyspark:
+    name: "Build modules: ${{ matrix.modules }}"
+    runs-on: ubuntu-20.04
+    container:
+       image: dongjoon/apache-spark-github-action-image:20201025
+    strategy:
+      fail-fast: false
+      matrix:
+        modules:
+          - >-
+            pyspark-sql, pyspark-mllib
+          - >-
+            pyspark-core, pyspark-streaming, pyspark-ml
+    env:
+      MODULES_TO_TEST: ${{ matrix.modules }}
+      HADOOP_PROFILE: hadoop2.7
+      HIVE_PROFILE: hive2.3
+      # GitHub Actions' default miniconda to use in pip packaging test.
+      CONDA_PREFIX: /usr/share/miniconda
+      GITHUB_PREV_SHA: ${{ github.event.before }}
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+      # In order to fetch changed files
+      with:
+        fetch-depth: 0
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT, Maven and Zinc
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/zinc-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Ivy local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.ivy2/cache
+        key: pyspark-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          pyspark-ivy-
+    - name: Install Python 2.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 2.7
+        architecture: x64
+    - name: Install Python packages (Python 2.7 )
+      run: |
+        python2.7 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
+        python2.7 -m pip list
+    # Run the tests.
+    - name: Run tests
+      run: |
+        ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST"
+    - name: Upload test results to report
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-results-${{ matrix.modules }}--8-hadoop2.7-hive2.3
+        path: "**/target/test-reports/*.xml"
+    - name: Upload unit tests log files
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: unit-tests-log-${{ matrix.modules }}--8-hadoop2.7-hive2.3
+        path: "**/target/unit-tests.log"
+
+  sparkr:
+    name: "Build modules: sparkr"
+    runs-on: ubuntu-20.04
+    container:
+       image: dongjoon/apache-spark-github-action-image:20201025
+    env:
+      HADOOP_PROFILE: hadoop2.7
+      HIVE_PROFILE: hive2.3
+      GITHUB_PREV_SHA: ${{ github.event.before }}
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+      # In order to fetch changed files
+      with:
+        fetch-depth: 0
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT, Maven and Zinc
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/zinc-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Ivy local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.ivy2/cache
+        key: sparkr-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          sparkr-ivy-
+    - name: Run tests
+      run: |
+        # The followings are also used by `r-lib/actions/setup-r` to avoid
+        # R issues at docker environment
+        export TZ=UTC
+        export _R_CHECK_SYSTEM_CLOCK_=FALSE
+        ./dev/run-tests --parallelism 2 --modules sparkr
+    - name: Upload test results to report
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-results-sparkr--8-hadoop2.7-hive2.3
+        path: "**/target/test-reports/*.xml"
+    - name: Upload unit tests log files
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: unit-tests-log-sparkr--8-hadoop2.7-hive2.3
+        path: "**/target/unit-tests.log"
+
+  # Static analysis, and documentation build
+  lint:
+    name: Linters, licenses, dependencies and documentation generation
+    runs-on: ubuntu-20.04
+    container:
+      image: dongjoon/apache-spark-github-action-image:20201025
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT, Maven and Zinc
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/zinc-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Ivy local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.ivy2/cache
+        key: docs-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          docs-ivy-
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: docs-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          docs-maven-
+    - name: Install Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+        architecture: x64
+    - name: Install Python linter dependencies
+      run: |
+        python3.6 -m pip install install flake8 sphinx numpy
+    - name: Install R linter dependencies and SparkR
+      run: |
+        apt-get install -y libcurl4-openssl-dev libgit2-dev libssl-dev libxml2-dev
+        Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')"
+        Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
+        ./R/install-dev.sh
+    - name: Install dependencies for documentation generation
+      run: |
+        apt-get install -y libcurl4-openssl-dev pandoc
+        python3.6 -m pip install sphinx mkdocs numpy
+        apt-get update -y
+        apt-get install -y ruby ruby-dev
+        gem install jekyll jekyll-redirect-from rouge
+        Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')"
+    - name: Scala linter
+      run: ./dev/lint-scala
+    - name: Java linter
+      run: ./dev/lint-java
+    - name: Python linter
+      run: ./dev/lint-python
+    - name: R linter
+      run: ./dev/lint-r
+    - name: License test
+      run: ./dev/check-license
+    - name: Dependencies test
+      run: ./dev/test-dependencies.sh
+    - name: Run documentation build
+      run: |
+        cd docs
+        export LC_ALL=C.UTF-8
+        export LANG=C.UTF-8
+        jekyll build
+
+  java-11:
+    name: Java 11 build with Maven
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: java11-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          java11-maven-
+    - name: Install Java 11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 11
+    - name: Build with Maven
+      run: |
+        export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
+        export MAVEN_CLI_OPTS="--no-transfer-progress"
+        # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414.
+        ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=11 install
+        rm -rf ~/.m2/repository/org/apache/spark
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
deleted file mode 100644
index d53119ad75599..0000000000000
--- a/.github/workflows/master.yml
+++ /dev/null
@@ -1,119 +0,0 @@
-name: master
-
-on:
-  push:
-    branches:
-    - master
-  pull_request:
-    branches:
-    - master
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        java: [ '1.8', '11' ]
-        hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ]
-        hive: [ 'hive-1.2', 'hive-2.3' ]
-        exclude:
-        - java: '11'
-          hive: 'hive-1.2'
-        - hadoop: 'hadoop-3.2'
-          hive: 'hive-1.2'
-    name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }}
-
-    steps:
-    - uses: actions/checkout@master
-    # We split caches because GitHub Action Cache has a 400MB-size limit.
-    - uses: actions/cache@v1
-      with:
-        path: build
-        key: build-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          build-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/com
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/org
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/net
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/io
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-
-    - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v1
-      with:
-        java-version: ${{ matrix.java }}
-    - name: Build with Maven
-      run: |
-        export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
-        export MAVEN_CLI_OPTS="--no-transfer-progress"
-        mkdir -p ~/.m2
-        ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install
-        rm -rf ~/.m2/repository/org/apache/spark
-
-
-  lint:
-    runs-on: ubuntu-latest
-    name: Linters (Java/Scala/Python), licenses, dependencies
-    steps:
-    - uses: actions/checkout@master
-    - uses: actions/setup-java@v1
-      with:
-        java-version: '11'
-    - uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-        architecture: 'x64'
-    - name: Scala
-      run: ./dev/lint-scala
-    - name: Java
-      run: ./dev/lint-java
-    - name: Python
-      run: |
-        pip install flake8 sphinx numpy
-        ./dev/lint-python
-    - name: License
-      run: ./dev/check-license
-    - name: Dependencies
-      run: ./dev/test-dependencies.sh
-
-  lintr:
-    runs-on: ubuntu-latest
-    name: Linter (R)
-    steps:
-    - uses: actions/checkout@master
-    - uses: actions/setup-java@v1
-      with:
-        java-version: '11'
-    - name: install R
-      run: |
-        echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' | sudo tee -a /etc/apt/sources.list
-        curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add
-        sudo apt-get update
-        sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev
-    - name: install R packages
-      run: |
-        sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
-        sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
-    - name: package and install SparkR
-      run: ./R/install-dev.sh
-    - name: lint-r
-      run: ./dev/lint-r
diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml
new file mode 100644
index 0000000000000..5d4fdf4c97ded
--- /dev/null
+++ b/.github/workflows/publish_snapshot.yml
@@ -0,0 +1,33 @@
+name: Publish Snapshot
+
+on:
+  push:
+    branches:
+    - branch-3.0
+
+jobs:
+  publish-snapshot:
+    if: github.repository == 'apache/spark'
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@master
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: snapshot-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          snapshot-maven-
+    - name: Install Java 8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 8
+    - name: Publish snapshot
+      env:
+        ASF_USERNAME: ${{ secrets.NEXUS_USER }}
+        ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
+        GPG_KEY: "not_used"
+        GPG_PASSPHRASE: "not_used"
+        GIT_REF: "branch-3.0"
+      run: ./dev/create-release/release-build.sh publish-snapshot
diff --git a/.github/workflows/test_report.yml b/.github/workflows/test_report.yml
new file mode 100644
index 0000000000000..93cdb86687261
--- /dev/null
+++ b/.github/workflows/test_report.yml
@@ -0,0 +1,24 @@
+name: Report test results
+on:
+  workflow_run:
+    workflows: ["Build and test"]
+    types:
+      - completed
+
+jobs:
+  test_report:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Download test results to report
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        workflow: ${{ github.event.workflow_run.workflow_id }}
+        commit: ${{ github.event.workflow_run.head_commit.id }}
+    - name: Publish test report
+      uses: scacap/action-surefire-report@v1
+      with:
+        check_name: Report test results
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        report_paths: "**/target/test-reports/*.xml"
+        commit: ${{ github.event.workflow_run.head_commit.id }}
diff --git a/.gitignore b/.gitignore
index 798e8acc4d43b..1f4b323c926ac 100644
--- a/.gitignore
+++ b/.gitignore
@@ -72,7 +72,6 @@ scalastyle-on-compile.generated.xml
 scalastyle-output.xml
 scalastyle.txt
 spark-*-bin-*.tgz
-spark-resources/
 spark-tests.log
 src_managed/
 streaming-tests.log
@@ -80,6 +79,7 @@ target/
 unit-tests.log
 work/
 docs/.jekyll-metadata
+docs/.jekyll-cache
 
 # For Hive
 TempStatsStore/
diff --git a/R/WINDOWS.md b/R/WINDOWS.md
index dbc27178bdb8c..9fe4a22bf22b2 100644
--- a/R/WINDOWS.md
+++ b/R/WINDOWS.md
@@ -22,8 +22,8 @@ To build SparkR on Windows, the following steps are required
 
 1. Make sure `bash` is available and in `PATH` if you already have a built-in `bash` on Windows. If you do not have, install [Cygwin](https://www.cygwin.com/).
 
-2. Install R (>= 3.1) and [Rtools](https://cloud.r-project.org/bin/windows/Rtools/). Make sure to
-include Rtools and R in `PATH`. Note that support for R prior to version 3.4 is deprecated as of Spark 3.0.0.
+2. Install R (>= 3.5) and [Rtools](https://cloud.r-project.org/bin/windows/Rtools/). Make sure to
+include Rtools and R in `PATH`.
 
 3. Install JDK that SparkR supports (see `R/pkg/DESCRIPTION`), and set `JAVA_HOME` in the system environment variables.
 
diff --git a/R/create-rd.sh b/R/create-rd.sh
index ff622a41a46c0..aaad3b1aafa0a 100755
--- a/R/create-rd.sh
+++ b/R/create-rd.sh
@@ -34,4 +34,4 @@ pushd "$FWDIR" > /dev/null
 . "$FWDIR/find-r.sh"
 
 # Generate Rd files if devtools is installed
-"$R_SCRIPT_PATH/Rscript" -e ' if("devtools" %in% rownames(installed.packages())) { library(devtools); devtools::document(pkg="./pkg", roclets=c("rd")) }'
+"$R_SCRIPT_PATH/Rscript" -e ' if("devtools" %in% rownames(installed.packages())) { library(devtools); setwd("'$FWDIR'"); devtools::document(pkg="./pkg", roclets=c("rd")) }'
diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 95d3e52bef3a9..a17dea9093771 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.0.0
+Version: 3.0.2
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
@@ -15,7 +15,7 @@ URL: https://www.apache.org/ https://spark.apache.org/
 BugReports: https://spark.apache.org/contributing.html
 SystemRequirements: Java (>= 8, < 12)
 Depends:
-    R (>= 3.1),
+    R (>= 3.5),
     methods
 Suggests:
     knitr,
@@ -23,7 +23,7 @@ Suggests:
     testthat,
     e1071,
     survival,
-    arrow
+    arrow (>= 0.15.1)
 Collate:
     'schema.R'
     'generics.R'
@@ -62,3 +62,4 @@ Collate:
 RoxygenNote: 5.0.1
 VignetteBuilder: knitr
 NeedsCompilation: no
+Encoding: UTF-8
diff --git a/R/pkg/NAMESPACE b/R/pkg/NAMESPACE
index 7ed2e36d59531..9fd7bb4c6ff2b 100644
--- a/R/pkg/NAMESPACE
+++ b/R/pkg/NAMESPACE
@@ -28,6 +28,7 @@ importFrom("utils", "download.file", "object.size", "packageVersion", "tail", "u
 
 # S3 methods exported
 export("sparkR.session")
+export("sparkR.init")
 export("sparkR.session.stop")
 export("sparkR.stop")
 export("sparkR.conf")
@@ -41,6 +42,9 @@ export("sparkR.callJStatic")
 
 export("install.spark")
 
+export("sparkRSQL.init",
+       "sparkRHive.init")
+
 # MLlib integration
 exportMethods("glm",
               "spark.glm",
@@ -148,6 +152,7 @@ exportMethods("arrange",
               "printSchema",
               "randomSplit",
               "rbind",
+              "registerTempTable",
               "rename",
               "repartition",
               "repartitionByRange",
@@ -420,8 +425,10 @@ export("as.DataFrame",
        "cacheTable",
        "clearCache",
        "createDataFrame",
+       "createExternalTable",
        "createTable",
        "currentDatabase",
+       "dropTempTable",
        "dropTempView",
        "listColumns",
        "listDatabases",
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 593d3ca16220d..4195ec8f07b8b 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -271,7 +271,7 @@ setMethod("show", "SparkDataFrame",
                 paste(l, collapse = ":")
               })
               s <- paste(cols, collapse = ", ")
-              cat(paste(class(object), "[", s, "]\n", sep = ""))
+              cat(paste0(class(object), "[", s, "]\n"))
             }
           })
 
@@ -431,7 +431,7 @@ setMethod("coltypes",
                 if (is.null(type)) {
                   specialtype <- specialtypeshandle(x)
                   if (is.null(specialtype)) {
-                    stop(paste("Unsupported data type: ", x))
+                    stop("Unsupported data type: ", x)
                   }
                   type <- PRIMITIVE_TYPES[[specialtype]]
                 }
@@ -521,6 +521,32 @@ setMethod("createOrReplaceTempView",
               invisible(callJMethod(x@sdf, "createOrReplaceTempView", viewName))
           })
 
+#' (Deprecated) Register Temporary Table
+#'
+#' Registers a SparkDataFrame as a Temporary Table in the SparkSession
+#' @param x A SparkDataFrame
+#' @param tableName A character vector containing the name of the table
+#'
+#' @seealso \link{createOrReplaceTempView}
+#' @rdname registerTempTable-deprecated
+#' @name registerTempTable
+#' @aliases registerTempTable,SparkDataFrame,character-method
+#' @examples
+#'\dontrun{
+#' sparkR.session()
+#' path <- "path/to/file.json"
+#' df <- read.json(path)
+#' registerTempTable(df, "json_df")
+#' new_df <- sql("SELECT * FROM json_df")
+#'}
+#' @note registerTempTable since 1.4.0
+setMethod("registerTempTable",
+          signature(x = "SparkDataFrame", tableName = "character"),
+          function(x, tableName) {
+              .Deprecated("createOrReplaceTempView")
+              invisible(callJMethod(x@sdf, "createOrReplaceTempView", tableName))
+          })
+
 #' insertInto
 #'
 #' Insert the contents of a SparkDataFrame into a table registered in the current SparkSession.
@@ -803,8 +829,8 @@ setMethod("repartitionByRange",
                 jcol <- lapply(cols, function(c) { c@jc })
                 sdf <- callJMethod(x@sdf, "repartitionByRange", numToInt(numPartitions), jcol)
               } else {
-                stop(paste("numPartitions and col must be numeric and Column; however, got",
-                           class(numPartitions), "and", class(col)))
+                stop("numPartitions and col must be numeric and Column; however, got ",
+                     class(numPartitions), " and ", class(col))
               }
             } else if (!is.null(col))  {
               # only columns are specified
@@ -813,7 +839,7 @@ setMethod("repartitionByRange",
                 jcol <- lapply(cols, function(c) { c@jc })
                 sdf <- callJMethod(x@sdf, "repartitionByRange", jcol)
               } else {
-                stop(paste("col must be Column; however, got", class(col)))
+                stop("col must be Column; however, got ", class(col))
               }
             } else if (!is.null(numPartitions)) {
               # only numPartitions is specified
@@ -1042,10 +1068,10 @@ setMethod("sample",
           signature(x = "SparkDataFrame"),
           function(x, withReplacement = FALSE, fraction, seed) {
             if (!is.numeric(fraction)) {
-              stop(paste("fraction must be numeric; however, got", class(fraction)))
+              stop("fraction must be numeric; however, got ", class(fraction))
             }
             if (!is.logical(withReplacement)) {
-              stop(paste("withReplacement must be logical; however, got", class(withReplacement)))
+              stop("withReplacement must be logical; however, got ", class(withReplacement))
             }
 
             if (!missing(seed)) {
@@ -1185,11 +1211,10 @@ setMethod("collect",
                 checkSchemaInArrow(schema(x))
                 TRUE
               }, error = function(e) {
-                warning(paste0("The conversion from Spark DataFrame to R DataFrame was attempted ",
-                               "with Arrow optimization because ",
-                               "'spark.sql.execution.arrow.sparkr.enabled' is set to true; ",
-                               "however, failed, attempting non-optimization. Reason: ",
-                               e))
+                warning("The conversion from Spark DataFrame to R DataFrame was attempted ",
+                        "with Arrow optimization because ",
+                        "'spark.sql.execution.arrow.sparkr.enabled' is set to true; ",
+                        "however, failed, attempting non-optimization. Reason: ", e)
                 FALSE
               })
             }
@@ -1200,27 +1225,23 @@ setMethod("collect",
               # empty data.frame with 0 columns and 0 rows
               data.frame()
             } else if (useArrow) {
-              requireNamespace1 <- requireNamespace
-              if (requireNamespace1("arrow", quietly = TRUE)) {
-                read_arrow <- get("read_arrow", envir = asNamespace("arrow"), inherits = FALSE)
-                # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190.
-                useAsTibble <- exists("as_tibble", envir = asNamespace("arrow"))
-
+              if (requireNamespace("arrow", quietly = TRUE)) {
                 portAuth <- callJMethod(x@sdf, "collectAsArrowToR")
                 port <- portAuth[[1]]
                 authSecret <- portAuth[[2]]
                 conn <- socketConnection(
                   port = port, blocking = TRUE, open = "wb", timeout = connectionTimeout)
+                version <- packageVersion("arrow")
                 output <- tryCatch({
                   doServerAuth(conn, authSecret)
-                  arrowTable <- read_arrow(readRaw(conn))
-                  if (useAsTibble) {
-                    as_tibble <- get("as_tibble", envir = asNamespace("arrow"))
-                    as.data.frame(as_tibble(arrowTable), stringsAsFactors = stringsAsFactors)
+                  if (version$minor >= 17 || version$major >= 1) {
+                    arrowTable <- arrow::read_ipc_stream(readRaw(conn))
                   } else {
-                    as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors)
+                    arrowTable <- arrow::read_arrow(readRaw(conn))
                   }
-                }, finally = {
+                  as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors)
+                },
+                finally = {
                   close(conn)
                 })
                 return(output)
@@ -1487,8 +1508,8 @@ dapplyInternal <- function(x, func, schema) {
     if (inherits(schema, "structType")) {
       checkSchemaInArrow(schema)
     } else if (is.null(schema)) {
-      stop(paste0("Arrow optimization does not support 'dapplyCollect' yet. Please disable ",
-                  "Arrow optimization or use 'collect' and 'dapply' APIs instead."))
+      stop("Arrow optimization does not support 'dapplyCollect' yet. Please disable ",
+           "Arrow optimization or use 'collect' and 'dapply' APIs instead.")
     } else {
       stop("'schema' should be DDL-formatted string or structType.")
     }
@@ -1631,9 +1652,7 @@ setMethod("dapplyCollect",
 #'
 #' @param cols grouping columns.
 #' @param func a function to be applied to each group partition specified by grouping
-#'             column of the SparkDataFrame. The function \code{func} takes as argument
-#'             a key - grouping columns and a data frame - a local R data.frame.
-#'             The output of \code{func} is a local R data.frame.
+#'             column of the SparkDataFrame. See Details.
 #' @param schema the schema of the resulting SparkDataFrame after the function is applied.
 #'               The schema must match to output of \code{func}. It has to be defined for each
 #'               output column with preferred output column name and corresponding data type.
@@ -1643,29 +1662,43 @@ setMethod("dapplyCollect",
 #' @aliases gapply,SparkDataFrame-method
 #' @rdname gapply
 #' @name gapply
+#' @details
+#' \code{func} is a function of two arguments. The first, usually named \code{key}
+#' (though this is not enforced) corresponds to the grouping key, will be an
+#' unnamed \code{list} of \code{length(cols)} length-one objects corresponding
+#' to the grouping columns' values for the current group.
+#'
+#' The second, herein \code{x}, will be a local \code{\link{data.frame}} with the
+#' columns of the input not in \code{cols} for the rows corresponding to \code{key}.
+#'
+#' The output of \code{func} must be a \code{data.frame} matching \code{schema} --
+#' in particular this means the names of the output \code{data.frame} are irrelevant
+#'
 #' @seealso \link{gapplyCollect}
 #' @examples
 #'
 #' \dontrun{
-#' Computes the arithmetic mean of the second column by grouping
-#' on the first and third columns. Output the grouping values and the average.
+#' # Computes the arithmetic mean of the second column by grouping
+#' # on the first and third columns. Output the grouping values and the average.
 #'
 #' df <- createDataFrame (
 #' list(list(1L, 1, "1", 0.1), list(1L, 2, "1", 0.2), list(3L, 3, "3", 0.3)),
 #'   c("a", "b", "c", "d"))
 #'
-#' Here our output contains three columns, the key which is a combination of two
-#' columns with data types integer and string and the mean which is a double.
+#' # Here our output contains three columns, the key which is a combination of two
+#' # columns with data types integer and string and the mean which is a double.
 #' schema <- structType(structField("a", "integer"), structField("c", "string"),
 #'   structField("avg", "double"))
 #' result <- gapply(
 #'   df,
 #'   c("a", "c"),
 #'   function(key, x) {
+#'     # key will either be list(1L, '1') (for the group where a=1L,c='1') or
+#'     #   list(3L, '3') (for the group where a=3L,c='3')
 #'     y <- data.frame(key, mean(x$b), stringsAsFactors = FALSE)
 #' }, schema)
 #'
-#' The schema also can be specified in a DDL-formatted string.
+#' # The schema also can be specified in a DDL-formatted string.
 #' schema <- "a INT, c STRING, avg DOUBLE"
 #' result <- gapply(
 #'   df,
@@ -1674,8 +1707,8 @@ setMethod("dapplyCollect",
 #'     y <- data.frame(key, mean(x$b), stringsAsFactors = FALSE)
 #' }, schema)
 #'
-#' We can also group the data and afterwards call gapply on GroupedData.
-#' For Example:
+#' # We can also group the data and afterwards call gapply on GroupedData.
+#' # For example:
 #' gdf <- group_by(df, "a", "c")
 #' result <- gapply(
 #'   gdf,
@@ -1684,15 +1717,15 @@ setMethod("dapplyCollect",
 #' }, schema)
 #' collect(result)
 #'
-#' Result
-#' ------
-#' a c avg
-#' 3 3 3.0
-#' 1 1 1.5
+#' # Result
+#' # ------
+#' # a c avg
+#' # 3 3 3.0
+#' # 1 1 1.5
 #'
-#' Fits linear models on iris dataset by grouping on the 'Species' column and
-#' using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
-#' and 'Petal_Width' as training features.
+#' # Fits linear models on iris dataset by grouping on the 'Species' column and
+#' # using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
+#' # and 'Petal_Width' as training features.
 #'
 #' df <- createDataFrame (iris)
 #' schema <- structType(structField("(Intercept)", "double"),
@@ -1708,12 +1741,12 @@ setMethod("dapplyCollect",
 #'   }, schema)
 #' collect(df1)
 #'
-#' Result
-#' ---------
-#' Model  (Intercept)  Sepal_Width  Petal_Length  Petal_Width
-#' 1        0.699883    0.3303370    0.9455356    -0.1697527
-#' 2        1.895540    0.3868576    0.9083370    -0.6792238
-#' 3        2.351890    0.6548350    0.2375602     0.2521257
+#' # Result
+#' # ---------
+#' # Model  (Intercept)  Sepal_Width  Petal_Length  Petal_Width
+#' # 1        0.699883    0.3303370    0.9455356    -0.1697527
+#' # 2        1.895540    0.3868576    0.9083370    -0.6792238
+#' # 3        2.351890    0.6548350    0.2375602     0.2521257
 #'
 #'}
 #' @note gapply(SparkDataFrame) since 2.0.0
@@ -1731,20 +1764,30 @@ setMethod("gapply",
 #'
 #' @param cols grouping columns.
 #' @param func a function to be applied to each group partition specified by grouping
-#'             column of the SparkDataFrame. The function \code{func} takes as argument
-#'             a key - grouping columns and a data frame - a local R data.frame.
-#'             The output of \code{func} is a local R data.frame.
+#'             column of the SparkDataFrame. See Details.
 #' @return A data.frame.
 #' @family SparkDataFrame functions
 #' @aliases gapplyCollect,SparkDataFrame-method
 #' @rdname gapplyCollect
 #' @name gapplyCollect
+#' @details
+#' \code{func} is a function of two arguments. The first, usually named \code{key}
+#' (though this is not enforced) corresponds to the grouping key, will be an
+#' unnamed \code{list} of \code{length(cols)} length-one objects corresponding
+#' to the grouping columns' values for the current group.
+#'
+#' The second, herein \code{x}, will be a local \code{\link{data.frame}} with the
+#' columns of the input not in \code{cols} for the rows corresponding to \code{key}.
+#'
+#' The output of \code{func} must be a \code{data.frame} matching \code{schema} --
+#' in particular this means the names of the output \code{data.frame} are irrelevant
+#'
 #' @seealso \link{gapply}
 #' @examples
 #'
 #' \dontrun{
-#' Computes the arithmetic mean of the second column by grouping
-#' on the first and third columns. Output the grouping values and the average.
+#' # Computes the arithmetic mean of the second column by grouping
+#' # on the first and third columns. Output the grouping values and the average.
 #'
 #' df <- createDataFrame (
 #' list(list(1L, 1, "1", 0.1), list(1L, 2, "1", 0.2), list(3L, 3, "3", 0.3)),
@@ -1759,8 +1802,8 @@ setMethod("gapply",
 #'     y
 #'   })
 #'
-#' We can also group the data and afterwards call gapply on GroupedData.
-#' For Example:
+#' # We can also group the data and afterwards call gapply on GroupedData.
+#' # For example:
 #' gdf <- group_by(df, "a", "c")
 #' result <- gapplyCollect(
 #'   gdf,
@@ -1770,15 +1813,15 @@ setMethod("gapply",
 #'     y
 #'   })
 #'
-#' Result
-#' ------
-#' key_a key_c mean_b
-#' 3 3 3.0
-#' 1 1 1.5
+#' # Result
+#' # ------
+#' # key_a key_c mean_b
+#' # 3 3 3.0
+#' # 1 1 1.5
 #'
-#' Fits linear models on iris dataset by grouping on the 'Species' column and
-#' using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
-#' and 'Petal_Width' as training features.
+#' # Fits linear models on iris dataset by grouping on the 'Species' column and
+#' # using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
+#' # and 'Petal_Width' as training features.
 #'
 #' df <- createDataFrame (iris)
 #' result <- gapplyCollect(
@@ -1790,12 +1833,12 @@ setMethod("gapply",
 #'     data.frame(t(coef(m)))
 #'   })
 #'
-#' Result
-#'---------
-#' Model  X.Intercept.  Sepal_Width  Petal_Length  Petal_Width
-#' 1        0.699883    0.3303370    0.9455356    -0.1697527
-#' 2        1.895540    0.3868576    0.9083370    -0.6792238
-#' 3        2.351890    0.6548350    0.2375602     0.2521257
+#' # Result
+#' # ---------
+#' # Model  X.Intercept.  Sepal_Width  Petal_Length  Petal_Width
+#' # 1        0.699883    0.3303370    0.9455356    -0.1697527
+#' # 2        1.895540    0.3868576    0.9083370    -0.6792238
+#' # 3        2.351890    0.6548350    0.2375602     0.2521257
 #'
 #'}
 #' @note gapplyCollect(SparkDataFrame) since 2.0.0
@@ -1969,8 +2012,8 @@ setMethod("[", signature(x = "SparkDataFrame"),
               x
             } else {
               if (class(i) != "Column") {
-                stop(paste0("Expressions other than filtering predicates are not supported ",
-                      "in the first parameter of extract operator [ or subset() method."))
+                stop("Expressions other than filtering predicates are not supported ",
+                      "in the first parameter of extract operator [ or subset() method.")
               }
               filter(x, i)
             }
@@ -2561,18 +2604,17 @@ setMethod("join",
               if (is.null(joinType)) {
                 sdf <- callJMethod(x@sdf, "join", y@sdf, joinExpr@jc)
               } else {
-                if (joinType %in% c("inner", "cross",
+                validJoinTypes <- c("inner", "cross",
                     "outer", "full", "fullouter", "full_outer",
                     "left", "leftouter", "left_outer",
                     "right", "rightouter", "right_outer",
-                    "semi", "left_semi", "leftsemi", "anti", "left_anti", "leftanti")) {
-                  joinType <- gsub("_", "", joinType)
+                    "semi", "leftsemi", "left_semi", "anti", "leftanti", "left_anti")
+                if (joinType %in% validJoinTypes) {
+                  joinType <- gsub("_", "", joinType, fixed = TRUE)
                   sdf <- callJMethod(x@sdf, "join", y@sdf, joinExpr@jc, joinType)
                 } else {
-                  stop(paste("joinType must be one of the following types:",
-                       "'inner', 'cross', 'outer', 'full', 'fullouter', 'full_outer',",
-                       "'left', 'leftouter', 'left_outer', 'right', 'rightouter', 'right_outer',",
-                       "'semi', 'leftsemi', 'left_semi', 'anti', 'leftanti' or 'left_anti'."))
+                  stop("joinType must be one of the following types: ",
+                       "'", paste(validJoinTypes, collapse = "', '"), "'")
                 }
               }
             }
@@ -2707,10 +2749,10 @@ setMethod("merge",
               colY <- joinY[[i]]
 
               if (colX %in% by) {
-                colX <- paste(colX, suffixes[1], sep = "")
+                colX <- paste0(colX, suffixes[1])
               }
               if (colY %in% by) {
-                colY <- paste(colY, suffixes[2], sep = "")
+                colY <- paste0(colY, suffixes[2])
               }
 
               colX <- getColumn(xsel, colX)
@@ -2725,7 +2767,7 @@ setMethod("merge",
 
             # sorts the result by 'by' columns if sort = TRUE
             if (sort && length(by) > 0) {
-              colNameWithSuffix <- paste(by, suffixes[2], sep = "")
+              colNameWithSuffix <- paste0(by, suffixes[2])
               joinRes <- do.call("arrange", c(joinRes, colNameWithSuffix, decreasing = FALSE))
             }
 
@@ -2748,7 +2790,7 @@ genAliasesForIntersectedCols <- function(x, intersectedColNames, suffix) {
   cols <- lapply(allColNames, function(colName) {
     col <- getColumn(x, colName)
     if (colName %in% intersectedColNames) {
-      newJoin <- paste(colName, suffix, sep = "")
+      newJoin <- paste0(colName, suffix)
       if (newJoin %in% allColNames) {
         stop("The following column name: ", newJoin, " occurs more than once in the 'DataFrame'.",
           "Please use different suffixes for the intersected columns.")
diff --git a/R/pkg/R/RDD.R b/R/pkg/R/RDD.R
index 6e89b4bb4d964..7a1d157bb8a36 100644
--- a/R/pkg/R/RDD.R
+++ b/R/pkg/R/RDD.R
@@ -69,7 +69,7 @@ setMethod("initialize", "RDD", function(.Object, jrdd, serializedMode,
 
 setMethod("showRDD", "RDD",
           function(object) {
-              cat(paste(callJMethod(getJRDD(object), "toString"), "\n", sep = ""))
+              cat(paste0(callJMethod(getJRDD(object), "toString"), "\n"))
           })
 
 setMethod("initialize", "PipelinedRDD", function(.Object, prev, func, jrdd_val) {
@@ -947,7 +947,7 @@ setMethod("takeSample", signature(x = "RDD", withReplacement = "logical",
             MAXINT <- .Machine$integer.max
 
             if (num < 0)
-              stop(paste("Negative number of elements requested"))
+              stop("Negative number of elements requested")
 
             if (initialCount > MAXINT - 1) {
               maxSelected <- MAXINT - 1
diff --git a/R/pkg/R/SQLContext.R b/R/pkg/R/SQLContext.R
index f48a334ed6766..c0ac68332ec41 100644
--- a/R/pkg/R/SQLContext.R
+++ b/R/pkg/R/SQLContext.R
@@ -34,7 +34,7 @@ getInternalType <- function(x) {
          Date = "date",
          POSIXlt = "timestamp",
          POSIXct = "timestamp",
-         stop(paste("Unsupported type for SparkDataFrame:", class(x))))
+         stop("Unsupported type for SparkDataFrame: ", class(x)))
 }
 
 #' return the SparkSession
@@ -110,10 +110,11 @@ sparkR.conf <- function(key, defaultValue) {
     value <- if (missing(defaultValue)) {
       tryCatch(callJMethod(conf, "get", key),
               error = function(e) {
-                if (any(grep("java.util.NoSuchElementException", as.character(e)))) {
-                  stop(paste0("Config '", key, "' is not set"))
+                estr <- as.character(e)
+                if (any(grepl("java.util.NoSuchElementException", estr, fixed = TRUE))) {
+                  stop("Config '", key, "' is not set")
                 } else {
-                  stop(paste0("Unknown error: ", as.character(e)))
+                  stop("Unknown error: ", estr)
                 }
               })
     } else {
@@ -205,9 +206,9 @@ getSchema <- function(schema, firstRow = NULL, rdd = NULL) {
     # SPAKR-SQL does not support '.' in column name, so replace it with '_'
     # TODO(davies): remove this once SPARK-2775 is fixed
     names <- lapply(names, function(n) {
-      nn <- gsub("[.]", "_", n)
+      nn <- gsub(".", "_", n, fixed = TRUE)
       if (nn != n) {
-        warning(paste("Use", nn, "instead of", n, "as column name"))
+        warning("Use ", nn, " instead of ", n, " as column name")
       }
       nn
     })
@@ -289,10 +290,9 @@ createDataFrame <- function(data, schema = NULL, samplingRatio = 1.0,
         TRUE
       },
       error = function(e) {
-        warning(paste0("createDataFrame attempted Arrow optimization because ",
-                       "'spark.sql.execution.arrow.sparkr.enabled' is set to true; however, ",
-                       "failed, attempting non-optimization. Reason: ",
-                       e))
+        warning("createDataFrame attempted Arrow optimization because ",
+                "'spark.sql.execution.arrow.sparkr.enabled' is set to true; however, ",
+                "failed, attempting non-optimization. Reason: ", e)
         FALSE
       })
     }
@@ -325,7 +325,7 @@ createDataFrame <- function(data, schema = NULL, samplingRatio = 1.0,
   } else if (inherits(data, "RDD")) {
     rdd <- data
   } else {
-    stop(paste("unexpected type:", class(data)))
+    stop("unexpected type: ", class(data))
   }
 
   schema <- getSchema(schema, firstRow, rdd)
@@ -556,7 +556,6 @@ tableToDF <- function(tableName) {
 #' stringSchema <- "name STRING, info MAP<STRING, DOUBLE>"
 #' df4 <- read.df(mapTypeJsonPath, "json", stringSchema, multiLine = TRUE)
 #' }
-#' @name read.df
 #' @note read.df since 1.4.0
 read.df <- function(path = NULL, source = NULL, schema = NULL, na.strings = "NA", ...) {
   if (!is.null(path) && !is.character(path)) {
@@ -687,7 +686,6 @@ read.jdbc <- function(url, tableName,
 #' stringSchema <- "name STRING, info MAP<STRING, DOUBLE>"
 #' df1 <- read.stream("json", path = jsonDir, schema = stringSchema, maxFilesPerTrigger = 1)
 #' }
-#' @name read.stream
 #' @note read.stream since 2.2.0
 #' @note experimental
 read.stream <- function(source = NULL, schema = NULL, ...) {
diff --git a/R/pkg/R/catalog.R b/R/pkg/R/catalog.R
index 7641f8a7a0432..275737f804bde 100644
--- a/R/pkg/R/catalog.R
+++ b/R/pkg/R/catalog.R
@@ -17,6 +17,35 @@
 
 # catalog.R: SparkSession catalog functions
 
+#' (Deprecated) Create an external table
+#'
+#' Creates an external table based on the dataset in a data source,
+#' Returns a SparkDataFrame associated with the external table.
+#'
+#' The data source is specified by the \code{source} and a set of options(...).
+#' If \code{source} is not specified, the default data source configured by
+#' "spark.sql.sources.default" will be used.
+#'
+#' @param tableName a name of the table.
+#' @param path the path of files to load.
+#' @param source the name of external data source.
+#' @param schema the schema of the data required for some data sources.
+#' @param ... additional argument(s) passed to the method.
+#' @return A SparkDataFrame.
+#' @rdname createExternalTable-deprecated
+#' @seealso \link{createTable}
+#' @examples
+#'\dontrun{
+#' sparkR.session()
+#' df <- createExternalTable("myjson", path="path/to/json", source="json", schema)
+#' }
+#' @name createExternalTable
+#' @note createExternalTable since 1.4.0
+createExternalTable <- function(tableName, path = NULL, source = NULL, schema = NULL, ...) {
+  .Deprecated("createTable", old = "createExternalTable")
+  createTable(tableName, path, source, schema, ...)
+}
+
 #' Creates a table based on the dataset in a data source
 #'
 #' Creates a table based on the dataset in a data source. Returns a SparkDataFrame associated with
@@ -130,6 +159,31 @@ clearCache <- function() {
   invisible(callJMethod(catalog, "clearCache"))
 }
 
+#' (Deprecated) Drop Temporary Table
+#'
+#' Drops the temporary table with the given table name in the catalog.
+#' If the table has been cached/persisted before, it's also unpersisted.
+#'
+#' @param tableName The name of the SparkSQL table to be dropped.
+#' @seealso \link{dropTempView}
+#' @rdname dropTempTable-deprecated
+#' @examples
+#' \dontrun{
+#' sparkR.session()
+#' df <- read.df(path, "parquet")
+#' createOrReplaceTempView(df, "table")
+#' dropTempTable("table")
+#' }
+#' @name dropTempTable
+#' @note dropTempTable since 1.4.0
+dropTempTable <- function(tableName) {
+  .Deprecated("dropTempView", old = "dropTempTable")
+  if (class(tableName) != "character") {
+    stop("tableName must be a string.")
+  }
+  dropTempView(tableName)
+}
+
 #' Drops the temporary view with the given view name in the catalog.
 #'
 #' Drops the temporary view with the given view name in the catalog.
diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R
index 2ff68ab7b9d77..797a5c7da1549 100644
--- a/R/pkg/R/client.R
+++ b/R/pkg/R/client.R
@@ -65,8 +65,8 @@ checkJavaVersion <- function() {
   javaHome <- Sys.getenv("JAVA_HOME")
   javaReqs <- utils::packageDescription(utils::packageName(), fields = c("SystemRequirements"))
   sparkJavaVersions <- strsplit(javaReqs, "[(,)]")[[1]]
-  minJavaVersion <- as.numeric(strsplit(sparkJavaVersions[[2]], ">= ")[[1]][[2]])
-  maxJavaVersion <- as.numeric(strsplit(sparkJavaVersions[[3]], "< ")[[1]][[2]])
+  minJavaVersion <- as.numeric(strsplit(sparkJavaVersions[[2]], ">= ", fixed = TRUE)[[1]][[2]])
+  maxJavaVersion <- as.numeric(strsplit(sparkJavaVersions[[3]], "< ", fixed = TRUE)[[1]][[2]])
   if (javaHome != "") {
     javaBin <- file.path(javaHome, "bin", javaBin)
   }
@@ -89,23 +89,22 @@ checkJavaVersion <- function() {
     })
   javaVersionFilter <- Filter(
       function(x) {
-        grepl(" version", x)
+        grepl(" version", x, fixed = TRUE)
       }, javaVersionOut)
 
-  javaVersionStr <- strsplit(javaVersionFilter[[1]], "[\"]")[[1L]][2]
+  javaVersionStr <- strsplit(javaVersionFilter[[1]], '"', fixed = TRUE)[[1L]][2]
   # javaVersionStr is of the form 1.8.0_92/9.0.x/11.0.x.
   # We are using 8, 9, 10, 11 for sparkJavaVersion.
-  versions <- strsplit(javaVersionStr, "[.]")[[1L]]
+  versions <- strsplit(javaVersionStr, ".", fixed = TRUE)[[1L]]
   if ("1" == versions[1]) {
     javaVersionNum <- as.integer(versions[2])
   } else {
     javaVersionNum <- as.integer(versions[1])
   }
   if (javaVersionNum < minJavaVersion || javaVersionNum >= maxJavaVersion) {
-    stop(paste0("Java version, greater than or equal to ", minJavaVersion,
-                " and less than ", maxJavaVersion,
-                ", is required for this package; found version: ",
-                javaVersionStr))
+    stop("Java version, greater than or equal to ", minJavaVersion,
+         " and less than ", maxJavaVersion, ", is required for this ",
+         "package; found version: ", javaVersionStr)
   }
   return(javaVersionNum)
 }
diff --git a/R/pkg/R/context.R b/R/pkg/R/context.R
index d96a287f818a2..e3c9d9f8793d6 100644
--- a/R/pkg/R/context.R
+++ b/R/pkg/R/context.R
@@ -144,13 +144,13 @@ parallelize <- function(sc, coll, numSlices = 1) {
   if ((!is.list(coll) && !is.vector(coll)) || is.data.frame(coll)) {
   # nolint end
     if (is.data.frame(coll)) {
-      message(paste("context.R: A data frame is parallelized by columns."))
+      message("context.R: A data frame is parallelized by columns.")
     } else {
       if (is.matrix(coll)) {
-        message(paste("context.R: A matrix is parallelized by elements."))
+        message("context.R: A matrix is parallelized by elements.")
       } else {
-        message(paste("context.R: parallelize() currently only supports lists and vectors.",
-                      "Calling as.list() to coerce coll into a list."))
+        message("context.R: parallelize() currently only supports lists and vectors. ",
+                "Calling as.list() to coerce coll into a list.")
       }
     }
     coll <- as.list(coll)
diff --git a/R/pkg/R/deserialize.R b/R/pkg/R/deserialize.R
index ca4a6e342d772..5d22340fb62a0 100644
--- a/R/pkg/R/deserialize.R
+++ b/R/pkg/R/deserialize.R
@@ -57,7 +57,7 @@ readTypedObject <- function(con, type) {
     "s" = readStruct(con),
     "n" = NULL,
     "j" = getJobj(readString(con)),
-    stop(paste("Unsupported type for deserialization", type)))
+    stop("Unsupported type for deserialization ", type))
 }
 
 readStringData <- function(con, len) {
@@ -233,24 +233,13 @@ readMultipleObjectsWithKeys <- function(inputCon) {
 
 readDeserializeInArrow <- function(inputCon) {
   if (requireNamespace("arrow", quietly = TRUE)) {
-    # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190.
-    useAsTibble <- exists("as_tibble", envir = asNamespace("arrow"))
-
-
     # Currently, there looks no way to read batch by batch by socket connection in R side,
     # See ARROW-4512. Therefore, it reads the whole Arrow streaming-formatted binary at once
     # for now.
     dataLen <- readInt(inputCon)
     arrowData <- readBin(inputCon, raw(), as.integer(dataLen), endian = "big")
     batches <- arrow::RecordBatchStreamReader$create(arrowData)$batches()
-
-    if (useAsTibble) {
-      as_tibble <- get("as_tibble", envir = asNamespace("arrow"))
-      # Read all groupped batches. Tibble -> data.frame is cheap.
-      lapply(batches, function(batch) as.data.frame(as_tibble(batch)))
-    } else {
-      lapply(batches, function(batch) as.data.frame(batch))
-    }
+    lapply(batches, function(batch) as.data.frame(batch))
   } else {
     stop("'arrow' package should be installed.")
   }
diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R
index 48f69d5769620..4cb23d72ef51a 100644
--- a/R/pkg/R/functions.R
+++ b/R/pkg/R/functions.R
@@ -77,7 +77,13 @@ NULL
 #'          days to be added to or subtracted from \code{y}. For class \code{character}, it is
 #'          \itemize{
 #'          \item \code{date_format}: date format specification.
-#'          \item \code{from_utc_timestamp}, \code{to_utc_timestamp}: time zone to use.
+#'          \item \code{from_utc_timestamp}, \code{to_utc_timestamp}: A string detailing
+#'            the time zone ID that the input should be adjusted to. It should be in the format
+#'            of either region-based zone IDs or zone offsets. Region IDs must have the form
+#'            'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format
+#'            (+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported
+#'            as aliases of '+00:00'. Other short names are not recommended to use
+#'            because they can be ambiguous.
 #'          \item \code{next_day}: day of the week string.
 #'          }
 #' @param ... additional argument(s).
@@ -1792,7 +1798,7 @@ setMethod("radians",
 #' @details
 #' \code{to_date}: Converts the column into a DateType. You may optionally specify
 #' a format according to the rules in:
-#' \url{https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html}.
+#' \href{https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html}{Datetime Pattern}
 #' If the string cannot be parsed according to the specified format (or default),
 #' the value of the column will be null.
 #' By default, it follows casting rules to a DateType if the format is omitted
@@ -1888,7 +1894,7 @@ setMethod("to_csv", signature(x = "Column"),
 #' @details
 #' \code{to_timestamp}: Converts the column into a TimestampType. You may optionally specify
 #' a format according to the rules in:
-#' \url{https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html}.
+#' \href{https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html}{Datetime Pattern}
 #' If the string cannot be parsed according to the specified format (or default),
 #' the value of the column will be null.
 #' By default, it follows casting rules to a TimestampType if the format is omitted
@@ -2193,7 +2199,7 @@ setMethod("pmod", signature(y = "Column"),
             column(jc)
           })
 
-#' @param rsd maximum estimation error allowed (default = 0.05).
+#' @param rsd maximum relative standard deviation allowed (default = 0.05).
 #'
 #' @rdname column_aggregate_functions
 #' @aliases approx_count_distinct,Column-method
@@ -2760,8 +2766,8 @@ setMethod("format_string", signature(format = "character", x = "Column"),
 #' \code{from_unixtime}: Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC)
 #' to a string representing the timestamp of that moment in the current system time zone in the JVM
 #' in the given format.
-#' See \href{https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html}{
-#' Customizing Formats} for available options.
+#' See \href{https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html}{
+#' Datetime Pattern} for available options.
 #'
 #' @rdname column_datetime_functions
 #'
@@ -2776,7 +2782,7 @@ setMethod("format_string", signature(format = "character", x = "Column"),
 #' head(tmp)}
 #' @note from_unixtime since 1.5.0
 setMethod("from_unixtime", signature(x = "Column"),
-          function(x, format = "uuuu-MM-dd HH:mm:ss") {
+          function(x, format = "yyyy-MM-dd HH:mm:ss") {
             jc <- callJStatic("org.apache.spark.sql.functions",
                               "from_unixtime",
                               x@jc, format)
@@ -2882,7 +2888,7 @@ setMethod("lpad", signature(x = "Column", len = "numeric", pad = "character"),
 
 #' @details
 #' \code{rand}: Generates a random column with independent and identically distributed (i.i.d.)
-#' samples from U[0.0, 1.0].
+#' samples uniformly distributed in [0.0, 1.0).
 #' Note: the function is non-deterministic in general case.
 #'
 #' @rdname column_nonaggregate_functions
@@ -3062,7 +3068,7 @@ setMethod("unix_timestamp", signature(x = "Column", format = "missing"),
 #' @aliases unix_timestamp,Column,character-method
 #' @note unix_timestamp(Column, character) since 1.5.0
 setMethod("unix_timestamp", signature(x = "Column", format = "character"),
-          function(x, format = "uuuu-MM-dd HH:mm:ss") {
+          function(x, format = "yyyy-MM-dd HH:mm:ss") {
             jc <- callJStatic("org.apache.spark.sql.functions", "unix_timestamp", x@jc, format)
             column(jc)
           })
@@ -4022,7 +4028,8 @@ setMethod("date_trunc",
           })
 
 #' @details
-#' \code{current_date}: Returns the current date as a date column.
+#' \code{current_date}: Returns the current date at the start of query evaluation as a date column.
+#' All calls of current_date within the same query return the same value.
 #'
 #' @rdname column_datetime_functions
 #' @aliases current_date current_date,missing-method
@@ -4038,7 +4045,8 @@ setMethod("current_date",
           })
 
 #' @details
-#' \code{current_timestamp}: Returns the current timestamp as a timestamp column.
+#' \code{current_timestamp}: Returns the current timestamp at the start of query evaluation as
+#' a timestamp column. All calls of current_timestamp within the same query return the same value.
 #'
 #' @rdname column_datetime_functions
 #' @aliases current_timestamp current_timestamp,missing-method
diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R
index 4134d5cecc888..89e0982429b5f 100644
--- a/R/pkg/R/generics.R
+++ b/R/pkg/R/generics.R
@@ -528,6 +528,9 @@ setGeneric("persist", function(x, newLevel) { standardGeneric("persist") })
 #' @rdname printSchema
 setGeneric("printSchema", function(x) { standardGeneric("printSchema") })
 
+#' @rdname registerTempTable-deprecated
+setGeneric("registerTempTable", function(x, tableName) { standardGeneric("registerTempTable") })
+
 #' @rdname rename
 setGeneric("rename", function(x, ...) { standardGeneric("rename") })
 
diff --git a/R/pkg/R/group.R b/R/pkg/R/group.R
index 2b7995e1e37f6..99d62240a3b2a 100644
--- a/R/pkg/R/group.R
+++ b/R/pkg/R/group.R
@@ -234,8 +234,8 @@ gapplyInternal <- function(x, func, schema) {
     if (inherits(schema, "structType")) {
       checkSchemaInArrow(schema)
     } else if (is.null(schema)) {
-      stop(paste0("Arrow optimization does not support 'gapplyCollect' yet. Please disable ",
-                  "Arrow optimization or use 'collect' and 'gapply' APIs instead."))
+      stop("Arrow optimization does not support 'gapplyCollect' yet. Please disable ",
+           "Arrow optimization or use 'collect' and 'gapply' APIs instead.")
     } else {
       stop("'schema' should be DDL-formatted string or structType.")
     }
diff --git a/R/pkg/R/install.R b/R/pkg/R/install.R
index 6d1edf6b6f3cf..ea2c0b4c0f42f 100644
--- a/R/pkg/R/install.R
+++ b/R/pkg/R/install.R
@@ -89,8 +89,8 @@ install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
   }
 
   if (overwrite) {
-    message(paste0("Overwrite = TRUE: download and overwrite the tar file",
-                   "and Spark package directory if they exist."))
+    message("Overwrite = TRUE: download and overwrite the tar file",
+            "and Spark package directory if they exist.")
   }
 
   releaseUrl <- Sys.getenv("SPARKR_RELEASE_DOWNLOAD_URL")
@@ -103,12 +103,11 @@ install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
   # can use dir.exists(packageLocalDir) under R 3.2.0 or later
   if (!is.na(file.info(packageLocalDir)$isdir) && !overwrite) {
     if (releaseUrl != "") {
-      message(paste(packageName, "found, setting SPARK_HOME to", packageLocalDir))
+      message(packageName, " found, setting SPARK_HOME to ", packageLocalDir)
     } else {
-      fmt <- "%s for Hadoop %s found, setting SPARK_HOME to %s"
-      msg <- sprintf(fmt, version, ifelse(hadoopVersion == "without", "Free build", hadoopVersion),
-                     packageLocalDir)
-      message(msg)
+      message(version, " for Hadoop ",
+              if (hadoopVersion == "without") "Free build" else hadoopVersion,
+              " found, setting SPARK_HOME to ", packageLocalDir)
     }
     Sys.setenv(SPARK_HOME = packageLocalDir)
     return(invisible(packageLocalDir))
@@ -127,26 +126,23 @@ install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
       success <- downloadUrl(releaseUrl, packageLocalPath)
       if (!success) {
         unlink(packageLocalPath)
-        stop(paste0("Fetch failed from ", releaseUrl))
+        stop("Fetch failed from ", releaseUrl)
       }
     } else {
       robustDownloadTar(mirrorUrl, version, hadoopVersion, packageName, packageLocalPath)
     }
   }
 
-  message(sprintf("Installing to %s", localDir))
+  message("Installing to ", localDir)
   # There are two ways untar can fail - untar could stop() on errors like incomplete block on file
   # or, tar command can return failure code
   success <- tryCatch(untar(tarfile = packageLocalPath, exdir = localDir) == 0,
                      error = function(e) {
-                       message(e)
-                       message()
+                       message(e, "\n")
                        FALSE
                      },
                      warning = function(w) {
-                       # Treat warning as error, add an empty line with message()
-                       message(w)
-                       message()
+                       message(w, "\n")
                        FALSE
                      })
   if (!tarExists || overwrite || !success) {
@@ -160,7 +156,7 @@ install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
   if (!success) stop("Extract archive failed.")
   message("DONE.")
   Sys.setenv(SPARK_HOME = packageLocalDir)
-  message(paste("SPARK_HOME set to", packageLocalDir))
+  message("SPARK_HOME set to ", packageLocalDir)
   invisible(packageLocalDir)
 }
 
@@ -173,7 +169,7 @@ robustDownloadTar <- function(mirrorUrl, version, hadoopVersion, packageName, pa
     if (success) {
       return()
     } else {
-      message(paste0("Unable to download from mirrorUrl: ", mirrorUrl))
+      message("Unable to download from mirrorUrl: ", mirrorUrl)
     }
   } else {
     message("MirrorUrl not provided.")
@@ -201,11 +197,9 @@ robustDownloadTar <- function(mirrorUrl, version, hadoopVersion, packageName, pa
     # remove any partially downloaded file
     unlink(packageLocalPath)
     message("Unable to download from default mirror site: ", mirrorUrl)
-    msg <- sprintf(paste("Unable to download Spark %s for Hadoop %s.",
-                         "Please check network connection, Hadoop version,",
-                         "or provide other mirror sites."),
-                   version, ifelse(hadoopVersion == "without", "Free build", hadoopVersion))
-    stop(msg)
+    stop("Unable to download Spark ", version,
+         " for Hadoop ", if (hadoopVersion == "without") "Free build" else hadoopVersion,
+         ". Please check network connection, Hadoop version, or provide other mirror sites.")
   }
 }
 
@@ -214,15 +208,15 @@ getPreferredMirror <- function(version, packageName) {
                         file.path("spark", version, packageName),
                         ".tgz&as_json=1")
   textLines <- readLines(jsonUrl, warn = FALSE)
-  rowNum <- grep("\"preferred\"", textLines)
+  rowNum <- grep('"preferred"', textLines, fixed = TRUE)
   linePreferred <- textLines[rowNum]
-  matchInfo <- regexpr("\"[A-Za-z][A-Za-z0-9+-.]*://.+\"", linePreferred)
+  matchInfo <- regexpr('"[A-Za-z][A-Za-z0-9+-.]*://.+"', linePreferred)
   if (matchInfo != -1) {
     startPos <- matchInfo + 1
     endPos <- matchInfo + attr(matchInfo, "match.length") - 2
     mirrorPreferred <- base::substr(linePreferred, startPos, endPos)
     mirrorPreferred <- paste0(mirrorPreferred, "spark")
-    message(sprintf("Preferred mirror site found: %s", mirrorPreferred))
+    message("Preferred mirror site found: ", mirrorPreferred)
   } else {
     mirrorPreferred <- NULL
   }
@@ -231,24 +225,20 @@ getPreferredMirror <- function(version, packageName) {
 
 directDownloadTar <- function(mirrorUrl, version, hadoopVersion, packageName, packageLocalPath) {
   packageRemotePath <- paste0(file.path(mirrorUrl, version, packageName), ".tgz")
-  fmt <- "Downloading %s for Hadoop %s from:\n- %s"
-  msg <- sprintf(fmt, version, ifelse(hadoopVersion == "without", "Free build", hadoopVersion),
-                 packageRemotePath)
-  message(msg)
+  message("Downloading ", version, " for Hadoop ",
+          if (hadoopVersion == "without") "Free build" else hadoopVersion,
+          " from:\n- ", packageRemotePath)
   downloadUrl(packageRemotePath, packageLocalPath)
 }
 
 downloadUrl <- function(remotePath, localPath) {
   isFail <- tryCatch(download.file(remotePath, localPath),
                      error = function(e) {
-                       message(e)
-                       message()
+                       message(e, "\n")
                        TRUE
                      },
                      warning = function(w) {
-                       # Treat warning as error, add an empty line with message()
-                       message(w)
-                       message()
+                       message(w, "\n")
                        TRUE
                      })
   !isFail
@@ -279,9 +269,9 @@ sparkCachePath <- function() {
       winAppPath <- Sys.getenv("USERPROFILE", unset = NA)
     }
     if (is.na(winAppPath)) {
-      stop(paste("%LOCALAPPDATA% and %USERPROFILE% not found.",
-                   "Please define the environment variable",
-                   "or restart and enter an installation path in localDir."))
+      stop("%LOCALAPPDATA% and %USERPROFILE% not found. ",
+           "Please define the environment variable ",
+           "or restart and enter an installation path in localDir.")
     } else {
       path <- file.path(winAppPath, "Apache", "Spark", "Cache")
     }
@@ -293,7 +283,7 @@ sparkCachePath <- function() {
         Sys.getenv("XDG_CACHE_HOME", file.path(Sys.getenv("HOME"), ".cache")), "spark")
     }
   } else {
-    stop(sprintf("Unknown OS: %s", .Platform$OS.type))
+    stop("Unknown OS: ", .Platform$OS.type)
   }
   normalizePath(path, mustWork = FALSE)
 }
@@ -322,7 +312,7 @@ installInstruction <- function(mode) {
            "If you need further help, ",
            "contact the administrators of the cluster.")
   } else {
-    stop(paste0("No instruction found for ", mode, " mode."))
+    stop("No instruction found for mode ", mode)
   }
 }
 
diff --git a/R/pkg/R/mllib_classification.R b/R/pkg/R/mllib_classification.R
index 3ad824e1e634a..53c9775963d23 100644
--- a/R/pkg/R/mllib_classification.R
+++ b/R/pkg/R/mllib_classification.R
@@ -331,8 +331,8 @@ setMethod("spark.logit", signature(data = "SparkDataFrame", formula = "formula")
 
               if (!is.null(lowerBoundsOnCoefficients) && (row != nrow(upperBoundsOnCoefficients)
                 || col != ncol(upperBoundsOnCoefficients))) {
-                stop(paste0("dimension of upperBoundsOnCoefficients ",
-                           "is not the same as lowerBoundsOnCoefficients", sep = ""))
+                stop("dimension of upperBoundsOnCoefficients ",
+                     "is not the same as lowerBoundsOnCoefficients")
               }
 
               if (is.null(lowerBoundsOnCoefficients)) {
diff --git a/R/pkg/R/mllib_stat.R b/R/pkg/R/mllib_stat.R
index f8c3329359961..6db4d5d4831dd 100644
--- a/R/pkg/R/mllib_stat.R
+++ b/R/pkg/R/mllib_stat.R
@@ -69,8 +69,7 @@ setMethod("spark.kstest", signature(data = "SparkDataFrame"),
           function(data, testCol = "test", nullHypothesis = c("norm"), distParams = c(0, 1)) {
             tryCatch(match.arg(nullHypothesis),
                      error = function(e) {
-                       msg <- paste("Distribution", nullHypothesis, "is not supported.")
-                       stop(msg)
+                       stop("Distribution ", nullHypothesis, " is not supported.")
                      })
             if (nullHypothesis == "norm") {
               distParams <- as.numeric(distParams)
diff --git a/R/pkg/R/pairRDD.R b/R/pkg/R/pairRDD.R
index 9c2e57d3067db..b29381bb900fb 100644
--- a/R/pkg/R/pairRDD.R
+++ b/R/pkg/R/pairRDD.R
@@ -906,7 +906,7 @@ setMethod("sampleByKey",
 
             for (elem in fractions) {
               if (elem < 0.0) {
-                stop(paste("Negative fraction value ", fractions[which(fractions == elem)]))
+                stop("Negative fraction value ", fractions[which(fractions == elem)])
               }
             }
 
diff --git a/R/pkg/R/schema.R b/R/pkg/R/schema.R
index 9831fc3cc6d01..7044ede0cc58b 100644
--- a/R/pkg/R/schema.R
+++ b/R/pkg/R/schema.R
@@ -99,10 +99,9 @@ print.structType <- function(x, ...) {
   cat("StructType\n",
       sapply(x$fields(),
              function(field) {
-               paste("|-", "name = \"", field$name(),
-                     "\", type = \"", field$dataType.toString(),
-                     "\", nullable = ", field$nullable(), "\n",
-                     sep = "")
+               paste0("|-", "name = \"", field$name(),
+                      "\", type = \"", field$dataType.toString(),
+                      "\", nullable = ", field$nullable(), "\n")
              }),
       sep = "")
 }
@@ -183,7 +182,7 @@ checkType <- function(type) {
                 # strsplit does not return the final empty string, so check if
                 # the final char is ","
                 if (substr(fieldsString, nchar(fieldsString), nchar(fieldsString)) != ",") {
-                  fields <- strsplit(fieldsString, ",")[[1]]
+                  fields <- strsplit(fieldsString, ",", fixed = TRUE)[[1]]
                   for (field in fields) {
                     m <- regexec("^(.+):(.+)$", field)
                     matchedStrings <- regmatches(field, m)
@@ -200,7 +199,7 @@ checkType <- function(type) {
             })
   }
 
-  stop(paste("Unsupported type for SparkDataframe:", type))
+  stop("Unsupported type for SparkDataframe: ", type)
 }
 
 #' @param type The data type of the field
diff --git a/R/pkg/R/serialize.R b/R/pkg/R/serialize.R
index cb3c1c59d12ed..7760d9be16f0b 100644
--- a/R/pkg/R/serialize.R
+++ b/R/pkg/R/serialize.R
@@ -84,7 +84,7 @@ writeObject <- function(con, object, writeType = TRUE) {
          Date = writeDate(con, object),
          POSIXlt = writeTime(con, object),
          POSIXct = writeTime(con, object),
-         stop(paste("Unsupported type for serialization", type)))
+         stop("Unsupported type for serialization ", type))
 }
 
 writeVoid <- function(con) {
@@ -158,7 +158,7 @@ writeType <- function(con, class) {
                  Date = "D",
                  POSIXlt = "t",
                  POSIXct = "t",
-                 stop(paste("Unsupported type for serialization", class)))
+                 stop("Unsupported type for serialization ", class))
   writeBin(charToRaw(type), con)
 }
 
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
index cdb59093781fb..e4a11a5f78a71 100644
--- a/R/pkg/R/sparkR.R
+++ b/R/pkg/R/sparkR.R
@@ -88,6 +88,49 @@ sparkR.stop <- function() {
   sparkR.session.stop()
 }
 
+#' (Deprecated) Initialize a new Spark Context
+#'
+#' This function initializes a new SparkContext.
+#'
+#' @param master The Spark master URL
+#' @param appName Application name to register with cluster manager
+#' @param sparkHome Spark Home directory
+#' @param sparkEnvir Named list of environment variables to set on worker nodes
+#' @param sparkExecutorEnv Named list of environment variables to be used when launching executors
+#' @param sparkJars Character vector of jar files to pass to the worker nodes
+#' @param sparkPackages Character vector of package coordinates
+#' @seealso \link{sparkR.session}
+#' @rdname sparkR.init-deprecated
+#' @examples
+#'\dontrun{
+#' sc <- sparkR.init("local[2]", "SparkR", "/home/spark")
+#' sc <- sparkR.init("local[2]", "SparkR", "/home/spark",
+#'                  list(spark.executor.memory="1g"))
+#' sc <- sparkR.init("yarn-client", "SparkR", "/home/spark",
+#'                  list(spark.executor.memory="4g"),
+#'                  list(LD_LIBRARY_PATH="/directory of JVM libraries (libjvm.so) on workers/"),
+#'                  c("one.jar", "two.jar", "three.jar"),
+#'                  c("com.databricks:spark-avro_2.11:2.0.1"))
+#'}
+#' @note sparkR.init since 1.4.0
+sparkR.init <- function(
+  master = "",
+  appName = "SparkR",
+  sparkHome = Sys.getenv("SPARK_HOME"),
+  sparkEnvir = list(),
+  sparkExecutorEnv = list(),
+  sparkJars = "",
+  sparkPackages = "") {
+  .Deprecated("sparkR.session")
+  sparkR.sparkContext(master,
+     appName,
+     sparkHome,
+     convertNamedListToEnv(sparkEnvir),
+     convertNamedListToEnv(sparkExecutorEnv),
+     sparkJars,
+     sparkPackages)
+}
+
 # Internal function to handle creating the SparkContext.
 sparkR.sparkContext <- function(
   master = "",
@@ -111,8 +154,8 @@ sparkR.sparkContext <- function(
   connectionTimeout <- as.numeric(Sys.getenv("SPARKR_BACKEND_CONNECTION_TIMEOUT", "6000"))
   if (existingPort != "") {
     if (length(packages) != 0) {
-      warning(paste("sparkPackages has no effect when using spark-submit or sparkR shell",
-                    " please use the --packages commandline instead", sep = ","))
+      warning("sparkPackages has no effect when using spark-submit or sparkR shell, ",
+              "please use the --packages commandline instead")
     }
     backendPort <- existingPort
     authSecret <- Sys.getenv("SPARKR_BACKEND_AUTH_SECRET")
@@ -201,7 +244,7 @@ sparkR.sparkContext <- function(
     uriSep <- "////"
   }
   localJarPaths <- lapply(jars,
-                          function(j) { utils::URLencode(paste("file:", uriSep, j, sep = "")) })
+                          function(j) { utils::URLencode(paste0("file:", uriSep, j)) })
 
   # Set the start time to identify jobjs
   # Seconds resolution is good enough for this purpose, so use ints
@@ -229,6 +272,61 @@ sparkR.sparkContext <- function(
   sc
 }
 
+#' (Deprecated) Initialize a new SQLContext
+#'
+#' This function creates a SparkContext from an existing JavaSparkContext and
+#' then uses it to initialize a new SQLContext
+#'
+#' Starting SparkR 2.0, a SparkSession is initialized and returned instead.
+#' This API is deprecated and kept for backward compatibility only.
+#'
+#' @param jsc The existing JavaSparkContext created with SparkR.init()
+#' @seealso \link{sparkR.session}
+#' @rdname sparkRSQL.init-deprecated
+#' @examples
+#'\dontrun{
+#' sc <- sparkR.init()
+#' sqlContext <- sparkRSQL.init(sc)
+#'}
+#' @note sparkRSQL.init since 1.4.0
+sparkRSQL.init <- function(jsc = NULL) {
+  .Deprecated("sparkR.session")
+
+  if (exists(".sparkRsession", envir = .sparkREnv)) {
+    return(get(".sparkRsession", envir = .sparkREnv))
+  }
+
+  # Default to without Hive support for backward compatibility.
+  sparkR.session(enableHiveSupport = FALSE)
+}
+
+#' (Deprecated) Initialize a new HiveContext
+#'
+#' This function creates a HiveContext from an existing JavaSparkContext
+#'
+#' Starting SparkR 2.0, a SparkSession is initialized and returned instead.
+#' This API is deprecated and kept for backward compatibility only.
+#'
+#' @param jsc The existing JavaSparkContext created with SparkR.init()
+#' @seealso \link{sparkR.session}
+#' @rdname sparkRHive.init-deprecated
+#' @examples
+#'\dontrun{
+#' sc <- sparkR.init()
+#' sqlContext <- sparkRHive.init(sc)
+#'}
+#' @note sparkRHive.init since 1.4.0
+sparkRHive.init <- function(jsc = NULL) {
+  .Deprecated("sparkR.session")
+
+  if (exists(".sparkRsession", envir = .sparkREnv)) {
+    return(get(".sparkRsession", envir = .sparkREnv))
+  }
+
+  # Default to without Hive support for backward compatibility.
+  sparkR.session(enableHiveSupport = TRUE)
+}
+
 #' Get the existing SparkSession or initialize a new SparkSession.
 #'
 #' SparkSession is the entry point into SparkR. \code{sparkR.session} gets the existing
@@ -337,12 +435,13 @@ sparkR.session <- function(
   # Check if version number of SparkSession matches version number of SparkR package
   jvmVersion <- callJMethod(sparkSession, "version")
   # Remove -SNAPSHOT from jvm versions
-  jvmVersionStrip <- gsub("-SNAPSHOT", "", jvmVersion)
+  jvmVersionStrip <- gsub("-SNAPSHOT", "", jvmVersion, fixed = TRUE)
   rPackageVersion <- paste0(packageVersion("SparkR"))
 
   if (jvmVersionStrip != rPackageVersion) {
-    warning(paste("Version mismatch between Spark JVM and SparkR package. JVM version was",
-                  jvmVersion, ", while R package version was", rPackageVersion))
+    warning("Version mismatch between Spark JVM and SparkR package. ",
+            "JVM version was ", jvmVersion,
+            ", while R package version was ", rPackageVersion)
   }
 
   sparkSession
@@ -508,7 +607,7 @@ getClientModeSparkSubmitOpts <- function(submitOps, sparkEnvirMap) {
     # process only if --option is not already specified
     if (!is.null(opsValue) &&
         nchar(opsValue) > 1 &&
-        !grepl(sparkConfToSubmitOps[[conf]], submitOps)) {
+        !grepl(sparkConfToSubmitOps[[conf]], submitOps, fixed = TRUE)) {
       # put "" around value in case it has spaces
       paste0(sparkConfToSubmitOps[[conf]], " \"", opsValue, "\" ")
     } else {
diff --git a/R/pkg/R/types.R b/R/pkg/R/types.R
index 55f75508e88ff..5d48a9eee2799 100644
--- a/R/pkg/R/types.R
+++ b/R/pkg/R/types.R
@@ -88,33 +88,27 @@ specialtypeshandle <- function(type) {
 checkSchemaInArrow <- function(schema) {
   stopifnot(inherits(schema, "structType"))
 
-  requireNamespace1 <- requireNamespace
-  if (!requireNamespace1("arrow", quietly = TRUE)) {
+  if (!requireNamespace("arrow", quietly = TRUE)) {
     stop("'arrow' package should be installed.")
   }
 
   # Both cases below produce a corrupt value for unknown reason. It needs to be investigated.
-  if (any(sapply(schema$fields(), function(x) x$dataType.toString() == "FloatType"))) {
+  field_strings <- sapply(schema$fields(), function(x) x$dataType.toString())
+  if (any(field_strings == "FloatType")) {
     stop("Arrow optimization in R does not support float type yet.")
   }
-  if (any(sapply(schema$fields(), function(x) x$dataType.toString() == "BinaryType"))) {
+  if (any(field_strings == "BinaryType")) {
     stop("Arrow optimization in R does not support binary type yet.")
   }
-  if (any(sapply(schema$fields(),
-                 function(x) startsWith(x$dataType.toString(),
-                 "ArrayType")))) {
+  if (any(startsWith(field_strings, "ArrayType"))) {
     stop("Arrow optimization in R does not support array type yet.")
   }
 
   # Arrow optimization in Spark does not yet support both cases below.
-  if (any(sapply(schema$fields(),
-                 function(x) startsWith(x$dataType.toString(),
-                 "StructType")))) {
+  if (any(startsWith(field_strings, "StructType"))) {
     stop("Arrow optimization in R does not support nested struct type yet.")
   }
-  if (any(sapply(schema$fields(),
-                 function(x) startsWith(x$dataType.toString(),
-                 "MapType")))) {
+  if (any(startsWith(field_strings, "MapType"))) {
     stop("Arrow optimization in R does not support map type yet.")
   }
 }
diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R
index a8c1ddb3dd20b..cef2fa9b47440 100644
--- a/R/pkg/R/utils.R
+++ b/R/pkg/R/utils.R
@@ -46,9 +46,9 @@ convertJListToRList <- function(jList, flatten, logicalUpperBound = NULL,
                 res <- list(unserialize(keyBytes),
                   unserialize(valBytes))
               } else {
-                stop(paste("utils.R: convertJListToRList only supports",
-                  "RDD[Array[Byte]] and",
-                  "JavaPairRDD[Array[Byte], Array[Byte]] for now"))
+                stop("utils.R: convertJListToRList only supports ",
+                  "RDD[Array[Byte]] and ",
+                  "JavaPairRDD[Array[Byte], Array[Byte]] for now")
               }
             } else {
               if (inherits(obj, "raw")) {
@@ -137,7 +137,7 @@ hashCode <- function(key) {
       as.integer(hashC)
     }
   } else {
-    warning(paste("Could not hash object, returning 0", sep = ""))
+    warning("Could not hash object, returning 0")
     as.integer(0)
   }
 }
@@ -354,8 +354,8 @@ varargsToStrEnv <- function(...) {
       } else {
         value <- pairs[[name]]
         if (!(is.logical(value) || is.numeric(value) || is.character(value) || is.null(value))) {
-          stop(paste0("Unsupported type for ", name, " : ", class(value),
-               ". Supported types are logical, numeric, character and NULL."), call. = FALSE)
+          stop("Unsupported type for ", name, " : ", toString(class(value)), ". ",
+               "Supported types are logical, numeric, character and NULL.", call. = FALSE)
         }
         if (is.logical(value)) {
           env[[name]] <- tolower(as.character(value))
@@ -369,8 +369,7 @@ varargsToStrEnv <- function(...) {
   }
 
   if (length(ignoredNames) != 0) {
-    warning(paste0("Unnamed arguments ignored: ", paste(ignoredNames, collapse = ", "), "."),
-            call. = FALSE)
+    warning("Unnamed arguments ignored: ", toString(ignoredNames), ".", call. = FALSE)
   }
   env
 }
@@ -449,7 +448,7 @@ storageLevelToString <- function(levelObj) {
 # the user to type (for example) `5` instead of `5L` to avoid a confusing error message.
 numToInt <- function(num) {
   if (as.integer(num) != num) {
-    warning(paste("Coercing", as.list(sys.call())[[2]], "to integer."))
+    warning("Coercing ", as.list(sys.call())[[2L]], " to integer.")
   }
   as.integer(num)
 }
@@ -530,7 +529,10 @@ processClosure <- function(node, oldEnv, defVars, checkedFuncs, newEnv) {
         # Namespaces other than "SparkR" will not be searched.
         if (!isNamespace(func.env) ||
             (getNamespaceName(func.env) == "SparkR" &&
-               !(nodeChar %in% getNamespaceExports("SparkR")))) {
+               !(nodeChar %in% getNamespaceExports("SparkR")) &&
+                  # Note that generic S4 methods should not be set to the environment of
+                  # cleaned closure. It does not work with R 4.0.0+. See also SPARK-31918.
+                  nodeChar != "" && !methods::isGeneric(nodeChar, func.env))) {
           # Only include SparkR internals.
 
           # Set parameter 'inherits' to FALSE since we do not need to search in
@@ -650,8 +652,8 @@ mergePartitions <- function(rdd, zip) {
         # For zip operation, check if corresponding partitions
         # of both RDDs have the same number of elements.
         if (zip && lengthOfKeys != lengthOfValues) {
-          stop(paste("Can only zip RDDs with same number of elements",
-                     "in each pair of corresponding partitions."))
+          stop("Can only zip RDDs with same number of elements ",
+               "in each pair of corresponding partitions.")
         }
 
         if (lengthOfKeys > 1) {
@@ -804,7 +806,7 @@ handledCallJMethod <- function(obj, method, ...) {
 
 captureJVMException <- function(e, method) {
   rawmsg <- as.character(e)
-  if (any(grep("^Error in .*?: ", rawmsg))) {
+  if (any(grepl("^Error in .*?: ", rawmsg))) {
     # If the exception message starts with "Error in ...", this is possibly
     # "Error in invokeJava(...)". Here, it replaces the characters to
     # `paste("Error in", method, ":")` in order to identify which function
@@ -818,54 +820,58 @@ captureJVMException <- function(e, method) {
   }
 
   # StreamingQueryException could wrap an IllegalArgumentException, so look for that first
-  if (any(grep("org.apache.spark.sql.streaming.StreamingQueryException: ", stacktrace))) {
+  if (any(grepl("org.apache.spark.sql.streaming.StreamingQueryException: ",
+                stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.streaming.StreamingQueryException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "streaming query error - ", first), call. = FALSE)
-  } else if (any(grep("java.lang.IllegalArgumentException: ", stacktrace))) {
+    stop(rmsg, "streaming query error - ", first, call. = FALSE)
+  } else if (any(grepl("java.lang.IllegalArgumentException: ", stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "java.lang.IllegalArgumentException: ", fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "illegal argument - ", first), call. = FALSE)
-  } else if (any(grep("org.apache.spark.sql.AnalysisException: ", stacktrace))) {
+    stop(rmsg, "illegal argument - ", first, call. = FALSE)
+  } else if (any(grepl("org.apache.spark.sql.AnalysisException: ", stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.AnalysisException: ", fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "analysis error - ", first), call. = FALSE)
+    stop(rmsg, "analysis error - ", first, call. = FALSE)
   } else
-    if (any(grep("org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: ", stacktrace))) {
+    if (any(grepl("org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: ",
+                  stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "no such database - ", first), call. = FALSE)
+    stop(rmsg, "no such database - ", first, call. = FALSE)
   } else
-    if (any(grep("org.apache.spark.sql.catalyst.analysis.NoSuchTableException: ", stacktrace))) {
+    if (any(grepl("org.apache.spark.sql.catalyst.analysis.NoSuchTableException: ",
+                  stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.catalyst.analysis.NoSuchTableException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "no such table - ", first), call. = FALSE)
-  } else if (any(grep("org.apache.spark.sql.catalyst.parser.ParseException: ", stacktrace))) {
+    stop(rmsg, "no such table - ", first, call. = FALSE)
+  } else if (any(grepl("org.apache.spark.sql.catalyst.parser.ParseException: ",
+                       stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.catalyst.parser.ParseException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "parse error - ", first), call. = FALSE)
+    stop(rmsg, "parse error - ", first, call. = FALSE)
   } else {
     stop(stacktrace, call. = FALSE)
   }
diff --git a/R/pkg/inst/profile/general.R b/R/pkg/inst/profile/general.R
index 3efb460846fc2..8c75c19ca7ac3 100644
--- a/R/pkg/inst/profile/general.R
+++ b/R/pkg/inst/profile/general.R
@@ -16,10 +16,6 @@
 #
 
 .First <- function() {
-  if (utils::compareVersion(paste0(R.version$major, ".", R.version$minor), "3.4.0") == -1) {
-    warning("Support for R prior to version 3.4 is deprecated since Spark 3.0.0")
-  }
-
   packageDir <- Sys.getenv("SPARKR_PACKAGE_DIR")
   dirs <- strsplit(packageDir, ",")[[1]]
   .libPaths(c(dirs, .libPaths()))
diff --git a/R/pkg/inst/profile/shell.R b/R/pkg/inst/profile/shell.R
index e4e0d032997de..f6c20e1a5ebc3 100644
--- a/R/pkg/inst/profile/shell.R
+++ b/R/pkg/inst/profile/shell.R
@@ -16,10 +16,6 @@
 #
 
 .First <- function() {
-  if (utils::compareVersion(paste0(R.version$major, ".", R.version$minor), "3.4.0") == -1) {
-    warning("Support for R prior to version 3.4 is deprecated since Spark 3.0.0")
-  }
-
   home <- Sys.getenv("SPARK_HOME")
   .libPaths(c(file.path(home, "R", "lib"), .libPaths()))
   Sys.setenv(NOAWT = 1)
diff --git a/R/pkg/tests/fulltests/test_context.R b/R/pkg/tests/fulltests/test_context.R
index b9139154bc165..f86872d727a1d 100644
--- a/R/pkg/tests/fulltests/test_context.R
+++ b/R/pkg/tests/fulltests/test_context.R
@@ -25,7 +25,10 @@ test_that("Check masked functions", {
   namesOfMasked <- c("describe", "cov", "filter", "lag", "na.omit", "predict", "sd", "var",
                      "colnames", "colnames<-", "intersect", "rank", "rbind", "sample", "subset",
                      "summary", "transform", "drop", "window", "as.data.frame", "union", "not")
-  if (as.numeric(R.version$major) >= 3 && as.numeric(R.version$minor) >= 3) {
+  version <- packageVersion("base")
+  is33Above <- as.numeric(version$major) >= 3 && as.numeric(version$minor) >= 3
+  is40Above <- as.numeric(version$major) >= 4
+  if (is33Above || is40Above) {
     namesOfMasked <- c("endsWith", "startsWith", namesOfMasked)
   }
   masked <- conflicts(detail = TRUE)$`package:SparkR`
diff --git a/R/pkg/tests/fulltests/test_includePackage.R b/R/pkg/tests/fulltests/test_includePackage.R
index 916361ff4c797..1d16b260c4c52 100644
--- a/R/pkg/tests/fulltests/test_includePackage.R
+++ b/R/pkg/tests/fulltests/test_includePackage.R
@@ -27,8 +27,8 @@ rdd <- parallelize(sc, nums, 2L)
 
 test_that("include inside function", {
   # Only run the test if plyr is installed.
-  if ("plyr" %in% rownames(installed.packages())) {
-    suppressPackageStartupMessages(library(plyr))
+  if ("plyr" %in% rownames(installed.packages()) &&
+      suppressPackageStartupMessages(suppressWarnings(library(plyr, logical.return = TRUE)))) {
     generateData <- function(x) {
       suppressPackageStartupMessages(library(plyr))
       attach(airquality)
@@ -44,8 +44,8 @@ test_that("include inside function", {
 
 test_that("use include package", {
   # Only run the test if plyr is installed.
-  if ("plyr" %in% rownames(installed.packages())) {
-    suppressPackageStartupMessages(library(plyr))
+  if ("plyr" %in% rownames(installed.packages()) &&
+      suppressPackageStartupMessages(suppressWarnings(library(plyr, logical.return = TRUE)))) {
     generateData <- function(x) {
       attach(airquality)
       result <- transform(Ozone, logOzone = log(Ozone))
diff --git a/R/pkg/tests/fulltests/test_mllib_classification.R b/R/pkg/tests/fulltests/test_mllib_classification.R
index 2da3a022f8941..2a09c23bc3617 100644
--- a/R/pkg/tests/fulltests/test_mllib_classification.R
+++ b/R/pkg/tests/fulltests/test_mllib_classification.R
@@ -34,7 +34,7 @@ test_that("spark.svmLinear", {
   summary <- summary(model)
 
   # test summary coefficients return matrix type
-  expect_true(class(summary$coefficients) == "matrix")
+  expect_true(any(class(summary$coefficients) == "matrix"))
   expect_true(class(summary$coefficients[, 1]) == "numeric")
 
   coefs <- summary$coefficients[, "Estimate"]
@@ -130,7 +130,7 @@ test_that("spark.logit", {
   summary <- summary(model)
 
   # test summary coefficients return matrix type
-  expect_true(class(summary$coefficients) == "matrix")
+  expect_true(any(class(summary$coefficients) == "matrix"))
   expect_true(class(summary$coefficients[, 1]) == "numeric")
 
   versicolorCoefsR <- c(1.52, 0.03, -0.53, 0.04, 0.00)
@@ -242,8 +242,8 @@ test_that("spark.logit", {
   # Test binomial logistic regression against two classes with upperBoundsOnCoefficients
   # and upperBoundsOnIntercepts
   u <- matrix(c(1.0, 0.0, 1.0, 0.0), nrow = 1, ncol = 4)
-  model <- spark.logit(training, Species ~ ., upperBoundsOnCoefficients = u,
-                       upperBoundsOnIntercepts = 1.0)
+  model <- suppressWarnings(spark.logit(training, Species ~ ., upperBoundsOnCoefficients = u,
+                                        upperBoundsOnIntercepts = 1.0))
   summary <- summary(model)
   coefsR <- c(-11.13331, 1.00000, 0.00000, 1.00000, 0.00000)
   coefs <- summary$coefficients[, "Estimate"]
@@ -255,8 +255,8 @@ test_that("spark.logit", {
   # Test binomial logistic regression against two classes with lowerBoundsOnCoefficients
   # and lowerBoundsOnIntercepts
   l <- matrix(c(0.0, -1.0, 0.0, -1.0), nrow = 1, ncol = 4)
-  model <- spark.logit(training, Species ~ ., lowerBoundsOnCoefficients = l,
-                       lowerBoundsOnIntercepts = 0.0)
+  model <- suppressWarnings(spark.logit(training, Species ~ ., lowerBoundsOnCoefficients = l,
+                                        lowerBoundsOnIntercepts = 0.0))
   summary <- summary(model)
   coefsR <- c(0, 0, -1, 0, 1.902192)
   coefs <- summary$coefficients[, "Estimate"]
@@ -268,9 +268,9 @@ test_that("spark.logit", {
   # Test multinomial logistic regression with lowerBoundsOnCoefficients
   # and lowerBoundsOnIntercepts
   l <- matrix(c(0.0, -1.0, 0.0, -1.0, 0.0, -1.0, 0.0, -1.0), nrow = 2, ncol = 4)
-  model <- spark.logit(training, Species ~ ., family = "multinomial",
-                       lowerBoundsOnCoefficients = l,
-                       lowerBoundsOnIntercepts = as.array(c(0.0, 0.0)))
+  model <- suppressWarnings(spark.logit(training, Species ~ ., family = "multinomial",
+                                        lowerBoundsOnCoefficients = l,
+                                        lowerBoundsOnIntercepts = as.array(c(0.0, 0.0))))
   summary <- summary(model)
   versicolorCoefsR <- c(42.639465, 7.258104, 14.330814, 16.298243, 11.716429)
   virginicaCoefsR <- c(0.0002970796, 4.79274, 7.65047, 25.72793, 30.0021)
diff --git a/R/pkg/tests/fulltests/test_mllib_clustering.R b/R/pkg/tests/fulltests/test_mllib_clustering.R
index 028ad574b8134..f180aeea28150 100644
--- a/R/pkg/tests/fulltests/test_mllib_clustering.R
+++ b/R/pkg/tests/fulltests/test_mllib_clustering.R
@@ -171,7 +171,7 @@ test_that("spark.kmeans", {
   expect_equal(sort(collect(distinct(select(cluster, "prediction")))$prediction), c(0, 1))
 
   # test summary coefficients return matrix type
-  expect_true(class(summary.model$coefficients) == "matrix")
+  expect_true(any(class(summary.model$coefficients) == "matrix"))
   expect_true(class(summary.model$coefficients[1, ]) == "numeric")
 
   # Test model save/load
diff --git a/R/pkg/tests/fulltests/test_mllib_regression.R b/R/pkg/tests/fulltests/test_mllib_regression.R
index b40c4cb9a9694..929cd27211fdc 100644
--- a/R/pkg/tests/fulltests/test_mllib_regression.R
+++ b/R/pkg/tests/fulltests/test_mllib_regression.R
@@ -116,7 +116,7 @@ test_that("spark.glm summary", {
   rStats <- summary(glm(Sepal.Width ~ Sepal.Length + Species, data = dataset))
 
   # test summary coefficients return matrix type
-  expect_true(class(stats$coefficients) == "matrix")
+  expect_true(any(class(stats$coefficients) == "matrix"))
   expect_true(class(stats$coefficients[, 1]) == "numeric")
 
   coefs <- stats$coefficients
diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R
index 3b3768f7e2715..fb0ed7451cdb9 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL.R
@@ -106,6 +106,15 @@ if (is_windows()) {
   Sys.setenv(TZ = "GMT")
 }
 
+test_that("calling sparkRSQL.init returns existing SQL context", {
+  sqlContext <- suppressWarnings(sparkRSQL.init(sc))
+  expect_equal(suppressWarnings(sparkRSQL.init(sc)), sqlContext)
+})
+
+test_that("calling sparkRSQL.init returns existing SparkSession", {
+  expect_equal(suppressWarnings(sparkRSQL.init(sc)), sparkSession)
+})
+
 test_that("calling sparkR.session returns existing SparkSession", {
   expect_equal(sparkR.session(), sparkSession)
 })
@@ -656,10 +665,10 @@ test_that("test tableNames and tables", {
   expect_true("tableName" %in% colnames(tables()))
   expect_true(all(c("tableName", "database", "isTemporary") %in% colnames(tables())))
 
-  createOrReplaceTempView(df, "table2")
+  suppressWarnings(registerTempTable(df, "table2"))
   tables <- listTables()
   expect_equal(count(tables), count + 2)
-  dropTempView("table1")
+  suppressWarnings(dropTempTable("table1"))
   expect_true(dropTempView("table2"))
 
   tables <- listTables()
@@ -1810,7 +1819,8 @@ test_that("string operators", {
   expect_true(first(select(df, endsWith(df$name, "el")))[[1]])
   expect_equal(first(select(df, substr(df$name, 1, 2)))[[1]], "Mi")
   expect_equal(first(select(df, substr(df$name, 4, 6)))[[1]], "hae")
-  if (as.numeric(R.version$major) >= 3 && as.numeric(R.version$minor) >= 3) {
+  version <- packageVersion("base")
+  if (as.numeric(version$major) >= 3 && as.numeric(version$minor) >= 3) {
     expect_true(startsWith("Hello World", "Hello"))
     expect_false(endsWith("Hello World", "a"))
   }
@@ -2462,7 +2472,7 @@ test_that("join(), crossJoin() and merge() on a DataFrame", {
   error_msg <- paste("joinType must be one of the following types:",
                  "'inner', 'cross', 'outer', 'full', 'fullouter', 'full_outer',",
                  "'left', 'leftouter', 'left_outer', 'right', 'rightouter', 'right_outer',",
-                 "'semi', 'leftsemi', 'left_semi', 'anti', 'leftanti' or 'left_anti'.")
+                 "'semi', 'leftsemi', 'left_semi', 'anti', 'leftanti', 'left_anti'")
   expect_error(join(df2, df, df2$name == df$name, "invalid"), error_msg)
 
   merged <- merge(df, df2, by.x = "name", by.y = "name", all.x = TRUE, all.y = TRUE)
@@ -2497,8 +2507,8 @@ test_that("join(), crossJoin() and merge() on a DataFrame", {
   writeLines(mockLines3, jsonPath3)
   df3 <- read.json(jsonPath3)
   expect_error(merge(df, df3),
-               paste("The following column name: name_y occurs more than once in the 'DataFrame'.",
-                     "Please use different suffixes for the intersected columns.", sep = ""))
+               paste0("The following column name: name_y occurs more than once in the 'DataFrame'.",
+                      "Please use different suffixes for the intersected columns."))
 
   unlink(jsonPath2)
   unlink(jsonPath3)
@@ -2541,20 +2551,20 @@ test_that("toJSON() on DataFrame", {
 
 test_that("showDF()", {
   df <- read.json(jsonPath)
-  expected <- paste("+----+-------+\n",
-                    "| age|   name|\n",
-                    "+----+-------+\n",
-                    "|null|Michael|\n",
-                    "|  30|   Andy|\n",
-                    "|  19| Justin|\n",
-                    "+----+-------+\n", sep = "")
-  expected2 <- paste("+---+----+\n",
-                     "|age|name|\n",
-                     "+---+----+\n",
-                     "|nul| Mic|\n",
-                     "| 30| And|\n",
-                     "| 19| Jus|\n",
-                     "+---+----+\n", sep = "")
+  expected <- paste("+----+-------+",
+                    "| age|   name|",
+                    "+----+-------+",
+                    "|null|Michael|",
+                    "|  30|   Andy|",
+                    "|  19| Justin|",
+                    "+----+-------+\n", sep = "\n")
+  expected2 <- paste("+---+----+",
+                     "|age|name|",
+                     "+---+----+",
+                     "|nul| Mic|",
+                     "| 30| And|",
+                     "| 19| Jus|",
+                     "+---+----+\n", sep = "\n")
   expect_output(showDF(df), expected)
   expect_output(showDF(df, truncate = 3), expected2)
 })
diff --git a/R/pkg/tests/fulltests/test_sparkSQL_arrow.R b/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
index 97972753a78fa..06743488fdf11 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
@@ -19,7 +19,10 @@ library(testthat)
 
 context("SparkSQL Arrow optimization")
 
-sparkSession <- sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE)
+sparkSession <- sparkR.session(
+  master = sparkRTestMaster,
+  enableHiveSupport = FALSE,
+  sparkConfig = list(spark.sql.execution.arrow.sparkr.enabled = "true"))
 
 test_that("createDataFrame/collect Arrow optimization", {
   skip_if_not_installed("arrow")
@@ -35,29 +38,13 @@ test_that("createDataFrame/collect Arrow optimization", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_equal(collect(createDataFrame(mtcars)), expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_equal(collect(createDataFrame(mtcars)), expected)
 })
 
 test_that("createDataFrame/collect Arrow optimization - many partitions (partition order test)", {
   skip_if_not_installed("arrow")
-
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_equal(collect(createDataFrame(mtcars, numPartitions = 32)),
-                 collect(createDataFrame(mtcars, numPartitions = 1)))
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_equal(collect(createDataFrame(mtcars, numPartitions = 32)),
+               collect(createDataFrame(mtcars, numPartitions = 1)))
 })
 
 test_that("createDataFrame/collect Arrow optimization - type specification", {
@@ -81,13 +68,7 @@ test_that("createDataFrame/collect Arrow optimization - type specification", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_equal(collect(createDataFrame(rdf)), expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_equal(collect(createDataFrame(rdf)), expected)
 })
 
 test_that("dapply() Arrow optimization", {
@@ -98,36 +79,30 @@ test_that("dapply() Arrow optimization", {
   arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
 
   callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "false")
-  tryCatch({
-    ret <- dapply(df,
-    function(rdf) {
-      stopifnot(is.data.frame(rdf))
-      rdf
-    },
-    schema(df))
-    expected <- collect(ret)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
   tryCatch({
     ret <- dapply(df,
                   function(rdf) {
                     stopifnot(is.data.frame(rdf))
-                    # mtcars' hp is more then 50.
-                    stopifnot(all(rdf$hp > 50))
                     rdf
                   },
                   schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-    expect_equal(count(ret), nrow(mtcars))
+    expected <- collect(ret)
   },
   finally = {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
+
+  ret <- dapply(df,
+                function(rdf) {
+                  stopifnot(is.data.frame(rdf))
+                  # mtcars' hp is more then 50.
+                  stopifnot(all(rdf$hp > 50))
+                  rdf
+                },
+                schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
+  expect_equal(count(ret), nrow(mtcars))
 })
 
 test_that("dapply() Arrow optimization - type specification", {
@@ -154,15 +129,9 @@ test_that("dapply() Arrow optimization - type specification", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- dapply(df, function(rdf) { rdf }, schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- dapply(df, function(rdf) { rdf }, schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
 })
 
 test_that("dapply() Arrow optimization - type specification (date and timestamp)", {
@@ -170,18 +139,8 @@ test_that("dapply() Arrow optimization - type specification (date and timestamp)
   rdf <- data.frame(list(list(a = as.Date("1990-02-24"),
                               b = as.POSIXct("1990-02-24 12:34:56"))))
   df <- createDataFrame(rdf)
-
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- dapply(df, function(rdf) { rdf }, schema(df))
-    expect_equal(collect(ret), rdf)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- dapply(df, function(rdf) { rdf }, schema(df))
+  expect_equal(collect(ret), rdf)
 })
 
 test_that("gapply() Arrow optimization", {
@@ -209,28 +168,22 @@ test_that("gapply() Arrow optimization", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- gapply(df,
-                 "gear",
-                 function(key, grouped) {
-                   if (length(key) > 0) {
-                     stopifnot(is.numeric(key[[1]]))
-                   }
-                   stopifnot(is.data.frame(grouped))
-                   stopifnot(length(colnames(grouped)) == 11)
-                   # mtcars' hp is more then 50.
-                   stopifnot(all(grouped$hp > 50))
-                   grouped
-                 },
-                 schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-    expect_equal(count(ret), nrow(mtcars))
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- gapply(df,
+               "gear",
+               function(key, grouped) {
+                 if (length(key) > 0) {
+                   stopifnot(is.numeric(key[[1]]))
+                 }
+                 stopifnot(is.data.frame(grouped))
+                 stopifnot(length(colnames(grouped)) == 11)
+                 # mtcars' hp is more then 50.
+                 stopifnot(all(grouped$hp > 50))
+                 grouped
+               },
+               schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
+  expect_equal(count(ret), nrow(mtcars))
 })
 
 test_that("gapply() Arrow optimization - type specification", {
@@ -250,26 +203,19 @@ test_that("gapply() Arrow optimization - type specification", {
   callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "false")
   tryCatch({
     ret <- gapply(df,
-                 "a",
-                 function(key, grouped) { grouped }, schema(df))
+                  "a",
+                  function(key, grouped) { grouped }, schema(df))
     expected <- collect(ret)
   },
   finally = {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- gapply(df,
-                 "a",
-                 function(key, grouped) { grouped }, schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- gapply(df,
+                "a",
+                function(key, grouped) { grouped }, schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
 })
 
 test_that("gapply() Arrow optimization - type specification (date and timestamp)", {
@@ -277,39 +223,30 @@ test_that("gapply() Arrow optimization - type specification (date and timestamp)
   rdf <- data.frame(list(list(a = as.Date("1990-02-24"),
                               b = as.POSIXct("1990-02-24 12:34:56"))))
   df <- createDataFrame(rdf)
+  ret <- gapply(df,
+                "a",
+                function(key, grouped) { grouped }, schema(df))
+  expect_equal(collect(ret), rdf)
+})
 
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
+test_that("Arrow optimization - unsupported types", {
+  skip_if_not_installed("arrow")
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- gapply(df,
-                  "a",
-                  function(key, grouped) { grouped }, schema(df))
-    expect_equal(collect(ret), rdf)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_error(checkSchemaInArrow(structType("a FLOAT")), "not support float type")
+  expect_error(checkSchemaInArrow(structType("a BINARY")), "not support binary type")
+  expect_error(checkSchemaInArrow(structType("a ARRAY<INT>")), "not support array type")
+  expect_error(checkSchemaInArrow(structType("a MAP<INT, INT>")), "not support map type")
+  expect_error(checkSchemaInArrow(structType("a STRUCT<a: INT>")),
+               "not support nested struct type")
 })
 
-test_that("Arrow optimization - unsupported types", {
+test_that("SPARK-32478: gapply() Arrow optimization - error message for schema mismatch", {
   skip_if_not_installed("arrow")
+  df <- createDataFrame(list(list(a = 1L, b = "a")))
 
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_error(checkSchemaInArrow(structType("a FLOAT")), "not support float type")
-    expect_error(checkSchemaInArrow(structType("a BINARY")), "not support binary type")
-    expect_error(checkSchemaInArrow(structType("a ARRAY<INT>")), "not support array type")
-    expect_error(checkSchemaInArrow(structType("a MAP<INT, INT>")), "not support map type")
-    expect_error(checkSchemaInArrow(structType("a STRUCT<a: INT>")),
-                 "not support nested struct type")
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_error(
+    count(gapply(df, "a", function(key, group) { group }, structType("a int, b int"))),
+    "expected IntegerType, IntegerType, got IntegerType, StringType")
 })
 
 sparkR.session.stop()
diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R
index bf02ecdad66ff..0a6a6c4ea3d1d 100644
--- a/R/pkg/tests/run-all.R
+++ b/R/pkg/tests/run-all.R
@@ -60,22 +60,49 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) {
     # set random seed for predictable results. mostly for base's sample() in tree and classification
     set.seed(42)
 
-    # TODO (SPARK-30663) To be removed once testthat 1.x is removed from all builds
-    if (grepl("^1\\..*", packageVersion("testthat"))) {
+    test_runner <- if (packageVersion("testthat")$major <= 1) {
       # testthat 1.x
-      test_runner <- testthat:::run_tests
-      reporter <- "summary"
+      function(path, package, reporter, filter) {
+        testthat:::run_tests(
+          test_path = path,
+          package = package,
+          filter = filter,
+          reporter = reporter
+        )
+      }
+    } else if (packageVersion("testthat")$major == 2) {
+      # testthat >= 2.0.0, < 3.0.0
+      function(path, package, reporter, filter) {
+        testthat:::test_package_dir(
+          test_path = path,
+          package = package,
+          filter = filter,
+          reporter = reporter
+        )
+      }
+    } else {
+      # testthat >= 3.0.0
+      testthat::test_dir
+    }
 
+    reporter <- if (packageVersion("testthat")$major <= 1) {
+      "summary"
     } else {
-      # testthat >= 2.0.0
-      test_runner <- testthat:::test_package_dir
-      reporter <- testthat::default_reporter()
+      dir.create("target/test-reports", showWarnings = FALSE)
+      MultiReporter$new(list(
+        SummaryReporter$new(),
+        JunitReporter$new(
+          file = file.path(getwd(), "target/test-reports/test-results.xml")
+        )
+      ))
     }
 
-    test_runner("SparkR",
-                file.path(sparkRDir, "pkg", "tests", "fulltests"),
-                NULL,
-                reporter)
+    test_runner(
+      path = file.path(sparkRDir, "pkg", "tests", "fulltests"),
+      package = "SparkR",
+      reporter = reporter,
+      filter = NULL
+    )
   }
 
   SparkR:::uninstallDownloadedSpark()
diff --git a/appveyor.yml b/appveyor.yml
index 5d98260265b1a..e0d2396bcbed4 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -41,15 +41,15 @@ cache:
 install:
   # Install maven and dependencies
   - ps: .\dev\appveyor-install-dependencies.ps1
-  # Required package for R unit tests
-  - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'e1071', 'survival', 'arrow'), repos='https://cloud.r-project.org/')"
-  - cmd: R -e "install.packages(c('crayon', 'praise', 'R6', 'testthat'), repos='https://cloud.r-project.org/')"
+  # Required package for R unit tests. xml2 is required to use jUnit reporter in testthat.
+  - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', 'xml2'), repos='https://cloud.r-project.org/')"
   - cmd: R -e "packageVersion('knitr'); packageVersion('rmarkdown'); packageVersion('testthat'); packageVersion('e1071'); packageVersion('survival'); packageVersion('arrow')"
 
 build_script:
   # '-Djna.nosys=true' is required to avoid kernel32.dll load failure.
   # See SPARK-28759.
-  - cmd: mvn -DskipTests -Psparkr -Phive -Djna.nosys=true package
+  # Ideally we should check the tests related to Hive in SparkR as well (SPARK-31745).
+  - cmd: mvn -DskipTests -Psparkr -Djna.nosys=true package
 
 environment:
   NOT_CRAN: true
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 193ad3d671bcf..e265bb948d9d4 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh
index 68fafbb848001..6d74f8328aea2 100755
--- a/bin/docker-image-tool.sh
+++ b/bin/docker-image-tool.sh
@@ -172,6 +172,7 @@ function build {
   local BASEDOCKERFILE=${BASEDOCKERFILE:-"kubernetes/dockerfiles/spark/Dockerfile"}
   local PYDOCKERFILE=${PYDOCKERFILE:-false}
   local RDOCKERFILE=${RDOCKERFILE:-false}
+  local ARCHS=${ARCHS:-"--platform linux/amd64,linux/arm64"}
 
   (cd $(img_ctx_dir base) && docker build $NOCACHEARG "${BUILD_ARGS[@]}" \
     -t $(image_ref spark) \
@@ -179,6 +180,11 @@ function build {
   if [ $? -ne 0 ]; then
     error "Failed to build Spark JVM Docker image, please refer to Docker build output for details."
   fi
+  if [ "${CROSS_BUILD}" != "false" ]; then
+  (cd $(img_ctx_dir base) && docker buildx build $ARCHS $NOCACHEARG "${BUILD_ARGS[@]}" \
+    -t $(image_ref spark) \
+    -f "$BASEDOCKERFILE" .)
+  fi
 
   if [ "${PYDOCKERFILE}" != "false" ]; then
     (cd $(img_ctx_dir pyspark) && docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
@@ -187,6 +193,11 @@ function build {
       if [ $? -ne 0 ]; then
         error "Failed to build PySpark Docker image, please refer to Docker build output for details."
       fi
+      if [ "${CROSS_BUILD}" != "false" ]; then
+        (cd $(img_ctx_dir pyspark) && docker buildx build $ARCHS $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
+          -t $(image_ref spark-py) \
+          -f "$PYDOCKERFILE" .)
+      fi
   fi
 
   if [ "${RDOCKERFILE}" != "false" ]; then
@@ -196,6 +207,11 @@ function build {
     if [ $? -ne 0 ]; then
       error "Failed to build SparkR Docker image, please refer to Docker build output for details."
     fi
+    if [ "${CROSS_BUILD}" != "false" ]; then
+      (cd $(img_ctx_dir sparkr) && docker buildx build $ARCHS $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
+        -t $(image_ref spark-r) \
+        -f "$RDOCKERFILE" .)
+    fi
   fi
 }
 
@@ -227,6 +243,8 @@ Options:
   -n                    Build docker image with --no-cache
   -u uid                UID to use in the USER directive to set the user the main Spark process runs as inside the
                         resulting container
+  -X                    Use docker buildx to cross build. Automatically pushes.
+                        See https://docs.docker.com/buildx/working-with-buildx/ for steps to setup buildx.
   -b arg                Build arg to build or push the image. For multiple build args, this option needs to
                         be used separately for each build arg.
 
@@ -248,6 +266,16 @@ Examples:
   - Build and push image with tag "v2.3.0" to docker.io/myrepo
     $0 -r docker.io/myrepo -t v2.3.0 build
     $0 -r docker.io/myrepo -t v2.3.0 push
+
+  - Build and push JDK11-based image with tag "v3.0.0" to docker.io/myrepo
+    $0 -r docker.io/myrepo -t v3.0.0 -b java_image_tag=11-jre-slim build
+    $0 -r docker.io/myrepo -t v3.0.0 push
+
+  - Build and push JDK11-based image for multiple archs to docker.io/myrepo
+    $0 -r docker.io/myrepo -t v3.0.0 -X -b java_image_tag=11-jre-slim build
+    # Note: buildx, which does cross building, needs to do the push during build
+    # So there is no seperate push step with -X
+
 EOF
 }
 
@@ -264,7 +292,8 @@ RDOCKERFILE=
 NOCACHEARG=
 BUILD_PARAMS=
 SPARK_UID=
-while getopts f:p:R:mr:t:nb:u: option
+CROSS_BUILD="false"
+while getopts f:p:R:mr:t:Xnb:u: option
 do
  case "${option}"
  in
@@ -275,6 +304,7 @@ do
  t) TAG=${OPTARG};;
  n) NOCACHEARG="--no-cache";;
  b) BUILD_PARAMS=${BUILD_PARAMS}" --build-arg "${OPTARG};;
+ X) CROSS_BUILD=1;;
  m)
    if ! which minikube 1>/dev/null; then
      error "Cannot find minikube."
diff --git a/bin/load-spark-env.cmd b/bin/load-spark-env.cmd
index 5f98cc34b6bab..7e2cf43af3c7d 100644
--- a/bin/load-spark-env.cmd
+++ b/bin/load-spark-env.cmd
@@ -29,10 +29,7 @@ if [%SPARK_ENV_LOADED%] == [] (
     set SPARK_CONF_DIR=%~dp0..\conf
   )
 
-  set SPARK_ENV_CMD=%SPARK_CONF_DIR%\%SPARK_ENV_CMD%
-  if exist %SPARK_ENV_CMD% (
-    call %SPARK_ENV_CMD%
-  )
+  call :LoadSparkEnv
 )
 
 rem Setting SPARK_SCALA_VERSION if not already set.
@@ -60,3 +57,8 @@ rem     set SPARK_SCALA_VERSION=%SCALA_VERSION_2%
 rem   )
 rem )
 exit /b 0
+
+:LoadSparkEnv
+if exist "%SPARK_CONF_DIR%\spark-env.cmd" (
+  call "%SPARK_CONF_DIR%\spark-env.cmd"
+)
diff --git a/bin/pyspark b/bin/pyspark
index 44891aee2e0a3..ad4132fb59eb0 100755
--- a/bin/pyspark
+++ b/bin/pyspark
@@ -50,7 +50,7 @@ export PYSPARK_DRIVER_PYTHON_OPTS
 
 # Add the PySpark classes to the Python path:
 export PYTHONPATH="${SPARK_HOME}/python/:$PYTHONPATH"
-export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.8.1-src.zip:$PYTHONPATH"
+export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9-src.zip:$PYTHONPATH"
 
 # Load the PySpark shell.py script when ./pyspark is used interactively:
 export OLD_PYTHONSTARTUP="$PYTHONSTARTUP"
diff --git a/bin/pyspark2.cmd b/bin/pyspark2.cmd
index 479fd464c7d3e..dc34be1a41706 100644
--- a/bin/pyspark2.cmd
+++ b/bin/pyspark2.cmd
@@ -30,7 +30,7 @@ if "x%PYSPARK_DRIVER_PYTHON%"=="x" (
 )
 
 set PYTHONPATH=%SPARK_HOME%\python;%PYTHONPATH%
-set PYTHONPATH=%SPARK_HOME%\python\lib\py4j-0.10.8.1-src.zip;%PYTHONPATH%
+set PYTHONPATH=%SPARK_HOME%\python\lib\py4j-0.10.9-src.zip;%PYTHONPATH%
 
 set OLD_PYTHONSTARTUP=%PYTHONSTARTUP%
 set PYTHONSTARTUP=%SPARK_HOME%\python\pyspark\shell.py
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index a1c8a8e6582eb..f433764648ccd 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
index b33c53871c32f..431c7e42774e4 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
@@ -163,6 +163,13 @@ public void clear() {
     }
   }
 
+  /**
+   * An alias class for the type "{@literal ConcurrentHashMap<Comparable<Object>, Boolean>}",
+   * which is used as a concurrent hashset for storing natural keys
+   * and the boolean value doesn't matter.
+   */
+  private static class NaturalKeys extends ConcurrentHashMap<Comparable<Object>, Boolean> {}
+
   private static class InstanceList<T> {
 
     /**
@@ -171,7 +178,7 @@ private static class InstanceList<T> {
      * iterators.  https://bugs.openjdk.java.net/browse/JDK-8078645
      */
     private static class CountingRemoveIfForEach<T> implements BiConsumer<Comparable<Object>, T> {
-      private final ConcurrentMap<Comparable<Object>, T> data;
+      private final InstanceList<T> instanceList;
       private final Predicate<? super T> filter;
 
       /**
@@ -183,17 +190,15 @@ private static class CountingRemoveIfForEach<T> implements BiConsumer<Comparable
        */
       private int count = 0;
 
-      CountingRemoveIfForEach(
-          ConcurrentMap<Comparable<Object>, T> data,
-          Predicate<? super T> filter) {
-        this.data = data;
+      CountingRemoveIfForEach(InstanceList<T> instanceList, Predicate<? super T> filter) {
+        this.instanceList = instanceList;
         this.filter = filter;
       }
 
       @Override
       public void accept(Comparable<Object> key, T value) {
         if (filter.test(value)) {
-          if (data.remove(key, value)) {
+          if (instanceList.delete(key, value)) {
             count++;
           }
         }
@@ -205,11 +210,19 @@ public void accept(Comparable<Object> key, T value) {
     private final KVTypeInfo ti;
     private final KVTypeInfo.Accessor naturalKey;
     private final ConcurrentMap<Comparable<Object>, T> data;
+    private final String naturalParentIndexName;
+    private final Boolean hasNaturalParentIndex;
+    // A mapping from parent to the natural keys of its children.
+    // For example, a mapping from a stage ID to all the task IDs in the stage.
+    private final ConcurrentMap<Comparable<Object>, NaturalKeys> parentToChildrenMap;
 
     private InstanceList(Class<?> klass) {
       this.ti = new KVTypeInfo(klass);
       this.naturalKey = ti.getAccessor(KVIndex.NATURAL_INDEX_NAME);
       this.data = new ConcurrentHashMap<>();
+      this.naturalParentIndexName = ti.getParentIndexName(KVIndex.NATURAL_INDEX_NAME);
+      this.parentToChildrenMap = new ConcurrentHashMap<>();
+      this.hasNaturalParentIndex = !naturalParentIndexName.isEmpty();
     }
 
     KVTypeInfo.Accessor getIndexAccessor(String indexName) {
@@ -217,11 +230,35 @@ KVTypeInfo.Accessor getIndexAccessor(String indexName) {
     }
 
     int countingRemoveAllByIndexValues(String index, Collection<?> indexValues) {
-      Predicate<? super T> filter = getPredicate(ti.getAccessor(index), indexValues);
-      CountingRemoveIfForEach<T> callback = new CountingRemoveIfForEach<>(data, filter);
+      int count = 0;
+      if (KVIndex.NATURAL_INDEX_NAME.equals(index)) {
+        for (Object naturalKey : indexValues) {
+          count += delete(asKey(naturalKey)) ? 1 : 0;
+        }
+        return count;
+      } else if (hasNaturalParentIndex && naturalParentIndexName.equals(index)) {
+        // If there is a parent index for the natural index and `index` happens to be it,
+        // Spark can use the `parentToChildrenMap` to get the related natural keys, and then
+        // delete them from `data`.
+        for (Object indexValue : indexValues) {
+          Comparable<Object> parentKey = asKey(indexValue);
+          NaturalKeys children = parentToChildrenMap.getOrDefault(parentKey, new NaturalKeys());
+          for (Comparable<Object> naturalKey : children.keySet()) {
+            data.remove(naturalKey);
+            count ++;
+          }
+          parentToChildrenMap.remove(parentKey);
+        }
+        return count;
+      } else {
+        Predicate<? super T> filter = getPredicate(ti.getAccessor(index), indexValues);
+        CountingRemoveIfForEach<T> callback = new CountingRemoveIfForEach<>(this, filter);
 
-      data.forEach(callback);
-      return callback.count();
+        // Go through all the values in `data` and delete objects that meets the predicate `filter`.
+        // This can be slow when there is a large number of entries in `data`.
+        data.forEach(callback);
+        return callback.count();
+      }
     }
 
     public T get(Object key) {
@@ -230,10 +267,42 @@ public T get(Object key) {
 
     public void put(T value) throws Exception {
       data.put(asKey(naturalKey.get(value)), value);
+      if (hasNaturalParentIndex) {
+        Comparable<Object> parentKey = asKey(getIndexAccessor(naturalParentIndexName).get(value));
+        NaturalKeys children =
+          parentToChildrenMap.computeIfAbsent(parentKey, k -> new NaturalKeys());
+        children.put(asKey(naturalKey.get(value)), true);
+      }
+    }
+
+    public boolean delete(Object key) {
+      boolean entryExists = data.remove(asKey(key)) != null;
+      if (entryExists) {
+        deleteParentIndex(key);
+      }
+      return entryExists;
     }
 
-    public void delete(Object key) {
-      data.remove(asKey(key));
+    public boolean delete(Object key, T value) {
+      boolean entryExists = data.remove(asKey(key), value);
+      if (entryExists) {
+        deleteParentIndex(key);
+      }
+      return entryExists;
+    }
+
+    private void deleteParentIndex(Object key) {
+      if (hasNaturalParentIndex) {
+        for (NaturalKeys v : parentToChildrenMap.values()) {
+          if (v.remove(asKey(key)) != null) {
+            // `v` can be empty after removing the natural key and we can remove it from
+            // `parentToChildrenMap`. However, `parentToChildrenMap` is a ConcurrentMap and such
+            // checking and deleting can be slow.
+            // This method is to delete one object with certain key, let's make it simple here.
+            break;
+          }
+        }
+      }
     }
 
     public int size() {
@@ -241,7 +310,7 @@ public int size() {
     }
 
     public InMemoryView<T> view() {
-      return new InMemoryView<>(data.values(), ti);
+      return new InMemoryView<>(data, ti, naturalParentIndexName, parentToChildrenMap);
     }
 
     private static <T> Predicate<? super T> getPredicate(
@@ -271,22 +340,32 @@ private static Object indexValueForEntity(KVTypeInfo.Accessor getter, Object ent
 
   private static class InMemoryView<T> extends KVStoreView<T> {
     private static final InMemoryView<?> EMPTY_VIEW =
-      new InMemoryView<>(Collections.emptyList(), null);
+      new InMemoryView<>(new ConcurrentHashMap<>(), null, "", new ConcurrentHashMap<>());
 
-    private final Collection<T> elements;
+    private final ConcurrentMap<Comparable<Object>, T> data;
     private final KVTypeInfo ti;
     private final KVTypeInfo.Accessor natural;
-
-    InMemoryView(Collection<T> elements, KVTypeInfo ti) {
-      this.elements = elements;
+    private final ConcurrentMap<Comparable<Object>, NaturalKeys> parentToChildrenMap;
+    private final String naturalParentIndexName;
+    private final Boolean hasNaturalParentIndex;
+
+    InMemoryView(
+        ConcurrentMap<Comparable<Object>, T> data,
+        KVTypeInfo ti,
+        String naturalParentIndexName,
+        ConcurrentMap<Comparable<Object>, NaturalKeys> parentToChildrenMap) {
+      this.data = data;
       this.ti = ti;
       this.natural = ti != null ? ti.getAccessor(KVIndex.NATURAL_INDEX_NAME) : null;
+      this.naturalParentIndexName = naturalParentIndexName;
+      this.parentToChildrenMap = parentToChildrenMap;
+      this.hasNaturalParentIndex = !naturalParentIndexName.isEmpty();
     }
 
     @Override
     public Iterator<T> iterator() {
-      if (elements.isEmpty()) {
-        return new InMemoryIterator<>(elements.iterator());
+      if (data.isEmpty()) {
+        return new InMemoryIterator<>(Collections.emptyIterator());
       }
 
       KVTypeInfo.Accessor getter = index != null ? ti.getAccessor(index) : null;
@@ -322,15 +401,31 @@ public Iterator<T> iterator() {
      */
     private List<T> copyElements() {
       if (parent != null) {
-        KVTypeInfo.Accessor parentGetter = ti.getParentAccessor(index);
-        Preconditions.checkArgument(parentGetter != null, "Parent filter for non-child index.");
-        Comparable<?> parentKey = asKey(parent);
-
-        return elements.stream()
-          .filter(e -> compare(e, parentGetter, parentKey) == 0)
-          .collect(Collectors.toList());
+        Comparable<Object> parentKey = asKey(parent);
+        if (hasNaturalParentIndex && naturalParentIndexName.equals(ti.getParentIndexName(index))) {
+          // If there is a parent index for the natural index and the parent of `index` happens to
+          // be it, Spark can use the `parentToChildrenMap` to get the related natural keys, and
+          // then copy them from `data`.
+          NaturalKeys children = parentToChildrenMap.getOrDefault(parentKey, new NaturalKeys());
+          ArrayList<T> elements = new ArrayList<>();
+          for (Comparable<Object> naturalKey : children.keySet()) {
+            data.computeIfPresent(naturalKey, (k, v) -> {
+              elements.add(v);
+              return v;
+            });
+          }
+          return elements;
+        } else {
+          // Go through all the values in `data` and collect all the objects has certain parent
+          // value. This can be slow when there is a large number of entries in `data`.
+          KVTypeInfo.Accessor parentGetter = ti.getParentAccessor(index);
+          Preconditions.checkArgument(parentGetter != null, "Parent filter for non-child index.");
+          return data.values().stream()
+            .filter(e -> compare(e, parentGetter, parentKey) == 0)
+            .collect(Collectors.toList());
+        }
       } else {
-        return new ArrayList<>(elements);
+        return new ArrayList<>(data.values());
       }
     }
 
diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/KVTypeInfo.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/KVTypeInfo.java
index d2a26982d8703..5404d33dba5fb 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/KVTypeInfo.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/KVTypeInfo.java
@@ -68,8 +68,6 @@ public KVTypeInfo(Class<?> type) {
 
     Preconditions.checkArgument(indices.containsKey(KVIndex.NATURAL_INDEX_NAME),
         "No natural index defined for type %s.", type.getName());
-    Preconditions.checkArgument(indices.get(KVIndex.NATURAL_INDEX_NAME).parent().isEmpty(),
-        "Natural index of %s cannot have a parent.", type.getName());
 
     for (KVIndex idx : indices.values()) {
       if (!idx.parent().isEmpty()) {
@@ -117,6 +115,11 @@ Accessor getParentAccessor(String indexName) {
     return index.parent().isEmpty() ? null : getAccessor(index.parent());
   }
 
+  String getParentIndexName(String indexName) {
+    KVIndex index = indices.get(indexName);
+    return index.parent();
+  }
+
   /**
    * Abstracts the difference between invoking a Field and a Method.
    */
diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java
index f4d359234cb9e..d7423537ddfcf 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java
@@ -133,12 +133,13 @@ class LevelDBTypeInfo {
 
     // First create the parent indices, then the child indices.
     ti.indices().forEach(idx -> {
-      if (idx.parent().isEmpty()) {
+      // In LevelDB, there is no parent index for the NUTURAL INDEX.
+      if (idx.parent().isEmpty() || idx.value().equals(KVIndex.NATURAL_INDEX_NAME)) {
         indices.put(idx.value(), new Index(idx, ti.getAccessor(idx.value()), null));
       }
     });
     ti.indices().forEach(idx -> {
-      if (!idx.parent().isEmpty()) {
+      if (!idx.parent().isEmpty() && !idx.value().equals(KVIndex.NATURAL_INDEX_NAME)) {
         indices.put(idx.value(), new Index(idx, ti.getAccessor(idx.value()),
           indices.get(idx.parent())));
       }
diff --git a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/CustomType2.java b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/CustomType2.java
new file mode 100644
index 0000000000000..3bb66bb3ec700
--- /dev/null
+++ b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/CustomType2.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util.kvstore;
+
+public class CustomType2 {
+
+  @KVIndex(parent = "parentId")
+  public String key;
+
+  @KVIndex("id")
+  public String id;
+
+  @KVIndex("parentId")
+  public String parentId;
+
+  @Override
+  public boolean equals(Object o) {
+    if (o instanceof CustomType2) {
+      CustomType2 other = (CustomType2) o;
+      return id.equals(other.id) && parentId.equals(other.parentId);
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return id.hashCode() ^ parentId.hashCode();
+  }
+
+  @Override
+  public String toString() {
+    return "CustomType2[key=" + key + ",id=" + id + ",parentId=" + parentId;
+  }
+
+}
diff --git a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
index 9e34225e14e18..35656fb12238a 100644
--- a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
+++ b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
@@ -158,23 +158,29 @@ public void testRemoveAll() throws Exception {
 
     assertEquals(9, store.count(ArrayKeyIndexType.class));
 
+    // Try removing non-existing keys
+    assert(!store.removeAllByIndexValues(
+      ArrayKeyIndexType.class,
+      KVIndex.NATURAL_INDEX_NAME,
+      ImmutableSet.of(new int[] {10, 10, 10}, new int[] { 3, 3, 3 })));
+    assertEquals(9, store.count(ArrayKeyIndexType.class));
 
-    store.removeAllByIndexValues(
+    assert(store.removeAllByIndexValues(
       ArrayKeyIndexType.class,
       KVIndex.NATURAL_INDEX_NAME,
-      ImmutableSet.of(new int[] {0, 0, 0}, new int[] { 2, 2, 2 }));
+      ImmutableSet.of(new int[] {0, 0, 0}, new int[] { 2, 2, 2 })));
     assertEquals(7, store.count(ArrayKeyIndexType.class));
 
-    store.removeAllByIndexValues(
+    assert(store.removeAllByIndexValues(
       ArrayKeyIndexType.class,
       "id",
-      ImmutableSet.of(new String [] { "things" }));
+      ImmutableSet.of(new String [] { "things" })));
     assertEquals(4, store.count(ArrayKeyIndexType.class));
 
-    store.removeAllByIndexValues(
+    assert(store.removeAllByIndexValues(
       ArrayKeyIndexType.class,
       "id",
-      ImmutableSet.of(new String [] { "more things" }));
+      ImmutableSet.of(new String [] { "more things" })));
     assertEquals(0, store.count(ArrayKeyIndexType.class));
   }
 
@@ -204,4 +210,46 @@ public void testBasicIteration() throws Exception {
     assertFalse(store.view(t1.getClass()).first(t2.id).skip(1).iterator().hasNext());
   }
 
+  @Test
+  public void testDeleteParentIndex() throws Exception {
+    KVStore store = new InMemoryStore();
+
+    CustomType2 t1 = new CustomType2();
+    t1.key = "key1";
+    t1.id = "id1";
+    t1.parentId = "parentId1";
+    store.write(t1);
+
+    CustomType2 t2 = new CustomType2();
+    t2.key = "key2";
+    t2.id = "id2";
+    t2.parentId = "parentId1";
+    store.write(t2);
+
+    CustomType2 t3 = new CustomType2();
+    t3.key = "key3";
+    t3.id = "id1";
+    t3.parentId = "parentId2";
+    store.write(t3);
+
+    CustomType2 t4 = new CustomType2();
+    t4.key = "key4";
+    t4.id = "id2";
+    t4.parentId = "parentId2";
+    store.write(t4);
+
+    assertEquals(4, store.count(CustomType2.class));
+
+    store.delete(t1.getClass(), t1.key);
+    assertEquals(3, store.count(CustomType2.class));
+
+    store.delete(t2.getClass(), t2.key);
+    assertEquals(2, store.count(CustomType2.class));
+
+    store.delete(t3.getClass(), t3.key);
+    assertEquals(1, store.count(CustomType2.class));
+
+    store.delete(t4.getClass(), t4.key);
+    assertEquals(0, store.count(CustomType2.class));
+  }
 }
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 163c250054e4d..4f31606fddf94 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java b/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java
index d99b9bdbce392..a0de9df1986f5 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java
@@ -123,7 +123,7 @@ public TransportContext(
 
     if (conf.getModuleName() != null &&
         conf.getModuleName().equalsIgnoreCase("shuffle") &&
-        !isClientOnly) {
+        !isClientOnly && conf.separateChunkFetchRequest()) {
       chunkFetchWorkers = NettyUtils.createEventLoop(
           IOMode.valueOf(conf.ioMode()),
           conf.chunkFetchHandlerThreads(),
@@ -187,8 +187,6 @@ public TransportChannelHandler initializePipeline(
       RpcHandler channelRpcHandler) {
     try {
       TransportChannelHandler channelHandler = createChannelHandler(channel, channelRpcHandler);
-      ChunkFetchRequestHandler chunkFetchHandler =
-        createChunkFetchHandler(channelHandler, channelRpcHandler);
       ChannelPipeline pipeline = channel.pipeline()
         .addLast("encoder", ENCODER)
         .addLast(TransportFrameDecoder.HANDLER_NAME, NettyUtils.createFrameDecoder())
@@ -200,6 +198,9 @@ public TransportChannelHandler initializePipeline(
         .addLast("handler", channelHandler);
       // Use a separate EventLoopGroup to handle ChunkFetchRequest messages for shuffle rpcs.
       if (chunkFetchWorkers != null) {
+        ChunkFetchRequestHandler chunkFetchHandler = new ChunkFetchRequestHandler(
+          channelHandler.getClient(), rpcHandler.getStreamManager(),
+          conf.maxChunksBeingTransferred(), true /* syncModeEnabled */);
         pipeline.addLast(chunkFetchWorkers, "chunkFetchHandler", chunkFetchHandler);
       }
       return channelHandler;
@@ -217,19 +218,17 @@ public TransportChannelHandler initializePipeline(
   private TransportChannelHandler createChannelHandler(Channel channel, RpcHandler rpcHandler) {
     TransportResponseHandler responseHandler = new TransportResponseHandler(channel);
     TransportClient client = new TransportClient(channel, responseHandler);
+    boolean separateChunkFetchRequest = conf.separateChunkFetchRequest();
+    ChunkFetchRequestHandler chunkFetchRequestHandler = null;
+    if (!separateChunkFetchRequest) {
+      chunkFetchRequestHandler = new ChunkFetchRequestHandler(
+        client, rpcHandler.getStreamManager(),
+        conf.maxChunksBeingTransferred(), false /* syncModeEnabled */);
+    }
     TransportRequestHandler requestHandler = new TransportRequestHandler(channel, client,
-      rpcHandler, conf.maxChunksBeingTransferred());
+      rpcHandler, conf.maxChunksBeingTransferred(), chunkFetchRequestHandler);
     return new TransportChannelHandler(client, responseHandler, requestHandler,
-      conf.connectionTimeoutMs(), closeIdleConnections, this);
-  }
-
-  /**
-   * Creates the dedicated ChannelHandler for ChunkFetchRequest messages.
-   */
-  private ChunkFetchRequestHandler createChunkFetchHandler(TransportChannelHandler channelHandler,
-      RpcHandler rpcHandler) {
-    return new ChunkFetchRequestHandler(channelHandler.getClient(),
-      rpcHandler.getStreamManager(), conf.maxChunksBeingTransferred());
+      conf.connectionTimeoutMs(), separateChunkFetchRequest, closeIdleConnections, this);
   }
 
   public TransportConf getConf() { return conf; }
diff --git a/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthRpcHandler.java b/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthRpcHandler.java
index 821cc7a849504..dd31c955350f1 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthRpcHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthRpcHandler.java
@@ -29,12 +29,11 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.spark.network.client.RpcResponseCallback;
-import org.apache.spark.network.client.StreamCallbackWithID;
 import org.apache.spark.network.client.TransportClient;
 import org.apache.spark.network.sasl.SecretKeyHolder;
 import org.apache.spark.network.sasl.SaslRpcHandler;
+import org.apache.spark.network.server.AbstractAuthRpcHandler;
 import org.apache.spark.network.server.RpcHandler;
-import org.apache.spark.network.server.StreamManager;
 import org.apache.spark.network.util.TransportConf;
 
 /**
@@ -46,7 +45,7 @@
  * The delegate will only receive messages if the given connection has been successfully
  * authenticated. A connection may be authenticated at most once.
  */
-class AuthRpcHandler extends RpcHandler {
+class AuthRpcHandler extends AbstractAuthRpcHandler {
   private static final Logger LOG = LoggerFactory.getLogger(AuthRpcHandler.class);
 
   /** Transport configuration. */
@@ -55,36 +54,31 @@ class AuthRpcHandler extends RpcHandler {
   /** The client channel. */
   private final Channel channel;
 
-  /**
-   * RpcHandler we will delegate to for authenticated connections. When falling back to SASL
-   * this will be replaced with the SASL RPC handler.
-   */
-  @VisibleForTesting
-  RpcHandler delegate;
-
   /** Class which provides secret keys which are shared by server and client on a per-app basis. */
   private final SecretKeyHolder secretKeyHolder;
 
-  /** Whether auth is done and future calls should be delegated. */
+  /** RPC handler for auth handshake when falling back to SASL auth. */
   @VisibleForTesting
-  boolean doDelegate;
+  SaslRpcHandler saslHandler;
 
   AuthRpcHandler(
       TransportConf conf,
       Channel channel,
       RpcHandler delegate,
       SecretKeyHolder secretKeyHolder) {
+    super(delegate);
     this.conf = conf;
     this.channel = channel;
-    this.delegate = delegate;
     this.secretKeyHolder = secretKeyHolder;
   }
 
   @Override
-  public void receive(TransportClient client, ByteBuffer message, RpcResponseCallback callback) {
-    if (doDelegate) {
-      delegate.receive(client, message, callback);
-      return;
+  protected boolean doAuthChallenge(
+      TransportClient client,
+      ByteBuffer message,
+      RpcResponseCallback callback) {
+    if (saslHandler != null) {
+      return saslHandler.doAuthChallenge(client, message, callback);
     }
 
     int position = message.position();
@@ -98,18 +92,17 @@ public void receive(TransportClient client, ByteBuffer message, RpcResponseCallb
       if (conf.saslFallback()) {
         LOG.warn("Failed to parse new auth challenge, reverting to SASL for client {}.",
           channel.remoteAddress());
-        delegate = new SaslRpcHandler(conf, channel, delegate, secretKeyHolder);
+        saslHandler = new SaslRpcHandler(conf, channel, null, secretKeyHolder);
         message.position(position);
         message.limit(limit);
-        delegate.receive(client, message, callback);
-        doDelegate = true;
+        return saslHandler.doAuthChallenge(client, message, callback);
       } else {
         LOG.debug("Unexpected challenge message from client {}, closing channel.",
           channel.remoteAddress());
         callback.onFailure(new IllegalArgumentException("Unknown challenge message."));
         channel.close();
       }
-      return;
+      return false;
     }
 
     // Here we have the client challenge, so perform the new auth protocol and set up the channel.
@@ -131,7 +124,7 @@ public void receive(TransportClient client, ByteBuffer message, RpcResponseCallb
       LOG.debug("Authentication failed for client {}, closing channel.", channel.remoteAddress());
       callback.onFailure(new IllegalArgumentException("Authentication failed."));
       channel.close();
-      return;
+      return false;
     } finally {
       if (engine != null) {
         try {
@@ -143,40 +136,6 @@ public void receive(TransportClient client, ByteBuffer message, RpcResponseCallb
     }
 
     LOG.debug("Authorization successful for client {}.", channel.remoteAddress());
-    doDelegate = true;
-  }
-
-  @Override
-  public void receive(TransportClient client, ByteBuffer message) {
-    delegate.receive(client, message);
-  }
-
-  @Override
-  public StreamCallbackWithID receiveStream(
-      TransportClient client,
-      ByteBuffer message,
-      RpcResponseCallback callback) {
-    return delegate.receiveStream(client, message, callback);
+    return true;
   }
-
-  @Override
-  public StreamManager getStreamManager() {
-    return delegate.getStreamManager();
-  }
-
-  @Override
-  public void channelActive(TransportClient client) {
-    delegate.channelActive(client);
-  }
-
-  @Override
-  public void channelInactive(TransportClient client) {
-    delegate.channelInactive(client);
-  }
-
-  @Override
-  public void exceptionCaught(Throwable cause, TransportClient client) {
-    delegate.exceptionCaught(cause, client);
-  }
-
 }
diff --git a/common/network-common/src/main/java/org/apache/spark/network/sasl/SaslRpcHandler.java b/common/network-common/src/main/java/org/apache/spark/network/sasl/SaslRpcHandler.java
index 355a3def8cc22..cc9e88fcf98e7 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/sasl/SaslRpcHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/sasl/SaslRpcHandler.java
@@ -28,10 +28,9 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.spark.network.client.RpcResponseCallback;
-import org.apache.spark.network.client.StreamCallbackWithID;
 import org.apache.spark.network.client.TransportClient;
+import org.apache.spark.network.server.AbstractAuthRpcHandler;
 import org.apache.spark.network.server.RpcHandler;
-import org.apache.spark.network.server.StreamManager;
 import org.apache.spark.network.util.JavaUtils;
 import org.apache.spark.network.util.TransportConf;
 
@@ -43,7 +42,7 @@
  * Note that the authentication process consists of multiple challenge-response pairs, each of
  * which are individual RPCs.
  */
-public class SaslRpcHandler extends RpcHandler {
+public class SaslRpcHandler extends AbstractAuthRpcHandler {
   private static final Logger logger = LoggerFactory.getLogger(SaslRpcHandler.class);
 
   /** Transport configuration. */
@@ -52,37 +51,28 @@ public class SaslRpcHandler extends RpcHandler {
   /** The client channel. */
   private final Channel channel;
 
-  /** RpcHandler we will delegate to for authenticated connections. */
-  private final RpcHandler delegate;
-
   /** Class which provides secret keys which are shared by server and client on a per-app basis. */
   private final SecretKeyHolder secretKeyHolder;
 
   private SparkSaslServer saslServer;
-  private boolean isComplete;
-  private boolean isAuthenticated;
 
   public SaslRpcHandler(
       TransportConf conf,
       Channel channel,
       RpcHandler delegate,
       SecretKeyHolder secretKeyHolder) {
+    super(delegate);
     this.conf = conf;
     this.channel = channel;
-    this.delegate = delegate;
     this.secretKeyHolder = secretKeyHolder;
     this.saslServer = null;
-    this.isComplete = false;
-    this.isAuthenticated = false;
   }
 
   @Override
-  public void receive(TransportClient client, ByteBuffer message, RpcResponseCallback callback) {
-    if (isComplete) {
-      // Authentication complete, delegate to base handler.
-      delegate.receive(client, message, callback);
-      return;
-    }
+  public boolean doAuthChallenge(
+      TransportClient client,
+      ByteBuffer message,
+      RpcResponseCallback callback) {
     if (saslServer == null || !saslServer.isComplete()) {
       ByteBuf nettyBuf = Unpooled.wrappedBuffer(message);
       SaslMessage saslMessage;
@@ -118,43 +108,21 @@ public void receive(TransportClient client, ByteBuffer message, RpcResponseCallb
       if (!SparkSaslServer.QOP_AUTH_CONF.equals(saslServer.getNegotiatedProperty(Sasl.QOP))) {
         logger.debug("SASL authentication successful for channel {}", client);
         complete(true);
-        return;
+        return true;
       }
 
       logger.debug("Enabling encryption for channel {}", client);
       SaslEncryption.addToChannel(channel, saslServer, conf.maxSaslEncryptedBlockSize());
       complete(false);
-      return;
+      return true;
     }
-  }
-
-  @Override
-  public void receive(TransportClient client, ByteBuffer message) {
-    delegate.receive(client, message);
-  }
-
-  @Override
-  public StreamCallbackWithID receiveStream(
-      TransportClient client,
-      ByteBuffer message,
-      RpcResponseCallback callback) {
-    return delegate.receiveStream(client, message, callback);
-  }
-
-  @Override
-  public StreamManager getStreamManager() {
-    return delegate.getStreamManager();
-  }
-
-  @Override
-  public void channelActive(TransportClient client) {
-    delegate.channelActive(client);
+    return false;
   }
 
   @Override
   public void channelInactive(TransportClient client) {
     try {
-      delegate.channelInactive(client);
+      super.channelInactive(client);
     } finally {
       if (saslServer != null) {
         saslServer.dispose();
@@ -162,11 +130,6 @@ public void channelInactive(TransportClient client) {
     }
   }
 
-  @Override
-  public void exceptionCaught(Throwable cause, TransportClient client) {
-    delegate.exceptionCaught(cause, client);
-  }
-
   private void complete(boolean dispose) {
     if (dispose) {
       try {
@@ -177,7 +140,6 @@ private void complete(boolean dispose) {
     }
 
     saslServer = null;
-    isComplete = true;
   }
 
 }
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/AbstractAuthRpcHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/AbstractAuthRpcHandler.java
new file mode 100644
index 0000000000000..92eb886283448
--- /dev/null
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/AbstractAuthRpcHandler.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.server;
+
+import java.nio.ByteBuffer;
+
+import org.apache.spark.network.client.RpcResponseCallback;
+import org.apache.spark.network.client.StreamCallbackWithID;
+import org.apache.spark.network.client.TransportClient;
+
+/**
+ * RPC Handler which performs authentication, and when it's successful, delegates further
+ * calls to another RPC handler. The authentication handshake itself should be implemented
+ * by subclasses.
+ */
+public abstract class AbstractAuthRpcHandler extends RpcHandler {
+  /** RpcHandler we will delegate to for authenticated connections. */
+  private final RpcHandler delegate;
+
+  private boolean isAuthenticated;
+
+  protected AbstractAuthRpcHandler(RpcHandler delegate) {
+    this.delegate = delegate;
+  }
+
+  /**
+   * Responds to an authentication challenge.
+   *
+   * @return Whether the client is authenticated.
+   */
+  protected abstract boolean doAuthChallenge(
+      TransportClient client,
+      ByteBuffer message,
+      RpcResponseCallback callback);
+
+  @Override
+  public final void receive(
+      TransportClient client,
+      ByteBuffer message,
+      RpcResponseCallback callback) {
+    if (isAuthenticated) {
+      delegate.receive(client, message, callback);
+    } else {
+      isAuthenticated = doAuthChallenge(client, message, callback);
+    }
+  }
+
+  @Override
+  public final void receive(TransportClient client, ByteBuffer message) {
+    if (isAuthenticated) {
+      delegate.receive(client, message);
+    } else {
+      throw new SecurityException("Unauthenticated call to receive().");
+    }
+  }
+
+  @Override
+  public final StreamCallbackWithID receiveStream(
+      TransportClient client,
+      ByteBuffer message,
+      RpcResponseCallback callback) {
+    if (isAuthenticated) {
+      return delegate.receiveStream(client, message, callback);
+    } else {
+      throw new SecurityException("Unauthenticated call to receiveStream().");
+    }
+  }
+
+  @Override
+  public StreamManager getStreamManager() {
+    return delegate.getStreamManager();
+  }
+
+  @Override
+  public void channelActive(TransportClient client) {
+    delegate.channelActive(client);
+  }
+
+  @Override
+  public void channelInactive(TransportClient client) {
+    delegate.channelInactive(client);
+  }
+
+  @Override
+  public void exceptionCaught(Throwable cause, TransportClient client) {
+    delegate.exceptionCaught(cause, client);
+  }
+
+  public boolean isAuthenticated() {
+    return isAuthenticated;
+  }
+}
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java
index 94412c4db559f..82810dacdad84 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java
@@ -55,14 +55,17 @@ public class ChunkFetchRequestHandler extends SimpleChannelInboundHandler<ChunkF
   private final StreamManager streamManager;
   /** The max number of chunks being transferred and not finished yet. */
   private final long maxChunksBeingTransferred;
+  private final boolean syncModeEnabled;
 
   public ChunkFetchRequestHandler(
       TransportClient client,
       StreamManager streamManager,
-      Long maxChunksBeingTransferred) {
+      Long maxChunksBeingTransferred,
+      boolean syncModeEnabled) {
     this.client = client;
     this.streamManager = streamManager;
     this.maxChunksBeingTransferred = maxChunksBeingTransferred;
+    this.syncModeEnabled = syncModeEnabled;
   }
 
   @Override
@@ -76,6 +79,11 @@ protected void channelRead0(
       ChannelHandlerContext ctx,
       final ChunkFetchRequest msg) throws Exception {
     Channel channel = ctx.channel();
+    processFetchRequest(channel, msg);
+  }
+
+  public void processFetchRequest(
+      final Channel channel, final ChunkFetchRequest msg) throws Exception {
     if (logger.isTraceEnabled()) {
       logger.trace("Received req from {} to fetch block {}", getRemoteAddress(channel),
         msg.streamChunkId);
@@ -112,19 +120,26 @@ protected void channelRead0(
    * channel will be handled by the EventLoop the channel is registered to. So even
    * though we are processing the ChunkFetchRequest in a separate thread pool, the actual I/O,
    * which is the potentially blocking call that could deplete server handler threads, is still
-   * being processed by TransportServer's default EventLoopGroup. In order to throttle the max
-   * number of threads that channel I/O for sending response to ChunkFetchRequest, the thread
-   * calling channel.writeAndFlush will wait for the completion of sending response back to
-   * client by invoking await(). This will throttle the rate at which threads from
-   * ChunkFetchRequest dedicated EventLoopGroup submit channel I/O requests to TransportServer's
-   * default EventLoopGroup, thus making sure that we can reserve some threads in
-   * TransportServer's default EventLoopGroup for handling other RPC messages.
+   * being processed by TransportServer's default EventLoopGroup.
+   *
+   * When syncModeEnabled is true, Spark will throttle the max number of threads that channel I/O
+   * for sending response to ChunkFetchRequest, the thread calling channel.writeAndFlush will wait
+   * for the completion of sending response back to client by invoking await(). This will throttle
+   * the rate at which threads from ChunkFetchRequest dedicated EventLoopGroup submit channel I/O
+   * requests to TransportServer's default EventLoopGroup, thus making sure that we can reserve
+   * some threads in TransportServer's default EventLoopGroup for handling other RPC messages.
    */
   private ChannelFuture respond(
       final Channel channel,
       final Encodable result) throws InterruptedException {
     final SocketAddress remoteAddress = channel.remoteAddress();
-    return channel.writeAndFlush(result).await().addListener((ChannelFutureListener) future -> {
+    ChannelFuture channelFuture;
+    if (syncModeEnabled) {
+      channelFuture = channel.writeAndFlush(result).await();
+    } else {
+      channelFuture = channel.writeAndFlush(result);
+    }
+    return channelFuture.addListener((ChannelFutureListener) future -> {
       if (future.isSuccess()) {
         logger.trace("Sent result {} to client {}", result, remoteAddress);
       } else {
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java
index 31371f6970ffe..e53a0c1a0852e 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java
@@ -58,6 +58,7 @@ public class TransportChannelHandler extends SimpleChannelInboundHandler<Message
   private final TransportRequestHandler requestHandler;
   private final long requestTimeoutNs;
   private final boolean closeIdleConnections;
+  private final boolean skipChunkFetchRequest;
   private final TransportContext transportContext;
 
   public TransportChannelHandler(
@@ -65,12 +66,14 @@ public TransportChannelHandler(
       TransportResponseHandler responseHandler,
       TransportRequestHandler requestHandler,
       long requestTimeoutMs,
+      boolean skipChunkFetchRequest,
       boolean closeIdleConnections,
       TransportContext transportContext) {
     this.client = client;
     this.responseHandler = responseHandler;
     this.requestHandler = requestHandler;
     this.requestTimeoutNs = requestTimeoutMs * 1000L * 1000;
+    this.skipChunkFetchRequest = skipChunkFetchRequest;
     this.closeIdleConnections = closeIdleConnections;
     this.transportContext = transportContext;
   }
@@ -124,7 +127,7 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
    */
   @Override
   public boolean acceptInboundMessage(Object msg) throws Exception {
-    if (msg instanceof ChunkFetchRequest) {
+    if (skipChunkFetchRequest && msg instanceof ChunkFetchRequest) {
       return false;
     } else {
       return super.acceptInboundMessage(msg);
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java
index 0792b58304247..f178928006902 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java
@@ -62,16 +62,21 @@ public class TransportRequestHandler extends MessageHandler<RequestMessage> {
   /** The max number of chunks being transferred and not finished yet. */
   private final long maxChunksBeingTransferred;
 
+  /** The dedicated ChannelHandler for ChunkFetchRequest messages. */
+  private final ChunkFetchRequestHandler chunkFetchRequestHandler;
+
   public TransportRequestHandler(
       Channel channel,
       TransportClient reverseClient,
       RpcHandler rpcHandler,
-      Long maxChunksBeingTransferred) {
+      Long maxChunksBeingTransferred,
+      ChunkFetchRequestHandler chunkFetchRequestHandler) {
     this.channel = channel;
     this.reverseClient = reverseClient;
     this.rpcHandler = rpcHandler;
     this.streamManager = rpcHandler.getStreamManager();
     this.maxChunksBeingTransferred = maxChunksBeingTransferred;
+    this.chunkFetchRequestHandler = chunkFetchRequestHandler;
   }
 
   @Override
@@ -97,8 +102,10 @@ public void channelInactive() {
   }
 
   @Override
-  public void handle(RequestMessage request) {
-    if (request instanceof RpcRequest) {
+  public void handle(RequestMessage request) throws Exception {
+    if (request instanceof ChunkFetchRequest) {
+      chunkFetchRequestHandler.processFetchRequest(channel, (ChunkFetchRequest) request);
+    } else if (request instanceof RpcRequest) {
       processRpcRequest((RpcRequest) request);
     } else if (request instanceof OneWayMessage) {
       processOneWayMessage((OneWayMessage) request);
diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
index cc0f2919568ac..646e4278811f4 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
@@ -290,7 +290,7 @@ public boolean sharedByteBufAllocators() {
   }
 
   /**
-  * If enabled then off-heap byte buffers will be prefered for the shared ByteBuf allocators.
+  * If enabled then off-heap byte buffers will be preferred for the shared ByteBuf allocators.
   */
   public boolean preferDirectBufsForSharedByteBufAllocators() {
     return conf.getBoolean("spark.network.io.preferDirectBufs", true);
@@ -316,7 +316,8 @@ public long maxChunksBeingTransferred() {
 
   /**
    * Percentage of io.serverThreads used by netty to process ChunkFetchRequest.
-   * Shuffle server will use a separate EventLoopGroup to process ChunkFetchRequest messages.
+   * When the config `spark.shuffle.server.chunkFetchHandlerThreadsPercent` is set,
+   * shuffle server will use a separate EventLoopGroup to process ChunkFetchRequest messages.
    * Although when calling the async writeAndFlush on the underlying channel to send
    * response back to client, the I/O on the channel is still being handled by
    * {@link org.apache.spark.network.server.TransportServer}'s default EventLoopGroup
@@ -339,12 +340,20 @@ public int chunkFetchHandlerThreads() {
       return 0;
     }
     int chunkFetchHandlerThreadsPercent =
-      conf.getInt("spark.shuffle.server.chunkFetchHandlerThreadsPercent", 100);
+      Integer.parseInt(conf.get("spark.shuffle.server.chunkFetchHandlerThreadsPercent"));
     int threads =
       this.serverThreads() > 0 ? this.serverThreads() : 2 * NettyRuntime.availableProcessors();
     return (int) Math.ceil(threads * (chunkFetchHandlerThreadsPercent / 100.0));
   }
 
+  /**
+   * Whether to use a separate EventLoopGroup to process ChunkFetchRequest messages, it is decided
+   * by the config `spark.shuffle.server.chunkFetchHandlerThreadsPercent` is set or not.
+   */
+  public boolean separateChunkFetchRequest() {
+    return conf.getInt("spark.shuffle.server.chunkFetchHandlerThreadsPercent", 0) > 0;
+  }
+
   /**
    * Whether to use the old protocol while doing the shuffle block fetching.
    * It is only enabled while we need the compatibility in the scenario of new spark version
diff --git a/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchRequestHandlerSuite.java b/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchRequestHandlerSuite.java
index 7e30ed4048ca8..addb4ff332746 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchRequestHandlerSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchRequestHandlerSuite.java
@@ -22,7 +22,6 @@
 import java.util.List;
 
 import io.netty.channel.Channel;
-import org.apache.spark.network.server.ChunkFetchRequestHandler;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -33,6 +32,7 @@
 import org.apache.spark.network.buffer.ManagedBuffer;
 import org.apache.spark.network.client.TransportClient;
 import org.apache.spark.network.protocol.*;
+import org.apache.spark.network.server.ChunkFetchRequestHandler;
 import org.apache.spark.network.server.NoOpRpcHandler;
 import org.apache.spark.network.server.OneForOneStreamManager;
 import org.apache.spark.network.server.RpcHandler;
@@ -68,7 +68,7 @@ public void handleChunkFetchRequest() throws Exception {
     long streamId = streamManager.registerStream("test-app", managedBuffers.iterator(), channel);
     TransportClient reverseClient = mock(TransportClient.class);
     ChunkFetchRequestHandler requestHandler = new ChunkFetchRequestHandler(reverseClient,
-      rpcHandler.getStreamManager(), 2L);
+      rpcHandler.getStreamManager(), 2L, false);
 
     RequestMessage request0 = new ChunkFetchRequest(new StreamChunkId(streamId, 0));
     requestHandler.channelRead(context, request0);
diff --git a/common/network-common/src/test/java/org/apache/spark/network/TransportRequestHandlerSuite.java b/common/network-common/src/test/java/org/apache/spark/network/TransportRequestHandlerSuite.java
index a43a659048686..0a6447176237a 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/TransportRequestHandlerSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/TransportRequestHandlerSuite.java
@@ -39,7 +39,7 @@
 public class TransportRequestHandlerSuite {
 
   @Test
-  public void handleStreamRequest() {
+  public void handleStreamRequest() throws Exception {
     RpcHandler rpcHandler = new NoOpRpcHandler();
     OneForOneStreamManager streamManager = (OneForOneStreamManager) (rpcHandler.getStreamManager());
     Channel channel = mock(Channel.class);
@@ -66,7 +66,7 @@ public void handleStreamRequest() {
 
     TransportClient reverseClient = mock(TransportClient.class);
     TransportRequestHandler requestHandler = new TransportRequestHandler(channel, reverseClient,
-      rpcHandler, 2L);
+      rpcHandler, 2L, null);
 
     RequestMessage request0 = new StreamRequest(String.format("%d_%d", streamId, 0));
     requestHandler.handle(request0);
diff --git a/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthIntegrationSuite.java b/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthIntegrationSuite.java
index 2f9dd629df1ba..a87a6aae21092 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthIntegrationSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthIntegrationSuite.java
@@ -34,7 +34,6 @@
 import org.apache.spark.network.client.RpcResponseCallback;
 import org.apache.spark.network.client.TransportClient;
 import org.apache.spark.network.client.TransportClientBootstrap;
-import org.apache.spark.network.sasl.SaslRpcHandler;
 import org.apache.spark.network.sasl.SaslServerBootstrap;
 import org.apache.spark.network.sasl.SecretKeyHolder;
 import org.apache.spark.network.server.RpcHandler;
@@ -65,8 +64,7 @@ public void testNewAuth() throws Exception {
 
     ByteBuffer reply = ctx.client.sendRpcSync(JavaUtils.stringToBytes("Ping"), 5000);
     assertEquals("Pong", JavaUtils.bytesToString(reply));
-    assertTrue(ctx.authRpcHandler.doDelegate);
-    assertFalse(ctx.authRpcHandler.delegate instanceof SaslRpcHandler);
+    assertNull(ctx.authRpcHandler.saslHandler);
   }
 
   @Test
@@ -78,7 +76,7 @@ public void testAuthFailure() throws Exception {
       ctx.createClient("client");
       fail("Should have failed to create client.");
     } catch (Exception e) {
-      assertFalse(ctx.authRpcHandler.doDelegate);
+      assertFalse(ctx.authRpcHandler.isAuthenticated());
       assertFalse(ctx.serverChannel.isActive());
     }
   }
@@ -91,6 +89,8 @@ public void testSaslServerFallback() throws Exception {
 
     ByteBuffer reply = ctx.client.sendRpcSync(JavaUtils.stringToBytes("Ping"), 5000);
     assertEquals("Pong", JavaUtils.bytesToString(reply));
+    assertNotNull(ctx.authRpcHandler.saslHandler);
+    assertTrue(ctx.authRpcHandler.isAuthenticated());
   }
 
   @Test
@@ -120,7 +120,7 @@ public void testAuthReplay() throws Exception {
       ctx.client.sendRpcSync(JavaUtils.stringToBytes("Ping"), 5000);
       fail("Should have failed unencrypted RPC.");
     } catch (Exception e) {
-      assertTrue(ctx.authRpcHandler.doDelegate);
+      assertTrue(ctx.authRpcHandler.isAuthenticated());
     }
   }
 
@@ -151,7 +151,7 @@ public StreamManager getStreamManager() {
       ctx.client.sendRpcSync(JavaUtils.stringToBytes("Ping"), 5000);
       fail("Should have failed unencrypted RPC.");
     } catch (Exception e) {
-      assertTrue(ctx.authRpcHandler.doDelegate);
+      assertTrue(ctx.authRpcHandler.isAuthenticated());
       assertTrue(e.getMessage() + " is not an expected error", e.getMessage().contains("DDDDD"));
       // Verify we receive the complete error message
       int messageStart = e.getMessage().indexOf("DDDDD");
diff --git a/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java b/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java
index cf2d72f71e8de..ecaeec98da182 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java
@@ -357,7 +357,8 @@ public void testRpcHandlerDelegate() throws Exception {
   public void testDelegates() throws Exception {
     Method[] rpcHandlerMethods = RpcHandler.class.getDeclaredMethods();
     for (Method m : rpcHandlerMethods) {
-      SaslRpcHandler.class.getDeclaredMethod(m.getName(), m.getParameterTypes());
+      Method delegate = SaslRpcHandler.class.getMethod(m.getName(), m.getParameterTypes());
+      assertNotEquals(delegate.getDeclaringClass(), RpcHandler.class);
     }
   }
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index a6d99813a8501..a3ac6611a3127 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java
index 13f6046dd856b..6549cac011feb 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java
@@ -23,11 +23,19 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.spark.network.util.JavaUtils;
 
 public class ExecutorDiskUtils {
 
-  private static final Pattern MULTIPLE_SEPARATORS = Pattern.compile(File.separator + "{2,}");
+  private static final Pattern MULTIPLE_SEPARATORS;
+  static {
+    if (SystemUtils.IS_OS_WINDOWS) {
+      MULTIPLE_SEPARATORS = Pattern.compile("[/\\\\]+");
+    } else {
+      MULTIPLE_SEPARATORS = Pattern.compile("/{2,}");
+    }
+  }
 
   /**
    * Hashes a filename into the corresponding local directory, in a manner consistent with
@@ -50,14 +58,18 @@ public static File getFile(String[] localDirs, int subDirsPerLocalDir, String fi
    * the internal code in java.io.File would normalize it later, creating a new "foo/bar"
    * String copy. Unfortunately, we cannot just reuse the normalization code that java.io.File
    * uses, since it is in the package-private class java.io.FileSystem.
+   *
+   * On Windows, separator "\" is used instead of "/".
+   *
+   * "\\" is a legal character in path name on Unix-like OS, but illegal on Windows.
    */
   @VisibleForTesting
   static String createNormalizedInternedPathname(String dir1, String dir2, String fname) {
     String pathname = dir1 + File.separator + dir2 + File.separator + fname;
     Matcher m = MULTIPLE_SEPARATORS.matcher(pathname);
-    pathname = m.replaceAll("/");
+    pathname = m.replaceAll(Matcher.quoteReplacement(File.separator));
     // A single trailing slash needs to be taken care of separately
-    if (pathname.length() > 1 && pathname.endsWith("/")) {
+    if (pathname.length() > 1 && pathname.charAt(pathname.length() - 1) == File.separatorChar) {
       pathname = pathname.substring(0, pathname.length() - 1);
     }
     return pathname.intern();
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
index 8c05288fb4111..33865a21ea914 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
@@ -229,8 +229,6 @@ public class ShuffleMetrics implements MetricSet {
     private final Meter blockTransferRateBytes = new Meter();
     // Number of active connections to the shuffle service
     private Counter activeConnections = new Counter();
-    // Number of registered connections to the shuffle service
-    private Counter registeredConnections = new Counter();
     // Number of exceptions caught in connections to the shuffle service
     private Counter caughtExceptions = new Counter();
 
@@ -242,7 +240,6 @@ public ShuffleMetrics() {
       allMetrics.put("registeredExecutorsSize",
                      (Gauge<Integer>) () -> blockManager.getRegisteredExecutorsSize());
       allMetrics.put("numActiveConnections", activeConnections);
-      allMetrics.put("numRegisteredConnections", registeredConnections);
       allMetrics.put("numCaughtExceptions", caughtExceptions);
     }
 
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java
index d6185f089d3c0..98247f24a036b 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java
@@ -175,8 +175,6 @@ public void onSuccess(ByteBuffer response) {
           logger.warn("Error trying to remove RDD blocks " + Arrays.toString(blockIds) +
             " via external shuffle service from executor: " + execId, t);
           numRemovedBlocksFuture.complete(0);
-        } finally {
-          client.close();
         }
       }
 
@@ -185,7 +183,6 @@ public void onFailure(Throwable e) {
         logger.warn("Error trying to remove RDD blocks " + Arrays.toString(blockIds) +
           " via external shuffle service from executor: " + execId, e);
         numRemovedBlocksFuture.complete(0);
-        client.close();
       }
     });
     return numRemovedBlocksFuture;
@@ -212,8 +209,6 @@ public void onSuccess(ByteBuffer response) {
               Arrays.toString(getLocalDirsMessage.execIds) + " via external shuffle service",
               t.getCause());
             hostLocalDirsCompletable.completeExceptionally(t);
-          } finally {
-            client.close();
           }
         }
 
@@ -223,7 +218,6 @@ public void onFailure(Throwable t) {
             Arrays.toString(getLocalDirsMessage.execIds) + " via external shuffle service",
             t.getCause());
           hostLocalDirsCompletable.completeExceptionally(t);
-          client.close();
         }
       });
     } catch (IOException | InterruptedException e) {
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
index 09b31430b1eb9..6515b6ca035f7 100644
--- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
@@ -25,6 +25,7 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.io.CharStreams;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo;
 import org.apache.spark.network.util.MapConfigProvider;
 import org.apache.spark.network.util.TransportConf;
@@ -146,12 +147,19 @@ public void jsonSerializationOfExecutorRegistration() throws IOException {
 
   @Test
   public void testNormalizeAndInternPathname() {
-    assertPathsMatch("/foo", "bar", "baz", "/foo/bar/baz");
-    assertPathsMatch("//foo/", "bar/", "//baz", "/foo/bar/baz");
-    assertPathsMatch("foo", "bar", "baz///", "foo/bar/baz");
-    assertPathsMatch("/foo/", "/bar//", "/baz", "/foo/bar/baz");
-    assertPathsMatch("/", "", "", "/");
-    assertPathsMatch("/", "/", "/", "/");
+    String sep = File.separator;
+    String expectedPathname = sep + "foo" + sep + "bar" + sep + "baz";
+    assertPathsMatch("/foo", "bar", "baz", expectedPathname);
+    assertPathsMatch("//foo/", "bar/", "//baz", expectedPathname);
+    assertPathsMatch("/foo/", "/bar//", "/baz", expectedPathname);
+    assertPathsMatch("foo", "bar", "baz///", "foo" + sep + "bar" + sep + "baz");
+    assertPathsMatch("/", "", "", sep);
+    assertPathsMatch("/", "/", "/", sep);
+    if (SystemUtils.IS_OS_WINDOWS) {
+      assertPathsMatch("/foo\\/", "bar", "baz", expectedPathname);
+    } else {
+      assertPathsMatch("/foo\\/", "bar", "baz", sep + "foo\\" + sep + "bar" + sep + "baz");
+    }
   }
 
   private void assertPathsMatch(String p1, String p2, String p3, String expectedPathname) {
@@ -160,6 +168,6 @@ private void assertPathsMatch(String p1, String p2, String p3, String expectedPa
     assertEquals(expectedPathname, normPathname);
     File file = new File(normPathname);
     String returnedPath = file.getPath();
-    assertTrue(normPathname == returnedPath);
+    assertEquals(normPathname, returnedPath);
   }
 }
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 76a402bb2bd31..5c5dfbabe862e 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
index 815a56d765b6a..c41efbad8ffec 100644
--- a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
+++ b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
@@ -188,7 +188,7 @@ protected void serviceInit(Configuration conf) throws Exception {
 
       int port = conf.getInt(
         SPARK_SHUFFLE_SERVICE_PORT_KEY, DEFAULT_SPARK_SHUFFLE_SERVICE_PORT);
-      transportContext = new TransportContext(transportConf, blockHandler);
+      transportContext = new TransportContext(transportConf, blockHandler, true);
       shuffleServer = transportContext.createServer(port, bootstraps);
       // the port should normally be fixed, but for tests its useful to find an open port
       port = shuffleServer.getPort();
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 3c3c0d2d96a1c..8041d7a093da9 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 883b73a69c9de..e3a550288d71a 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala b/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java
similarity index 72%
rename from sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala
rename to common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java
index 5fc7123c9097b..e3fed3d656d20 100644
--- a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala
+++ b/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java
@@ -15,15 +15,13 @@
  * limitations under the License.
  */
 
-package org.apache.spark.status.api.v1.sql
+package org.apache.spark.tags;
 
-import javax.ws.rs.Path
+import java.lang.annotation.*;
 
-import org.apache.spark.status.api.v1.ApiRequestContext
+import org.scalatest.TagAnnotation;
 
-@Path("/v1")
-private[v1] class ApiSqlRootResource extends ApiRequestContext {
-
-  @Path("applications/{appId}/sql")
-  def sqlList(): Class[SqlResource] = classOf[SqlResource]
-}
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface ChromeUITest { }
diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala b/common/tags/src/test/java/org/apache/spark/tags/GitHubActionsUnstableTest.java
similarity index 63%
rename from sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala
rename to common/tags/src/test/java/org/apache/spark/tags/GitHubActionsUnstableTest.java
index 7ace66ffb06e1..a602656bb22f8 100644
--- a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala
+++ b/common/tags/src/test/java/org/apache/spark/tags/GitHubActionsUnstableTest.java
@@ -14,20 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.status.api.v1.sql
 
-import java.util.Date
+package org.apache.spark.tags;
 
-class ExecutionData private[spark] (
-    val id: Long,
-    val status: String,
-    val description: String,
-    val planDescription: String,
-    val metrics: Seq[Metrics],
-    val submissionTime: Date,
-    val duration: Long,
-    val runningJobIds: Seq[Int],
-    val successJobIds: Seq[Int],
-    val failedJobIds: Seq[Int])
+import org.scalatest.TagAnnotation;
 
-case class Metrics private[spark] (metricName: String, metricValue: String)
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface GitHubActionsUnstableTest { }
diff --git a/common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java b/common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java
new file mode 100644
index 0000000000000..a7e6f352667d7
--- /dev/null
+++ b/common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.tags;
+
+import org.scalatest.TagAnnotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface SlowHiveTest { }
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index 93a4f67fd23f2..09965b573706c 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/DateTimeConstants.java b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/DateTimeConstants.java
index 84a0156ebfb66..0ae238564d591 100644
--- a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/DateTimeConstants.java
+++ b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/DateTimeConstants.java
@@ -19,15 +19,9 @@
 
 public class DateTimeConstants {
 
-  public static final int YEARS_PER_DECADE = 10;
-  public static final int YEARS_PER_CENTURY = 100;
-  public static final int YEARS_PER_MILLENNIUM = 1000;
-
-  public static final byte MONTHS_PER_QUARTER = 3;
   public static final int MONTHS_PER_YEAR = 12;
 
   public static final byte DAYS_PER_WEEK = 7;
-  public static final long DAYS_PER_MONTH = 30L;
 
   public static final long HOURS_PER_DAY = 24L;
 
@@ -47,9 +41,6 @@ public class DateTimeConstants {
   public static final long MICROS_PER_MINUTE = SECONDS_PER_MINUTE * MICROS_PER_SECOND;
   public static final long MICROS_PER_HOUR = MINUTES_PER_HOUR * MICROS_PER_MINUTE;
   public static final long MICROS_PER_DAY = HOURS_PER_DAY * MICROS_PER_HOUR;
-  public static final long MICROS_PER_MONTH = DAYS_PER_MONTH * MICROS_PER_DAY;
-  /* 365.25 days per year assumes leap year every four years */
-  public static final long MICROS_PER_YEAR = (36525L * MICROS_PER_DAY) / 100;
 
   public static final long NANOS_PER_MICROS = 1000L;
   public static final long NANOS_PER_MILLIS = MICROS_PER_MILLIS * NANOS_PER_MICROS;
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/UnsafeAlignedOffset.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/UnsafeAlignedOffset.java
index 546e8780a6606..d399e66aa2a5f 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/UnsafeAlignedOffset.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/UnsafeAlignedOffset.java
@@ -28,12 +28,20 @@ public class UnsafeAlignedOffset {
 
   private static final int UAO_SIZE = Platform.unaligned() ? 4 : 8;
 
+  private static int TEST_UAO_SIZE = 0;
+
+  // used for test only
+  public static void setUaoSize(int size) {
+    assert size == 0 || size == 4 || size == 8;
+    TEST_UAO_SIZE = size;
+  }
+
   public static int getUaoSize() {
-    return UAO_SIZE;
+    return TEST_UAO_SIZE == 0 ? UAO_SIZE : TEST_UAO_SIZE;
   }
 
   public static int getSize(Object object, long offset) {
-    switch (UAO_SIZE) {
+    switch (getUaoSize()) {
       case 4:
         return Platform.getInt(object, offset);
       case 8:
@@ -46,7 +54,7 @@ public static int getSize(Object object, long offset) {
   }
 
   public static void putSize(Object object, long offset, int value) {
-    switch (UAO_SIZE) {
+    switch (getUaoSize()) {
       case 4:
         Platform.putInt(object, offset, value);
         break;
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index c5384669eb922..b8dda22240042 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -341,8 +341,17 @@ public UTF8String substringSQL(int pos, int length) {
     // to the -ith element before the end of the sequence. If a start index i is 0, it
     // refers to the first element.
     int len = numChars();
+    // `len + pos` does not overflow as `len >= 0`.
     int start = (pos > 0) ? pos -1 : ((pos < 0) ? len + pos : 0);
-    int end = (length == Integer.MAX_VALUE) ? len : start + length;
+
+    int end;
+    if ((long) start + length > Integer.MAX_VALUE) {
+      end = Integer.MAX_VALUE;
+    } else if ((long) start + length < Integer.MIN_VALUE) {
+      end = Integer.MIN_VALUE;
+    } else {
+      end = start + length;
+    }
     return substring(start, end);
   }
 
@@ -554,7 +563,7 @@ public UTF8String trim() {
   }
 
   /**
-   * Trims whitespaces (<= ASCII 32) from both ends of this string.
+   * Trims whitespaces ({@literal <=} ASCII 32) from both ends of this string.
    *
    * Note that, this method is the same as java's {@link String#trim}, and different from
    * {@link UTF8String#trim()} which remove only spaces(= ASCII 32) from both ends.
@@ -566,14 +575,14 @@ public UTF8String trim() {
   public UTF8String trimAll() {
     int s = 0;
     // skip all of the whitespaces (<=0x20) in the left side
-    while (s < this.numBytes && getByte(s) <= ' ') s++;
+    while (s < this.numBytes && Character.isWhitespace(getByte(s))) s++;
     if (s == this.numBytes) {
       // Everything trimmed
       return EMPTY_UTF8;
     }
     // skip all of the whitespaces (<=0x20) in the right side
     int e = this.numBytes - 1;
-    while (e > s && getByte(e) <= ' ') e--;
+    while (e > s && Character.isWhitespace(getByte(e))) e--;
     if (s == 0 && e == numBytes - 1) {
       // Nothing trimmed
       return this;
@@ -1105,12 +1114,16 @@ public static class IntWrapper implements Serializable {
    * @return true if the parsing was successful else false
    */
   public boolean toLong(LongWrapper toLongResult) {
+    return toLong(toLongResult, true);
+  }
+
+  private boolean toLong(LongWrapper toLongResult, boolean allowDecimal) {
     int offset = 0;
-    while (offset < this.numBytes && getByte(offset) <= ' ') offset++;
+    while (offset < this.numBytes && Character.isWhitespace(getByte(offset))) offset++;
     if (offset == this.numBytes) return false;
 
     int end = this.numBytes - 1;
-    while (end > offset && getByte(end) <= ' ') end--;
+    while (end > offset && Character.isWhitespace(getByte(end))) end--;
 
     byte b = getByte(offset);
     final boolean negative = b == '-';
@@ -1129,7 +1142,7 @@ public boolean toLong(LongWrapper toLongResult) {
     while (offset <= end) {
       b = getByte(offset);
       offset++;
-      if (b == separator) {
+      if (b == separator && allowDecimal) {
         // We allow decimals and will return a truncated integral in that case.
         // Therefore we won't throw an exception here (checking the fractional
         // part happens below.)
@@ -1198,12 +1211,16 @@ public boolean toLong(LongWrapper toLongResult) {
    * @return true if the parsing was successful else false
    */
   public boolean toInt(IntWrapper intWrapper) {
+    return toInt(intWrapper, true);
+  }
+
+  private boolean toInt(IntWrapper intWrapper, boolean allowDecimal) {
     int offset = 0;
-    while (offset < this.numBytes && getByte(offset) <= ' ') offset++;
+    while (offset < this.numBytes && Character.isWhitespace(getByte(offset))) offset++;
     if (offset == this.numBytes) return false;
 
     int end = this.numBytes - 1;
-    while (end > offset && getByte(end) <= ' ') end--;
+    while (end > offset && Character.isWhitespace(getByte(end))) end--;
 
     byte b = getByte(offset);
     final boolean negative = b == '-';
@@ -1222,7 +1239,7 @@ public boolean toInt(IntWrapper intWrapper) {
     while (offset <= end) {
       b = getByte(offset);
       offset++;
-      if (b == separator) {
+      if (b == separator && allowDecimal) {
         // We allow decimals and will return a truncated integral in that case.
         // Therefore we won't throw an exception here (checking the fractional
         // part happens below.)
@@ -1276,9 +1293,7 @@ public boolean toShort(IntWrapper intWrapper) {
     if (toInt(intWrapper)) {
       int intValue = intWrapper.value;
       short result = (short) intValue;
-      if (result == intValue) {
-        return true;
-      }
+      return result == intValue;
     }
     return false;
   }
@@ -1287,9 +1302,7 @@ public boolean toByte(IntWrapper intWrapper) {
     if (toInt(intWrapper)) {
       int intValue = intWrapper.value;
       byte result = (byte) intValue;
-      if (result == intValue) {
-        return true;
-      }
+      return result == intValue;
     }
     return false;
   }
@@ -1302,7 +1315,7 @@ public boolean toByte(IntWrapper intWrapper) {
    */
   public long toLongExact() {
     LongWrapper result = new LongWrapper();
-    if (toLong(result)) {
+    if (toLong(result, false)) {
       return result.value;
     }
     throw new NumberFormatException("invalid input syntax for type numeric: " + this);
@@ -1316,7 +1329,7 @@ public long toLongExact() {
    */
   public int toIntExact() {
     IntWrapper result = new IntWrapper();
-    if (toInt(result)) {
+    if (toInt(result, false)) {
       return result.value;
     }
     throw new NumberFormatException("invalid input syntax for type numeric: " + this);
diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
index 8f933877f82e6..70e276f7e5a8b 100644
--- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
+++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
@@ -390,6 +390,10 @@ public void substringSQL() {
     assertEquals(fromString("example"), e.substringSQL(0, Integer.MAX_VALUE));
     assertEquals(fromString("example"), e.substringSQL(1, Integer.MAX_VALUE));
     assertEquals(fromString("xample"), e.substringSQL(2, Integer.MAX_VALUE));
+    assertEquals(EMPTY_UTF8, e.substringSQL(-100, -100));
+    assertEquals(EMPTY_UTF8, e.substringSQL(-1207959552, -1207959552));
+    assertEquals(fromString("pl"), e.substringSQL(-3, 2));
+    assertEquals(EMPTY_UTF8, e.substringSQL(Integer.MIN_VALUE, 6));
   }
 
   @Test
diff --git a/core/pom.xml b/core/pom.xml
index 9d54d21b95ba3..19b5b30051602 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
@@ -414,7 +414,7 @@
     <dependency>
       <groupId>net.sf.py4j</groupId>
       <artifactId>py4j</artifactId>
-      <version>0.10.8.1</version>
+      <version>0.10.9</version>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
diff --git a/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java b/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java
index 0c0d0df8ae682..1d676ff781c70 100644
--- a/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java
+++ b/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java
@@ -41,7 +41,7 @@ public interface DriverPlugin {
    * initialization.
    * <p>
    * It's recommended that plugins be careful about what operations are performed in this call,
-   * preferrably performing expensive operations in a separate thread, or postponing them until
+   * preferably performing expensive operations in a separate thread, or postponing them until
    * the application has fully started.
    *
    * @param sc The SparkContext loading the plugin.
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
index d09282e61a9c7..3eee1e478616d 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
@@ -18,6 +18,7 @@
 package org.apache.spark.shuffle.sort;
 
 import java.nio.channels.Channels;
+import java.util.Arrays;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import java.io.*;
@@ -274,6 +275,8 @@ private long[] mergeSpills(SpillInfo[] spills) throws IOException {
         // Here, we don't need to perform any metrics updates because the bytes written to this
         // output file would have already been counted as shuffle bytes written.
         partitionLengths = spills[0].partitionLengths;
+        logger.debug("Merge shuffle spills for mapId {} with length {}", mapId,
+            partitionLengths.length);
         maybeSingleFileWriter.get().transferMapSpillFile(spills[0].file, partitionLengths);
       } else {
         partitionLengths = mergeSpillsUsingStandardWriter(spills);
@@ -360,6 +363,7 @@ private void mergeSpillsWithFileStream(
       SpillInfo[] spills,
       ShuffleMapOutputWriter mapWriter,
       @Nullable CompressionCodec compressionCodec) throws IOException {
+    logger.debug("Merge shuffle spills with FileStream for mapId {}", mapId);
     final int numPartitions = partitioner.numPartitions();
     final InputStream[] spillInputStreams = new InputStream[spills.length];
 
@@ -369,6 +373,11 @@ private void mergeSpillsWithFileStream(
         spillInputStreams[i] = new NioBufferedFileInputStream(
           spills[i].file,
           inputBufferSizeInBytes);
+        // Only convert the partitionLengths when debug level is enabled.
+        if (logger.isDebugEnabled()) {
+          logger.debug("Partition lengths for mapId {} in Spill {}: {}", mapId, i,
+              Arrays.toString(spills[i].partitionLengths));
+        }
       }
       for (int partition = 0; partition < numPartitions; partition++) {
         boolean copyThrewException = true;
@@ -431,6 +440,7 @@ private void mergeSpillsWithFileStream(
   private void mergeSpillsWithTransferTo(
       SpillInfo[] spills,
       ShuffleMapOutputWriter mapWriter) throws IOException {
+    logger.debug("Merge shuffle spills with TransferTo for mapId {}", mapId);
     final int numPartitions = partitioner.numPartitions();
     final FileChannel[] spillInputChannels = new FileChannel[spills.length];
     final long[] spillInputChannelPositions = new long[spills.length];
@@ -439,6 +449,11 @@ private void mergeSpillsWithTransferTo(
     try {
       for (int i = 0; i < spills.length; i++) {
         spillInputChannels[i] = new FileInputStream(spills[i].file).getChannel();
+        // Only convert the partitionLengths when debug level is enabled.
+        if (logger.isDebugEnabled()) {
+          logger.debug("Partition lengths for mapId {} in Spill {}: {}", mapId, i,
+              Arrays.toString(spills[i].partitionLengths));
+        }
       }
       for (int partition = 0; partition < numPartitions; partition++) {
         boolean copyThrewException = true;
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
index a6529fd76188a..1c3eb34f9be37 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
@@ -112,6 +112,8 @@ public long[] commitAllPartitions() throws IOException {
     }
     cleanUp();
     File resolvedTmp = outputTempFile != null && outputTempFile.isFile() ? outputTempFile : null;
+    log.debug("Writing shuffle index file for mapId {} with length {}", mapId,
+        partitionLengths.length);
     blockResolver.writeIndexFileAndCommit(shuffleId, mapId, partitionLengths, resolvedTmp);
     return partitionLengths;
   }
@@ -210,14 +212,14 @@ public long getNumBytesWritten() {
 
   private class PartitionWriterStream extends OutputStream {
     private final int partitionId;
-    private int count = 0;
+    private long count = 0;
     private boolean isClosed = false;
 
     PartitionWriterStream(int partitionId) {
       this.partitionId = partitionId;
     }
 
-    public int getCount() {
+    public long getCount() {
       return count;
     }
 
diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 7bdd89488d119..888cc8c3a6cd6 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -54,13 +54,13 @@
  * probably be using sorting instead of hashing for better cache locality.
  *
  * The key and values under the hood are stored together, in the following format:
- *   Bytes 0 to 4: len(k) (key length in bytes) + len(v) (value length in bytes) + 4
- *   Bytes 4 to 8: len(k)
- *   Bytes 8 to 8 + len(k): key data
- *   Bytes 8 + len(k) to 8 + len(k) + len(v): value data
- *   Bytes 8 + len(k) + len(v) to 8 + len(k) + len(v) + 8: pointer to next pair
+ *   First uaoSize bytes: len(k) (key length in bytes) + len(v) (value length in bytes) + uaoSize
+ *   Next uaoSize bytes: len(k)
+ *   Next len(k) bytes: key data
+ *   Next len(v) bytes: value data
+ *   Last 8 bytes: pointer to next pair
  *
- * This means that the first four bytes store the entire record (key + value) length. This format
+ * It means first uaoSize bytes store the entire record (key + value + uaoSize) length. This format
  * is compatible with {@link org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter},
  * so we can pass records from this map directly into the sorter to sort records in place.
  */
@@ -96,8 +96,7 @@ public final class BytesToBytesMap extends MemoryConsumer {
    * since that's the largest power-of-2 that's less than Integer.MAX_VALUE. We need two long array
    * entries per key, giving us a maximum capacity of (1 &lt;&lt; 29).
    */
-  @VisibleForTesting
-  static final int MAX_CAPACITY = (1 << 29);
+  public static final int MAX_CAPACITY = (1 << 29);
 
   // This choice of page table size and page size means that we can address up to 500 gigabytes
   // of memory.
@@ -394,10 +393,12 @@ public void remove() {
     }
 
     private void handleFailedDelete() {
-      // remove the spill file from disk
-      File file = spillWriters.removeFirst().getFile();
-      if (file != null && file.exists() && !file.delete()) {
-        logger.error("Was unable to delete spill file {}", file.getAbsolutePath());
+      if (spillWriters.size() > 0) {
+        // remove the spill file from disk
+        File file = spillWriters.removeFirst().getFile();
+        if (file != null && file.exists() && !file.delete()) {
+          logger.error("Was unable to delete spill file {}", file.getAbsolutePath());
+        }
       }
     }
   }
@@ -407,17 +408,10 @@ private void handleFailedDelete() {
    *
    * For efficiency, all calls to `next()` will return the same {@link Location} object.
    *
-   * If any other lookups or operations are performed on this map while iterating over it, including
-   * `lookup()`, the behavior of the returned iterator is undefined.
+   * The returned iterator is thread-safe. However if the map is modified while iterating over it,
+   * the behavior of the returned iterator is undefined.
    */
   public MapIterator iterator() {
-    return new MapIterator(numValues, loc, false);
-  }
-
-  /**
-   * Returns a thread safe iterator that iterates of the entries of this map.
-   */
-  public MapIterator safeIterator() {
     return new MapIterator(numValues, new Location(), false);
   }
 
@@ -428,19 +422,20 @@ public MapIterator safeIterator() {
    *
    * For efficiency, all calls to `next()` will return the same {@link Location} object.
    *
-   * If any other lookups or operations are performed on this map while iterating over it, including
-   * `lookup()`, the behavior of the returned iterator is undefined.
+   * The returned iterator is thread-safe. However if the map is modified while iterating over it,
+   * the behavior of the returned iterator is undefined.
    */
   public MapIterator destructiveIterator() {
     updatePeakMemoryUsed();
-    return new MapIterator(numValues, loc, true);
+    return new MapIterator(numValues, new Location(), true);
   }
 
   /**
    * Looks up a key, and return a {@link Location} handle that can be used to test existence
    * and read/write values.
    *
-   * This function always return the same {@link Location} instance to avoid object allocation.
+   * This function always returns the same {@link Location} instance to avoid object allocation.
+   * This function is not thread-safe.
    */
   public Location lookup(Object keyBase, long keyOffset, int keyLength) {
     safeLookup(keyBase, keyOffset, keyLength, loc,
@@ -452,7 +447,8 @@ public Location lookup(Object keyBase, long keyOffset, int keyLength) {
    * Looks up a key, and return a {@link Location} handle that can be used to test existence
    * and read/write values.
    *
-   * This function always return the same {@link Location} instance to avoid object allocation.
+   * This function always returns the same {@link Location} instance to avoid object allocation.
+   * This function is not thread-safe.
    */
   public Location lookup(Object keyBase, long keyOffset, int keyLength, int hash) {
     safeLookup(keyBase, keyOffset, keyLength, loc, hash);
@@ -707,7 +703,7 @@ public boolean append(Object kbase, long koff, int klen, Object vbase, long voff
       // Here, we'll copy the data into our data pages. Because we only store a relative offset from
       // the key address instead of storing the absolute address of the value, the key and value
       // must be stored in the same memory page.
-      // (8 byte key length) (key) (value) (8 byte pointer to next value)
+      // (total length) (key length) (key) (value) (8 byte pointer to next value)
       int uaoSize = UnsafeAlignedOffset.getUaoSize();
       final long recordLength = (2L * uaoSize) + klen + vlen + 8;
       if (currentPage == null || currentPage.size() - pageCursor < recordLength) {
@@ -744,12 +740,21 @@ public boolean append(Object kbase, long koff, int klen, Object vbase, long voff
         longArray.set(pos * 2 + 1, keyHashcode);
         isDefined = true;
 
-        // We use two array entries per key, so the array size is twice the capacity.
-        // We should compare the current capacity of the array, instead of its size.
-        if (numKeys >= growthThreshold && longArray.size() / 2 < MAX_CAPACITY) {
-          try {
-            growAndRehash();
-          } catch (SparkOutOfMemoryError oom) {
+        // If the map has reached its growth threshold, try to grow it.
+        if (numKeys >= growthThreshold) {
+          // We use two array entries per key, so the array size is twice the capacity.
+          // We should compare the current capacity of the array, instead of its size.
+          if (longArray.size() / 2 < MAX_CAPACITY) {
+            try {
+              growAndRehash();
+            } catch (SparkOutOfMemoryError oom) {
+              canGrowArray = false;
+            }
+          } else {
+            // The map is already at MAX_CAPACITY and cannot grow. Instead, we prevent it from
+            // accepting any more new elements to make sure we don't exceed the load factor. If we
+            // need to spill later, this allows UnsafeKVExternalSorter to reuse the array for
+            // sorting.
             canGrowArray = false;
           }
         }
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
index 55e4e609c3c7b..e4e369baf9dfa 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
@@ -104,11 +104,14 @@ public static UnsafeExternalSorter createWithExistingInMemorySorter(
       int initialSize,
       long pageSizeBytes,
       int numElementsForSpillThreshold,
-      UnsafeInMemorySorter inMemorySorter) throws IOException {
+      UnsafeInMemorySorter inMemorySorter,
+      long existingMemoryConsumption) throws IOException {
     UnsafeExternalSorter sorter = new UnsafeExternalSorter(taskMemoryManager, blockManager,
       serializerManager, taskContext, recordComparatorSupplier, prefixComparator, initialSize,
         pageSizeBytes, numElementsForSpillThreshold, inMemorySorter, false /* ignored */);
     sorter.spill(Long.MAX_VALUE, sorter);
+    taskContext.taskMetrics().incMemoryBytesSpilled(existingMemoryConsumption);
+    sorter.totalSpillBytes += existingMemoryConsumption;
     // The external sorter will be used to insert records, in-memory sorter is not needed.
     sorter.inMemSorter = null;
     return sorter;
@@ -203,6 +206,10 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
     }
 
     if (inMemSorter == null || inMemSorter.numRecords() <= 0) {
+      // There could still be some memory allocated when there are no records in the in-memory
+      // sorter. We will not spill it however, to ensure that we can always process at least one
+      // record before spilling. See the comments in `allocateMemoryForRecordIfNecessary` for why
+      // this is necessary.
       return 0L;
     }
 
@@ -224,7 +231,7 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
     // Note that this is more-or-less going to be a multiple of the page size, so wasted space in
     // pages will currently be counted as memory spilled even though that space isn't actually
     // written to disk. This also counts the space needed to store the sorter's pointer array.
-    inMemSorter.reset();
+    inMemSorter.freeMemory();
     // Reset the in-memory sorter's pointer array only after freeing up the memory pages holding the
     // records. Otherwise, if the task is over allocated memory, then without freeing the memory
     // pages, we might not be able to get memory for the pointer array.
@@ -325,7 +332,7 @@ public void cleanupResources() {
       deleteSpillFiles();
       freeMemory();
       if (inMemSorter != null) {
-        inMemSorter.free();
+        inMemSorter.freeMemory();
         inMemSorter = null;
       }
     }
@@ -339,40 +346,53 @@ public void cleanupResources() {
   private void growPointerArrayIfNecessary() throws IOException {
     assert(inMemSorter != null);
     if (!inMemSorter.hasSpaceForAnotherRecord()) {
+      if (inMemSorter.numRecords() <= 0) {
+        // Spilling was triggered just before this method was called. The pointer array was freed
+        // during the spill, so a new pointer array needs to be allocated here.
+        LongArray array = allocateArray(inMemSorter.getInitialSize());
+        inMemSorter.expandPointerArray(array);
+        return;
+      }
+
       long used = inMemSorter.getMemoryUsage();
-      LongArray array;
+      LongArray array = null;
       try {
         // could trigger spilling
         array = allocateArray(used / 8 * 2);
       } catch (TooLargePageException e) {
         // The pointer array is too big to fix in a single page, spill.
         spill();
-        return;
       } catch (SparkOutOfMemoryError e) {
-        // should have trigger spilling
-        if (!inMemSorter.hasSpaceForAnotherRecord()) {
+        if (inMemSorter.numRecords() > 0) {
           logger.error("Unable to grow the pointer array");
           throw e;
         }
-        return;
+        // The new array could not be allocated, but that is not an issue as it is longer needed,
+        // as all records were spilled.
       }
-      // check if spilling is triggered or not
-      if (inMemSorter.hasSpaceForAnotherRecord()) {
-        freeArray(array);
-      } else {
-        inMemSorter.expandPointerArray(array);
+
+      if (inMemSorter.numRecords() <= 0) {
+        // Spilling was triggered while trying to allocate the new array.
+        if (array != null) {
+          // We succeeded in allocating the new array, but, since all records were spilled, a
+          // smaller array would also suffice.
+          freeArray(array);
+        }
+        // The pointer array was freed during the spill, so a new pointer array needs to be
+        // allocated here.
+        array = allocateArray(inMemSorter.getInitialSize());
       }
+      inMemSorter.expandPointerArray(array);
     }
   }
 
   /**
-   * Allocates more memory in order to insert an additional record. This will request additional
-   * memory from the memory manager and spill if the requested memory can not be obtained.
+   * Allocates an additional page in order to insert an additional record. This will request
+   * additional memory from the memory manager and spill if the requested memory can not be
+   * obtained.
    *
    * @param required the required space in the data page, in bytes, including space for storing
-   *                      the record size. This must be less than or equal to the page size (records
-   *                      that exceed the page size are handled via a different code path which uses
-   *                      special overflow pages).
+   *                 the record size.
    */
   private void acquireNewPageIfNecessary(int required) {
     if (currentPage == null ||
@@ -384,6 +404,37 @@ private void acquireNewPageIfNecessary(int required) {
     }
   }
 
+  /**
+   * Allocates more memory in order to insert an additional record. This will request additional
+   * memory from the memory manager and spill if the requested memory can not be obtained.
+   *
+   * @param required the required space in the data page, in bytes, including space for storing
+   *                 the record size.
+   */
+  private void allocateMemoryForRecordIfNecessary(int required) throws IOException {
+    // Step 1:
+    // Ensure that the pointer array has space for another record. This may cause a spill.
+    growPointerArrayIfNecessary();
+    // Step 2:
+    // Ensure that the last page has space for another record. This may cause a spill.
+    acquireNewPageIfNecessary(required);
+    // Step 3:
+    // The allocation in step 2 could have caused a spill, which would have freed the pointer
+    // array allocated in step 1. Therefore we need to check again whether we have to allocate
+    // a new pointer array.
+    //
+    // If the allocation in this step causes a spill event then it will not cause the page
+    // allocated in the previous step to be freed. The function `spill` only frees memory if at
+    // least one record has been inserted in the in-memory sorter. This will not be the case if
+    // we have spilled in the previous step.
+    //
+    // If we did not spill in the previous step then `growPointerArrayIfNecessary` will be a
+    // no-op that does not allocate any memory, and therefore can't cause a spill event.
+    //
+    // Thus there is no need to call `acquireNewPageIfNecessary` again after this step.
+    growPointerArrayIfNecessary();
+  }
+
   /**
    * Write a record to the sorter.
    */
@@ -398,11 +449,10 @@ public void insertRecord(
       spill();
     }
 
-    growPointerArrayIfNecessary();
-    int uaoSize = UnsafeAlignedOffset.getUaoSize();
+    final int uaoSize = UnsafeAlignedOffset.getUaoSize();
     // Need 4 or 8 bytes to store the record length.
     final int required = length + uaoSize;
-    acquireNewPageIfNecessary(required);
+    allocateMemoryForRecordIfNecessary(required);
 
     final Object base = currentPage.getBaseObject();
     final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor);
@@ -425,10 +475,9 @@ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen,
       Object valueBase, long valueOffset, int valueLen, long prefix, boolean prefixIsNull)
     throws IOException {
 
-    growPointerArrayIfNecessary();
-    int uaoSize = UnsafeAlignedOffset.getUaoSize();
+    final int uaoSize = UnsafeAlignedOffset.getUaoSize();
     final int required = keyLen + valueLen + (2 * uaoSize);
-    acquireNewPageIfNecessary(required);
+    allocateMemoryForRecordIfNecessary(required);
 
     final Object base = currentPage.getBaseObject();
     final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor);
@@ -450,6 +499,7 @@ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen,
    */
   public void merge(UnsafeExternalSorter other) throws IOException {
     other.spill();
+    totalSpillBytes += other.totalSpillBytes;
     spillWriters.addAll(other.spillWriters);
     // remove them from `spillWriters`, or the files will be deleted in `cleanupResources`.
     other.spillWriters.clear();
@@ -501,11 +551,15 @@ private static void spillIterator(UnsafeSorterIterator inMemIterator,
    */
   class SpillableIterator extends UnsafeSorterIterator {
     private UnsafeSorterIterator upstream;
-    private UnsafeSorterIterator nextUpstream = null;
     private MemoryBlock lastPage = null;
     private boolean loaded = false;
     private int numRecords = 0;
 
+    private Object currentBaseObject;
+    private long currentBaseOffset;
+    private int currentRecordLength;
+    private long currentKeyPrefix;
+
     SpillableIterator(UnsafeSorterIterator inMemIterator) {
       this.upstream = inMemIterator;
       this.numRecords = inMemIterator.getNumRecords();
@@ -516,23 +570,26 @@ public int getNumRecords() {
       return numRecords;
     }
 
+    @Override
+    public long getCurrentPageNumber() {
+      throw new UnsupportedOperationException();
+    }
+
     public long spill() throws IOException {
       synchronized (this) {
-        if (!(upstream instanceof UnsafeInMemorySorter.SortedIterator && nextUpstream == null
-          && numRecords > 0)) {
+        if (inMemSorter == null || numRecords <= 0) {
           return 0L;
         }
 
-        UnsafeInMemorySorter.SortedIterator inMemIterator =
-          ((UnsafeInMemorySorter.SortedIterator) upstream).clone();
+        long currentPageNumber = upstream.getCurrentPageNumber();
 
-       ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics();
+        ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics();
         // Iterate over the records that have not been returned and spill them.
         final UnsafeSorterSpillWriter spillWriter =
           new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes, writeMetrics, numRecords);
-        spillIterator(inMemIterator, spillWriter);
+        spillIterator(upstream, spillWriter);
         spillWriters.add(spillWriter);
-        nextUpstream = spillWriter.getReader(serializerManager);
+        upstream = spillWriter.getReader(serializerManager);
 
         long released = 0L;
         synchronized (UnsafeExternalSorter.this) {
@@ -540,8 +597,7 @@ public long spill() throws IOException {
           // is accessing the current record. We free this page in that caller's next loadNext()
           // call.
           for (MemoryBlock page : allocatedPages) {
-            if (!loaded || page.pageNumber !=
-                    ((UnsafeInMemorySorter.SortedIterator)upstream).getCurrentPageNumber()) {
+            if (!loaded || page.pageNumber != currentPageNumber) {
               released += page.size();
               freePage(page);
             } else {
@@ -555,7 +611,7 @@ public long spill() throws IOException {
         assert(inMemSorter != null);
         released += inMemSorter.getMemoryUsage();
         totalSortTimeNanos += inMemSorter.getSortTimeNanos();
-        inMemSorter.free();
+        inMemSorter.freeMemory();
         inMemSorter = null;
         taskContext.taskMetrics().incMemoryBytesSpilled(released);
         taskContext.taskMetrics().incDiskBytesSpilled(writeMetrics.bytesWritten());
@@ -575,22 +631,26 @@ public void loadNext() throws IOException {
       try {
         synchronized (this) {
           loaded = true;
-          if (nextUpstream != null) {
-            // Just consumed the last record from in memory iterator
-            if(lastPage != null) {
-              // Do not free the page here, while we are locking `SpillableIterator`. The `freePage`
-              // method locks the `TaskMemoryManager`, and it's a bad idea to lock 2 objects in
-              // sequence. We may hit dead lock if another thread locks `TaskMemoryManager` and
-              // `SpillableIterator` in sequence, which may happen in
-              // `TaskMemoryManager.acquireExecutionMemory`.
-              pageToFree = lastPage;
-              lastPage = null;
-            }
-            upstream = nextUpstream;
-            nextUpstream = null;
+          // Just consumed the last record from in memory iterator
+          if (lastPage != null) {
+            // Do not free the page here, while we are locking `SpillableIterator`. The `freePage`
+            // method locks the `TaskMemoryManager`, and it's a bad idea to lock 2 objects in
+            // sequence. We may hit dead lock if another thread locks `TaskMemoryManager` and
+            // `SpillableIterator` in sequence, which may happen in
+            // `TaskMemoryManager.acquireExecutionMemory`.
+            pageToFree = lastPage;
+            lastPage = null;
           }
           numRecords--;
           upstream.loadNext();
+
+          // Keep track of the current base object, base offset, record length, and key prefix,
+          // so that the current record can still be read in case a spill is triggered and we
+          // switch to the spill writer's iterator.
+          currentBaseObject = upstream.getBaseObject();
+          currentBaseOffset = upstream.getBaseOffset();
+          currentRecordLength = upstream.getRecordLength();
+          currentKeyPrefix = upstream.getKeyPrefix();
         }
       } finally {
         if (pageToFree != null) {
@@ -601,22 +661,22 @@ public void loadNext() throws IOException {
 
     @Override
     public Object getBaseObject() {
-      return upstream.getBaseObject();
+      return currentBaseObject;
     }
 
     @Override
     public long getBaseOffset() {
-      return upstream.getBaseOffset();
+      return currentBaseOffset;
     }
 
     @Override
     public int getRecordLength() {
-      return upstream.getRecordLength();
+      return currentRecordLength;
     }
 
     @Override
     public long getKeyPrefix() {
-      return upstream.getKeyPrefix();
+      return currentKeyPrefix;
     }
   }
 
@@ -646,7 +706,7 @@ public UnsafeSorterIterator getIterator(int startIndex) throws IOException {
         }
         i += spillWriter.recordsSpilled();
       }
-      if (inMemSorter != null) {
+      if (inMemSorter != null && inMemSorter.numRecords() > 0) {
         UnsafeSorterIterator iter = inMemSorter.getSortedIterator();
         moveOver(iter, startIndex - i);
         queue.add(iter);
@@ -693,6 +753,11 @@ public int getNumRecords() {
       return numRecords;
     }
 
+    @Override
+    public long getCurrentPageNumber() {
+      return current.getCurrentPageNumber();
+    }
+
     @Override
     public boolean hasNext() {
       while (!current.hasNext() && !iterators.isEmpty()) {
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
index e14964d68119b..33be899b6b438 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
@@ -159,32 +159,26 @@ private int getUsableCapacity() {
     return (int) (array.size() / (radixSortSupport != null ? 2 : 1.5));
   }
 
+  public long getInitialSize() {
+    return initialSize;
+  }
+
   /**
    * Free the memory used by pointer array.
    */
-  public void free() {
+  public void freeMemory() {
     if (consumer != null) {
       if (array != null) {
         consumer.freeArray(array);
       }
-      array = null;
-    }
-  }
 
-  public void reset() {
-    if (consumer != null) {
-      consumer.freeArray(array);
-      // the call to consumer.allocateArray may trigger a spill which in turn access this instance
-      // and eventually re-enter this method and try to free the array again.  by setting the array
-      // to null and its length to 0 we effectively make the spill code-path a no-op.  setting the
-      // array to null also indicates that it has already been de-allocated which prevents a double
-      // de-allocation in free().
+      // Set the array to null instead of allocating a new array. Allocating an array could have
+      // triggered another spill and this method already is called from UnsafeExternalSorter when
+      // spilling. Attempting to allocate while spilling is dangerous, as we could be holding onto
+      // a large partially complete allocation, which may prevent other memory from being allocated.
+      // Instead we will allocate the new array when it is necessary.
       array = null;
       usableCapacity = 0;
-      pos = 0;
-      nullBoundaryPos = 0;
-      array = consumer.allocateArray(initialSize);
-      usableCapacity = getUsableCapacity();
     }
     pos = 0;
     nullBoundaryPos = 0;
@@ -217,25 +211,27 @@ public boolean hasSpaceForAnotherRecord() {
   }
 
   public void expandPointerArray(LongArray newArray) {
-    if (newArray.size() < array.size()) {
-      // checkstyle.off: RegexpSinglelineJava
-      throw new SparkOutOfMemoryError("Not enough memory to grow pointer array");
-      // checkstyle.on: RegexpSinglelineJava
+    if (array != null) {
+      if (newArray.size() < array.size()) {
+        // checkstyle.off: RegexpSinglelineJava
+        throw new SparkOutOfMemoryError("Not enough memory to grow pointer array");
+        // checkstyle.on: RegexpSinglelineJava
+      }
+      Platform.copyMemory(
+        array.getBaseObject(),
+        array.getBaseOffset(),
+        newArray.getBaseObject(),
+        newArray.getBaseOffset(),
+        pos * 8L);
+      consumer.freeArray(array);
     }
-    Platform.copyMemory(
-      array.getBaseObject(),
-      array.getBaseOffset(),
-      newArray.getBaseObject(),
-      newArray.getBaseOffset(),
-      pos * 8L);
-    consumer.freeArray(array);
     array = newArray;
     usableCapacity = getUsableCapacity();
   }
 
   /**
    * Inserts a record to be sorted. Assumes that the record pointer points to a record length
-   * stored as a 4-byte integer, followed by the record's bytes.
+   * stored as a uaoSize(4 or 8) bytes integer, followed by the record's bytes.
    *
    * @param recordPointer pointer to a record in a data page, encoded by {@link TaskMemoryManager}.
    * @param keyPrefix a user-defined key prefix
@@ -330,6 +326,7 @@ public void loadNext() {
     @Override
     public long getBaseOffset() { return baseOffset; }
 
+    @Override
     public long getCurrentPageNumber() {
       return currentPageNumber;
     }
@@ -346,6 +343,11 @@ public long getCurrentPageNumber() {
    * {@code next()} will return the same mutable object.
    */
   public UnsafeSorterIterator getSortedIterator() {
+    if (numRecords() == 0) {
+      // `array` might be null, so make sure that it is not accessed by returning early.
+      return new SortedIterator(0, 0);
+    }
+
     int offset = 0;
     long start = System.nanoTime();
     if (sortComparator != null) {
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java
index 1b3167fcc250c..d9f22311d07c2 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java
@@ -34,4 +34,6 @@ public abstract class UnsafeSorterIterator {
   public abstract long getKeyPrefix();
 
   public abstract int getNumRecords();
+
+  public abstract long getCurrentPageNumber();
 }
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java
index ab800288dcb43..f8603c5799e9b 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java
@@ -70,6 +70,11 @@ public int getNumRecords() {
         return numRecords;
       }
 
+      @Override
+      public long getCurrentPageNumber() {
+        throw new UnsupportedOperationException();
+      }
+
       @Override
       public boolean hasNext() {
         return !priorityQueue.isEmpty() || (spillReader != null && spillReader.hasNext());
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
index a524c4790407d..db79efd008530 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
@@ -89,6 +89,11 @@ public int getNumRecords() {
     return numRecords;
   }
 
+  @Override
+  public long getCurrentPageNumber() {
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public boolean hasNext() {
     return (numRecordsRemaining > 0);
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
index 33eb7bfa671bc..7e9927d40c191 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
@@ -77,12 +77,12 @@
   <tbody>
   {{#applications}}
     <tr>
+      {{#attempts}}
       <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}>{{version}}</td>
-      <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}><span title="{{id}}"><a href="{{uiroot}}/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td>
+      <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}><span title="{{id}}"><a href="{{uiroot}}/history/{{id}}{{#hasAttemptId}}/{{attemptId}}{{/hasAttemptId}}/jobs/">{{id}}</a></span></td>
       <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}>{{name}}</td>
-      {{#attempts}}
       {{#hasMultipleAttempts}}
-      <td><a href="{{uiroot}}/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td>
+      <td><a href="{{uiroot}}/history/{{id}}{{#hasAttemptId}}/{{attemptId}}{{/hasAttemptId}}/jobs/">{{attemptId}}</a></td>
       {{/hasMultipleAttempts}}
       <td>{{startTime}}</td>
       {{#showCompletedColumns}}
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
index 4df5f07f077d7..3a4c815029f6d 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
@@ -130,7 +130,7 @@ $(document).ready(function() {
         if (app["attempts"].length > 1) {
             hasMultipleAttempts = true;
         }
-        var num = app["attempts"].length;
+
         for (var j in app["attempts"]) {
           var attempt = app["attempts"][j];
           attempt["startTime"] = formatTimeMillis(attempt["startTimeEpoch"]);
@@ -140,7 +140,8 @@ $(document).ready(function() {
             (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs";
           attempt["durationMillisec"] = attempt["duration"];
           attempt["duration"] = formatDuration(attempt["duration"]);
-          var app_clone = {"id" : id, "name" : name, "version": version, "num" : num, "attempts" : [attempt]};
+          var hasAttemptId = attempt.hasOwnProperty("attemptId");
+          var app_clone = {"id" : id, "name" : name, "version": version, "hasAttemptId" : hasAttemptId, "attempts" : [attempt]};
           array.push(app_clone);
         }
       }
diff --git a/core/src/main/resources/org/apache/spark/ui/static/sorttable.js b/core/src/main/resources/org/apache/spark/ui/static/sorttable.js
index ecd580e5c64aa..3f98a0379dc3c 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/sorttable.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/sorttable.js
@@ -99,12 +99,12 @@ sorttable = {
                                                     'sorttable_sorted_reverse');
             rowlists = this.parentNode.getElementsByTagName("span");
             for (var j=0; j < rowlists.length; j++) {
-                if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/)) {
+                if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/) != -1) {
                     rowlists[j].parentNode.removeChild(rowlists[j]);
                 }
             }
             sortrevind = document.createElement('span');
-            sortrevind.class = "sorttable_sortrevind";
+            sortrevind.className = "sorttable_sortrevind";
             sortrevind.innerHTML = stIsIE ? '&nbsp<font face="webdings">5</font>' : '&nbsp;&#x25BE;';
             this.appendChild(sortrevind);
             return;
@@ -117,12 +117,12 @@ sorttable = {
                                                   'sorttable_sorted');
             rowlists = this.parentNode.getElementsByTagName("span");
             for (var j=0; j < rowlists.length; j++) {
-                if (rowlists[j].className.search(/\sorttable_sortrevind\b/)) {
+                if (rowlists[j].className.search(/\bsorttable_sortrevind\b/) != -1) {
                     rowlists[j].parentNode.removeChild(rowlists[j]);
                 }
             }
             sortfwdind = document.createElement('span');
-            sortfwdind.class = "sorttable_sortfwdind";
+            sortfwdind.className = "sorttable_sortfwdind";
             sortfwdind.innerHTML = stIsIE ? '&nbsp<font face="webdings">6</font>' : '&nbsp;&#x25B4;';
             this.appendChild(sortfwdind);
             return;
@@ -138,15 +138,15 @@ sorttable = {
           });
           rowlists = this.parentNode.getElementsByTagName("span");
           for (var j=0; j < rowlists.length; j++) {
-              if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/)
-                  || rowlists[j].className.search(/\sorttable_sortrevind\b/) ) {
+              if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/) != -1
+                  || rowlists[j].className.search(/\bsorttable_sortrevind\b/) != -1) {
                   rowlists[j].parentNode.removeChild(rowlists[j]);
               }
           }
 
           this.className += ' sorttable_sorted';
           sortfwdind = document.createElement('span');
-          sortfwdind.class = "sorttable_sortfwdind";
+          sortfwdind.className = "sorttable_sortfwdind";
           sortfwdind.innerHTML = stIsIE ? '&nbsp<font face="webdings">6</font>' : '&nbsp;&#x25B4;';
           this.appendChild(sortfwdind);
 
diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css
index 1fbc90b832bc9..e44a724c202f7 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css
@@ -22,6 +22,7 @@
 #dag-viz-graph .label {
   font-weight: normal;
   text-shadow: none;
+  color: #333;
 }
 
 #dag-viz-graph svg path {
diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
index 25dec9d3788ba..474c453643365 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
@@ -173,9 +173,11 @@ function renderDagViz(forJob) {
   });
 
   metadataContainer().selectAll(".barrier-rdd").each(function() {
-    var rddId = d3.select(this).text().trim()
-    var clusterId = VizConstants.clusterPrefix + rddId
-    svg.selectAll("g." + clusterId).classed("barrier", true)
+    var opId = d3.select(this).text().trim();
+    var opClusterId = VizConstants.clusterPrefix + opId;
+    var stageId = $(this).parents(".stage-metadata").attr("stage-id");
+    var stageClusterId = VizConstants.graphPrefix + stageId;
+    svg.selectAll("g[id=" + stageClusterId + "] g." + opClusterId).classed("barrier", true)
   });
 
   resizeSvg(svg);
@@ -216,7 +218,7 @@ function renderDagVizForJob(svgContainer) {
     var dot = metadata.select(".dot-file").text();
     var stageId = metadata.attr("stage-id");
     var containerId = VizConstants.graphPrefix + stageId;
-    var isSkipped = metadata.attr("skipped") == "true";
+    var isSkipped = metadata.attr("skipped") === "true";
     var container;
     if (isSkipped) {
       container = svgContainer
@@ -225,11 +227,8 @@ function renderDagVizForJob(svgContainer) {
         .attr("skipped", "true");
     } else {
       // Link each graph to the corresponding stage page (TODO: handle stage attempts)
-      // Use the link from the stage table so it also works for the history server
       var attemptId = 0;
-      var stageLink = d3.select("#stage-" + stageId + "-" + attemptId)
-        .select("a.name-link")
-        .attr("href");
+      var stageLink = uiRoot + appBasePath + "/stages/stage/?id=" + stageId + "&attempt=" + attemptId;
       container = svgContainer
         .append("a")
         .attr("xlink:href", stageLink)
@@ -282,11 +281,7 @@ function renderDagVizForJob(svgContainer) {
 
 /* Render the dot file as an SVG in the given container. */
 function renderDot(dot, container, forJob) {
-  var escaped_dot = dot
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, "\"");
-  var g = graphlibDot.read(escaped_dot);
+  var g = graphlibDot.read(dot);
   var renderer = new dagreD3.render();
   preprocessGraphLayout(g, forJob);
   renderer(container, g);
@@ -498,18 +493,11 @@ function connectRDDs(fromRDDId, toRDDId, edgesContainer, svgContainer) {
   edgesContainer.append("path").datum(points).attr("d", line);
 }
 
-/*
- * Replace `/n` with `<br/>`
- */
-function replaceLineBreak(str) {
-    return str.replace("\\n", "<br/>");
-}
-
 /* (Job page only) Helper function to add tooltips for RDDs. */
 function addTooltipsForRDDs(svgContainer) {
   svgContainer.selectAll("g.node").each(function() {
     var node = d3.select(this);
-    var tooltipText = replaceLineBreak(node.attr("name"));
+    var tooltipText = node.attr("name");
     if (tooltipText) {
       node.select("circle")
         .attr("data-toggle", "tooltip")
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
index ee2b7b353d62e..67d6d741e5537 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
@@ -70,7 +70,7 @@ function stageEndPoint(appId) {
             return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId;
         }
     }
-    return location.origin + "/api/v1/applications/" + appId + "/stages/" + stageId;
+    return uiRoot + "/api/v1/applications/" + appId + "/stages/" + stageId;
 }
 
 function getColumnNameForTaskMetricSummary(columnKey) {
@@ -821,7 +821,8 @@ $(document).ready(function () {
                         },
                         {
                             data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics && row.taskMetrics.shuffleReadMetrics.localBytesRead > 0) {
+                                if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics &&
+                                    (row.taskMetrics.shuffleReadMetrics.localBytesRead > 0 || row.taskMetrics.shuffleReadMetrics.remoteBytesRead > 0)) {
                                     var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead);
                                     if (type === 'display') {
                                         return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
index 5b75bc3011b6d..9a01c3ba83696 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
@@ -33,6 +33,8 @@ var yValueFormat = d3.format(",.2f");
 
 var unitLabelYOffset = -10;
 
+var onClickTimeline = function() {};
+
 // Show a tooltip "text" for "node"
 function showBootstrapTooltip(node, text) {
     $(node).tooltip({title: text, trigger: "manual", container: "body"});
@@ -44,6 +46,45 @@ function hideBootstrapTooltip(node) {
     $(node).tooltip("destroy");
 }
 
+// Return the function to scroll to the corresponding
+// row on clicking a point of batch in the timeline.
+function getOnClickTimelineFunction() {
+    // If the user click one point in the graphs, jump to the batch row and highlight it. And
+    // recovery the batch row after 3 seconds if necessary.
+    // We need to remember the last clicked batch so that we can recovery it.
+    var lastClickedBatch = null;
+    var lastTimeout = null;
+
+    return function(d) {
+        var batchSelector = $("#batch-" + d.x);
+        // If there is a corresponding batch row, scroll down to it and highlight it.
+        if (batchSelector.length > 0) {
+            if (lastTimeout != null) {
+                window.clearTimeout(lastTimeout);
+            }
+            if (lastClickedBatch != null) {
+                clearBatchRow(lastClickedBatch);
+                lastClickedBatch = null;
+            }
+            lastClickedBatch = d.x;
+            highlightBatchRow(lastClickedBatch);
+            lastTimeout = window.setTimeout(function () {
+                lastTimeout = null;
+                if (lastClickedBatch != null) {
+                    clearBatchRow(lastClickedBatch);
+                    lastClickedBatch = null;
+                }
+            }, 3000); // Clean up after 3 seconds
+
+            var topOffset = batchSelector.offset().top - 15;
+            if (topOffset < 0) {
+                topOffset = 0;
+            }
+            $('html,body').animate({scrollTop: topOffset}, 200);
+        }
+    }
+}
+
 // Register a timeline graph. All timeline graphs should be register before calling any
 // "drawTimeline" so that we can determine the max margin left for all timeline graphs.
 function registerTimeline(minY, maxY) {
@@ -171,7 +212,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
             .attr("cy", function(d) { return y(d.y); })
             .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";})
             .on('mouseover', function(d) {
-                var tip = formatYValue(d.y) + " " + unitY + " at " + timeFormat[d.x];
+                var tip = formatYValue(d.y) + " " + unitY + " at " + timeTipStrings[d.x];
                 showBootstrapTooltip(d3.select(this).node(), tip);
                 // show the point
                 d3.select(this)
@@ -189,31 +230,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
                     .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";})
                     .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";});
             })
-            .on("click", function(d) {
-                if (lastTimeout != null) {
-                    window.clearTimeout(lastTimeout);
-                }
-                if (lastClickedBatch != null) {
-                    clearBatchRow(lastClickedBatch);
-                    lastClickedBatch = null;
-                }
-                lastClickedBatch = d.x;
-                highlightBatchRow(lastClickedBatch);
-                lastTimeout = window.setTimeout(function () {
-                    lastTimeout = null;
-                    if (lastClickedBatch != null) {
-                        clearBatchRow(lastClickedBatch);
-                        lastClickedBatch = null;
-                    }
-                }, 3000); // Clean up after 3 seconds
-
-                var batchSelector = $("#batch-" + d.x);
-                var topOffset = batchSelector.offset().top - 15;
-                if (topOffset < 0) {
-                    topOffset = 0;
-                }
-                $('html,body').animate({scrollTop: topOffset}, 200);
-            });
+            .on("click", onClickTimeline);
 }
 
 /**
diff --git a/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
index 70250fdbd2d0c..c92226b408b6c 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
@@ -106,12 +106,12 @@ function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
         .on('mouseover', function(d) {
             var tip = '';
             var idx = 0;
-            var _values = timeToValues[d._x]
+            var _values = formattedTimeToValues[d._x];
             _values.forEach(function (k) {
                 tip += labels[idx] + ': ' + k + '   ';
                 idx += 1;
             });
-            tip += " at " + d._x
+            tip += " at " + formattedTimeTipStrings[d._x];
             showBootstrapTooltip(d3.select(this).node(), tip);
         })
         .on('mouseout',  function() {
diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
index 3f31403eaeef3..c9bf83ca98b4f 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
@@ -238,18 +238,6 @@ tr.corresponding-item-hover > td, tr.corresponding-item-hover > th {
   background-color: #D6FFE4 !important;
 }
 
-#application-timeline.collapsed {
-  display: none;
-}
-
-#job-timeline.collapsed {
-  display: none;
-}
-
-#task-assignment-timeline.collapsed {
-  display: none;
-}
-
 .control-panel {
   margin-bottom: 5px;
 }
diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
index b2cd616791734..220b76a0f1b27 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
@@ -26,8 +26,9 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
     editable: false,
     align: 'left',
     showCurrentTime: false,
-    min: startTime,
+    start: startTime,
     zoomable: false,
+    locale: "en",
     moment: function (date) {
       return vis.moment(date).utcOffset(offset);
     }
@@ -41,26 +42,31 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
   setupZoomable("#application-timeline-zoom-lock", applicationTimeline);
   setupExecutorEventAction();
 
+  function getIdForJobEntry(baseElem) {
+    var jobIdText = $($(baseElem).find(".application-timeline-content")[0]).text();
+    var jobId = jobIdText.match("\\(Job (\\d+)\\)$")[1];
+    return jobId;
+  }
+
+  function getSelectorForJobEntry(jobId) {
+    return "#job-" + jobId;
+  }
+
   function setupJobEventAction() {
     $(".vis-item.vis-range.job.application-timeline-object").each(function() {
-      var getSelectorForJobEntry = function(baseElem) {
-        var jobIdText = $($(baseElem).find(".application-timeline-content")[0]).text();
-        var jobId = jobIdText.match("\\(Job (\\d+)\\)$")[1];
-       return "#job-" + jobId;
-      };
-
       $(this).click(function() {
-        var jobPagePath = $(getSelectorForJobEntry(this)).find("a.name-link").attr("href")
-          window.location.href = jobPagePath
+        var jobId = getIdForJobEntry(this);
+        var jobPagePath = uiRoot + appBasePath + "/jobs/job/?id=" + jobId;
+        window.location.href = jobPagePath;
       });
 
       $(this).hover(
         function() {
-          $(getSelectorForJobEntry(this)).addClass("corresponding-item-hover");
+          $(getSelectorForJobEntry(getIdForJobEntry(this))).addClass("corresponding-item-hover");
           $($(this).find("div.application-timeline-content")[0]).tooltip("show");
         },
         function() {
-          $(getSelectorForJobEntry(this)).removeClass("corresponding-item-hover");
+          $(getSelectorForJobEntry(getIdForJobEntry(this))).removeClass("corresponding-item-hover");
           $($(this).find("div.application-timeline-content")[0]).tooltip("hide");
         }
       );
@@ -75,6 +81,9 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
 
     $("#application-timeline").toggleClass('collapsed');
 
+    var visibilityState = status ? "" : "none";
+    $("#application-timeline").css("display", visibilityState);
+
     // Switch the class of the arrow from open to closed.
     $(this).find('.expand-application-timeline-arrow').toggleClass('arrow-open');
     $(this).find('.expand-application-timeline-arrow').toggleClass('arrow-closed');
@@ -89,6 +98,8 @@ $(function () {
     // Set it to false so that the click function can revert it
     window.localStorage.setItem("expand-application-timeline", "false");
     $("span.expand-application-timeline").trigger('click');
+  } else {
+    $("#application-timeline").css("display", "none");
   }
 });
 
@@ -103,8 +114,9 @@ function drawJobTimeline(groupArray, eventObjArray, startTime, offset) {
     editable: false,
     align: 'left',
     showCurrentTime: false,
-    min: startTime,
+    start: startTime,
     zoomable: false,
+    locale: "en",
     moment: function (date) {
       return vis.moment(date).utcOffset(offset);
     }
@@ -118,26 +130,34 @@ function drawJobTimeline(groupArray, eventObjArray, startTime, offset) {
   setupZoomable("#job-timeline-zoom-lock", jobTimeline);
   setupExecutorEventAction();
 
+  function getStageIdAndAttemptForStageEntry(baseElem) {
+    var stageIdText = $($(baseElem).find(".job-timeline-content")[0]).text();
+    var stageIdAndAttempt = stageIdText.match("\\(Stage (\\d+\\.\\d+)\\)$")[1].split(".");
+    return stageIdAndAttempt;
+  }
+
+  function getSelectorForStageEntry(stageIdAndAttempt) {
+    return "#stage-" + stageIdAndAttempt[0] + "-" + stageIdAndAttempt[1];
+  }
+
   function setupStageEventAction() {
     $(".vis-item.vis-range.stage.job-timeline-object").each(function() {
-      var getSelectorForStageEntry = function(baseElem) {
-        var stageIdText = $($(baseElem).find(".job-timeline-content")[0]).text();
-        var stageIdAndAttempt = stageIdText.match("\\(Stage (\\d+\\.\\d+)\\)$")[1].split(".");
-        return "#stage-" + stageIdAndAttempt[0] + "-" + stageIdAndAttempt[1];
-      };
-
       $(this).click(function() {
-        var stagePagePath = $(getSelectorForStageEntry(this)).find("a.name-link").attr("href")
-        window.location.href = stagePagePath
+        var stageIdAndAttempt = getStageIdAndAttemptForStageEntry(this);
+        var stagePagePath = uiRoot + appBasePath +
+          "/stages/stage/?id=" + stageIdAndAttempt[0] + "&attempt=" + stageIdAndAttempt[1];
+        window.location.href = stagePagePath;
       });
 
       $(this).hover(
         function() {
-          $(getSelectorForStageEntry(this)).addClass("corresponding-item-hover");
+          $(getSelectorForStageEntry(getStageIdAndAttemptForStageEntry(this)))
+            .addClass("corresponding-item-hover");
           $($(this).find("div.job-timeline-content")[0]).tooltip("show");
         },
         function() {
-          $(getSelectorForStageEntry(this)).removeClass("corresponding-item-hover");
+          $(getSelectorForStageEntry(getStageIdAndAttemptForStageEntry(this)))
+            .removeClass("corresponding-item-hover");
           $($(this).find("div.job-timeline-content")[0]).tooltip("hide");
         }
       );
@@ -152,6 +172,9 @@ function drawJobTimeline(groupArray, eventObjArray, startTime, offset) {
 
     $("#job-timeline").toggleClass('collapsed');
 
+    var visibilityState = status ? "" : "none";
+    $("#job-timeline").css("display", visibilityState);
+
     // Switch the class of the arrow from open to closed.
     $(this).find('.expand-job-timeline-arrow').toggleClass('arrow-open');
     $(this).find('.expand-job-timeline-arrow').toggleClass('arrow-closed');
@@ -166,13 +189,15 @@ $(function () {
     // Set it to false so that the click function can revert it
     window.localStorage.setItem("expand-job-timeline", "false");
     $("span.expand-job-timeline").trigger('click');
+  } else {
+    $("#job-timeline").css("display", "none");
   }
 });
 
 function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, maxFinishTime, offset) {
   var groups = new vis.DataSet(groupArray);
   var items = new vis.DataSet(eventObjArray);
-  var container = $("#task-assignment-timeline")[0]
+  var container = $("#task-assignment-timeline")[0];
   var options = {
     groupOrder: function(a, b) {
       return a.value - b.value
@@ -181,15 +206,16 @@ function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, ma
     align: 'left',
     selectable: false,
     showCurrentTime: false,
-    min: minLaunchTime,
-    max: maxFinishTime,
+    start: minLaunchTime,
+    end: maxFinishTime,
     zoomable: false,
+    locale: "en",
     moment: function (date) {
       return vis.moment(date).utcOffset(offset);
     }
   };
 
-  var taskTimeline = new vis.Timeline(container)
+  var taskTimeline = new vis.Timeline(container);
   taskTimeline.setOptions(options);
   taskTimeline.setGroups(groups);
   taskTimeline.setItems(items);
@@ -220,6 +246,9 @@ function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, ma
 
     $("#task-assignment-timeline").toggleClass("collapsed");
 
+    var visibilityState = status ? "" : "none";
+    $("#task-assignment-timeline").css("display", visibilityState);
+
      // Switch the class of the arrow from open to closed.
     $(this).find(".expand-task-assignment-timeline-arrow").toggleClass("arrow-open");
     $(this).find(".expand-task-assignment-timeline-arrow").toggleClass("arrow-closed");
@@ -234,6 +263,8 @@ $(function () {
     // Set it to false so that the click function can revert it
     window.localStorage.setItem("expand-task-assignment-timeline", "false");
     $("span.expand-task-assignment-timeline").trigger('click');
+  } else {
+    $("#task-assignment-timeline").css("display", "none");
   }
 });
 
diff --git a/core/src/main/resources/org/apache/spark/ui/static/utils.js b/core/src/main/resources/org/apache/spark/ui/static/utils.js
index 6fc34a9e1f7ea..d15d0035844de 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/utils.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/utils.js
@@ -56,13 +56,17 @@ function formatTimeMillis(timeMillis) {
     return "-";
   } else {
     var dt = new Date(timeMillis);
+    return formatDateString(dt);
+  }
+}
+
+function formatDateString(dt) {
     return dt.getFullYear() + "-" +
       padZeroes(dt.getMonth() + 1) + "-" +
       padZeroes(dt.getDate()) + " " +
       padZeroes(dt.getHours()) + ":" +
       padZeroes(dt.getMinutes()) + ":" +
       padZeroes(dt.getSeconds());
-  }
 }
 
 function getTimeZone() {
@@ -101,7 +105,7 @@ function getStandAloneAppId(cb) {
   }
   // Looks like Web UI is running in standalone mode
   // Let's get application-id using REST End Point
-  $.getJSON(location.origin + "/api/v1/applications", function(response, status, jqXHR) {
+  $.getJSON(uiRoot + "/api/v1/applications", function(response, status, jqXHR) {
     if (response && response.length > 0) {
       var appId = response[0].id;
       cb(appId);
@@ -148,7 +152,7 @@ function createTemplateURI(appId, templateName) {
     var baseURI = words.slice(0, ind).join('/') + '/static/' + templateName + '-template.html';
     return baseURI;
   }
-  return location.origin + "/static/" + templateName + "-template.html";
+  return uiRoot + "/static/" + templateName + "-template.html";
 }
 
 function setDataTableDefaults() {
@@ -161,7 +165,10 @@ function setDataTableDefaults() {
 
 function formatDate(date) {
   if (date <= 0) return "-";
-  else return date.split(".")[0].replace("T", " ");
+  else {
+     var dt = new Date(date.replace("GMT", "Z"))
+     return formatDateString(dt);
+  }
 }
 
 function createRESTEndPointForExecutorsPage(appId) {
@@ -183,5 +190,5 @@ function createRESTEndPointForExecutorsPage(appId) {
             return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors";
         }
     }
-    return location.origin + "/api/v1/applications/" + appId + "/allexecutors";
+    return uiRoot + "/api/v1/applications/" + appId + "/allexecutors";
 }
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
new file mode 100644
index 0000000000000..eeacbd802d305
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
@@ -0,0 +1 @@
+.vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}.vis [class*=span]{min-height:0;width:auto}div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper::after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:rgba(0,0,0,0);width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(to bottom,#dedede 0,#c8c8c8 99%);border:1px solid #999;box-shadow:#aaa 0 0 3px 0;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2 0,#385380 100%);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(to bottom,#3876c2 0,#385380 100%);box-shadow:#111927 0 0 1px 0;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:0}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(to bottom,#9d9d9d 0,#c8c8c8 99%)}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(to bottom,#dedede 0,#c8c8c8 99%);border:1px solid #999;box-shadow:#aaa 0 0 3px 0;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:1px solid #fff;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:0 0;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0);border-left-color:rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0);border-left-color:#f2faff;border-width:12px;margin-top:-12px}div.vis-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none;z-index:5}.vis-current-time{background-color:#ff7f6e;width:2px;z-index:1;pointer-events:none}.vis-rolling-mode-btn{height:40px;width:40px;position:absolute;top:7px;right:20px;border-radius:50%;font-size:28px;cursor:pointer;opacity:.8;color:#fff;font-weight:700;text-align:center;background:#3876c2}.vis-rolling-mode-btn:before{content:"\26F6"}.vis-rolling-mode-btn:hover{opacity:1}.vis-custom-time{background-color:#6e94ff;width:2px;cursor:move;z-index:1}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px rgba(154,154,154,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}.vis-item{position:absolute;color:#1a1a1a;border-color:#97b0f8;border-width:1px;background-color:#d5ddf6;display:inline-block;z-index:1}.vis-item.vis-selected{border-color:#ffc200;background-color:#fff785;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-point.vis-selected{background-color:#fff785}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:0 0}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item-visible-frame{white-space:nowrap}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-onUpdateTime-tooltip{position:absolute;background:#4f81bd;color:#fff;width:200px;text-align:center;white-space:nowrap;padding:5px;border-radius:1px;transition:.4s;-o-transition:.4s;-moz-transition:.4s;-webkit-transition:.4s}.vis-item .vis-delete,.vis-item .vis-delete-rtl{position:absolute;top:0;width:24px;height:24px;box-sizing:border-box;padding:0 5px;cursor:pointer;-webkit-transition:background .2s linear;-moz-transition:background .2s linear;-ms-transition:background .2s linear;-o-transition:background .2s linear;transition:background .2s linear}.vis-item .vis-delete{right:-24px}.vis-item .vis-delete-rtl{left:-24px}.vis-item .vis-delete-rtl:after,.vis-item .vis-delete:after{content:"\00D7";color:red;font-family:arial,sans-serif;font-size:22px;font-weight:700;-webkit-transition:color .2s linear;-moz-transition:color .2s linear;-ms-transition:color .2s linear;-o-transition:color .2s linear;transition:color .2s linear}.vis-item .vis-delete-rtl:hover,.vis-item .vis-delete:hover{background:red}.vis-item .vis-delete-rtl:hover:after,.vis-item .vis-delete:hover:after{color:#fff}.vis-item .vis-drag-center{position:absolute;width:100%;height:100%;top:0;left:0;cursor:move}.vis-item.vis-range .vis-drag-left{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0;left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0;right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-itemset{position:relative;padding:0;margin:0;box-sizing:border-box}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-nesting-group{cursor:pointer}.vis-nested-group{background:#f5f5f5}.vis-label.vis-nesting-group.expanded:before{content:"\25BC"}.vis-label.vis-nesting-group.collapsed-rtl:before{content:"\25C0"}.vis-label.vis-nesting-group.collapsed:before{content:"\25B6"}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-labelset{position:relative;overflow:hidden;box-sizing:border-box}.vis-labelset .vis-label{position:relative;left:0;top:0;width:100%;color:#4d4d4d;box-sizing:border-box}.vis-labelset .vis-label{border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}.vis-panel{position:absolute;padding:0;margin:0;box-sizing:border-box}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-left.vis-panel.vis-vertical-scroll,.vis-right.vis-panel.vis-vertical-scroll{height:100%;overflow-x:hidden;overflow-y:scroll}.vis-left.vis-panel.vis-vertical-scroll{direction:rtl}.vis-left.vis-panel.vis-vertical-scroll .vis-content{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll .vis-content{direction:rtl}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-background{overflow:hidden}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}.vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}.vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}.vis-timeline{position:relative;border:1px solid #bfbfbf;overflow:hidden;padding:0;margin:0;box-sizing:border-box}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
new file mode 100644
index 0000000000000..5cfa0da3d4f38
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
@@ -0,0 +1,30 @@
+/**
+ * vis.js
+ * https://github.com/almende/vis
+ *
+ * A dynamic, browser-based visualization library.
+ *
+ * @version 4.21.0
+ * @date    2017-10-12
+ *
+ * @license
+ * Copyright (C) 2011-2017 Almende B.V, http://almende.com
+ *
+ * Vis.js is dual licensed under both
+ *
+ * * The Apache 2.0 License
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * and
+ *
+ * * The MIT License
+ *   http://opensource.org/licenses/MIT
+ *
+ * Vis.js may be distributed under either license.
+ */
+"use strict";!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.vis=e():t.vis=e()}(this,function(){return function(t){function e(o){if(i[o])return i[o].exports;var n=i[o]={i:o,l:!1,exports:{}};return t[o].call(n.exports,n,n.exports,e),n.l=!0,n.exports}var i={};return e.m=t,e.c=i,e.d=function(t,i,o){e.o(t,i)||Object.defineProperty(t,i,{configurable:!1,enumerable:!0,get:o})},e.n=function(t){var i=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(i,"a",i),i},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="",e(e.s=85)}([function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e,i,o){var n=!1;!0===o&&(n=null===e[i]&&void 0!==t[i]),n?delete t[i]:t[i]=e[i]}var s=i(86),r=o(s),a=i(43),h=o(a),d=i(4),l=o(d),u=i(1),p=o(u),c=i(3),m=i(126);e.isNumber=function(t){return t instanceof Number||"number"==typeof t},e.recursiveDOMDelete=function(t){if(t)for(;!0===t.hasChildNodes();)e.recursiveDOMDelete(t.firstChild),t.removeChild(t.firstChild)},e.giveRange=function(t,e,i,o){if(e==t)return.5;var n=1/(e-t);return Math.max(0,(o-t)*n)},e.isString=function(t){return t instanceof String||"string"==typeof t},e.isDate=function(t){if(t instanceof Date)return!0;if(e.isString(t)){if(f.exec(t))return!0;if(!isNaN(Date.parse(t)))return!0}return!1},e.randomUUID=function(){return m.v4()},e.assignAllKeys=function(t,e){for(var i in t)t.hasOwnProperty(i)&&"object"!==(0,p.default)(t[i])&&(t[i]=e)},e.fillIfDefined=function(t,i){var o=arguments.length>2&&void 0!==arguments[2]&&arguments[2];for(var s in t)void 0!==i[s]&&(null===i[s]||"object"!==(0,p.default)(i[s])?n(t,i,s,o):"object"===(0,p.default)(t[s])&&e.fillIfDefined(t[s],i[s],o))},e.extend=function(t,e){for(var i=1;i<arguments.length;i++){var o=arguments[i];for(var n in o)o.hasOwnProperty(n)&&(t[n]=o[n])}return t},e.selectiveExtend=function(t,e,i){if(!Array.isArray(t))throw new Error("Array with property names expected as first argument");for(var o=2;o<arguments.length;o++)for(var n=arguments[o],s=0;s<t.length;s++){var r=t[s];n&&n.hasOwnProperty(r)&&(e[r]=n[r])}return e},e.selectiveDeepExtend=function(t,i,o){var s=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Array.isArray(o))throw new TypeError("Arrays are not supported by deepExtend");for(var r=0;r<t.length;r++){var a=t[r];if(o.hasOwnProperty(a))if(o[a]&&o[a].constructor===Object)void 0===i[a]&&(i[a]={}),i[a].constructor===Object?e.deepExtend(i[a],o[a],!1,s):n(i,o,a,s);else{if(Array.isArray(o[a]))throw new TypeError("Arrays are not supported by deepExtend");n(i,o,a,s)}}return i},e.selectiveNotDeepExtend=function(t,i,o){var s=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Array.isArray(o))throw new TypeError("Arrays are not supported by deepExtend");for(var r in o)if(o.hasOwnProperty(r)&&-1===t.indexOf(r))if(o[r]&&o[r].constructor===Object)void 0===i[r]&&(i[r]={}),i[r].constructor===Object?e.deepExtend(i[r],o[r]):n(i,o,r,s);else if(Array.isArray(o[r])){i[r]=[];for(var a=0;a<o[r].length;a++)i[r].push(o[r][a])}else n(i,o,r,s);return i},e.deepExtend=function(t,i){var o=arguments.length>2&&void 0!==arguments[2]&&arguments[2],s=arguments.length>3&&void 0!==arguments[3]&&arguments[3];for(var r in i)if(i.hasOwnProperty(r)||!0===o)if(i[r]&&i[r].constructor===Object)void 0===t[r]&&(t[r]={}),t[r].constructor===Object?e.deepExtend(t[r],i[r],o):n(t,i,r,s);else if(Array.isArray(i[r])){t[r]=[];for(var a=0;a<i[r].length;a++)t[r].push(i[r][a])}else n(t,i,r,s);return t},e.equalArray=function(t,e){if(t.length!=e.length)return!1;for(var i=0,o=t.length;i<o;i++)if(t[i]!=e[i])return!1;return!0},e.convert=function(t,i){var o;if(void 0!==t){if(null===t)return null;if(!i)return t;if("string"!=typeof i&&!(i instanceof String))throw new Error("Type must be a string");switch(i){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return e.isString(t)&&!isNaN(Date.parse(t))?c(t).valueOf():Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(e.isNumber(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(c.isMoment(t))return new Date(t.valueOf());if(e.isString(t))return o=f.exec(t),o?new Date(Number(o[1])):c(new Date(t)).toDate();throw new Error("Cannot convert object of type "+e.getType(t)+" to type Date");case"Moment":if(e.isNumber(t))return c(t);if(t instanceof Date)return c(t.valueOf());if(c.isMoment(t))return c(t);if(e.isString(t))return o=f.exec(t),c(o?Number(o[1]):t);throw new Error("Cannot convert object of type "+e.getType(t)+" to type Date");case"ISODate":if(e.isNumber(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(c.isMoment(t))return t.toDate().toISOString();if(e.isString(t))return o=f.exec(t),o?new Date(Number(o[1])).toISOString():c(t).format();throw new Error("Cannot convert object of type "+e.getType(t)+" to type ISODate");case"ASPDate":if(e.isNumber(t))return"/Date("+t+")/";if(t instanceof Date)return"/Date("+t.valueOf()+")/";if(e.isString(t)){o=f.exec(t);return"/Date("+(o?new Date(Number(o[1])).valueOf():new Date(t).valueOf())+")/"}throw new Error("Cannot convert object of type "+e.getType(t)+" to type ASPDate");default:throw new Error('Unknown type "'+i+'"')}}};var f=/^\/?Date\((\-?\d+)/i;e.getType=function(t){var e=void 0===t?"undefined":(0,p.default)(t);return"object"==e?null===t?"null":t instanceof Boolean?"Boolean":t instanceof Number?"Number":t instanceof String?"String":Array.isArray(t)?"Array":t instanceof Date?"Date":"Object":"number"==e?"Number":"boolean"==e?"Boolean":"string"==e?"String":void 0===e?"undefined":e},e.copyAndExtendArray=function(t,e){for(var i=[],o=0;o<t.length;o++)i.push(t[o]);return i.push(e),i},e.copyArray=function(t){for(var e=[],i=0;i<t.length;i++)e.push(t[i]);return e},e.getAbsoluteLeft=function(t){return t.getBoundingClientRect().left},e.getAbsoluteRight=function(t){return t.getBoundingClientRect().right},e.getAbsoluteTop=function(t){return t.getBoundingClientRect().top},e.addClassName=function(t,e){var i=t.className.split(" "),o=e.split(" ");i=i.concat(o.filter(function(t){return i.indexOf(t)<0})),t.className=i.join(" ")},e.removeClassName=function(t,e){var i=t.className.split(" "),o=e.split(" ");i=i.filter(function(t){return o.indexOf(t)<0}),t.className=i.join(" ")},e.forEach=function(t,e){var i,o;if(Array.isArray(t))for(i=0,o=t.length;i<o;i++)e(t[i],i,t);else for(i in t)t.hasOwnProperty(i)&&e(t[i],i,t)},e.toArray=function(t){var e=[];for(var i in t)t.hasOwnProperty(i)&&e.push(t[i]);return e},e.updateProperty=function(t,e,i){return t[e]!==i&&(t[e]=i,!0)},e.throttle=function(t){var e=!1;return function(){e||(e=!0,requestAnimationFrame(function(){e=!1,t()}))}},e.addEventListener=function(t,e,i,o){t.addEventListener?(void 0===o&&(o=!1),"mousewheel"===e&&navigator.userAgent.indexOf("Firefox")>=0&&(e="DOMMouseScroll"),t.addEventListener(e,i,o)):t.attachEvent("on"+e,i)},e.removeEventListener=function(t,e,i,o){t.removeEventListener?(void 0===o&&(o=!1),"mousewheel"===e&&navigator.userAgent.indexOf("Firefox")>=0&&(e="DOMMouseScroll"),t.removeEventListener(e,i,o)):t.detachEvent("on"+e,i)},e.preventDefault=function(t){t||(t=window.event),t.preventDefault?t.preventDefault():t.returnValue=!1},e.getTarget=function(t){t||(t=window.event);var e;return t.target?e=t.target:t.srcElement&&(e=t.srcElement),void 0!=e.nodeType&&3==e.nodeType&&(e=e.parentNode),e},e.hasParent=function(t,e){for(var i=t;i;){if(i===e)return!0;i=i.parentNode}return!1},e.option={},e.option.asBoolean=function(t,e){return"function"==typeof t&&(t=t()),null!=t?0!=t:e||null},e.option.asNumber=function(t,e){return"function"==typeof t&&(t=t()),null!=t?Number(t)||e||null:e||null},e.option.asString=function(t,e){return"function"==typeof t&&(t=t()),null!=t?String(t):e||null},e.option.asSize=function(t,i){return"function"==typeof t&&(t=t()),e.isString(t)?t:e.isNumber(t)?t+"px":i||null},e.option.asElement=function(t,e){return"function"==typeof t&&(t=t()),t||e||null},e.hexToRGB=function(t){var e=/^#?([a-f\d])([a-f\d])([a-f\d])$/i;t=t.replace(e,function(t,e,i,o){return e+e+i+i+o+o});var i=/^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(t);return i?{r:parseInt(i[1],16),g:parseInt(i[2],16),b:parseInt(i[3],16)}:null},e.overrideOpacity=function(t,i){var o;return-1!=t.indexOf("rgba")?t:-1!=t.indexOf("rgb")?(o=t.substr(t.indexOf("(")+1).replace(")","").split(","),"rgba("+o[0]+","+o[1]+","+o[2]+","+i+")"):(o=e.hexToRGB(t),null==o?t:"rgba("+o.r+","+o.g+","+o.b+","+i+")")},e.RGBToHex=function(t,e,i){return"#"+((1<<24)+(t<<16)+(e<<8)+i).toString(16).slice(1)},e.parseColor=function(t){var i;if(!0===e.isString(t)){if(!0===e.isValidRGB(t)){var o=t.substr(4).substr(0,t.length-5).split(",").map(function(t){return parseInt(t)});t=e.RGBToHex(o[0],o[1],o[2])}if(!0===e.isValidHex(t)){var n=e.hexToHSV(t),s={h:n.h,s:.8*n.s,v:Math.min(1,1.02*n.v)},r={h:n.h,s:Math.min(1,1.25*n.s),v:.8*n.v},a=e.HSVToHex(r.h,r.s,r.v),h=e.HSVToHex(s.h,s.s,s.v);i={background:t,border:a,highlight:{background:h,border:a},hover:{background:h,border:a}}}else i={background:t,border:t,highlight:{background:t,border:t},hover:{background:t,border:t}}}else i={},i.background=t.background||void 0,i.border=t.border||void 0,e.isString(t.highlight)?i.highlight={border:t.highlight,background:t.highlight}:(i.highlight={},i.highlight.background=t.highlight&&t.highlight.background||void 0,i.highlight.border=t.highlight&&t.highlight.border||void 0),e.isString(t.hover)?i.hover={border:t.hover,background:t.hover}:(i.hover={},i.hover.background=t.hover&&t.hover.background||void 0,i.hover.border=t.hover&&t.hover.border||void 0);return i},e.RGBToHSV=function(t,e,i){t/=255,e/=255,i/=255;var o=Math.min(t,Math.min(e,i)),n=Math.max(t,Math.max(e,i));if(o==n)return{h:0,s:0,v:o};var s=t==o?e-i:i==o?t-e:i-t;return{h:60*((t==o?3:i==o?1:5)-s/(n-o))/360,s:(n-o)/n,v:n}};var g={split:function(t){var e={};return t.split(";").forEach(function(t){if(""!=t.trim()){var i=t.split(":"),o=i[0].trim(),n=i[1].trim();e[o]=n}}),e},join:function(t){return(0,l.default)(t).map(function(e){return e+": "+t[e]}).join("; ")}};e.addCssText=function(t,i){var o=g.split(t.style.cssText),n=g.split(i),s=e.extend(o,n);t.style.cssText=g.join(s)},e.removeCssText=function(t,e){var i=g.split(t.style.cssText),o=g.split(e);for(var n in o)o.hasOwnProperty(n)&&delete i[n];t.style.cssText=g.join(i)},e.HSVToRGB=function(t,e,i){var o,n,s,r=Math.floor(6*t),a=6*t-r,h=i*(1-e),d=i*(1-a*e),l=i*(1-(1-a)*e);switch(r%6){case 0:o=i,n=l,s=h;break;case 1:o=d,n=i,s=h;break;case 2:o=h,n=i,s=l;break;case 3:o=h,n=d,s=i;break;case 4:o=l,n=h,s=i;break;case 5:o=i,n=h,s=d}return{r:Math.floor(255*o),g:Math.floor(255*n),b:Math.floor(255*s)}},e.HSVToHex=function(t,i,o){var n=e.HSVToRGB(t,i,o);return e.RGBToHex(n.r,n.g,n.b)},e.hexToHSV=function(t){var i=e.hexToRGB(t);return e.RGBToHSV(i.r,i.g,i.b)},e.isValidHex=function(t){return/(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(t)},e.isValidRGB=function(t){return t=t.replace(" ",""),/rgb\((\d{1,3}),(\d{1,3}),(\d{1,3})\)/i.test(t)},e.isValidRGBA=function(t){return t=t.replace(" ",""),/rgba\((\d{1,3}),(\d{1,3}),(\d{1,3}),(.{1,3})\)/i.test(t)},e.selectiveBridgeObject=function(t,i){if(null!==i&&"object"===(void 0===i?"undefined":(0,p.default)(i))){for(var o=(0,h.default)(i),n=0;n<t.length;n++)i.hasOwnProperty(t[n])&&"object"==(0,p.default)(i[t[n]])&&(o[t[n]]=e.bridgeObject(i[t[n]]));return o}return null},e.bridgeObject=function(t){if(null!==t&&"object"===(void 0===t?"undefined":(0,p.default)(t))){var i=(0,h.default)(t);if(t instanceof Element)i=t;else{i=(0,h.default)(t);for(var o in t)t.hasOwnProperty(o)&&"object"==(0,p.default)(t[o])&&(i[o]=e.bridgeObject(t[o]))}return i}return null},e.insertSort=function(t,e){for(var i=0;i<t.length;i++){for(var o=t[i],n=i;n>0&&e(o,t[n-1])<0;n--)t[n]=t[n-1];t[n]=o}return t},e.mergeOptions=function(t,e,i){var o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{},n=function(t){return null!==t&&void 0!==t},s=function(t){return null!==t&&"object"===(void 0===t?"undefined":(0,p.default)(t))};if(!s(t))throw new Error("Parameter mergeTarget must be an object");if(!s(e))throw new Error("Parameter options must be an object");if(!n(i))throw new Error("Parameter option must have a value");if(!s(o))throw new Error("Parameter globalOptions must be an object");var r=e[i],a=s(o)&&!function(t){for(var e in t)if(t.hasOwnProperty(e))return!1;return!0}(o),d=a?o[i]:void 0,l=d?d.enabled:void 0;if(void 0!==r){if("boolean"==typeof r)return s(t[i])||(t[i]={}),void(t[i].enabled=r);if(null===r&&!s(t[i])){if(!n(d))return;t[i]=(0,h.default)(d)}if(s(r)){var u=!0;void 0!==r.enabled?u=r.enabled:void 0!==l&&(u=d.enabled),function(t,e,i){s(t[i])||(t[i]={});var o=e[i],n=t[i];for(var r in o)o.hasOwnProperty(r)&&(n[r]=o[r])}(t,e,i),t[i].enabled=u}}},e.binarySearchCustom=function(t,e,i,o){for(var n=0,s=0,r=t.length-1;s<=r&&n<1e4;){var a=Math.floor((s+r)/2),h=t[a],d=void 0===o?h[i]:h[i][o],l=e(d);if(0==l)return a;-1==l?s=a+1:r=a-1,n++}return-1},e.binarySearchValue=function(t,e,i,o,n){var s,r,a,h,d=0,l=0,u=t.length-1;for(n=void 0!=n?n:function(t,e){return t==e?0:t<e?-1:1};l<=u&&d<1e4;){if(h=Math.floor(.5*(u+l)),s=t[Math.max(0,h-1)][i],r=t[h][i],a=t[Math.min(t.length-1,h+1)][i],0==n(r,e))return h;if(n(s,e)<0&&n(r,e)>0)return"before"==o?Math.max(0,h-1):h;if(n(r,e)<0&&n(a,e)>0)return"before"==o?h:Math.min(t.length-1,h+1);n(r,e)<0?l=h+1:u=h-1,d++}return-1},e.easingFunctions={linear:function(t){return t},easeInQuad:function(t){return t*t},easeOutQuad:function(t){return t*(2-t)},easeInOutQuad:function(t){return t<.5?2*t*t:(4-2*t)*t-1},easeInCubic:function(t){return t*t*t},easeOutCubic:function(t){return--t*t*t+1},easeInOutCubic:function(t){return t<.5?4*t*t*t:(t-1)*(2*t-2)*(2*t-2)+1},easeInQuart:function(t){return t*t*t*t},easeOutQuart:function(t){return 1- --t*t*t*t},easeInOutQuart:function(t){return t<.5?8*t*t*t*t:1-8*--t*t*t*t},easeInQuint:function(t){return t*t*t*t*t},easeOutQuint:function(t){return 1+--t*t*t*t*t},easeInOutQuint:function(t){return t<.5?16*t*t*t*t*t:1+16*--t*t*t*t*t}},e.getScrollBarWidth=function(){var t=document.createElement("p");t.style.width="100%",t.style.height="200px";var e=document.createElement("div");e.style.position="absolute",e.style.top="0px",e.style.left="0px",e.style.visibility="hidden",e.style.width="200px",e.style.height="150px",e.style.overflow="hidden",e.appendChild(t),document.body.appendChild(e);var i=t.offsetWidth;e.style.overflow="scroll";var o=t.offsetWidth;return i==o&&(o=e.clientWidth),document.body.removeChild(e),i-o},e.topMost=function(t,e){var i=void 0;Array.isArray(e)||(e=[e]);var o=!0,n=!1,s=void 0;try{for(var a,h=(0,r.default)(t);!(o=(a=h.next()).done);o=!0){var d=a.value;if(d){i=d[e[0]];for(var l=1;l<e.length;l++)i&&(i=i[e[l]]);if(void 0!==i)break}}}catch(t){n=!0,s=t}finally{try{!o&&h.return&&h.return()}finally{if(n)throw s}}return i}},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}e.__esModule=!0;var n=i(110),s=o(n),r=i(112),a=o(r),h="function"==typeof a.default&&"symbol"==typeof s.default?function(t){return typeof t}:function(t){return t&&"function"==typeof a.default&&t.constructor===a.default&&t!==a.default.prototype?"symbol":typeof t};e.default="function"==typeof a.default&&"symbol"===h(s.default)?function(t){return void 0===t?"undefined":h(t)}:function(t){return t&&"function"==typeof a.default&&t.constructor===a.default&&t!==a.default.prototype?"symbol":void 0===t?"undefined":h(t)}},function(t,e){var i=t.exports={version:"2.5.1"};"number"==typeof __e&&(__e=i)},function(t,e,i){t.exports="undefined"!=typeof window&&window.moment||i(123)},function(t,e,i){t.exports={default:i(107),__esModule:!0}},function(t,e,i){function o(t,e){this.options=null,this.props=null}var n=i(0);o.prototype.setOptions=function(t){t&&n.extend(this.options,t)},o.prototype.redraw=function(){return!1},o.prototype.destroy=function(){},o.prototype._isResized=function(){var t=this.props._previousWidth!==this.props.width||this.props._previousHeight!==this.props.height;return this.props._previousWidth=this.props.width,this.props._previousHeight=this.props.height,t},t.exports=o},function(t,e){var i=t.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=i)},function(t,e,i){var o=i(40)("wks"),n=i(27),s=i(6).Symbol,r="function"==typeof s;(t.exports=function(t){return o[t]||(o[t]=r&&s[t]||(r?s:n)("Symbol."+t))}).store=o},function(t,e,i){var o=i(18),n=i(57),s=i(36),r=Object.defineProperty;e.f=i(9)?Object.defineProperty:function(t,e,i){if(o(t),e=s(e,!0),o(i),n)try{return r(t,e,i)}catch(t){}if("get"in i||"set"in i)throw TypeError("Accessors not supported!");return"value"in i&&(t[e]=i.value),t}},function(t,e,i){t.exports=!i(19)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(t,e){var i={}.hasOwnProperty;t.exports=function(t,e){return i.call(t,e)}},function(t,e,i){e.prepareElements=function(t){for(var e in t)t.hasOwnProperty(e)&&(t[e].redundant=t[e].used,t[e].used=[])},e.cleanupElements=function(t){for(var e in t)if(t.hasOwnProperty(e)&&t[e].redundant){for(var i=0;i<t[e].redundant.length;i++)t[e].redundant[i].parentNode.removeChild(t[e].redundant[i]);t[e].redundant=[]}},e.resetElements=function(t){e.prepareElements(t),e.cleanupElements(t),e.prepareElements(t)},e.getSVGElement=function(t,e,i){var o;return e.hasOwnProperty(t)?e[t].redundant.length>0?(o=e[t].redundant[0],e[t].redundant.shift()):(o=document.createElementNS("http://www.w3.org/2000/svg",t),i.appendChild(o)):(o=document.createElementNS("http://www.w3.org/2000/svg",t),e[t]={used:[],redundant:[]},i.appendChild(o)),e[t].used.push(o),o},e.getDOMElement=function(t,e,i,o){var n;return e.hasOwnProperty(t)?e[t].redundant.length>0?(n=e[t].redundant[0],e[t].redundant.shift()):(n=document.createElement(t),void 0!==o?i.insertBefore(n,o):i.appendChild(n)):(n=document.createElement(t),e[t]={used:[],redundant:[]},void 0!==o?i.insertBefore(n,o):i.appendChild(n)),e[t].used.push(n),n},e.drawPoint=function(t,i,o,n,s,r){var a;if("circle"==o.style?(a=e.getSVGElement("circle",n,s),a.setAttributeNS(null,"cx",t),a.setAttributeNS(null,"cy",i),a.setAttributeNS(null,"r",.5*o.size)):(a=e.getSVGElement("rect",n,s),a.setAttributeNS(null,"x",t-.5*o.size),a.setAttributeNS(null,"y",i-.5*o.size),a.setAttributeNS(null,"width",o.size),a.setAttributeNS(null,"height",o.size)),void 0!==o.styles&&a.setAttributeNS(null,"style",o.styles),a.setAttributeNS(null,"class",o.className+" vis-point"),r){var h=e.getSVGElement("text",n,s);r.xOffset&&(t+=r.xOffset),r.yOffset&&(i+=r.yOffset),r.content&&(h.textContent=r.content),r.className&&h.setAttributeNS(null,"class",r.className+" vis-label"),h.setAttributeNS(null,"x",t),h.setAttributeNS(null,"y",i)}return a},e.drawBar=function(t,i,o,n,s,r,a,h){if(0!=n){n<0&&(n*=-1,i-=n);var d=e.getSVGElement("rect",r,a);d.setAttributeNS(null,"x",t-.5*o),d.setAttributeNS(null,"y",i),d.setAttributeNS(null,"width",o),d.setAttributeNS(null,"height",n),d.setAttributeNS(null,"class",s),h&&d.setAttributeNS(null,"style",h)}}},function(t,e,i){function o(){var t=function(){};return{on:t,off:t,destroy:t,emit:t,get:function(e){return{set:t}}}}if("undefined"!=typeof window){var n=i(130),s=window.Hammer||i(131);t.exports=n(s,{preventDefault:"mouse"})}else t.exports=function(){return o()}},function(t,e,i){var o=i(91),n=i(34);t.exports=function(t){return o(n(t))}},function(t,e,i){var o=i(8),n=i(25);t.exports=i(9)?function(t,e,i){return o.f(t,e,n(1,i))}:function(t,e,i){return t[e]=i,t}},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e){if(t&&!Array.isArray(t)&&(e=t,t=null),this._options=e||{},this._data={},this.length=0,this._fieldId=this._options.fieldId||"id",this._type={},this._options.type)for(var i=(0,l.default)(this._options.type),o=0,n=i.length;o<n;o++){var s=i[o],r=this._options.type[s];this._type[s]="Date"==r||"ISODate"==r||"ASPDate"==r?"Date":r}this._subscribers={},t&&this.add(t),this.setOptions(e)}var s=i(16),r=o(s),a=i(1),h=o(a),d=i(4),l=o(d),u=i(0),p=i(65);n.prototype.setOptions=function(t){t&&void 0!==t.queue&&(!1===t.queue?this._queue&&(this._queue.destroy(),delete this._queue):(this._queue||(this._queue=p.extend(this,{replace:["add","update","remove"]})),"object"===(0,h.default)(t.queue)&&this._queue.setOptions(t.queue)))},n.prototype.on=function(t,e){var i=this._subscribers[t];i||(i=[],this._subscribers[t]=i),i.push({callback:e})},n.prototype.off=function(t,e){var i=this._subscribers[t];i&&(this._subscribers[t]=i.filter(function(t){return t.callback!=e}))},n.prototype._trigger=function(t,e,i){if("*"==t)throw new Error("Cannot trigger event *");var o=[];t in this._subscribers&&(o=o.concat(this._subscribers[t])),"*"in this._subscribers&&(o=o.concat(this._subscribers["*"]));for(var n=0,s=o.length;n<s;n++){var r=o[n];r.callback&&r.callback(t,e,i||null)}},n.prototype.add=function(t,e){var i,o=[],n=this;if(Array.isArray(t))for(var s=0,r=t.length;s<r;s++)i=n._addItem(t[s]),o.push(i);else{if(!t||"object"!==(void 0===t?"undefined":(0,h.default)(t)))throw new Error("Unknown dataType");i=n._addItem(t),o.push(i)}return o.length&&this._trigger("add",{items:o},e),o},n.prototype.update=function(t,e){var i=[],o=[],n=[],s=[],r=this,a=r._fieldId,d=function(t){var e=t[a];if(r._data[e]){var h=u.extend({},r._data[e]);e=r._updateItem(t),o.push(e),s.push(t),n.push(h)}else e=r._addItem(t),i.push(e)};if(Array.isArray(t))for(var l=0,p=t.length;l<p;l++)t[l]&&"object"===(0,h.default)(t[l])?d(t[l]):console.warn("Ignoring input item, which is not an object at index "+l);else{if(!t||"object"!==(void 0===t?"undefined":(0,h.default)(t)))throw new Error("Unknown dataType");d(t)}if(i.length&&this._trigger("add",{items:i},e),o.length){var c={items:o,oldData:n,data:s};this._trigger("update",c,e)}return i.concat(o)},n.prototype.get=function(t){var e,i,o,n=this,s=u.getType(arguments[0]);"String"==s||"Number"==s?(e=arguments[0],o=arguments[1]):"Array"==s?(i=arguments[0],o=arguments[1]):o=arguments[0];var r;if(o&&o.returnType){r=-1==["Array","Object"].indexOf(o.returnType)?"Array":o.returnType}else r="Array";var a,h,d,p,c,m=o&&o.type||this._options.type,f=o&&o.filter,g=[];if(void 0!=e)(a=n._getItem(e,m))&&f&&!f(a)&&(a=null);else if(void 0!=i)for(p=0,c=i.length;p<c;p++)a=n._getItem(i[p],m),f&&!f(a)||g.push(a);else for(h=(0,l.default)(this._data),p=0,c=h.length;p<c;p++)d=h[p],a=n._getItem(d,m),f&&!f(a)||g.push(a);if(o&&o.order&&void 0==e&&this._sort(g,o.order),o&&o.fields){var v=o.fields;if(void 0!=e)a=this._filterFields(a,v);else for(p=0,c=g.length;p<c;p++)g[p]=this._filterFields(g[p],v)}if("Object"==r){var y,b={};for(p=0,c=g.length;p<c;p++)y=g[p],b[y.id]=y;return b}return void 0!=e?a:g},n.prototype.getIds=function(t){var e,i,o,n,s,r=this._data,a=t&&t.filter,h=t&&t.order,d=t&&t.type||this._options.type,u=(0,l.default)(r),p=[];if(a)if(h){for(s=[],e=0,i=u.length;e<i;e++)o=u[e],n=this._getItem(o,d),a(n)&&s.push(n);for(this._sort(s,h),e=0,i=s.length;e<i;e++)p.push(s[e][this._fieldId])}else for(e=0,i=u.length;e<i;e++)o=u[e],n=this._getItem(o,d),a(n)&&p.push(n[this._fieldId]);else if(h){for(s=[],e=0,i=u.length;e<i;e++)o=u[e],s.push(r[o]);for(this._sort(s,h),e=0,i=s.length;e<i;e++)p.push(s[e][this._fieldId])}else for(e=0,i=u.length;e<i;e++)o=u[e],n=r[o],p.push(n[this._fieldId]);return p},n.prototype.getDataSet=function(){return this},n.prototype.forEach=function(t,e){var i,o,n,s,r=e&&e.filter,a=e&&e.type||this._options.type,h=this._data,d=(0,l.default)(h);if(e&&e.order){var u=this.get(e);for(i=0,o=u.length;i<o;i++)n=u[i],s=n[this._fieldId],t(n,s)}else for(i=0,o=d.length;i<o;i++)s=d[i],n=this._getItem(s,a),r&&!r(n)||t(n,s)},n.prototype.map=function(t,e){var i,o,n,s,r=e&&e.filter,a=e&&e.type||this._options.type,h=[],d=this._data,u=(0,l.default)(d);for(i=0,o=u.length;i<o;i++)n=u[i],s=this._getItem(n,a),r&&!r(s)||h.push(t(s,n));return e&&e.order&&this._sort(h,e.order),h},n.prototype._filterFields=function(t,e){if(!t)return t;var i,o,n={},s=(0,l.default)(t),r=s.length;if(Array.isArray(e))for(i=0;i<r;i++)o=s[i],-1!=e.indexOf(o)&&(n[o]=t[o]);else for(i=0;i<r;i++)o=s[i],e.hasOwnProperty(o)&&(n[e[o]]=t[o]);return n},n.prototype._sort=function(t,e){if(u.isString(e)){var i=e;t.sort(function(t,e){var o=t[i],n=e[i];return o>n?1:o<n?-1:0})}else{if("function"!=typeof e)throw new TypeError("Order must be a function or a string");t.sort(e)}},n.prototype.remove=function(t,e){var i,o,n,s,r=[],a=[],h=[];for(h=Array.isArray(t)?t:[t],i=0,o=h.length;i<o;i++)(s=this._remove(h[i]))&&void 0!=(n=s[this._fieldId])&&(r.push(n),a.push(s));return r.length&&this._trigger("remove",{items:r,oldData:a},e),r},n.prototype._remove=function(t){var e,i;return u.isNumber(t)||u.isString(t)?i=t:t&&"object"===(void 0===t?"undefined":(0,h.default)(t))&&(i=t[this._fieldId]),void 0!==i&&this._data[i]?(e=this._data[i],delete this._data[i],this.length--,e):null},n.prototype.clear=function(t){var e,i,o=(0,l.default)(this._data),n=[];for(e=0,i=o.length;e<i;e++)n.push(this._data[o[e]]);return this._data={},this.length=0,this._trigger("remove",{items:o,oldData:n},t),o},n.prototype.max=function(t){var e,i,o=this._data,n=(0,l.default)(o),s=null,r=null;for(e=0,i=n.length;e<i;e++){var a=n[e],h=o[a],d=h[t];null!=d&&(!s||d>r)&&(s=h,r=d)}return s},n.prototype.min=function(t){var e,i,o=this._data,n=(0,l.default)(o),s=null,r=null;for(e=0,i=n.length;e<i;e++){var a=n[e],h=o[a],d=h[t];null!=d&&(!s||d<r)&&(s=h,r=d)}return s},n.prototype.distinct=function(t){var e,i,o,n=this._data,s=(0,l.default)(n),r=[],a=this._options.type&&this._options.type[t]||null,h=0;for(e=0,o=s.length;e<o;e++){var d=s[e],p=n[d],c=p[t],m=!1;for(i=0;i<h;i++)if(r[i]==c){m=!0;break}m||void 0===c||(r[h]=c,h++)}if(a)for(e=0,o=r.length;e<o;e++)r[e]=u.convert(r[e],a);return r},n.prototype._addItem=function(t){var e=t[this._fieldId];if(void 0!=e){if(this._data[e])throw new Error("Cannot add item: item with id "+e+" already exists")}else e=u.randomUUID(),t[this._fieldId]=e;var i,o,n={},s=(0,l.default)(t);for(i=0,o=s.length;i<o;i++){var r=s[i],a=this._type[r];n[r]=u.convert(t[r],a)}return this._data[e]=n,this.length++,e},n.prototype._getItem=function(t,e){var i,o,n,s,r=this._data[t];if(!r)return null;var a={},h=(0,l.default)(r);if(e)for(n=0,s=h.length;n<s;n++)i=h[n],o=r[i],a[i]=u.convert(o,e[i]);else for(n=0,s=h.length;n<s;n++)i=h[n],o=r[i],a[i]=o;return a[this._fieldId]||(a[this._fieldId]=r.id),a},n.prototype._updateItem=function(t){var e=t[this._fieldId];if(void 0==e)throw new Error("Cannot update item: item has no id (item: "+(0,r.default)(t)+")");var i=this._data[e];if(!i)throw new Error("Cannot update item: no item with id "+e+" found");for(var o=(0,l.default)(t),n=0,s=o.length;n<s;n++){var a=o[n],h=this._type[a];i[a]=u.convert(t[a],h)}return e},t.exports=n},function(t,e,i){t.exports={default:i(128),__esModule:!0}},function(t,e,i){var o=i(6),n=i(2),s=i(92),r=i(14),a=function(t,e,i){var h,d,l,u=t&a.F,p=t&a.G,c=t&a.S,m=t&a.P,f=t&a.B,g=t&a.W,v=p?n:n[e]||(n[e]={}),y=v.prototype,b=p?o:c?o[e]:(o[e]||{}).prototype;p&&(i=e);for(h in i)(d=!u&&b&&void 0!==b[h])&&h in v||(l=d?b[h]:i[h],v[h]=p&&"function"!=typeof b[h]?i[h]:f&&d?s(l,o):g&&b[h]==l?function(t){var e=function(e,i,o){if(this instanceof t){switch(arguments.length){case 0:return new t;case 1:return new t(e);case 2:return new t(e,i)}return new t(e,i,o)}return t.apply(this,arguments)};return e.prototype=t.prototype,e}(l):m&&"function"==typeof l?s(Function.call,l):l,m&&((v.virtual||(v.virtual={}))[h]=l,t&a.R&&y&&!y[h]&&r(y,h,l)))};a.F=1,a.G=2,a.S=4,a.P=8,a.B=16,a.W=32,a.U=64,a.R=128,t.exports=a},function(t,e,i){var o=i(24);t.exports=function(t){if(!o(t))throw TypeError(t+" is not an object!");return t}},function(t,e){t.exports=function(t){try{return!!t()}catch(t){return!0}}},function(t,e,i){function o(t,e){this._data=null,this._ids={},this.length=0,this._options=e||{},this._fieldId="id",this._subscribers={};var i=this;this.listener=function(){i._onEvent.apply(i,arguments)},this.setData(t)}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(15);o.prototype.setData=function(t){var e,i,o,n,s;if(this._data){for(this._data.off&&this._data.off("*",this.listener),e=this._data.getIds({filter:this._options&&this._options.filter}),s=[],o=0,n=e.length;o<n;o++)s.push(this._data._data[e[o]]);this._ids={},this.length=0,this._trigger("remove",{items:e,oldData:s})}if(this._data=t,this._data){for(this._fieldId=this._options.fieldId||this._data&&this._data.options&&this._data.options.fieldId||"id",e=this._data.getIds({filter:this._options&&this._options.filter}),o=0,n=e.length;o<n;o++)i=e[o],this._ids[i]=!0;this.length=e.length,this._trigger("add",{items:e}),this._data.on&&this._data.on("*",this.listener)}},o.prototype.refresh=function(){var t,e,i,o=this._data.getIds({filter:this._options&&this._options.filter}),n=(0,s.default)(this._ids),r={},a=[],h=[],d=[];for(e=0,i=o.length;e<i;e++)t=o[e],r[t]=!0,this._ids[t]||(a.push(t),this._ids[t]=!0);for(e=0,i=n.length;e<i;e++)t=n[e],r[t]||(h.push(t),d.push(this._data._data[t]),delete this._ids[t]);this.length+=a.length-h.length,a.length&&this._trigger("add",{items:a}),h.length&&this._trigger("remove",{items:h,oldData:d})},o.prototype.get=function(t){var e,i,o,n=this,s=r.getType(arguments[0]);"String"==s||"Number"==s||"Array"==s?(e=arguments[0],i=arguments[1],o=arguments[2]):(i=arguments[0],o=arguments[1]);var a=r.extend({},this._options,i);this._options.filter&&i&&i.filter&&(a.filter=function(t){return n._options.filter(t)&&i.filter(t)});var h=[];return void 0!=e&&h.push(e),h.push(a),h.push(o),this._data&&this._data.get.apply(this._data,h)},o.prototype.getIds=function(t){var e;if(this._data){var i,o=this._options.filter;i=t&&t.filter?o?function(e){return o(e)&&t.filter(e)}:t.filter:o,e=this._data.getIds({filter:i,order:t&&t.order})}else e=[];return e},o.prototype.map=function(t,e){var i=[];if(this._data){var o,n=this._options.filter;o=e&&e.filter?n?function(t){return n(t)&&e.filter(t)}:e.filter:n,i=this._data.map(t,{filter:o,order:e&&e.order})}else i=[];return i},o.prototype.getDataSet=function(){for(var t=this;t instanceof o;)t=t._data;return t||null},o.prototype._onEvent=function(t,e,i){var o,n,s,r,a=e&&e.items,h=[],d=[],l=[],u=[],p=[],c=[];if(a&&this._data){switch(t){case"add":for(o=0,n=a.length;o<n;o++)s=a[o],(r=this.get(s))&&(this._ids[s]=!0,h.push(s));break;case"update":for(o=0,n=a.length;o<n;o++)s=a[o],r=this.get(s),r?this._ids[s]?(d.push(s),p.push(e.data[o]),u.push(e.oldData[o])):(this._ids[s]=!0,h.push(s)):this._ids[s]&&(delete this._ids[s],l.push(s),c.push(e.oldData[o]));break;case"remove":for(o=0,n=a.length;o<n;o++)s=a[o],this._ids[s]&&(delete this._ids[s],l.push(s),c.push(e.oldData[o]))}this.length+=h.length-l.length,h.length&&this._trigger("add",{items:h},i),d.length&&this._trigger("update",{items:d,oldData:u,data:p},i),l.length&&this._trigger("remove",{items:l,oldData:c},i)}},o.prototype.on=a.prototype.on,o.prototype.off=a.prototype.off,o.prototype._trigger=a.prototype._trigger,o.prototype.subscribe=o.prototype.on,o.prototype.unsubscribe=o.prototype.off,t.exports=o},function(t,e,i){e.convertHiddenOptions=function(t,i,o){if(o&&!Array.isArray(o))return e.convertHiddenOptions(t,i,[o]);if(i.hiddenDates=[],o&&1==Array.isArray(o)){for(var n=0;n<o.length;n++)if(void 0===o[n].repeat){var s={};s.start=t(o[n].start).toDate().valueOf(),s.end=t(o[n].end).toDate().valueOf(),i.hiddenDates.push(s)}i.hiddenDates.sort(function(t,e){return t.start-e.start})}},e.updateHiddenDates=function(t,i,o){if(o&&!Array.isArray(o))return e.updateHiddenDates(t,i,[o]);if(o&&void 0!==i.domProps.centerContainer.width){e.convertHiddenOptions(t,i,o);for(var n=t(i.range.start),s=t(i.range.end),r=i.range.end-i.range.start,a=r/i.domProps.centerContainer.width,h=0;h<o.length;h++)if(void 0!==o[h].repeat){var d=t(o[h].start),l=t(o[h].end);if("Invalid Date"==d._d)throw new Error("Supplied start date is not valid: "+o[h].start);if("Invalid Date"==l._d)throw new Error("Supplied end date is not valid: "+o[h].end);var u=l-d;if(u>=4*a){var p=0,c=s.clone();switch(o[h].repeat){case"daily":d.day()!=l.day()&&(p=1),d.dayOfYear(n.dayOfYear()),d.year(n.year()),d.subtract(7,"days"),l.dayOfYear(n.dayOfYear()),l.year(n.year()),l.subtract(7-p,"days"),c.add(1,"weeks");break;case"weekly":var m=l.diff(d,"days"),f=d.day();d.date(n.date()),d.month(n.month()),d.year(n.year()),l=d.clone(),d.day(f),l.day(f),l.add(m,"days"),d.subtract(1,"weeks"),l.subtract(1,"weeks"),c.add(1,"weeks");break;case"monthly":d.month()!=l.month()&&(p=1),d.month(n.month()),d.year(n.year()),d.subtract(1,"months"),l.month(n.month()),l.year(n.year()),l.subtract(1,"months"),l.add(p,"months"),c.add(1,"months");break;case"yearly":d.year()!=l.year()&&(p=1),d.year(n.year()),d.subtract(1,"years"),l.year(n.year()),l.subtract(1,"years"),l.add(p,"years"),c.add(1,"years");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",o[h].repeat)}for(;d<c;)switch(i.hiddenDates.push({start:d.valueOf(),end:l.valueOf()}),o[h].repeat){case"daily":d.add(1,"days"),l.add(1,"days");break;case"weekly":d.add(1,"weeks"),l.add(1,"weeks");break;case"monthly":d.add(1,"months"),l.add(1,"months");break;case"yearly":d.add(1,"y"),l.add(1,"y");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",o[h].repeat)}i.hiddenDates.push({start:d.valueOf(),end:l.valueOf()})}}e.removeDuplicates(i);var g=e.isHidden(i.range.start,i.hiddenDates),v=e.isHidden(i.range.end,i.hiddenDates),y=i.range.start,b=i.range.end;1==g.hidden&&(y=1==i.range.startToFront?g.startDate-1:g.endDate+1),1==v.hidden&&(b=1==i.range.endToFront?v.startDate-1:v.endDate+1),1!=g.hidden&&1!=v.hidden||i.range._applyRange(y,b)}},e.removeDuplicates=function(t){for(var e=t.hiddenDates,i=[],o=0;o<e.length;o++)for(var n=0;n<e.length;n++)o!=n&&1!=e[n].remove&&1!=e[o].remove&&(e[n].start>=e[o].start&&e[n].end<=e[o].end?e[n].remove=!0:e[n].start>=e[o].start&&e[n].start<=e[o].end?(e[o].end=e[n].end,e[n].remove=!0):e[n].end>=e[o].start&&e[n].end<=e[o].end&&(e[o].start=e[n].start,e[n].remove=!0));for(o=0;o<e.length;o++)!0!==e[o].remove&&i.push(e[o]);t.hiddenDates=i,t.hiddenDates.sort(function(t,e){return t.start-e.start})},e.printDates=function(t){for(var e=0;e<t.length;e++)console.log(e,new Date(t[e].start),new Date(t[e].end),t[e].start,t[e].end,t[e].remove)},e.stepOverHiddenDates=function(t,e,i){for(var o=!1,n=e.current.valueOf(),s=0;s<e.hiddenDates.length;s++){var r=e.hiddenDates[s].start,a=e.hiddenDates[s].end;if(n>=r&&n<a){o=!0;break}}if(1==o&&n<e._end.valueOf()&&n!=i){var h=t(i),d=t(a);h.year()!=d.year()?e.switchedYear=!0:h.month()!=d.month()?e.switchedMonth=!0:h.dayOfYear()!=d.dayOfYear()&&(e.switchedDay=!0),e.current=d}},e.toScreen=function(t,i,o){var n;if(0==t.body.hiddenDates.length)return n=t.range.conversion(o),(i.valueOf()-n.offset)*n.scale;var s=e.isHidden(i,t.body.hiddenDates);1==s.hidden&&(i=s.startDate);var r=e.getHiddenDurationBetween(t.body.hiddenDates,t.range.start,t.range.end);if(i<t.range.start){n=t.range.conversion(o,r);var a=e.getHiddenDurationBeforeStart(t.body.hiddenDates,i,n.offset);return i=t.options.moment(i).toDate().valueOf(),i+=a,-(n.offset-i.valueOf())*n.scale}if(i>t.range.end){var h={start:t.range.start,end:i};return i=e.correctTimeForHidden(t.options.moment,t.body.hiddenDates,h,i),n=t.range.conversion(o,r),(i.valueOf()-n.offset)*n.scale}return i=e.correctTimeForHidden(t.options.moment,t.body.hiddenDates,t.range,i),n=t.range.conversion(o,r),(i.valueOf()-n.offset)*n.scale},e.toTime=function(t,i,o){if(0==t.body.hiddenDates.length){var n=t.range.conversion(o);return new Date(i/n.scale+n.offset)}var s=e.getHiddenDurationBetween(t.body.hiddenDates,t.range.start,t.range.end),r=t.range.end-t.range.start-s,a=r*i/o,h=e.getAccumulatedHiddenDuration(t.body.hiddenDates,t.range,a);return new Date(h+a+t.range.start)},e.getHiddenDurationBetween=function(t,e,i){for(var o=0,n=0;n<t.length;n++){var s=t[n].start,r=t[n].end;s>=e&&r<i&&(o+=r-s)}return o},e.getHiddenDurationBeforeStart=function(t,e,i){for(var o=0,n=0;n<t.length;n++){var s=t[n].start,r=t[n].end;s>=e&&r<=i&&(o+=r-s)}return o},e.correctTimeForHidden=function(t,i,o,n){return n=t(n).toDate().valueOf(),n-=e.getHiddenDurationBefore(t,i,o,n)},e.getHiddenDurationBefore=function(t,e,i,o){var n=0;o=t(o).toDate().valueOf();for(var s=0;s<e.length;s++){var r=e[s].start,a=e[s].end;r>=i.start&&a<i.end&&o>=a&&(n+=a-r)}return n},e.getAccumulatedHiddenDuration=function(t,e,i){for(var o=0,n=0,s=e.start,r=0;r<t.length;r++){var a=t[r].start,h=t[r].end;if(a>=e.start&&h<e.end){if(n+=a-s,s=h,n>=i)break;o+=h-a}}return o},e.snapAwayFromHidden=function(t,i,o,n){var s=e.isHidden(i,t);return 1==s.hidden?o<0?1==n?s.startDate-(s.endDate-i)-1:s.startDate-1:1==n?s.endDate+(i-s.startDate)+1:s.endDate+1:i},e.isHidden=function(t,e){for(var i=0;i<e.length;i++){var o=e[i].start,n=e[i].end;if(t>=o&&t<n)return{hidden:!0,startDate:o,endDate:n}}return{hidden:!1,startDate:o,endDate:n}}},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e,i){this.id=null,this.parent=null,this.data=t,this.dom=null,this.conversion=e||{},this.options=i||{},this.selected=!1,this.displayed=!1,this.groupShowing=!0,this.dirty=!0,this.top=null,this.right=null,this.left=null,this.width=null,this.height=null,this.editable=null,this._updateEditStatus()}var s=i(1),r=o(s),a=i(4),h=o(a),d=i(12),l=i(0),u=i(3);n.prototype.stack=!0,n.prototype.select=function(){this.selected=!0,this.dirty=!0,this.displayed&&this.redraw()},n.prototype.unselect=function(){this.selected=!1,this.dirty=!0,this.displayed&&this.redraw()},n.prototype.setData=function(t){void 0!=t.group&&this.data.group!=t.group&&null!=this.parent&&this.parent.itemSet._moveToGroup(this,t.group),this.parent&&(this.parent.stackDirty=!0),void 0!=t.subgroup&&this.data.subgroup!=t.subgroup&&null!=this.parent&&this.parent.changeSubgroup(this,this.data.subgroup,t.subgroup),this.data=t,this._updateEditStatus(),this.dirty=!0,this.displayed&&this.redraw()},n.prototype.setParent=function(t){this.displayed?(this.hide(),this.parent=t,this.parent&&this.show()):this.parent=t},n.prototype.isVisible=function(t){return!1},n.prototype.show=function(){return!1},n.prototype.hide=function(){return!1},n.prototype.redraw=function(){},n.prototype.repositionX=function(){},n.prototype.repositionY=function(){},n.prototype._repaintDragCenter=function(){if(this.selected&&this.options.editable.updateTime&&!this.dom.dragCenter){var t=this,e=document.createElement("div");e.className="vis-drag-center",e.dragCenterItem=this;var i=new d(e);i.on("tap",function(e){t.parent.itemSet.body.emitter.emit("click",{event:e,item:t.id})}),i.on("doubletap",function(e){e.stopPropagation(),t.parent.itemSet._onUpdateItem(t),t.parent.itemSet.body.emitter.emit("doubleClick",{event:e,item:t.id})}),this.dom.box?this.dom.dragLeft?this.dom.box.insertBefore(e,this.dom.dragLeft):this.dom.box.appendChild(e):this.dom.point&&this.dom.point.appendChild(e),this.dom.dragCenter=e}else!this.selected&&this.dom.dragCenter&&(this.dom.dragCenter.parentNode&&this.dom.dragCenter.parentNode.removeChild(this.dom.dragCenter),this.dom.dragCenter=null)},n.prototype._repaintDeleteButton=function(t){var e=(this.options.editable.overrideItems||null==this.editable)&&this.options.editable.remove||!this.options.editable.overrideItems&&null!=this.editable&&this.editable.remove;if(this.selected&&e&&!this.dom.deleteButton){var i=this,o=document.createElement("div");this.options.rtl?o.className="vis-delete-rtl":o.className="vis-delete",o.title="Delete this item",new d(o).on("tap",function(t){t.stopPropagation(),i.parent.removeFromDataSet(i)}),t.appendChild(o),this.dom.deleteButton=o}else!this.selected&&this.dom.deleteButton&&(this.dom.deleteButton.parentNode&&this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton),this.dom.deleteButton=null)},n.prototype._repaintOnItemUpdateTimeTooltip=function(t){if(this.options.tooltipOnItemUpdateTime){var e=(this.options.editable.updateTime||!0===this.data.editable)&&!1!==this.data.editable;if(this.selected&&e&&!this.dom.onItemUpdateTimeTooltip){var i=document.createElement("div");i.className="vis-onUpdateTime-tooltip",t.appendChild(i),this.dom.onItemUpdateTimeTooltip=i}else!this.selected&&this.dom.onItemUpdateTimeTooltip&&(this.dom.onItemUpdateTimeTooltip.parentNode&&this.dom.onItemUpdateTimeTooltip.parentNode.removeChild(this.dom.onItemUpdateTimeTooltip),this.dom.onItemUpdateTimeTooltip=null);if(this.dom.onItemUpdateTimeTooltip){this.dom.onItemUpdateTimeTooltip.style.visibility=this.parent.itemSet.touchParams.itemIsDragging?"visible":"hidden",this.options.rtl?this.dom.onItemUpdateTimeTooltip.style.right=this.dom.content.style.right:this.dom.onItemUpdateTimeTooltip.style.left=this.dom.content.style.left;var o,n=this.parent.itemSet.body.domProps.scrollTop;o="top"==this.options.orientation.item?this.top:this.parent.height-this.top-this.height;o+this.parent.top-50<-n?(this.dom.onItemUpdateTimeTooltip.style.bottom="",this.dom.onItemUpdateTimeTooltip.style.top=this.height+2+"px"):(this.dom.onItemUpdateTimeTooltip.style.top="",this.dom.onItemUpdateTimeTooltip.style.bottom=this.height+2+"px");var s,r;this.options.tooltipOnItemUpdateTime&&this.options.tooltipOnItemUpdateTime.template?(r=this.options.tooltipOnItemUpdateTime.template.bind(this),s=r(this.data)):(s="start: "+u(this.data.start).format("MM/DD/YYYY hh:mm"),this.data.end&&(s+="<br> end: "+u(this.data.end).format("MM/DD/YYYY hh:mm"))),this.dom.onItemUpdateTimeTooltip.innerHTML=s}}},n.prototype._updateContents=function(t){var e,i,o,n,s=this.parent.itemSet.itemsData.get(this.id),r=this.dom.box||this.dom.point,a=r.getElementsByClassName("vis-item-visible-frame")[0];if(this.options.visibleFrameTemplate?(n=this.options.visibleFrameTemplate.bind(this),o=n(s,r)):o="",a)if(o instanceof Object&&!(o instanceof Element))n(s,a);else if(this._contentToString(this.itemVisibleFrameContent)!==this._contentToString(o)){if(o instanceof Element)a.innerHTML="",a.appendChild(o);else if(void 0!=o)a.innerHTML=o;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '+this.id);this.itemVisibleFrameContent=o}if(this.options.template?(i=this.options.template.bind(this),e=i(s,t,this.data)):e=this.data.content,e instanceof Object&&!(e instanceof Element))i(s,t);else if(this._contentToString(this.content)!==this._contentToString(e)){if(e instanceof Element)t.innerHTML="",t.appendChild(e);else if(void 0!=e)t.innerHTML=e;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '+this.id);this.content=e}},n.prototype._updateDataAttributes=function(t){if(this.options.dataAttributes&&this.options.dataAttributes.length>0){var e=[];if(Array.isArray(this.options.dataAttributes))e=this.options.dataAttributes;else{if("all"!=this.options.dataAttributes)return;e=(0,h.default)(this.data)}for(var i=0;i<e.length;i++){var o=e[i],n=this.data[o];null!=n?t.setAttribute("data-"+o,n):t.removeAttribute("data-"+o)}}},n.prototype._updateStyle=function(t){this.style&&(l.removeCssText(t,this.style),this.style=null),this.data.style&&(l.addCssText(t,this.data.style),this.style=this.data.style)},n.prototype._contentToString=function(t){return"string"==typeof t?t:t&&"outerHTML"in t?t.outerHTML:t},n.prototype._updateEditStatus=function(){this.options&&("boolean"==typeof this.options.editable?this.editable={updateTime:this.options.editable,updateGroup:this.options.editable,remove:this.options.editable}:"object"===(0,r.default)(this.options.editable)&&(this.editable={},l.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.options.editable))),this.options&&this.options.editable&&!0===this.options.editable.overrideItems||this.data&&("boolean"==typeof this.data.editable?this.editable={updateTime:this.data.editable,updateGroup:this.data.editable,remove:this.data.editable}:"object"===(0,r.default)(this.data.editable)&&(this.editable={},l.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.data.editable)))},n.prototype.getWidthLeft=function(){return 0},n.prototype.getWidthRight=function(){return 0},n.prototype.getTitle=function(){return this.data.title},t.exports=n},function(t,e){t.exports={}},function(t,e){t.exports=function(t){return"object"==typeof t?null!==t:"function"==typeof t}},function(t,e){t.exports=function(t,e){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:e}}},function(t,e,i){var o=i(60),n=i(41);t.exports=Object.keys||function(t){return o(t,n)}},function(t,e){var i=0,o=Math.random();t.exports=function(t){return"Symbol(".concat(void 0===t?"":t,")_",(++i+o).toString(36))}},function(t,e,i){function o(t,e){this.dom={foreground:null,lines:[],majorTexts:[],minorTexts:[],redundant:{lines:[],majorTexts:[],minorTexts:[]}},this.props={range:{start:0,end:0,minimumStep:0},lineTop:0},this.defaultOptions={orientation:{axis:"bottom"},showMinorLabels:!0,showMajorLabels:!0,maxMinorChars:7,format:h.FORMAT,moment:l,timeAxis:null},this.options=r.extend({},this.defaultOptions),this.body=t,this._create(),this.setOptions(e)}var n=i(1),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(5),h=i(49),d=i(21),l=i(3);o.prototype=new a,o.prototype.setOptions=function(t){t&&(r.selectiveExtend(["showMinorLabels","showMajorLabels","maxMinorChars","hiddenDates","timeAxis","moment","rtl"],this.options,t),r.selectiveDeepExtend(["format"],this.options,t),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.axis=t.orientation:"object"===(0,s.default)(t.orientation)&&"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis)),"locale"in t&&("function"==typeof l.locale?l.locale(t.locale):l.lang(t.locale)))},o.prototype._create=function(){this.dom.foreground=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.foreground.className="vis-time-axis vis-foreground",this.dom.background.className="vis-time-axis vis-background"},o.prototype.destroy=function(){this.dom.foreground.parentNode&&this.dom.foreground.parentNode.removeChild(this.dom.foreground),this.dom.background.parentNode&&this.dom.background.parentNode.removeChild(this.dom.background),this.body=null},o.prototype.redraw=function(){var t=this.props,e=this.dom.foreground,i=this.dom.background,o="top"==this.options.orientation.axis?this.body.dom.top:this.body.dom.bottom,n=e.parentNode!==o;this._calculateCharSize();var s=this.options.showMinorLabels&&"none"!==this.options.orientation.axis,r=this.options.showMajorLabels&&"none"!==this.options.orientation.axis;t.minorLabelHeight=s?t.minorCharHeight:0,t.majorLabelHeight=r?t.majorCharHeight:0,t.height=t.minorLabelHeight+t.majorLabelHeight,t.width=e.offsetWidth,t.minorLineHeight=this.body.domProps.root.height-t.majorLabelHeight-("top"==this.options.orientation.axis?this.body.domProps.bottom.height:this.body.domProps.top.height),t.minorLineWidth=1,t.majorLineHeight=t.minorLineHeight+t.majorLabelHeight,t.majorLineWidth=1;var a=e.nextSibling,h=i.nextSibling;return e.parentNode&&e.parentNode.removeChild(e),i.parentNode&&i.parentNode.removeChild(i),e.style.height=this.props.height+"px",this._repaintLabels(),a?o.insertBefore(e,a):o.appendChild(e),h?this.body.dom.backgroundVertical.insertBefore(i,h):this.body.dom.backgroundVertical.appendChild(i),this._isResized()||n},o.prototype._repaintLabels=function(){var t=this.options.orientation.axis,e=r.convert(this.body.range.start,"Number"),i=r.convert(this.body.range.end,"Number"),o=this.body.util.toTime((this.props.minorCharWidth||10)*this.options.maxMinorChars).valueOf(),n=o-d.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this.body.range,o);n-=this.body.util.toTime(0).valueOf();var s=new h(new Date(e),new Date(i),n,this.body.hiddenDates,this.options);s.setMoment(this.options.moment),this.options.format&&s.setFormat(this.options.format),this.options.timeAxis&&s.setScale(this.options.timeAxis),this.step=s;var a=this.dom;a.redundant.lines=a.lines,a.redundant.majorTexts=a.majorTexts,a.redundant.minorTexts=a.minorTexts,a.lines=[],a.majorTexts=[],a.minorTexts=[];var l,p,c,m,f,g,v,y,b,_=0,w=void 0,x=0;for(s.start(),l=s.getCurrent(),c=this.body.util.toScreen(l);s.hasNext()&&x<1e3;){switch(x++,m=s.isMajor(),b=s.getClassName(),y=s.getLabelMinor(),l,p=c,s.next(),l=s.getCurrent(),s.isMajor(),c=this.body.util.toScreen(l),g=_,_=c-p,s.scale){case"week":f=!0;break;default:f=_>=.4*g}if(this.options.showMinorLabels&&f){var D=this._repaintMinorText(p,y,t,b);D.style.width=_+"px"}m&&this.options.showMajorLabels?(p>0&&(void 0==w&&(w=p),D=this._repaintMajorText(p,s.getLabelMajor(),t,b)),v=this._repaintMajorLine(p,_,t,b)):f?v=this._repaintMinorLine(p,_,t,b):v&&(v.style.width=parseInt(v.style.width)+_+"px")}if(1e3!==x||u||(console.warn("Something is wrong with the Timeline scale. Limited drawing of grid lines to 1000 lines."),u=!0),this.options.showMajorLabels){var S=this.body.util.toTime(0),k=s.getLabelMajor(S),C=k.length*(this.props.majorCharWidth||10)+10;(void 0==w||C<w)&&this._repaintMajorText(0,k,t,b)}r.forEach(this.dom.redundant,function(t){for(;t.length;){var e=t.pop();e&&e.parentNode&&e.parentNode.removeChild(e)}})},o.prototype._repaintMinorText=function(t,e,i,o){var n=this.dom.redundant.minorTexts.shift();if(!n){var s=document.createTextNode("");n=document.createElement("div"),n.appendChild(s),this.dom.foreground.appendChild(n)}return this.dom.minorTexts.push(n),n.innerHTML=e,n.style.top="top"==i?this.props.majorLabelHeight+"px":"0",this.options.rtl?(n.style.left="",n.style.right=t+"px"):n.style.left=t+"px",n.className="vis-text vis-minor "+o,n},o.prototype._repaintMajorText=function(t,e,i,o){var n=this.dom.redundant.majorTexts.shift();if(!n){var s=document.createElement("div");n=document.createElement("div"),n.appendChild(s),this.dom.foreground.appendChild(n)}return n.childNodes[0].innerHTML=e,n.className="vis-text vis-major "+o,n.style.top="top"==i?"0":this.props.minorLabelHeight+"px",this.options.rtl?(n.style.left="",n.style.right=t+"px"):n.style.left=t+"px",this.dom.majorTexts.push(n),n},o.prototype._repaintMinorLine=function(t,e,i,o){var n=this.dom.redundant.lines.shift();n||(n=document.createElement("div"),this.dom.background.appendChild(n)),this.dom.lines.push(n);var s=this.props;return n.style.top="top"==i?s.majorLabelHeight+"px":this.body.domProps.top.height+"px",n.style.height=s.minorLineHeight+"px",this.options.rtl?(n.style.left="",n.style.right=t-s.minorLineWidth/2+"px",n.className="vis-grid vis-vertical-rtl vis-minor "+o):(n.style.left=t-s.minorLineWidth/2+"px",n.className="vis-grid vis-vertical vis-minor "+o),n.style.width=e+"px",n},o.prototype._repaintMajorLine=function(t,e,i,o){var n=this.dom.redundant.lines.shift();n||(n=document.createElement("div"),this.dom.background.appendChild(n)),this.dom.lines.push(n);var s=this.props;return n.style.top="top"==i?"0":this.body.domProps.top.height+"px",this.options.rtl?(n.style.left="",n.style.right=t-s.majorLineWidth/2+"px",n.className="vis-grid vis-vertical-rtl vis-major "+o):(n.style.left=t-s.majorLineWidth/2+"px",n.className="vis-grid vis-vertical vis-major "+o),n.style.height=s.majorLineHeight+"px",n.style.width=e+"px",n},o.prototype._calculateCharSize=function(){this.dom.measureCharMinor||(this.dom.measureCharMinor=document.createElement("DIV"),this.dom.measureCharMinor.className="vis-text vis-minor vis-measure",this.dom.measureCharMinor.style.position="absolute",this.dom.measureCharMinor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMinor)),this.props.minorCharHeight=this.dom.measureCharMinor.clientHeight,this.props.minorCharWidth=this.dom.measureCharMinor.clientWidth,this.dom.measureCharMajor||(this.dom.measureCharMajor=document.createElement("DIV"),this.dom.measureCharMajor.className="vis-text vis-major vis-measure",this.dom.measureCharMajor.style.position="absolute",this.dom.measureCharMajor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMajor)),this.props.majorCharHeight=this.dom.measureCharMajor.clientHeight,this.props.majorCharWidth=this.dom.measureCharMajor.clientWidth};var u=!1;t.exports=o},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={moment:a,locales:h,locale:"en",id:void 0,title:void 0},this.options=s.extend({},this.defaultOptions),e&&e.time?this.customTime=e.time:this.customTime=new Date,this.eventParams={},this.setOptions(e),this._create()}var n=i(12),s=i(0),r=i(5),a=i(3),h=i(69);o.prototype=new r,o.prototype.setOptions=function(t){t&&s.selectiveExtend(["moment","locale","locales","id"],this.options,t)},o.prototype._create=function(){function t(t){this.body.range._onMouseWheel(t)}var e=document.createElement("div");e["custom-time"]=this,e.className="vis-custom-time "+(this.options.id||""),e.style.position="absolute",e.style.top="0px",e.style.height="100%",this.bar=e;var i=document.createElement("div");i.style.position="relative",i.style.top="0px",i.style.left="-10px",i.style.height="100%",i.style.width="20px",i.addEventListener?(i.addEventListener("mousewheel",t.bind(this),!1),i.addEventListener("DOMMouseScroll",t.bind(this),!1)):i.attachEvent("onmousewheel",t.bind(this)),e.appendChild(i),this.hammer=new n(i),this.hammer.on("panstart",this._onDragStart.bind(this)),this.hammer.on("panmove",this._onDrag.bind(this)),this.hammer.on("panend",this._onDragEnd.bind(this)),this.hammer.get("pan").set({threshold:5,direction:n.DIRECTION_HORIZONTAL})},o.prototype.destroy=function(){this.hide(),this.hammer.destroy(),this.hammer=null,this.body=null},o.prototype.redraw=function(){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar));var e=this.body.util.toScreen(this.customTime),i=this.options.locales[this.options.locale];i||(this.warned||(console.log("WARNING: options.locales['"+this.options.locale+"'] not found. See http://visjs.org/docs/timeline/#Localization"),this.warned=!0),i=this.options.locales.en);var o=this.options.title;return void 0===o?(o=i.time+": "+this.options.moment(this.customTime).format("dddd, MMMM Do YYYY, H:mm:ss"),o=o.charAt(0).toUpperCase()+o.substring(1)):"function"==typeof o&&(o=o.call(this.customTime)),this.bar.style.left=e+"px",this.bar.title=o,!1},o.prototype.hide=function(){this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar)},o.prototype.setCustomTime=function(t){this.customTime=s.convert(t,"Date"),this.redraw()},o.prototype.getCustomTime=function(){return new Date(this.customTime.valueOf())},o.prototype.setCustomTitle=function(t){this.options.title=t},o.prototype._onDragStart=function(t){this.eventParams.dragging=!0,this.eventParams.customTime=this.customTime,t.stopPropagation()},o.prototype._onDrag=function(t){if(this.eventParams.dragging){var e=this.body.util.toScreen(this.eventParams.customTime)+t.deltaX,i=this.body.util.toTime(e);this.setCustomTime(i),this.body.emitter.emit("timechange",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation()}},o.prototype._onDragEnd=function(t){this.eventParams.dragging&&(this.body.emitter.emit("timechanged",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation())},o.customTimeFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("custom-time"))return e["custom-time"];e=e.parentNode}return null},t.exports=o},function(t,e,i){e.__esModule=!0,e.default=function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}},function(t,e,i){e.__esModule=!0;var o=i(134),n=function(t){return t&&t.__esModule?t:{default:t}}(o);e.default=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),(0,n.default)(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}()},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0}),e.printStyle=void 0;var n=i(16),s=o(n),r=i(1),a=o(r),h=i(4),d=o(h),l=i(30),u=o(l),p=i(31),c=o(p),m=i(0),f=!1,g=void 0,v="background: #FFeeee; color: #dd0000",y=function(){function t(){(0,u.default)(this,t)}return(0,c.default)(t,null,[{key:"validate",value:function(e,i,o){f=!1,g=i;var n=i;return void 0!==o&&(n=i[o]),t.parse(e,n,[]),f}},{key:"parse",value:function(e,i,o){for(var n in e)e.hasOwnProperty(n)&&t.check(n,e,i,o)}},{key:"check",value:function(e,i,o,n){if(void 0===o[e]&&void 0===o.__any__)return void t.getSuggestion(e,o,n);var s=e,r=!0;void 0===o[e]&&void 0!==o.__any__&&(s="__any__",r="object"===t.getType(i[e]));var a=o[s];r&&void 0!==a.__type__&&(a=a.__type__),t.checkFields(e,i,o,s,a,n)}},{key:"checkFields",value:function(e,i,o,n,s,r){var a=function(i){console.log("%c"+i+t.printLocation(r,e),v)},h=t.getType(i[e]),l=s[h];void 0!==l?"array"===t.getType(l)&&-1===l.indexOf(i[e])?(a('Invalid option detected in "'+e+'". Allowed values are:'+t.print(l)+' not "'+i[e]+'". '),f=!0):"object"===h&&"__any__"!==n&&(r=m.copyAndExtendArray(r,e),t.parse(i[e],o[n],r)):void 0===s.any&&(a('Invalid type received for "'+e+'". Expected: '+t.print((0,d.default)(s))+". Received ["+h+'] "'+i[e]+'"'),f=!0)}},{key:"getType",value:function(t){var e=void 0===t?"undefined":(0,a.default)(t);return"object"===e?null===t?"null":t instanceof Boolean?"boolean":t instanceof Number?"number":t instanceof String?"string":Array.isArray(t)?"array":t instanceof Date?"date":void 0!==t.nodeType?"dom":!0===t._isAMomentObject?"moment":"object":"number"===e?"number":"boolean"===e?"boolean":"string"===e?"string":void 0===e?"undefined":e}},{key:"getSuggestion",value:function(e,i,o){var n=t.findInOptions(e,i,o,!1),s=t.findInOptions(e,g,[],!0),r=void 0;r=void 0!==n.indexMatch?" in "+t.printLocation(n.path,e,"")+'Perhaps it was incomplete? Did you mean: "'+n.indexMatch+'"?\n\n':s.distance<=4&&n.distance>s.distance?" in "+t.printLocation(n.path,e,"")+"Perhaps it was misplaced? Matching option found at: "+t.printLocation(s.path,s.closestMatch,""):n.distance<=8?'. Did you mean "'+n.closestMatch+'"?'+t.printLocation(n.path,e):". Did you mean one of these: "+t.print((0,d.default)(i))+t.printLocation(o,e),console.log('%cUnknown option detected: "'+e+'"'+r,v),f=!0}},{key:"findInOptions",value:function(e,i,o){var n=arguments.length>3&&void 0!==arguments[3]&&arguments[3],s=1e9,r="",a=[],h=e.toLowerCase(),d=void 0;for(var l in i){var u=void 0;if(void 0!==i[l].__type__&&!0===n){var p=t.findInOptions(e,i[l],m.copyAndExtendArray(o,l));s>p.distance&&(r=p.closestMatch,a=p.path,s=p.distance,d=p.indexMatch)}else-1!==l.toLowerCase().indexOf(h)&&(d=l),u=t.levenshteinDistance(e,l),s>u&&(r=l,a=m.copyArray(o),s=u)}return{closestMatch:r,path:a,distance:s,indexMatch:d}}},{key:"printLocation",value:function(t,e){for(var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:"Problem value found at: \n",o="\n\n"+i+"options = {\n",n=0;n<t.length;n++){for(var s=0;s<n+1;s++)o+="  ";o+=t[n]+": {\n"}for(var r=0;r<t.length+1;r++)o+="  ";o+=e+"\n";for(var a=0;a<t.length+1;a++){for(var h=0;h<t.length-a;h++)o+="  ";o+="}\n"}return o+"\n\n"}},{key:"print",value:function(t){return(0,s.default)(t).replace(/(\")|(\[)|(\])|(,"__type__")/g,"").replace(/(\,)/g,", ")}},{key:"levenshteinDistance",value:function(t,e){if(0===t.length)return e.length;if(0===e.length)return t.length;var i,o=[];for(i=0;i<=e.length;i++)o[i]=[i];var n;for(n=0;n<=t.length;n++)o[0][n]=n;for(i=1;i<=e.length;i++)for(n=1;n<=t.length;n++)e.charAt(i-1)==t.charAt(n-1)?o[i][n]=o[i-1][n-1]:o[i][n]=Math.min(o[i-1][n-1]+1,Math.min(o[i][n-1]+1,o[i-1][n]+1));return o[e.length][t.length]}}]),t}();e.default=y,e.printStyle=v},function(t,e){var i={}.toString;t.exports=function(t){return i.call(t).slice(8,-1)}},function(t,e){t.exports=function(t){if(void 0==t)throw TypeError("Can't call method on  "+t);return t}},function(t,e){t.exports=!0},function(t,e,i){var o=i(24);t.exports=function(t,e){if(!o(t))return t;var i,n;if(e&&"function"==typeof(i=t.toString)&&!o(n=i.call(t)))return n;if("function"==typeof(i=t.valueOf)&&!o(n=i.call(t)))return n;if(!e&&"function"==typeof(i=t.toString)&&!o(n=i.call(t)))return n;throw TypeError("Can't convert object to primitive value")}},function(t,e,i){var o=i(18),n=i(95),s=i(41),r=i(39)("IE_PROTO"),a=function(){},h=function(){var t,e=i(58)("iframe"),o=s.length;for(e.style.display="none",i(99).appendChild(e),e.src="javascript:",t=e.contentWindow.document,t.open(),t.write("<script>document.F=Object<\/script>"),t.close(),h=t.F;o--;)delete h.prototype[s[o]];return h()};t.exports=Object.create||function(t,e){var i;return null!==t?(a.prototype=o(t),i=new a,a.prototype=null,i[r]=t):i=h(),void 0===e?i:n(i,e)}},function(t,e){var i=Math.ceil,o=Math.floor;t.exports=function(t){return isNaN(t=+t)?0:(t>0?o:i)(t)}},function(t,e,i){var o=i(40)("keys"),n=i(27);t.exports=function(t){return o[t]||(o[t]=n(t))}},function(t,e,i){var o=i(6),n=o["__core-js_shared__"]||(o["__core-js_shared__"]={});t.exports=function(t){return n[t]||(n[t]={})}},function(t,e){t.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(t,e,i){var o=i(8).f,n=i(10),s=i(7)("toStringTag");t.exports=function(t,e,i){t&&!n(t=i?t:t.prototype,s)&&o(t,s,{configurable:!0,value:e})}},function(t,e,i){t.exports={default:i(105),__esModule:!0}},function(t,e,i){e.f=i(7)},function(t,e,i){var o=i(6),n=i(2),s=i(35),r=i(44),a=i(8).f;t.exports=function(t){var e=n.Symbol||(n.Symbol=s?{}:o.Symbol||{});"_"==t.charAt(0)||t in e||a(e,t,{value:r.f(t)})}},function(t,e){e.f={}.propertyIsEnumerable},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e){var i=c().hours(0).minutes(0).seconds(0).milliseconds(0),o=i.clone().add(-3,"days").valueOf(),n=i.clone().add(3,"days").valueOf();this.millisecondsPerPixelCache=void 0,void 0===e?(this.start=o,this.end=n):(this.start=e.start||o,this.end=e.end||n),this.rolling=!1,this.body=t,this.deltaDifference=0,this.scaleOffset=0,this.startToFront=!1,this.endToFront=!0,this.defaultOptions={rtl:!1,start:null,end:null,moment:c,direction:"horizontal",moveable:!0,zoomable:!0,min:null,max:null,zoomMin:10,zoomMax:31536e10,rollingMode:{follow:!1,offset:.5}},this.options=p.extend({},this.defaultOptions),this.props={touch:{}},this.animationTimer=null,this.body.emitter.on("panstart",this._onDragStart.bind(this)),this.body.emitter.on("panmove",this._onDrag.bind(this)),this.body.emitter.on("panend",this._onDragEnd.bind(this)),this.body.emitter.on("mousewheel",this._onMouseWheel.bind(this)),this.body.emitter.on("touch",this._onTouch.bind(this)),this.body.emitter.on("pinch",this._onPinch.bind(this)),this.body.dom.rollingModeBtn.addEventListener("click",this.startRolling.bind(this)),this.setOptions(e)}function s(t){if("horizontal"!=t&&"vertical"!=t)throw new TypeError('Unknown direction "'+t+'". Choose "horizontal" or "vertical".')}var r=i(4),a=o(r),h=i(16),d=o(h),l=i(1),u=o(l),p=i(0),c=i(3),m=i(5),f=i(21);n.prototype=new m,n.prototype.setOptions=function(t){if(t){var e=["animation","direction","min","max","zoomMin","zoomMax","moveable","zoomable","moment","activate","hiddenDates","zoomKey","rtl","showCurrentTime","rollingMode","horizontalScroll"];p.selectiveExtend(e,this.options,t),t.rollingMode&&t.rollingMode.follow&&this.startRolling(),("start"in t||"end"in t)&&this.setRange(t.start,t.end)}},n.prototype.startRolling=function(){function t(){e.stopRolling(),e.rolling=!0;var i=e.end-e.start,o=p.convert(new Date,"Date").valueOf(),n=o-i*e.options.rollingMode.offset,s=o+i*(1-e.options.rollingMode.offset),r={animation:!1};e.setRange(n,s,r),i=1/e.conversion(e.body.domProps.center.width).scale/10,i<30&&(i=30),i>1e3&&(i=1e3),e.body.dom.rollingModeBtn.style.visibility="hidden",e.currentTimeTimer=setTimeout(t,i)}var e=this;t()},n.prototype.stopRolling=function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),this.rolling=!1,this.body.dom.rollingModeBtn.style.visibility="visible")},n.prototype.setRange=function(t,e,i,o,n){i||(i={}),!0!==i.byUser&&(i.byUser=!1);var s=this,r=void 0!=t?p.convert(t,"Date").valueOf():null,h=void 0!=e?p.convert(e,"Date").valueOf():null;if(this._cancelAnimation(),this.millisecondsPerPixelCache=void 0,i.animation){var l=this.start,c=this.end,m="object"===(0,u.default)(i.animation)&&"duration"in i.animation?i.animation.duration:500,g="object"===(0,u.default)(i.animation)&&"easingFunction"in i.animation?i.animation.easingFunction:"easeInOutQuad",v=p.easingFunctions[g];if(!v)throw new Error("Unknown easing function "+(0,d.default)(g)+". Choose from: "+(0,a.default)(p.easingFunctions).join(", "));var y=(new Date).valueOf(),b=!1;return function t(){if(!s.props.touch.dragging){var e=(new Date).valueOf(),a=e-y,d=v(a/m),u=a>m,p=u||null===r?r:l+(r-l)*d,g=u||null===h?h:c+(h-c)*d;_=s._applyRange(p,g),f.updateHiddenDates(s.options.moment,s.body,s.options.hiddenDates),b=b||_;var w={start:new Date(s.start),end:new Date(s.end),byUser:i.byUser,event:i.event};if(n&&n(d,_,u),_&&s.body.emitter.emit("rangechange",w),u){if(b&&(s.body.emitter.emit("rangechanged",w),o))return o()}else s.animationTimer=setTimeout(t,20)}}()}var _=this._applyRange(r,h);if(f.updateHiddenDates(this.options.moment,this.body,this.options.hiddenDates),_){var w={start:new Date(this.start),end:new Date(this.end),byUser:i.byUser,event:i.event};if(this.body.emitter.emit("rangechange",w),clearTimeout(s.timeoutID),s.timeoutID=setTimeout(function(){s.body.emitter.emit("rangechanged",w)},200),o)return o()}},n.prototype.getMillisecondsPerPixel=function(){return void 0===this.millisecondsPerPixelCache&&(this.millisecondsPerPixelCache=(this.end-this.start)/this.body.dom.center.clientWidth),this.millisecondsPerPixelCache},n.prototype._cancelAnimation=function(){this.animationTimer&&(clearTimeout(this.animationTimer),this.animationTimer=null)},n.prototype._applyRange=function(t,e){var i,o=null!=t?p.convert(t,"Date").valueOf():this.start,n=null!=e?p.convert(e,"Date").valueOf():this.end,s=null!=this.options.max?p.convert(this.options.max,"Date").valueOf():null,r=null!=this.options.min?p.convert(this.options.min,"Date").valueOf():null;if(isNaN(o)||null===o)throw new Error('Invalid start "'+t+'"');if(isNaN(n)||null===n)throw new Error('Invalid end "'+e+'"');if(n<o&&(n=o),null!==r&&o<r&&(i=r-o,o+=i,n+=i,null!=s&&n>s&&(n=s)),null!==s&&n>s&&(i=n-s,o-=i,n-=i,null!=r&&o<r&&(o=r)),null!==this.options.zoomMin){var a=parseFloat(this.options.zoomMin);if(a<0&&(a=0),n-o<a){this.end-this.start===a&&o>=this.start-.5&&n<=this.end?(o=this.start,n=this.end):(i=a-(n-o),o-=i/2,n+=i/2)}}if(null!==this.options.zoomMax){var h=parseFloat(this.options.zoomMax);h<0&&(h=0),n-o>h&&(this.end-this.start===h&&o<this.start&&n>this.end?(o=this.start,n=this.end):(i=n-o-h,o+=i/2,n-=i/2))}var d=this.start!=o||this.end!=n;return o>=this.start&&o<=this.end||n>=this.start&&n<=this.end||this.start>=o&&this.start<=n||this.end>=o&&this.end<=n||this.body.emitter.emit("checkRangedItems"),this.start=o,this.end=n,d},n.prototype.getRange=function(){return{start:this.start,end:this.end}},n.prototype.conversion=function(t,e){return n.conversion(this.start,this.end,t,e)},n.conversion=function(t,e,i,o){return void 0===o&&(o=0),0!=i&&e-t!=0?{offset:t,scale:i/(e-t-o)}:{offset:0,scale:1}},n.prototype._onDragStart=function(t){this.deltaDifference=0,this.previousDelta=0,this.options.moveable&&this._isInsideRange(t)&&this.props.touch.allowDragging&&(this.stopRolling(),this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.dragging=!0,this.body.dom.root&&(this.body.dom.root.style.cursor="move"))},n.prototype._onDrag=function(t){if(t&&this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging){var e=this.options.direction;s(e);var i="horizontal"==e?t.deltaX:t.deltaY;i-=this.deltaDifference;var o=this.props.touch.end-this.props.touch.start;o-=f.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end);var n,r="horizontal"==e?this.body.domProps.center.width:this.body.domProps.center.height;n=this.options.rtl?i/r*o:-i/r*o;var a=this.props.touch.start+n,h=this.props.touch.end+n,d=f.snapAwayFromHidden(this.body.hiddenDates,a,this.previousDelta-i,!0),l=f.snapAwayFromHidden(this.body.hiddenDates,h,this.previousDelta-i,!0);if(d!=a||l!=h)return this.deltaDifference+=i,this.props.touch.start=d,this.props.touch.end=l,void this._onDrag(t);this.previousDelta=i,this._applyRange(a,h);var u=new Date(this.start),p=new Date(this.end);this.body.emitter.emit("rangechange",{start:u,end:p,byUser:!0,event:t}),this.body.emitter.emit("panmove")}},n.prototype._onDragEnd=function(t){this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging&&(this.props.touch.dragging=!1,this.body.dom.root&&(this.body.dom.root.style.cursor="auto"),this.body.emitter.emit("rangechanged",{start:new Date(this.start),end:new Date(this.end),byUser:!0,event:t}))},n.prototype._onMouseWheel=function(t){var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail&&(e=-t.detail/3),!(this.options.zoomKey&&!t[this.options.zoomKey]&&this.options.zoomable||!this.options.zoomable&&this.options.moveable)&&this.options.zoomable&&this.options.moveable&&this._isInsideRange(t)&&e){var i;i=e<0?1-e/5:1/(1+e/5);var o;if(this.rolling)o=this.start+(this.end-this.start)*this.options.rollingMode.offset;else{var n=this.getPointer({x:t.clientX,y:t.clientY},this.body.dom.center);o=this._pointerToDate(n)}this.zoom(i,o,e,t),t.preventDefault()}},n.prototype._onTouch=function(t){this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.allowDragging=!0,this.props.touch.center=null,this.scaleOffset=0,this.deltaDifference=0,p.preventDefault(t)},n.prototype._onPinch=function(t){if(this.options.zoomable&&this.options.moveable){p.preventDefault(t),this.props.touch.allowDragging=!1,this.props.touch.center||(this.props.touch.center=this.getPointer(t.center,this.body.dom.center)),this.stopRolling();var e=1/(t.scale+this.scaleOffset),i=this._pointerToDate(this.props.touch.center),o=f.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end),n=f.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this,i),s=o-n,r=i-n+(this.props.touch.start-(i-n))*e,a=i+s+(this.props.touch.end-(i+s))*e;this.startToFront=1-e<=0,this.endToFront=e-1<=0;var h=f.snapAwayFromHidden(this.body.hiddenDates,r,1-e,!0),d=f.snapAwayFromHidden(this.body.hiddenDates,a,e-1,!0);h==r&&d==a||(this.props.touch.start=h,this.props.touch.end=d,this.scaleOffset=1-t.scale,r=h,a=d);var l={animation:!1,byUser:!0,event:t};this.setRange(r,a,l),this.startToFront=!1,this.endToFront=!0}},n.prototype._isInsideRange=function(t){var e,i=t.center?t.center.x:t.clientX;e=this.options.rtl?i-p.getAbsoluteLeft(this.body.dom.centerContainer):p.getAbsoluteRight(this.body.dom.centerContainer)-i;var o=this.body.util.toTime(e);return o>=this.start&&o<=this.end},n.prototype._pointerToDate=function(t){var e,i=this.options.direction;if(s(i),"horizontal"==i)return this.body.util.toTime(t.x).valueOf();var o=this.body.domProps.center.height;return e=this.conversion(o),t.y/e.scale+e.offset},n.prototype.getPointer=function(t,e){return this.options.rtl?{x:p.getAbsoluteRight(e)-t.x,y:t.y-p.getAbsoluteTop(e)}:{x:t.x-p.getAbsoluteLeft(e),y:t.y-p.getAbsoluteTop(e)}},n.prototype.zoom=function(t,e,i,o){null==e&&(e=(this.start+this.end)/2);var n=f.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end),s=f.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this,e),r=n-s,a=e-s+(this.start-(e-s))*t,h=e+r+(this.end-(e+r))*t;this.startToFront=!(i>0),this.endToFront=!(-i>0);var d=f.snapAwayFromHidden(this.body.hiddenDates,a,i,!0),l=f.snapAwayFromHidden(this.body.hiddenDates,h,-i,!0);d==a&&l==h||(a=d,h=l);var u={animation:!1,byUser:!0,event:o};this.setRange(a,h,u),this.startToFront=!1,this.endToFront=!0},n.prototype.move=function(t){var e=this.end-this.start,i=this.start+e*t,o=this.end+e*t;this.start=i,this.end=o},n.prototype.moveTo=function(t){var e=(this.start+this.end)/2,i=e-t,o=this.start-i,n=this.end-i,s={animation:!1,byUser:!0,event:null};this.setRange(o,n,s)},t.exports=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(){}var s=i(16),r=o(s),a=i(1),h=o(a),d=i(66),l=i(12),u=i(67),p=i(0),c=i(28),m=i(132),f=i(21),g=i(29);d(n.prototype),n.prototype._create=function(t){function e(t){this.isActive()&&this.emit("mousewheel",t);var e=0,i=0;if("detail"in t&&(i=-1*t.detail),"wheelDelta"in t&&(i=t.wheelDelta),"wheelDeltaY"in t&&(i=t.wheelDeltaY),"wheelDeltaX"in t&&(e=-1*t.wheelDeltaX),"axis"in t&&t.axis===t.HORIZONTAL_AXIS&&(e=-1*i,i=0),"deltaY"in t&&(i=-1*t.deltaY),"deltaX"in t&&(e=t.deltaX),this.options.zoomKey&&!t[this.options.zoomKey])if(t.preventDefault(),this.options.verticalScroll&&Math.abs(i)>=Math.abs(e)){var o=this.props.scrollTop,n=o+i;this.isActive()&&(this._setScrollTop(n),this._redraw(),this.emit("scroll",t))}else if(this.options.horizontalScroll){var s=Math.abs(e)>=Math.abs(i)?e:i,r=s/120*(this.range.end-this.range.start)/20,a=this.range.start+r,h=this.range.end+r,d={animation:!1,byUser:!0,event:t};this.range.setRange(a,h,d)}}function i(t){if(s.options.verticalScroll&&(t.preventDefault(),s.isActive())){var e=-t.target.scrollTop;s._setScrollTop(e),s._redraw(),s.emit("scrollSide",t)}}function o(t){if(t.preventDefault&&t.preventDefault(),!(!t.target.className.indexOf("vis")>-1||a))return t.dataTransfer.dropEffect="move",a=!0,!1}function n(t){t.preventDefault&&t.preventDefault(),t.stopPropagation&&t.stopPropagation();try{var e=JSON.parse(t.dataTransfer.getData("text"));if(!e||!e.content)return}catch(t){return!1}return a=!1,t.center={x:t.clientX,y:t.clientY},"item"!==e.target?s.itemSet._onAddItem(t):s.itemSet._onDropObjectOnItem(t),s.emit("drop",s.getEventProperties(t)),!1}this.dom={},this.dom.container=t,this.dom.root=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.backgroundVertical=document.createElement("div"),this.dom.backgroundHorizontal=document.createElement("div"),this.dom.centerContainer=document.createElement("div"),this.dom.leftContainer=document.createElement("div"),this.dom.rightContainer=document.createElement("div"),this.dom.center=document.createElement("div"),this.dom.left=document.createElement("div"),this.dom.right=document.createElement("div"),this.dom.top=document.createElement("div"),this.dom.bottom=document.createElement("div"),this.dom.shadowTop=document.createElement("div"),this.dom.shadowBottom=document.createElement("div"),this.dom.shadowTopLeft=document.createElement("div"),this.dom.shadowBottomLeft=document.createElement("div"),this.dom.shadowTopRight=document.createElement("div"),this.dom.shadowBottomRight=document.createElement("div"),this.dom.rollingModeBtn=document.createElement("div"),this.dom.root.className="vis-timeline",this.dom.background.className="vis-panel vis-background",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical",this.dom.backgroundHorizontal.className="vis-panel vis-background vis-horizontal",this.dom.centerContainer.className="vis-panel vis-center",this.dom.leftContainer.className="vis-panel vis-left",this.dom.rightContainer.className="vis-panel vis-right",this.dom.top.className="vis-panel vis-top",this.dom.bottom.className="vis-panel vis-bottom",this.dom.left.className="vis-content",this.dom.center.className="vis-content",this.dom.right.className="vis-content",this.dom.shadowTop.className="vis-shadow vis-top",this.dom.shadowBottom.className="vis-shadow vis-bottom",this.dom.shadowTopLeft.className="vis-shadow vis-top",this.dom.shadowBottomLeft.className="vis-shadow vis-bottom",this.dom.shadowTopRight.className="vis-shadow vis-top",this.dom.shadowBottomRight.className="vis-shadow vis-bottom",this.dom.rollingModeBtn.className="vis-rolling-mode-btn",this.dom.root.appendChild(this.dom.background),this.dom.root.appendChild(this.dom.backgroundVertical),this.dom.root.appendChild(this.dom.backgroundHorizontal),this.dom.root.appendChild(this.dom.centerContainer),this.dom.root.appendChild(this.dom.leftContainer),this.dom.root.appendChild(this.dom.rightContainer),this.dom.root.appendChild(this.dom.top),this.dom.root.appendChild(this.dom.bottom),this.dom.root.appendChild(this.dom.bottom),this.dom.root.appendChild(this.dom.rollingModeBtn),this.dom.centerContainer.appendChild(this.dom.center),this.dom.leftContainer.appendChild(this.dom.left),this.dom.rightContainer.appendChild(this.dom.right),this.dom.centerContainer.appendChild(this.dom.shadowTop),this.dom.centerContainer.appendChild(this.dom.shadowBottom),this.dom.leftContainer.appendChild(this.dom.shadowTopLeft),this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft),this.dom.rightContainer.appendChild(this.dom.shadowTopRight),this.dom.rightContainer.appendChild(this.dom.shadowBottomRight),this.props={root:{},background:{},centerContainer:{},leftContainer:{},rightContainer:{},center:{},left:{},right:{},top:{},bottom:{},border:{},scrollTop:0,scrollTopMin:0},this.on("rangechange",function(){!0===this.initialDrawDone&&this._redraw()}.bind(this)),this.on("rangechanged",function(){this.initialRangeChangeDone||(this.initialRangeChangeDone=!0)}.bind(this)),this.on("touch",this._onTouch.bind(this)),this.on("panmove",this._onDrag.bind(this));var s=this;this._origRedraw=this._redraw.bind(this),this._redraw=p.throttle(this._origRedraw),this.on("_change",function(t){s.itemSet&&s.itemSet.initialItemSetDrawn&&t&&1==t.queue?s._redraw():s._origRedraw()}),this.hammer=new l(this.dom.root);var r=this.hammer.get("pinch").set({enable:!0});u.disablePreventDefaultVertically(r),this.hammer.get("pan").set({threshold:5,direction:l.DIRECTION_HORIZONTAL}),this.listeners={},["tap","doubletap","press","pinch","pan","panstart","panmove","panend"].forEach(function(t){var e=function(e){s.isActive()&&s.emit(t,e)};s.hammer.on(t,e),s.listeners[t]=e}),u.onTouch(this.hammer,function(t){s.emit("touch",t)}.bind(this)),u.onRelease(this.hammer,function(t){s.emit("release",t)}.bind(this)),this.dom.centerContainer.addEventListener?(this.dom.centerContainer.addEventListener("mousewheel",e.bind(this),!1),this.dom.centerContainer.addEventListener("DOMMouseScroll",e.bind(this),!1)):this.dom.centerContainer.attachEvent("onmousewheel",e.bind(this)),this.dom.left.parentNode.addEventListener("scroll",i.bind(this)),this.dom.right.parentNode.addEventListener("scroll",i.bind(this));var a=!1;if(this.dom.center.addEventListener("dragover",o.bind(this),!1),this.dom.center.addEventListener("drop",n.bind(this),!1),this.customTimes=[],this.touch={},this.redrawCount=0,this.initialDrawDone=!1,this.initialRangeChangeDone=!1,!t)throw new Error("No container provided");t.appendChild(this.dom.root)},n.prototype.setOptions=function(t){if(t){var e=["width","height","minHeight","maxHeight","autoResize","start","end","clickToUse","dataAttributes","hiddenDates","locale","locales","moment","rtl","zoomKey","horizontalScroll","verticalScroll"];if(p.selectiveExtend(e,this.options,t),this.dom.rollingModeBtn.style.visibility="hidden",this.options.rtl&&(this.dom.container.style.direction="rtl",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical-rtl"),this.options.verticalScroll&&(this.options.rtl?this.dom.rightContainer.className="vis-panel vis-right vis-vertical-scroll":this.dom.leftContainer.className="vis-panel vis-left vis-vertical-scroll"),"object"!==(0,h.default)(this.options.orientation)&&(this.options.orientation={item:void 0,axis:void 0}),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation={item:t.orientation,axis:t.orientation}:"object"===(0,h.default)(t.orientation)&&("item"in t.orientation&&(this.options.orientation.item=t.orientation.item),"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis))),"both"===this.options.orientation.axis){if(!this.timeAxis2){var i=this.timeAxis2=new c(this.body);i.setOptions=function(t){var e=t?p.extend({},t):{};e.orientation="top",c.prototype.setOptions.call(i,e)},this.components.push(i)}}else if(this.timeAxis2){var o=this.components.indexOf(this.timeAxis2);-1!==o&&this.components.splice(o,1),this.timeAxis2.destroy(),this.timeAxis2=null}if("function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),"hiddenDates"in this.options&&f.convertHiddenOptions(this.options.moment,this.body,this.options.hiddenDates),"clickToUse"in t&&(t.clickToUse?this.activator||(this.activator=new m(this.dom.root)):this.activator&&(this.activator.destroy(),delete this.activator)),"showCustomTime"in t)throw new Error("Option `showCustomTime` is deprecated. Create a custom time bar via timeline.addCustomTime(time [, id])");this._initAutoResize()}if(this.components.forEach(function(e){return e.setOptions(t)}),"configure"in t){this.configurator||(this.configurator=this._createConfigurator()),this.configurator.setOptions(t.configure);var n=p.deepExtend({},this.options);this.components.forEach(function(t){p.deepExtend(n,t.options)}),this.configurator.setModuleOptions({global:n})}this._redraw()},n.prototype.isActive=function(){return!this.activator||this.activator.active},n.prototype.destroy=function(){this.setItems(null),this.setGroups(null),this.off(),this._stopAutoResize(),this.dom.root.parentNode&&this.dom.root.parentNode.removeChild(this.dom.root),this.dom=null,this.activator&&(this.activator.destroy(),delete this.activator);for(var t in this.listeners)this.listeners.hasOwnProperty(t)&&delete this.listeners[t];this.listeners=null,this.hammer=null,this.components.forEach(function(t){return t.destroy()}),this.body=null},n.prototype.setCustomTime=function(t,e){var i=this.customTimes.filter(function(t){return e===t.options.id});if(0===i.length)throw new Error("No custom time bar found with id "+(0,r.default)(e));i.length>0&&i[0].setCustomTime(t)},n.prototype.getCustomTime=function(t){var e=this.customTimes.filter(function(e){return e.options.id===t});if(0===e.length)throw new Error("No custom time bar found with id "+(0,r.default)(t));return e[0].getCustomTime()},n.prototype.setCustomTimeTitle=function(t,e){var i=this.customTimes.filter(function(t){return t.options.id===e});if(0===i.length)throw new Error("No custom time bar found with id "+(0,r.default)(e));if(i.length>0)return i[0].setCustomTitle(t)},n.prototype.getEventProperties=function(t){return{event:t}},n.prototype.addCustomTime=function(t,e){var i=void 0!==t?p.convert(t,"Date").valueOf():new Date;if(this.customTimes.some(function(t){return t.options.id===e}))throw new Error("A custom time with id "+(0,r.default)(e)+" already exists");var o=new g(this.body,p.extend({},this.options,{time:i,id:e}));return this.customTimes.push(o),this.components.push(o),this._redraw(),e},n.prototype.removeCustomTime=function(t){var e=this.customTimes.filter(function(e){return e.options.id===t});if(0===e.length)throw new Error("No custom time bar found with id "+(0,r.default)(t));e.forEach(function(t){this.customTimes.splice(this.customTimes.indexOf(t),1),this.components.splice(this.components.indexOf(t),1),t.destroy()}.bind(this))},n.prototype.getVisibleItems=function(){return this.itemSet&&this.itemSet.getVisibleItems()||[]},n.prototype.fit=function(t,e){var i=this.getDataRange();if(null!==i.min||null!==i.max){var o=i.max-i.min,n=new Date(i.min.valueOf()-.01*o),s=new Date(i.max.valueOf()+.01*o),r=!t||void 0===t.animation||t.animation;this.range.setRange(n,s,{animation:r},e)}},n.prototype.getDataRange=function(){throw new Error("Cannot invoke abstract method getDataRange")},n.prototype.setWindow=function(t,e,i,o){"function"==typeof arguments[2]&&(o=arguments[2],i={});var n,s;1==arguments.length?(s=arguments[0],n=void 0===s.animation||s.animation,this.range.setRange(s.start,s.end,{animation:n})):2==arguments.length&&"function"==typeof arguments[1]?(s=arguments[0],o=arguments[1],n=void 0===s.animation||s.animation,this.range.setRange(s.start,s.end,{animation:n},o)):(n=!i||void 0===i.animation||i.animation,this.range.setRange(t,e,{animation:n},o))},n.prototype.moveTo=function(t,e,i){"function"==typeof arguments[1]&&(i=arguments[1],e={});var o=this.range.end-this.range.start,n=p.convert(t,"Date").valueOf(),s=n-o/2,r=n+o/2,a=!e||void 0===e.animation||e.animation;this.range.setRange(s,r,{animation:a},i)},n.prototype.getWindow=function(){var t=this.range.getRange();return{start:new Date(t.start),end:new Date(t.end)}},n.prototype.zoomIn=function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var o=this.getWindow(),n=o.start.valueOf(),s=o.end.valueOf(),r=s-n,a=r/(1+t),h=(r-a)/2,d=n+h,l=s-h;this.setWindow(d,l,e,i)}},n.prototype.zoomOut=function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var o=this.getWindow(),n=o.start.valueOf(),s=o.end.valueOf(),r=s-n,a=n-r*t/2,h=s+r*t/2;this.setWindow(a,h,e,i)}},n.prototype.redraw=function(){this._redraw()},n.prototype._redraw=function(){this.redrawCount++;var t=!1,e=this.options,i=this.props,o=this.dom;if(o&&o.container&&0!=o.root.offsetWidth){f.updateHiddenDates(this.options.moment,this.body,this.options.hiddenDates),"top"==e.orientation?(p.addClassName(o.root,"vis-top"),p.removeClassName(o.root,"vis-bottom")):(p.removeClassName(o.root,"vis-top"),p.addClassName(o.root,"vis-bottom")),o.root.style.maxHeight=p.option.asSize(e.maxHeight,""),o.root.style.minHeight=p.option.asSize(e.minHeight,""),o.root.style.width=p.option.asSize(e.width,""),i.border.left=(o.centerContainer.offsetWidth-o.centerContainer.clientWidth)/2,i.border.right=i.border.left,i.border.top=(o.centerContainer.offsetHeight-o.centerContainer.clientHeight)/2,i.border.bottom=i.border.top,i.borderRootHeight=o.root.offsetHeight-o.root.clientHeight,i.borderRootWidth=o.root.offsetWidth-o.root.clientWidth,0===o.centerContainer.clientHeight&&(i.border.left=i.border.top,i.border.right=i.border.left),0===o.root.clientHeight&&(i.borderRootWidth=i.borderRootHeight),i.center.height=o.center.offsetHeight,i.left.height=o.left.offsetHeight,i.right.height=o.right.offsetHeight,i.top.height=o.top.clientHeight||-i.border.top,i.bottom.height=o.bottom.clientHeight||-i.border.bottom;var n=Math.max(i.left.height,i.center.height,i.right.height),s=i.top.height+n+i.bottom.height+i.borderRootHeight+i.border.top+i.border.bottom;o.root.style.height=p.option.asSize(e.height,s+"px"),i.root.height=o.root.offsetHeight,i.background.height=i.root.height-i.borderRootHeight;var r=i.root.height-i.top.height-i.bottom.height-i.borderRootHeight;i.centerContainer.height=r,i.leftContainer.height=r,i.rightContainer.height=i.leftContainer.height,i.root.width=o.root.offsetWidth,i.background.width=i.root.width-i.borderRootWidth,this.initialDrawDone||(i.scrollbarWidth=p.getScrollBarWidth()),e.verticalScroll?e.rtl?(i.left.width=o.leftContainer.clientWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth+i.scrollbarWidth||-i.border.right):(i.left.width=o.leftContainer.clientWidth+i.scrollbarWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth||-i.border.right):(i.left.width=o.leftContainer.clientWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth||-i.border.right),this._setDOM();var a=this._updateScrollTop();"top"!=e.orientation.item&&(a+=Math.max(i.centerContainer.height-i.center.height-i.border.top-i.border.bottom,0)),o.center.style.top=a+"px";var h=0==i.scrollTop?"hidden":"",d=i.scrollTop==i.scrollTopMin?"hidden":"";o.shadowTop.style.visibility=h,o.shadowBottom.style.visibility=d,o.shadowTopLeft.style.visibility=h,o.shadowBottomLeft.style.visibility=d,o.shadowTopRight.style.visibility=h,o.shadowBottomRight.style.visibility=d,e.verticalScroll&&(o.rightContainer.className="vis-panel vis-right vis-vertical-scroll",o.leftContainer.className="vis-panel vis-left vis-vertical-scroll",o.shadowTopRight.style.visibility="hidden",o.shadowBottomRight.style.visibility="hidden",o.shadowTopLeft.style.visibility="hidden",o.shadowBottomLeft.style.visibility="hidden",o.left.style.top="0px",o.right.style.top="0px"),(!e.verticalScroll||i.center.height<i.centerContainer.height)&&(o.left.style.top=a+"px",o.right.style.top=a+"px",o.rightContainer.className=o.rightContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),o.leftContainer.className=o.leftContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),i.left.width=o.leftContainer.clientWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth||-i.border.right,this._setDOM());var u=i.center.height>i.centerContainer.height;this.hammer.get("pan").set({direction:u?l.DIRECTION_ALL:l.DIRECTION_HORIZONTAL}),this.components.forEach(function(e){t=e.redraw()||t});if(t){if(this.redrawCount<5)return void this.body.emitter.emit("_change");console.log("WARNING: infinite loop in redraw?")}else this.redrawCount=0;this.body.emitter.emit("changed")}},n.prototype._setDOM=function(){var t=this.props,e=this.dom;t.leftContainer.width=t.left.width,t.rightContainer.width=t.right.width;var i=t.root.width-t.left.width-t.right.width-t.borderRootWidth;t.center.width=i,t.centerContainer.width=i,t.top.width=i,t.bottom.width=i,e.background.style.height=t.background.height+"px",e.backgroundVertical.style.height=t.background.height+"px",e.backgroundHorizontal.style.height=t.centerContainer.height+"px",e.centerContainer.style.height=t.centerContainer.height+"px",e.leftContainer.style.height=t.leftContainer.height+"px",e.rightContainer.style.height=t.rightContainer.height+"px",e.background.style.width=t.background.width+"px",e.backgroundVertical.style.width=t.centerContainer.width+"px",e.backgroundHorizontal.style.width=t.background.width+"px",e.centerContainer.style.width=t.center.width+"px",e.top.style.width=t.top.width+"px",e.bottom.style.width=t.bottom.width+"px",e.background.style.left="0",e.background.style.top="0",e.backgroundVertical.style.left=t.left.width+t.border.left+"px",e.backgroundVertical.style.top="0",e.backgroundHorizontal.style.left="0",e.backgroundHorizontal.style.top=t.top.height+"px",e.centerContainer.style.left=t.left.width+"px",e.centerContainer.style.top=t.top.height+"px",e.leftContainer.style.left="0",e.leftContainer.style.top=t.top.height+"px",e.rightContainer.style.left=t.left.width+t.center.width+"px",e.rightContainer.style.top=t.top.height+"px",e.top.style.left=t.left.width+"px",e.top.style.top="0",e.bottom.style.left=t.left.width+"px",e.bottom.style.top=t.top.height+t.centerContainer.height+"px",e.center.style.left="0",e.left.style.left="0",e.right.style.left="0"},n.prototype.repaint=function(){throw new Error("Function repaint is deprecated. Use redraw instead.")},n.prototype.setCurrentTime=function(t){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");this.currentTime.setCurrentTime(t)},n.prototype.getCurrentTime=function(){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");return this.currentTime.getCurrentTime()},n.prototype._toTime=function(t){return f.toTime(this,t,this.props.center.width)},n.prototype._toGlobalTime=function(t){return f.toTime(this,t,this.props.root.width)},n.prototype._toScreen=function(t){return f.toScreen(this,t,this.props.center.width)},n.prototype._toGlobalScreen=function(t){return f.toScreen(this,t,this.props.root.width)},n.prototype._initAutoResize=function(){1==this.options.autoResize?this._startAutoResize():this._stopAutoResize()},n.prototype._startAutoResize=function(){var t=this;this._stopAutoResize(),this._onResize=function(){if(1!=t.options.autoResize)return void t._stopAutoResize();t.dom.root&&(t.dom.root.offsetWidth==t.props.lastWidth&&t.dom.root.offsetHeight==t.props.lastHeight||(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight,t.props.scrollbarWidth=p.getScrollBarWidth(),t.body.emitter.emit("_change")))},p.addEventListener(window,"resize",this._onResize),t.dom.root&&(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight),this.watchTimer=setInterval(this._onResize,1e3)},n.prototype._stopAutoResize=function(){this.watchTimer&&(clearInterval(this.watchTimer),this.watchTimer=void 0),this._onResize&&(p.removeEventListener(window,"resize",this._onResize),this._onResize=null)},n.prototype._onTouch=function(t){this.touch.allowDragging=!0,this.touch.initialScrollTop=this.props.scrollTop},n.prototype._onPinch=function(t){this.touch.allowDragging=!1},n.prototype._onDrag=function(t){if(t&&this.touch.allowDragging){var e=t.deltaY,i=this._getScrollTop(),o=this._setScrollTop(this.touch.initialScrollTop+e);this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),o!=i&&this.emit("verticalDrag")}},n.prototype._setScrollTop=function(t){return this.props.scrollTop=t,this._updateScrollTop(),this.props.scrollTop},n.prototype._updateScrollTop=function(){var t=Math.min(this.props.centerContainer.height-this.props.center.height,0);return t!=this.props.scrollTopMin&&("top"!=this.options.orientation.item&&(this.props.scrollTop+=t-this.props.scrollTopMin),this.props.scrollTopMin=t),this.props.scrollTop>0&&(this.props.scrollTop=0),this.props.scrollTop<t&&(this.props.scrollTop=t),this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),this.props.scrollTop},n.prototype._getScrollTop=function(){return this.props.scrollTop},n.prototype._createConfigurator=function(){throw new Error("Cannot invoke abstract method _createConfigurator")},t.exports=n},function(t,e,i){function o(t,e,i,s,r){this.moment=n,this.current=this.moment(),this._start=this.moment(),this._end=this.moment(),this.autoScale=!0,this.scale="day",this.step=1,this.setRange(t,e,i),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,Array.isArray(s)?this.hiddenDates=s:this.hiddenDates=void 0!=s?[s]:[],this.format=o.FORMAT,this.options=r||{}}var n=i(3),s=i(21),r=i(0);o.FORMAT={minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}},o.prototype.setMoment=function(t){this.moment=t,this.current=this.moment(this.current.valueOf()),this._start=this.moment(this._start.valueOf()),this._end=this.moment(this._end.valueOf())},o.prototype.setFormat=function(t){var e=r.deepExtend({},o.FORMAT);this.format=r.deepExtend(e,t)},o.prototype.setRange=function(t,e,i){if(!(t instanceof Date&&e instanceof Date))throw"No legal start or end date in method setRange";this._start=void 0!=t?this.moment(t.valueOf()):new Date,this._end=void 0!=e?this.moment(e.valueOf()):new Date,this.autoScale&&this.setMinimumStep(i)},o.prototype.start=function(){this.current=this._start.clone(),this.roundToMinor()},o.prototype.roundToMinor=function(){switch("week"==this.scale&&this.current.weekday(0),this.scale){case"year":this.current.year(this.step*Math.floor(this.current.year()/this.step)),this.current.month(0);case"month":this.current.date(1);case"week":case"day":case"weekday":this.current.hours(0);case"hour":this.current.minutes(0);case"minute":this.current.seconds(0);case"second":this.current.milliseconds(0)}if(1!=this.step)switch(this.scale){case"millisecond":this.current.subtract(this.current.milliseconds()%this.step,"milliseconds");break;case"second":this.current.subtract(this.current.seconds()%this.step,"seconds");break;case"minute":this.current.subtract(this.current.minutes()%this.step,"minutes");break;case"hour":this.current.subtract(this.current.hours()%this.step,"hours");break;case"weekday":case"day":this.current.subtract((this.current.date()-1)%this.step,"day");break;case"week":this.current.subtract(this.current.week()%this.step,"week");break;case"month":this.current.subtract(this.current.month()%this.step,"month");break;case"year":this.current.subtract(this.current.year()%this.step,"year")}},o.prototype.hasNext=function(){return this.current.valueOf()<=this._end.valueOf()},o.prototype.next=function(){var t=this.current.valueOf();switch(this.scale){case"millisecond":this.current.add(this.step,"millisecond");break;case"second":this.current.add(this.step,"second");break;case"minute":this.current.add(this.step,"minute");break;case"hour":this.current.add(this.step,"hour"),this.current.month()<6?this.current.subtract(this.current.hours()%this.step,"hour"):this.current.hours()%this.step!=0&&this.current.add(this.step-this.current.hours()%this.step,"hour");break;case"weekday":case"day":this.current.add(this.step,"day");break;case"week":if(0!==this.current.weekday())this.current.weekday(0),this.current.add(this.step,"week");else if(!1===this.options.showMajorLabels)this.current.add(this.step,"week");else{var e=this.current.clone();e.add(1,"week"),e.isSame(this.current,"month")?this.current.add(this.step,"week"):(this.current.add(this.step,"week"),this.current.date(1))}break;case"month":this.current.add(this.step,"month");break;case"year":this.current.add(this.step,"year")}if(1!=this.step)switch(this.scale){case"millisecond":this.current.milliseconds()>0&&this.current.milliseconds()<this.step&&this.current.milliseconds(0);break;case"second":this.current.seconds()>0&&this.current.seconds()<this.step&&this.current.seconds(0);break;case"minute":this.current.minutes()>0&&this.current.minutes()<this.step&&this.current.minutes(0);break;case"hour":this.current.hours()>0&&this.current.hours()<this.step&&this.current.hours(0);break;case"weekday":case"day":this.current.date()<this.step+1&&this.current.date(1);break;case"week":this.current.week()<this.step&&this.current.week(1);break;case"month":this.current.month()<this.step&&this.current.month(0)}this.current.valueOf()==t&&(this.current=this._end.clone()),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,s.stepOverHiddenDates(this.moment,this,t)},o.prototype.getCurrent=function(){return this.current},o.prototype.setScale=function(t){t&&"string"==typeof t.scale&&(this.scale=t.scale,this.step=t.step>0?t.step:1,this.autoScale=!1)},o.prototype.setAutoScale=function(t){this.autoScale=t},o.prototype.setMinimumStep=function(t){if(void 0!=t){31104e9>t&&(this.scale="year",this.step=1e3),15552e9>t&&(this.scale="year",this.step=500),31104e8>t&&(this.scale="year",this.step=100),15552e8>t&&(this.scale="year",this.step=50),31104e7>t&&(this.scale="year",this.step=10),15552e7>t&&(this.scale="year",this.step=5),31104e6>t&&(this.scale="year",this.step=1),7776e6>t&&(this.scale="month",this.step=3),2592e6>t&&(this.scale="month",this.step=1),432e6>t&&(this.scale="day",this.step=5),1728e5>t&&(this.scale="day",this.step=2),864e5>t&&(this.scale="day",this.step=1),432e5>t&&(this.scale="weekday",this.step=1),144e5>t&&(this.scale="hour",this.step=4),36e5>t&&(this.scale="hour",this.step=1),9e5>t&&(this.scale="minute",this.step=15),6e5>t&&(this.scale="minute",this.step=10),3e5>t&&(this.scale="minute",this.step=5),6e4>t&&(this.scale="minute",this.step=1),15e3>t&&(this.scale="second",this.step=15),1e4>t&&(this.scale="second",this.step=10),5e3>t&&(this.scale="second",this.step=5),1e3>t&&(this.scale="second",this.step=1),200>t&&(this.scale="millisecond",this.step=200),100>t&&(this.scale="millisecond",this.step=100),50>t&&(this.scale="millisecond",this.step=50),10>t&&(this.scale="millisecond",this.step=10),5>t&&(this.scale="millisecond",this.step=5),1>t&&(this.scale="millisecond",this.step=1)}},o.snap=function(t,e,i){var o=n(t);if("year"==e){var s=o.year()+Math.round(o.month()/12);o.year(Math.round(s/i)*i),o.month(0),o.date(0),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("month"==e)o.date()>15?(o.date(1),o.add(1,"month")):o.date(1),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0);else if("week"==e)o.weekday()>2?(o.weekday(0),o.add(1,"week")):o.weekday(0),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0);else if("day"==e){switch(i){case 5:case 2:o.hours(24*Math.round(o.hours()/24));break;default:o.hours(12*Math.round(o.hours()/12))}o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("weekday"==e){switch(i){case 5:case 2:o.hours(12*Math.round(o.hours()/12));break;default:o.hours(6*Math.round(o.hours()/6))}o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("hour"==e){switch(i){case 4:o.minutes(60*Math.round(o.minutes()/60));break;default:o.minutes(30*Math.round(o.minutes()/30))}o.seconds(0),o.milliseconds(0)}else if("minute"==e){switch(i){case 15:case 10:o.minutes(5*Math.round(o.minutes()/5)),o.seconds(0);break;case 5:o.seconds(60*Math.round(o.seconds()/60));break;default:o.seconds(30*Math.round(o.seconds()/30))}o.milliseconds(0)}else if("second"==e)switch(i){case 15:case 10:o.seconds(5*Math.round(o.seconds()/5)),o.milliseconds(0);break;case 5:o.milliseconds(1e3*Math.round(o.milliseconds()/1e3));break;default:o.milliseconds(500*Math.round(o.milliseconds()/500))}else if("millisecond"==e){var r=i>5?i/2:1;o.milliseconds(Math.round(o.milliseconds()/r)*r)}return o},o.prototype.isMajor=function(){if(1==this.switchedYear)switch(this.scale){case"year":case"month":case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedMonth)switch(this.scale){case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedDay)switch(this.scale){case"millisecond":case"second":case"minute":case"hour":return!0;default:return!1}var t=this.moment(this.current);switch(this.scale){case"millisecond":return 0==t.milliseconds();case"second":return 0==t.seconds();case"minute":return 0==t.hours()&&0==t.minutes();case"hour":return 0==t.hours();case"weekday":case"day":case"week":return 1==t.date();case"month":return 0==t.month();case"year":default:return!1}},o.prototype.getLabelMinor=function(t){if(void 0==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.minorLabels)return this.format.minorLabels(t,this.scale,this.step);var e=this.format.minorLabels[this.scale];switch(this.scale){case"week":if(this.isMajor()&&0!==t.weekday())return"";default:return e&&e.length>0?this.moment(t).format(e):""}},o.prototype.getLabelMajor=function(t){if(void 0==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.majorLabels)return this.format.majorLabels(t,this.scale,this.step);var e=this.format.majorLabels[this.scale];return e&&e.length>0?this.moment(t).format(e):""},o.prototype.getClassName=function(){function t(t){return t/a%2==0?" vis-even":" vis-odd"}function e(t){return t.isSame(new Date,"day")?" vis-today":t.isSame(n().add(1,"day"),"day")?" vis-tomorrow":t.isSame(n().add(-1,"day"),"day")?" vis-yesterday":""}function i(t){return t.isSame(new Date,"week")?" vis-current-week":""}function o(t){return t.isSame(new Date,"month")?" vis-current-month":""}var n=this.moment,s=this.moment(this.current),r=s.locale?s.locale("en"):s.lang("en"),a=this.step,h=[];switch(this.scale){case"millisecond":h.push(e(r)),h.push(t(r.milliseconds()));break;case"second":h.push(e(r)),h.push(t(r.seconds()));break;case"minute":h.push(e(r)),h.push(t(r.minutes()));break;case"hour":h.push("vis-h"+r.hours()+(4==this.step?"-h"+(r.hours()+4):"")),h.push(e(r)),h.push(t(r.hours()));break;case"weekday":h.push("vis-"+r.format("dddd").toLowerCase()),h.push(e(r)),h.push(i(r)),h.push(t(r.date()));break;case"day":h.push("vis-day"+r.date()),h.push("vis-"+r.format("MMMM").toLowerCase()),h.push(e(r)),h.push(o(r)),h.push(this.step<=2?e(r):""),h.push(this.step<=2?"vis-"+r.format("dddd").toLowerCase():""),h.push(t(r.date()-1));break;case"week":h.push("vis-week"+r.format("w")),h.push(i(r)),h.push(t(r.week()));break;case"month":h.push("vis-"+r.format("MMMM").toLowerCase()),h.push(o(r)),h.push(t(r.month()));break;case"year":h.push("vis-year"+r.year()),h.push(function(t){return t.isSame(new Date,"year")?" vis-current-year":""}(r)),h.push(t(r.year()))}return h.filter(String).join(" ")},t.exports=o},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={rtl:!1,showCurrentTime:!0,moment:r,locales:a,locale:"en"},this.options=n.extend({},this.defaultOptions),this.offset=0,this._create(),this.setOptions(e)}var n=i(0),s=i(5),r=i(3),a=i(69);o.prototype=new s,o.prototype._create=function(){var t=document.createElement("div");t.className="vis-current-time",t.style.position="absolute",t.style.top="0px",t.style.height="100%",this.bar=t},o.prototype.destroy=function(){this.options.showCurrentTime=!1,this.redraw(),this.body=null},o.prototype.setOptions=function(t){t&&n.selectiveExtend(["rtl","showCurrentTime","moment","locale","locales"],this.options,t)},o.prototype.redraw=function(){if(this.options.showCurrentTime){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar),this.start());var e=this.options.moment((new Date).valueOf()+this.offset),i=this.body.util.toScreen(e),o=this.options.locales[this.options.locale];o||(this.warned||(console.log("WARNING: options.locales['"+this.options.locale+"'] not found. See http://visjs.org/docs/timeline/#Localization"),this.warned=!0),o=this.options.locales.en);var n=o.current+" "+o.time+": "+e.format("dddd, MMMM Do YYYY, H:mm:ss");n=n.charAt(0).toUpperCase()+n.substring(1),this.options.rtl?this.bar.style.right=i+"px":this.bar.style.left=i+"px",this.bar.title=n}else this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),this.stop();return!1},o.prototype.start=function(){function t(){e.stop();var i=e.body.range.conversion(e.body.domProps.center.width).scale,o=1/i/10;o<30&&(o=30),o>1e3&&(o=1e3),e.redraw(),e.body.emitter.emit("currentTimeTick"),e.currentTimeTimer=setTimeout(t,o)}var e=this;t()},o.prototype.stop=function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),delete this.currentTimeTimer)},o.prototype.setCurrentTime=function(t){var e=n.convert(t,"Date").valueOf(),i=(new Date).valueOf();this.offset=e-i,this.redraw()},o.prototype.getCurrentTime=function(){return new Date((new Date).valueOf()+this.offset)},t.exports=o},function(t,e,i){function o(t,e,i){if(this.groupId=t,this.subgroups={},this.subgroupStack={},this.subgroupStackAll=!1,this.doInnerStack=!1,this.subgroupIndex=0,this.subgroupOrderer=e&&e.subgroupOrder,this.itemSet=i,this.isVisible=null,this.stackDirty=!0,e&&e.nestedGroups&&(this.nestedGroups=e.nestedGroups,0==e.showNested?this.showNested=!1:this.showNested=!0),e&&e.subgroupStack)if("boolean"==typeof e.subgroupStack)this.doInnerStack=e.subgroupStack,this.subgroupStackAll=e.subgroupStack;else for(var o in e.subgroupStack)this.subgroupStack[o]=e.subgroupStack[o],this.doInnerStack=this.doInnerStack||e.subgroupStack[o];this.nestedInGroup=null,this.dom={},this.props={label:{width:0,height:0}},this.className=null,this.items={},this.visibleItems=[],this.itemsInRange=[],this.orderedItems={byStart:[],byEnd:[]},this.checkRangedItems=!1;var n=this;this.itemSet.body.emitter.on("checkRangedItems",function(){n.checkRangedItems=!0}),this._create(),this.setData(e)}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(71);o.prototype._create=function(){var t=document.createElement("div");this.itemSet.options.groupEditable.order?t.className="vis-label draggable":t.className="vis-label",this.dom.label=t;var e=document.createElement("div");e.className="vis-inner",t.appendChild(e),this.dom.inner=e;var i=document.createElement("div");i.className="vis-group",i["timeline-group"]=this,this.dom.foreground=i,this.dom.background=document.createElement("div"),this.dom.background.className="vis-group",this.dom.axis=document.createElement("div"),this.dom.axis.className="vis-group",this.dom.marker=document.createElement("div"),this.dom.marker.style.visibility="hidden",this.dom.marker.style.position="absolute",this.dom.marker.innerHTML="",this.dom.background.appendChild(this.dom.marker)},o.prototype.setData=function(t){var e,i;if(this.itemSet.options&&this.itemSet.options.groupTemplate?(i=this.itemSet.options.groupTemplate.bind(this),e=i(t,this.dom.inner)):e=t&&t.content,e instanceof Element){for(this.dom.inner.appendChild(e);this.dom.inner.firstChild;)this.dom.inner.removeChild(this.dom.inner.firstChild);this.dom.inner.appendChild(e)}else e instanceof Object?i(t,this.dom.inner):this.dom.inner.innerHTML=void 0!==e&&null!==e?e:this.groupId||"";if(this.dom.label.title=t&&t.title||"",this.dom.inner.firstChild?r.removeClassName(this.dom.inner,"vis-hidden"):r.addClassName(this.dom.inner,"vis-hidden"),t&&t.nestedGroups){this.nestedGroups&&this.nestedGroups==t.nestedGroups||(this.nestedGroups=t.nestedGroups),void 0===t.showNested&&void 0!==this.showNested||(0==t.showNested?this.showNested=!1:this.showNested=!0),r.addClassName(this.dom.label,"vis-nesting-group");var o=this.itemSet.options.rtl?"collapsed-rtl":"collapsed";this.showNested?(r.removeClassName(this.dom.label,o),r.addClassName(this.dom.label,"expanded")):(r.removeClassName(this.dom.label,"expanded"),r.addClassName(this.dom.label,o))}else this.nestedGroups&&(this.nestedGroups=null,o=this.itemSet.options.rtl?"collapsed-rtl":"collapsed",r.removeClassName(this.dom.label,o),r.removeClassName(this.dom.label,"expanded"),r.removeClassName(this.dom.label,"vis-nesting-group"));t&&t.nestedInGroup&&(r.addClassName(this.dom.label,"vis-nested-group"),this.itemSet.options&&this.itemSet.options.rtl?this.dom.inner.style.paddingRight="30px":this.dom.inner.style.paddingLeft="30px");var n=t&&t.className||null;n!=this.className&&(this.className&&(r.removeClassName(this.dom.label,this.className),r.removeClassName(this.dom.foreground,this.className),r.removeClassName(this.dom.background,this.className),r.removeClassName(this.dom.axis,this.className)),r.addClassName(this.dom.label,n),r.addClassName(this.dom.foreground,n),r.addClassName(this.dom.background,n),r.addClassName(this.dom.axis,n),this.className=n),this.style&&(r.removeCssText(this.dom.label,this.style),this.style=null),t&&t.style&&(r.addCssText(this.dom.label,t.style),this.style=t.style)},o.prototype.getLabelWidth=function(){return this.props.label.width},o.prototype._didMarkerHeightChange=function(){var t=this.dom.marker.clientHeight;if(t!=this.lastMarkerHeight){this.lastMarkerHeight=t;var e={},i=0;r.forEach(this.items,function(t,o){if(t.dirty=!0,t.displayed){e[o]=t.redraw(!0),i=e[o].length}});if(i>0)for(var o=0;o<i;o++)r.forEach(e,function(t){t[o]()});return!0}},o.prototype._calculateGroupSizeAndPosition=function(){var t=this.dom.foreground.offsetTop,e=this.dom.foreground.offsetLeft,i=this.dom.foreground.offsetWidth;this.top=t,this.right=e,this.width=i},o.prototype._redrawItems=function(t,e,i,o){if(t||this.stackDirty||this.isVisible&&!e){var n={},s=null;if("function"==typeof this.itemSet.options.order){var h=this,d={},l=0;if(r.forEach(this.items,function(t,e){t.displayed||(d[e]=t.redraw(!0),l=d[e].length,h.visibleItems.push(t))}),l>0)for(var u=0;u<l;u++)r.forEach(d,function(t){t[u]()});if(r.forEach(this.items,function(t){t.repositionX(!1)}),this.doInnerStack&&this.itemSet.options.stackSubgroups){for(s in this.subgroups)n[s]=this.subgroups[s].items.slice().sort(function(t,e){return h.itemSet.options.order(t.data,e.data)});a.stackSubgroupsWithInnerStack(n,i,this.subgroups)}else{var p=this.orderedItems.byStart.slice().sort(function(t,e){return h.itemSet.options.order(t.data,e.data)});a.stack(p,i,!0)}this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,o)}else if(this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,o),this.itemSet.options.stack)if(this.doInnerStack&&this.itemSet.options.stackSubgroups){for(s in this.subgroups)n[s]=this.subgroups[s].items;a.stackSubgroupsWithInnerStack(n,i,this.subgroups)}else a.stack(this.visibleItems,i,!0);else a.nostack(this.visibleItems,i,this.subgroups,this.itemSet.options.stackSubgroups);this.stackDirty=!1}},o.prototype._didResize=function(t,e){t=r.updateProperty(this,"height",e)||t;var i=this.dom.inner.clientWidth,o=this.dom.inner.clientHeight;return t=r.updateProperty(this.props.label,"width",i)||t,t=r.updateProperty(this.props.label,"height",o)||t},o.prototype._applyGroupHeight=function(t){this.dom.background.style.height=t+"px",this.dom.foreground.style.height=t+"px",this.dom.label.style.height=t+"px"},o.prototype._updateItemsVerticalPosition=function(t){for(var e=0,i=this.visibleItems.length;e<i;e++){var o=this.visibleItems[e];o.repositionY(t),this.isVisible||"__background__"==this.groupId||o.displayed&&o.hide()}},o.prototype.redraw=function(t,e,i,o){var n,s=!1,r=this.isVisible,a=[function(){i=this._didMarkerHeightChange.bind(this)}.bind(this),this._updateSubGroupHeights.bind(this,e),this._calculateGroupSizeAndPosition.bind(this),function(){this.isVisible=this._isGroupVisible.bind(this)(t,e)}.bind(this),function(){this._redrawItems.bind(this)(i,r,e,t)}.bind(this),this._updateSubgroupsSizes.bind(this),function(){n=this._calculateHeight.bind(this)(e)}.bind(this),this._calculateGroupSizeAndPosition.bind(this),function(){s=this._didResize.bind(this)(s,n)}.bind(this),function(){this._applyGroupHeight.bind(this)(n)}.bind(this),function(){this._updateItemsVerticalPosition.bind(this)(e)}.bind(this),function(){return!this.isVisible&&this.height&&(s=!1),s}];if(o)return a;var h;return a.forEach(function(t){h=t()}),h},o.prototype._updateSubGroupHeights=function(t){if((0,s.default)(this.subgroups).length>0){var e=this;this.resetSubgroups(),r.forEach(this.visibleItems,function(i){void 0!==i.data.subgroup&&(e.subgroups[i.data.subgroup].height=Math.max(e.subgroups[i.data.subgroup].height,i.height+t.item.vertical),e.subgroups[i.data.subgroup].visible=!0)})}},o.prototype._isGroupVisible=function(t,e){return this.top<=t.body.domProps.centerContainer.height-t.body.domProps.scrollTop+e.axis&&this.top+this.height+e.axis>=-t.body.domProps.scrollTop},o.prototype._calculateHeight=function(t){var e,i=this.visibleItems;if(i.length>0){var o=i[0].top,n=i[0].top+i[0].height;if(r.forEach(i,function(t){o=Math.min(o,t.top),n=Math.max(n,t.top+t.height)}),o>t.axis){var s=o-t.axis;n-=s,r.forEach(i,function(t){t.top-=s})}e=n+t.item.vertical/2}else e=0;return e=Math.max(e,this.props.label.height)},o.prototype.show=function(){this.dom.label.parentNode||this.itemSet.dom.labelSet.appendChild(this.dom.label),this.dom.foreground.parentNode||this.itemSet.dom.foreground.appendChild(this.dom.foreground),this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background),this.dom.axis.parentNode||this.itemSet.dom.axis.appendChild(this.dom.axis)},o.prototype.hide=function(){var t=this.dom.label;t.parentNode&&t.parentNode.removeChild(t);var e=this.dom.foreground;e.parentNode&&e.parentNode.removeChild(e);var i=this.dom.background;i.parentNode&&i.parentNode.removeChild(i);var o=this.dom.axis;o.parentNode&&o.parentNode.removeChild(o)},o.prototype.add=function(t){if(this.items[t.id]=t,t.setParent(this),this.stackDirty=!0,void 0!==t.data.subgroup&&(this._addToSubgroup(t),this.orderSubgroups()),-1==this.visibleItems.indexOf(t)){var e=this.itemSet.body.range;this._checkIfVisible(t,this.visibleItems,e)}},o.prototype._addToSubgroup=function(t,e){e=e||t.data.subgroup,void 0!=e&&void 0===this.subgroups[e]&&(this.subgroups[e]={height:0,top:0,start:t.data.start,end:t.data.end||t.data.start,visible:!1,index:this.subgroupIndex,items:[],stack:this.subgroupStackAll||this.subgroupStack[e]||!1},this.subgroupIndex++),new Date(t.data.start)<new Date(this.subgroups[e].start)&&(this.subgroups[e].start=t.data.start);var i=t.data.end||t.data.start;new Date(i)>new Date(this.subgroups[e].end)&&(this.subgroups[e].end=i),this.subgroups[e].items.push(t)},o.prototype._updateSubgroupsSizes=function(){var t=this;if(t.subgroups)for(var e in t.subgroups){var i=t.subgroups[e].items[0].data.end||t.subgroups[e].items[0].data.start,o=t.subgroups[e].items[0].data.start,n=i-1;t.subgroups[e].items.forEach(function(t){new Date(t.data.start)<new Date(o)&&(o=t.data.start);var e=t.data.end||t.data.start;new Date(e)>new Date(n)&&(n=e)}),t.subgroups[e].start=o,t.subgroups[e].end=new Date(n-1)}},o.prototype.orderSubgroups=function(){if(void 0!==this.subgroupOrderer){var t,e=[];if("string"==typeof this.subgroupOrderer){for(t in this.subgroups)e.push({subgroup:t,sortField:this.subgroups[t].items[0].data[this.subgroupOrderer]});e.sort(function(t,e){return t.sortField-e.sortField})}else if("function"==typeof this.subgroupOrderer){for(t in this.subgroups)e.push(this.subgroups[t].items[0].data);e.sort(this.subgroupOrderer)}if(e.length>0)for(var i=0;i<e.length;i++)this.subgroups[e[i].subgroup].index=i}},o.prototype.resetSubgroups=function(){for(var t in this.subgroups)this.subgroups.hasOwnProperty(t)&&(this.subgroups[t].visible=!1,this.subgroups[t].height=0)},o.prototype.remove=function(t){delete this.items[t.id],t.setParent(null),this.stackDirty=!0;var e=this.visibleItems.indexOf(t);-1!=e&&this.visibleItems.splice(e,1),void 0!==t.data.subgroup&&(this._removeFromSubgroup(t),this.orderSubgroups())},o.prototype._removeFromSubgroup=function(t,e){if(void 0!=(e=e||t.data.subgroup)){var i=this.subgroups[e];if(i){var o=i.items.indexOf(t);o>=0&&(i.items.splice(o,1),i.items.length?this._updateSubgroupsSizes():delete this.subgroups[e])}}},o.prototype.removeFromDataSet=function(t){this.itemSet.removeItem(t.id)},o.prototype.order=function(){for(var t=r.toArray(this.items),e=[],i=[],o=0;o<t.length;o++)void 0!==t[o].data.end&&i.push(t[o]),e.push(t[o]);this.orderedItems={byStart:e,byEnd:i},a.orderByStart(this.orderedItems.byStart),a.orderByEnd(this.orderedItems.byEnd)},o.prototype._updateItemsInRange=function(t,e,i){var o=[],n={},s=(i.end-i.start)/4,a=i.start-s,h=i.end+s,d=function(t){return t<a?-1:t<=h?0:1};if(e.length>0)for(var l=0;l<e.length;l++)this._checkIfVisibleWithReference(e[l],o,n,i);var u=r.binarySearchCustom(t.byStart,d,"data","start");if(this._traceVisible(u,t.byStart,o,n,function(t){return t.data.start<a||t.data.start>h}),1==this.checkRangedItems)for(this.checkRangedItems=!1,l=0;l<t.byEnd.length;l++)this._checkIfVisibleWithReference(t.byEnd[l],o,n,i);else{var p=r.binarySearchCustom(t.byEnd,d,"data","end");this._traceVisible(p,t.byEnd,o,n,function(t){return t.data.end<a||t.data.end>h})}var c={},m=0;for(l=0;l<o.length;l++){var f=o[l];if(!f.displayed){c[l]=f.redraw(!0),m=c[l].length}}if(m>0)for(var g=0;g<m;g++)r.forEach(c,function(t){t[g]()});for(l=0;l<o.length;l++)o[l].repositionX();return o},o.prototype._traceVisible=function(t,e,i,o,n){if(-1!=t){var s,r;for(s=t;s>=0&&(r=e[s],!n(r));s--)void 0===o[r.id]&&(o[r.id]=!0,i.push(r));for(s=t+1;s<e.length&&(r=e[s],!n(r));s++)void 0===o[r.id]&&(o[r.id]=!0,i.push(r))}},o.prototype._checkIfVisible=function(t,e,i){t.isVisible(i)?(t.displayed||t.show(),t.repositionX(),e.push(t)):t.displayed&&t.hide()},o.prototype._checkIfVisibleWithReference=function(t,e,i,o){t.isVisible(o)?void 0===i[t.id]&&(i[t.id]=!0,e.push(t)):t.displayed&&t.hide()},o.prototype.changeSubgroup=function(t,e,i){this._removeFromSubgroup(t,e),this._addToSubgroup(t,i),this.orderSubgroups()},t.exports=o},function(t,e,i){function o(t,e,i){r.call(this,t,e,i),this.width=0,this.height=0,this.top=0,this.left=0}var n=i(43),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(51);o.prototype=(0,s.default)(r.prototype),o.prototype.redraw=function(t,e,i){this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,t),this.width=this.dom.background.offsetWidth,this.dom.background.style.height="0";for(var o=0,n=this.visibleItems.length;o<n;o++){this.visibleItems[o].repositionY(e)}return!1},o.prototype.show=function(){this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background)},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={content:{width:0}},this.overflow=!1,this.options=i,t){if(void 0==t.start)throw new Error('Property "start" missing in item '+t.id);if(void 0==t.end)throw new Error('Property "end" missing in item '+t.id)}n.call(this,t,e,i)}var n=i(22);o.prototype=new n(null,null,null),o.prototype.baseClassName="vis-item vis-range",o.prototype.isVisible=function(t){return this.data.start<t.end&&this.data.end>t.start},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.visibleFrame=document.createElement("div"),this.dom.visibleFrame.className="vis-item-visible-frame",this.dom.box.appendChild(this.dom.visibleFrame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dom.box["timeline-item"]=this,this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className=this.baseClassName+e,this.dom.content.style.maxWidth="none"}},o.prototype._getDomComponentsSizes=function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.frame).overflow,{content:{width:this.dom.content.offsetWidth},box:{height:this.dom.box.offsetHeight}}},o.prototype._updateDomComponentsSizes=function(t){this.props.content.width=t.content.width,this.height=t.box.height,this.dom.content.style.maxWidth="",this.dirty=!1},o.prototype._repaintDomAdditionals=function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDeleteButton(this.dom.box),this._repaintDragCenter(),this._repaintDragLeft(),this._repaintDragRight()},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes.bind(this)())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){if(this.displayed){var t=this.dom.box;t.parentNode&&t.parentNode.removeChild(t),this.displayed=!1}},o.prototype.repositionX=function(t){var e,i,o=this.parent.width,n=this.conversion.toScreen(this.data.start),s=this.conversion.toScreen(this.data.end),r=void 0===this.data.align?this.options.align:this.data.align;!1===this.data.limitSize||void 0!==t&&!0!==t||(n<-o&&(n=-o),s>2*o&&(s=2*o));var a=Math.max(s-n+.5,1);switch(this.overflow?(this.options.rtl?this.right=n:this.left=n,this.width=a+this.props.content.width,i=this.props.content.width):(this.options.rtl?this.right=n:this.left=n,this.width=a,i=Math.min(s-n,this.props.content.width)),this.options.rtl?this.dom.box.style.right=this.right+"px":this.dom.box.style.left=this.left+"px",this.dom.box.style.width=a+"px",r){case"left":this.options.rtl?this.dom.content.style.right="0":this.dom.content.style.left="0";break;case"right":this.options.rtl?this.dom.content.style.right=Math.max(a-i,0)+"px":this.dom.content.style.left=Math.max(a-i,0)+"px";break;case"center":this.options.rtl?this.dom.content.style.right=Math.max((a-i)/2,0)+"px":this.dom.content.style.left=Math.max((a-i)/2,0)+"px";break;default:e=this.overflow?s>0?Math.max(-n,0):-i:n<0?-n:0,this.options.rtl?this.dom.content.style.right=e+"px":(this.dom.content.style.left=e+"px",this.dom.content.style.width="calc(100% - "+e+"px)")}},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.box;e.style.top="top"==t?this.top+"px":this.parent.height-this.top-this.height+"px"},o.prototype._repaintDragLeft=function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.options.editable.updateTime&&!this.dom.dragLeft){var t=document.createElement("div");t.className="vis-drag-left",t.dragLeftItem=this,this.dom.box.appendChild(t),this.dom.dragLeft=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragLeft||(this.dom.dragLeft.parentNode&&this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft),this.dom.dragLeft=null)},o.prototype._repaintDragRight=function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.options.editable.updateTime&&!this.dom.dragRight){var t=document.createElement("div");t.className="vis-drag-right",t.dragRightItem=this,this.dom.box.appendChild(t),this.dom.dragRight=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragRight||(this.dom.dragRight.parentNode&&this.dom.dragRight.parentNode.removeChild(this.dom.dragRight),this.dom.dragRight=null)},t.exports=o},function(t,e,i){function o(t,e){}function n(t,e){return e=void 0===e?{}:e,{style:e.style||t.options.drawPoints.style,styles:e.styles||t.options.drawPoints.styles,size:e.size||t.options.drawPoints.size,className:e.className||t.className}}function s(t,e){var i=void 0;return t.options&&t.options.drawPoints&&t.options.drawPoints.onRender&&"function"==typeof t.options.drawPoints.onRender&&(i=t.options.drawPoints.onRender),e.group.options&&e.group.options.drawPoints&&e.group.options.drawPoints.onRender&&"function"==typeof e.group.options.drawPoints.onRender&&(i=e.group.options.drawPoints.onRender),i}var r=i(1),a=function(t){return t&&t.__esModule?t:{default:t}}(r),h=i(11);o.draw=function(t,e,i,o){o=o||0;for(var r=s(i,e),d=0;d<t.length;d++)if(r){var l=r(t[d],e);!0!==l&&"object"!==(void 0===l?"undefined":(0,a.default)(l))||h.drawPoint(t[d].screen_x+o,t[d].screen_y,n(e,l),i.svgElements,i.svg,t[d].label)}else h.drawPoint(t[d].screen_x+o,t[d].screen_y,n(e),i.svgElements,i.svg,t[d].label)},o.drawIcon=function(t,e,i,o,s,r){var a=.5*s,d=h.getSVGElement("rect",r.svgElements,r.svg);d.setAttributeNS(null,"x",e),d.setAttributeNS(null,"y",i-a),d.setAttributeNS(null,"width",o),d.setAttributeNS(null,"height",2*a),d.setAttributeNS(null,"class","vis-outline"),h.drawPoint(e+.5*o,i,n(t),r.svgElements,r.svg)},t.exports=o},function(t,e,i){i(88);for(var o=i(6),n=i(14),s=i(23),r=i(7)("toStringTag"),a="CSSRuleList,CSSStyleDeclaration,CSSValueList,ClientRectList,DOMRectList,DOMStringList,DOMTokenList,DataTransferItemList,FileList,HTMLAllCollection,HTMLCollection,HTMLFormElement,HTMLSelectElement,MediaList,MimeTypeArray,NamedNodeMap,NodeList,PaintRequestList,Plugin,PluginArray,SVGLengthList,SVGNumberList,SVGPathSegList,SVGPointList,SVGStringList,SVGTransformList,SourceBufferList,StyleSheetList,TextTrackCueList,TextTrackList,TouchList".split(","),h=0;h<a.length;h++){var d=a[h],l=o[d],u=l&&l.prototype;u&&!u[r]&&n(u,r,d),s[d]=s.Array}},function(t,e,i){var o=i(35),n=i(17),s=i(59),r=i(14),a=i(10),h=i(23),d=i(94),l=i(42),u=i(100),p=i(7)("iterator"),c=!([].keys&&"next"in[].keys()),m=function(){return this};t.exports=function(t,e,i,f,g,v,y){d(i,e,f);var b,_,w,x=function(t){if(!c&&t in C)return C[t];switch(t){case"keys":case"values":return function(){return new i(this,t)}}return function(){return new i(this,t)}},D=e+" Iterator",S="values"==g,k=!1,C=t.prototype,T=C[p]||C["@@iterator"]||g&&C[g],M=T||x(g),O=g?S?x("entries"):M:void 0,E="Array"==e?C.entries||T:T;if(E&&(w=u(E.call(new t)))!==Object.prototype&&w.next&&(l(w,D,!0),o||a(w,p)||r(w,p,m)),S&&T&&"values"!==T.name&&(k=!0,M=function(){return T.call(this)}),o&&!y||!c&&!k&&C[p]||r(C,p,M),h[e]=M,h[D]=m,g)if(b={values:S?M:x("values"),keys:v?M:x("keys"),entries:O},y)for(_ in b)_ in C||s(C,_,b[_]);else n(n.P+n.F*(c||k),e,b);return b}},function(t,e,i){t.exports=!i(9)&&!i(19)(function(){return 7!=Object.defineProperty(i(58)("div"),"a",{get:function(){return 7}}).a})},function(t,e,i){var o=i(24),n=i(6).document,s=o(n)&&o(n.createElement);t.exports=function(t){return s?n.createElement(t):{}}},function(t,e,i){t.exports=i(14)},function(t,e,i){var o=i(10),n=i(13),s=i(96)(!1),r=i(39)("IE_PROTO");t.exports=function(t,e){var i,a=n(t),h=0,d=[];for(i in a)i!=r&&o(a,i)&&d.push(i);for(;e.length>h;)o(a,i=e[h++])&&(~s(d,i)||d.push(i));return d}},function(t,e,i){var o=i(34);t.exports=function(t){return Object(o(t))}},function(t,e,i){var o=i(101)(!0);i(56)(String,"String",function(t){this._t=String(t),this._i=0},function(){var t,e=this._t,i=this._i;return i>=e.length?{value:void 0,done:!0}:(t=o(e,i),this._i+=t.length,{value:t,done:!1})})},function(t,e){e.f=Object.getOwnPropertySymbols},function(t,e,i){var o=i(60),n=i(41).concat("length","prototype");e.f=Object.getOwnPropertyNames||function(t){return o(t,n)}},function(t,e,i){function o(t){this.delay=null,this.max=1/0,this._queue=[],this._timeout=null,this._extended=null,this.setOptions(t)}o.prototype.setOptions=function(t){t&&void 0!==t.delay&&(this.delay=t.delay),t&&void 0!==t.max&&(this.max=t.max),this._flushIfNeeded()},o.extend=function(t,e){var i=new o(e);if(void 0!==t.flush)throw new Error("Target object already has a property flush");t.flush=function(){i.flush()};var n=[{name:"flush",original:void 0}];if(e&&e.replace)for(var s=0;s<e.replace.length;s++){var r=e.replace[s];n.push({name:r,original:t[r]}),i.replace(t,r)}return i._extended={object:t,methods:n},i},o.prototype.destroy=function(){if(this.flush(),this._extended){for(var t=this._extended.object,e=this._extended.methods,i=0;i<e.length;i++){var o=e[i];o.original?t[o.name]=o.original:delete t[o.name]}this._extended=null}},o.prototype.replace=function(t,e){var i=this,o=t[e];if(!o)throw new Error("Method "+e+" undefined");t[e]=function(){for(var t=[],e=0;e<arguments.length;e++)t[e]=arguments[e];i.queue({args:t,fn:o,context:this})}},o.prototype.queue=function(t){"function"==typeof t?this._queue.push({fn:t}):this._queue.push(t),this._flushIfNeeded()},o.prototype._flushIfNeeded=function(){if(this._queue.length>this.max&&this.flush(),clearTimeout(this._timeout),this.queue.length>0&&"number"==typeof this.delay){var t=this;this._timeout=setTimeout(function(){t.flush()},this.delay)}},o.prototype.flush=function(){for(;this._queue.length>0;){var t=this._queue.shift();t.fn.apply(t.context||t.fn,t.args||[])}},t.exports=o},function(t,e){function i(t){if(t)return o(t)}function o(t){for(var e in i.prototype)t[e]=i.prototype[e];return t}t.exports=i,i.prototype.on=i.prototype.addEventListener=function(t,e){return this._callbacks=this._callbacks||{},(this._callbacks[t]=this._callbacks[t]||[]).push(e),this},i.prototype.once=function(t,e){function i(){o.off(t,i),e.apply(this,arguments)}var o=this;return this._callbacks=this._callbacks||{},i.fn=e,this.on(t,i),this},i.prototype.off=i.prototype.removeListener=i.prototype.removeAllListeners=i.prototype.removeEventListener=function(t,e){if(this._callbacks=this._callbacks||{},0==arguments.length)return this._callbacks={},this;var i=this._callbacks[t];if(!i)return this;if(1==arguments.length)return delete this._callbacks[t],this;for(var o,n=0;n<i.length;n++)if((o=i[n])===e||o.fn===e){i.splice(n,1);break}return this},i.prototype.emit=function(t){this._callbacks=this._callbacks||{};var e=[].slice.call(arguments,1),i=this._callbacks[t];if(i){i=i.slice(0);for(var o=0,n=i.length;o<n;++o)i[o].apply(this,e)}return this},i.prototype.listeners=function(t){return this._callbacks=this._callbacks||{},this._callbacks[t]||[]},i.prototype.hasListeners=function(t){return!!this.listeners(t).length}},function(t,e,i){e.onTouch=function(t,e){e.inputHandler=function(t){t.isFirst&&e(t)},t.on("hammer.input",e.inputHandler)},e.onRelease=function(t,e){return e.inputHandler=function(t){t.isFinal&&e(t)},t.on("hammer.input",e.inputHandler)},e.offTouch=function(t,e){t.off("hammer.input",e.inputHandler)},e.offRelease=e.offTouch,e.disablePreventDefaultVertically=function(t){return t.getTouchAction=function(){return["pan-y"]},t}},function(t,e,i){var o,n,s;!function(i,r){n=[],o=r,void 0!==(s="function"==typeof o?o.apply(e,n):o)&&(t.exports=s)}(0,function(){function t(t){var e,i=t&&t.preventDefault||!1,o=t&&t.container||window,n={},s={keydown:{},keyup:{}},r={};for(e=97;e<=122;e++)r[String.fromCharCode(e)]={code:e-97+65,shift:!1};for(e=65;e<=90;e++)r[String.fromCharCode(e)]={code:e,shift:!0};for(e=0;e<=9;e++)r[""+e]={code:48+e,shift:!1};for(e=1;e<=12;e++)r["F"+e]={code:111+e,shift:!1};for(e=0;e<=9;e++)r["num"+e]={code:96+e,shift:!1};r["num*"]={code:106,shift:!1},r["num+"]={code:107,shift:!1},r["num-"]={code:109,shift:!1},r["num/"]={code:111,shift:!1},r["num."]={code:110,shift:!1},r.left={code:37,shift:!1},r.up={code:38,shift:!1},r.right={code:39,shift:!1},r.down={code:40,shift:!1},r.space={code:32,shift:!1},r.enter={code:13,shift:!1},r.shift={code:16,shift:void 0},r.esc={code:27,shift:!1},r.backspace={code:8,shift:!1},r.tab={code:9,shift:!1},r.ctrl={code:17,shift:!1},r.alt={code:18,shift:!1},r.delete={code:46,shift:!1},r.pageup={code:33,shift:!1},r.pagedown={code:34,shift:!1},r["="]={code:187,shift:!1},r["-"]={code:189,shift:!1},r["]"]={code:221,shift:!1},r["["]={code:219,shift:!1};var a=function(t){d(t,"keydown")},h=function(t){d(t,"keyup")},d=function(t,e){if(void 0!==s[e][t.keyCode]){for(var o=s[e][t.keyCode],n=0;n<o.length;n++)void 0===o[n].shift?o[n].fn(t):1==o[n].shift&&1==t.shiftKey?o[n].fn(t):0==o[n].shift&&0==t.shiftKey&&o[n].fn(t);1==i&&t.preventDefault()}};return n.bind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);void 0===s[i][r[t].code]&&(s[i][r[t].code]=[]),s[i][r[t].code].push({fn:e,shift:r[t].shift})},n.bindAll=function(t,e){void 0===e&&(e="keydown");for(var i in r)r.hasOwnProperty(i)&&n.bind(i,t,e)},n.getKey=function(t){for(var e in r)if(r.hasOwnProperty(e)){if(1==t.shiftKey&&1==r[e].shift&&t.keyCode==r[e].code)return e;if(0==t.shiftKey&&0==r[e].shift&&t.keyCode==r[e].code)return e;if(t.keyCode==r[e].code&&"shift"==e)return e}return"unknown key, currently not supported"},n.unbind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);if(void 0!==e){var o=[],n=s[i][r[t].code];if(void 0!==n)for(var a=0;a<n.length;a++)n[a].fn==e&&n[a].shift==r[t].shift||o.push(s[i][r[t].code][a]);s[i][r[t].code]=o}else s[i][r[t].code]=[]},n.reset=function(){s={keydown:{},keyup:{}}},n.destroy=function(){s={keydown:{},keyup:{}},o.removeEventListener("keydown",a,!0),o.removeEventListener("keyup",h,!0)},o.addEventListener("keydown",a,!0),o.addEventListener("keyup",h,!0),n}return t})},function(t,e,i){e.en={current:"current",time:"time"},e.en_EN=e.en,e.en_US=e.en,e.it={current:"attuale",time:"tempo"},e.it_IT=e.it,e.it_CH=e.it,e.nl={current:"huidige",time:"tijd"},e.nl_NL=e.nl,e.nl_BE=e.nl,e.de={current:"Aktuelle",time:"Zeit"},e.de_DE=e.de,e.fr={current:"actuel",time:"heure"},e.fr_FR=e.fr,e.fr_CA=e.fr,e.fr_BE=e.fr,e.es={current:"corriente",time:"hora"},e.es_ES=e.es},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e){this.body=t,this.defaultOptions={type:null,orientation:{item:"bottom"},align:"auto",stack:!0,stackSubgroups:!0,groupOrderSwap:function(t,e,i){var o=e.order;e.order=t.order,t.order=o},groupOrder:"order",selectable:!0,multiselect:!1,itemsAlwaysDraggable:{item:!1,range:!1},editable:{updateTime:!1,updateGroup:!1,add:!1,remove:!1,overrideItems:!1},groupEditable:{order:!1,add:!1,remove:!1},snap:c.snap,onDropObjectOnItem:function(t,e,i){i(e)},onAdd:function(t,e){e(t)},onUpdate:function(t,e){e(t)},onMove:function(t,e){e(t)},onRemove:function(t,e){e(t)},onMoving:function(t,e){e(t)},onAddGroup:function(t,e){e(t)},onMoveGroup:function(t,e){e(t)},onRemoveGroup:function(t,e){e(t)},margin:{item:{horizontal:10,vertical:10},axis:20},showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip"},tooltipOnItemUpdateTime:!1},this.options=l.extend({},this.defaultOptions),this.options.rtl=e.rtl,this.itemOptions={type:{start:"Date",end:"Date"}},this.conversion={toScreen:t.util.toScreen,toTime:t.util.toTime},this.dom={},this.props={},this.hammer=null;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,o){i._onAdd(e.items)},update:function(t,e,o){i._onUpdate(e.items)},remove:function(t,e,o){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,o){if(i._onAddGroups(e.items),i.groupsData&&i.groupsData.length>0){var n=i.groupsData.getDataSet();n.get().forEach(function(t){if(t.nestedGroups){0!=t.showNested&&(t.showNested=!0);var e=[];t.nestedGroups.forEach(function(i){var o=n.get(i);o&&(o.nestedInGroup=t.id,0==t.showNested&&(o.visible=!1),e=e.concat(o))}),n.update(e,o)}})}},update:function(t,e,o){i._onUpdateGroups(e.items)},remove:function(t,e,o){i._onRemoveGroups(e.items)}},this.items={},this.groups={},this.groupIds=[],this.selection=[],this.popup=null,this.touchParams={},this.groupTouchParams={},this._create(),this.setOptions(e)}var s=i(43),r=o(s),a=i(1),h=o(a),d=i(12),l=i(0),u=i(15),p=i(20),c=i(49),m=i(5),f=i(51),g=i(52),v=i(72),y=i(73),b=i(53),_=i(74),w=i(133).default,x="__ungrouped__",D="__background__";n.prototype=new m,n.types={background:_,box:v,range:b,point:y},n.prototype._create=function(){var t=document.createElement("div");t.className="vis-itemset",t["timeline-itemset"]=this,this.dom.frame=t;var e=document.createElement("div");e.className="vis-background",t.appendChild(e),this.dom.background=e;var i=document.createElement("div");i.className="vis-foreground",t.appendChild(i),this.dom.foreground=i;var o=document.createElement("div");o.className="vis-axis",this.dom.axis=o;var n=document.createElement("div");n.className="vis-labelset",this.dom.labelSet=n,this._updateUngrouped();var s=new g(D,null,this);s.show(),this.groups[D]=s,this.hammer=new d(this.body.dom.centerContainer),this.hammer.on("hammer.input",function(t){t.isFirst&&this._onTouch(t)}.bind(this)),this.hammer.on("panstart",this._onDragStart.bind(this)),this.hammer.on("panmove",this._onDrag.bind(this)),this.hammer.on("panend",this._onDragEnd.bind(this)),this.hammer.get("pan").set({threshold:5,direction:d.DIRECTION_HORIZONTAL}),this.hammer.on("tap",this._onSelectItem.bind(this)),this.hammer.on("press",this._onMultiSelectItem.bind(this)),this.hammer.on("doubletap",this._onAddItem.bind(this)),this.options.rtl?this.groupHammer=new d(this.body.dom.rightContainer):this.groupHammer=new d(this.body.dom.leftContainer),this.groupHammer.on("tap",this._onGroupClick.bind(this)),this.groupHammer.on("panstart",this._onGroupDragStart.bind(this)),this.groupHammer.on("panmove",this._onGroupDrag.bind(this)),this.groupHammer.on("panend",this._onGroupDragEnd.bind(this)),this.groupHammer.get("pan").set({threshold:5,direction:d.DIRECTION_VERTICAL}),this.body.dom.centerContainer.addEventListener("mouseover",this._onMouseOver.bind(this)),this.body.dom.centerContainer.addEventListener("mouseout",this._onMouseOut.bind(this)),this.body.dom.centerContainer.addEventListener("mousemove",this._onMouseMove.bind(this)),this.body.dom.centerContainer.addEventListener("contextmenu",this._onDragEnd.bind(this)),this.body.dom.centerContainer.addEventListener("mousewheel",this._onMouseWheel.bind(this)),this.show()},n.prototype.setOptions=function(t){if(t){var e=["type","rtl","align","order","stack","stackSubgroups","selectable","multiselect","multiselectPerGroup","groupOrder","dataAttributes","template","groupTemplate","visibleFrameTemplate","hide","snap","groupOrderSwap","showTooltips","tooltip","tooltipOnItemUpdateTime"];l.selectiveExtend(e,this.options,t),"itemsAlwaysDraggable"in t&&("boolean"==typeof t.itemsAlwaysDraggable?(this.options.itemsAlwaysDraggable.item=t.itemsAlwaysDraggable,this.options.itemsAlwaysDraggable.range=!1):"object"===(0,h.default)(t.itemsAlwaysDraggable)&&(l.selectiveExtend(["item","range"],this.options.itemsAlwaysDraggable,t.itemsAlwaysDraggable),this.options.itemsAlwaysDraggable.item||(this.options.itemsAlwaysDraggable.range=!1))),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.item="top"===t.orientation?"top":"bottom":"object"===(0,h.default)(t.orientation)&&"item"in t.orientation&&(this.options.orientation.item=t.orientation.item)),"margin"in t&&("number"==typeof t.margin?(this.options.margin.axis=t.margin,this.options.margin.item.horizontal=t.margin,this.options.margin.item.vertical=t.margin):"object"===(0,h.default)(t.margin)&&(l.selectiveExtend(["axis"],this.options.margin,t.margin),"item"in t.margin&&("number"==typeof t.margin.item?(this.options.margin.item.horizontal=t.margin.item,this.options.margin.item.vertical=t.margin.item):"object"===(0,h.default)(t.margin.item)&&l.selectiveExtend(["horizontal","vertical"],this.options.margin.item,t.margin.item)))),"editable"in t&&("boolean"==typeof t.editable?(this.options.editable.updateTime=t.editable,this.options.editable.updateGroup=t.editable,this.options.editable.add=t.editable,this.options.editable.remove=t.editable,this.options.editable.overrideItems=!1):"object"===(0,h.default)(t.editable)&&l.selectiveExtend(["updateTime","updateGroup","add","remove","overrideItems"],this.options.editable,t.editable)),"groupEditable"in t&&("boolean"==typeof t.groupEditable?(this.options.groupEditable.order=t.groupEditable,this.options.groupEditable.add=t.groupEditable,this.options.groupEditable.remove=t.groupEditable):"object"===(0,h.default)(t.groupEditable)&&l.selectiveExtend(["order","add","remove"],this.options.groupEditable,t.groupEditable));["onDropObjectOnItem","onAdd","onUpdate","onRemove","onMove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup"].forEach(function(e){var i=t[e];if(i){if(!(i instanceof Function))throw new Error("option "+e+" must be a function "+e+"(item, callback)");this.options[e]=i}}.bind(this)),this.markDirty()}},n.prototype.markDirty=function(t){this.groupIds=[],t&&t.refreshItems&&l.forEach(this.items,function(t){t.dirty=!0,t.displayed&&t.redraw()})},n.prototype.destroy=function(){this.hide(),this.setItems(null),this.setGroups(null),this.hammer=null,this.body=null,this.conversion=null},n.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.axis.parentNode&&this.dom.axis.parentNode.removeChild(this.dom.axis),this.dom.labelSet.parentNode&&this.dom.labelSet.parentNode.removeChild(this.dom.labelSet)},n.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame),this.dom.axis.parentNode||this.body.dom.backgroundVertical.appendChild(this.dom.axis),this.dom.labelSet.parentNode||(this.options.rtl?this.body.dom.right.appendChild(this.dom.labelSet):this.body.dom.left.appendChild(this.dom.labelSet))},n.prototype.setSelection=function(t){var e,i,o,n;for(void 0==t&&(t=[]),Array.isArray(t)||(t=[t]),e=0,i=this.selection.length;e<i;e++)o=this.selection[e],(n=this.items[o])&&n.unselect();for(this.selection=[],e=0,i=t.length;e<i;e++)o=t[e],(n=this.items[o])&&(this.selection.push(o),n.select())},n.prototype.getSelection=function(){return this.selection.concat([])},n.prototype.getVisibleItems=function(){var t,e,i=this.body.range.getRange();this.options.rtl?(t=this.body.util.toScreen(i.start),e=this.body.util.toScreen(i.end)):(e=this.body.util.toScreen(i.start),t=this.body.util.toScreen(i.end));var o=[];for(var n in this.groups)if(this.groups.hasOwnProperty(n))for(var s=this.groups[n],r=s.isVisible?s.visibleItems:[],a=0;a<r.length;a++){var h=r[a];this.options.rtl?h.right<e&&h.right+h.width>t&&o.push(h.id):h.left<t&&h.left+h.width>e&&o.push(h.id)}return o},n.prototype._deselect=function(t){for(var e=this.selection,i=0,o=e.length;i<o;i++)if(e[i]==t){e.splice(i,1);break}},n.prototype.redraw=function(){var t=this.options.margin,e=this.body.range,i=l.option.asSize,o=this.options,n=o.orientation.item,s=!1,r=this.dom.frame;this.props.top=this.body.domProps.top.height+this.body.domProps.border.top,this.options.rtl?this.props.right=this.body.domProps.right.width+this.body.domProps.border.right:this.props.left=this.body.domProps.left.width+this.body.domProps.border.left,r.className="vis-itemset",s=this._orderGroups()||s;var a=e.end-e.start,h=a!=this.lastVisibleInterval||this.props.width!=this.props.lastWidth,d=e.start!=this.lastRangeStart,u=o.stack!=this.lastStack,p=o.stackSubgroups!=this.lastStackSubgroups,c=h||d||u||p;this.lastVisibleInterval=a,this.lastRangeStart=e.start,this.lastStack=o.stack,this.lastStackSubgroups=o.stackSubgroups,this.props.lastWidth=this.props.width;var m=this._firstGroup(),f={item:t.item,axis:t.axis},g={item:t.item,axis:t.item.vertical/2},v=0,y=t.axis+t.item.vertical;this.groups[D].redraw(e,g,c);var b={},_=0;if(l.forEach(this.groups,function(t,i){if(i!==D){var o=t==m?f:g;b[i]=t.redraw(e,o,c,!0),_=b[i].length}}),_>0){for(var w={},x=0;x<_;x++)l.forEach(b,function(t,e){w[e]=t[x]()});l.forEach(this.groups,function(t,e){if(e!==D){var i=w[e];s=i||s,v+=t.height}}),v=Math.max(v,y)}return v=Math.max(v,y),r.style.height=i(v),this.props.width=r.offsetWidth,this.props.height=v,this.dom.axis.style.top=i("top"==n?this.body.domProps.top.height+this.body.domProps.border.top:this.body.domProps.top.height+this.body.domProps.centerContainer.height),this.options.rtl?this.dom.axis.style.right="0":this.dom.axis.style.left="0",this.initialItemSetDrawn=!0,s=this._isResized()||s},n.prototype._firstGroup=function(){var t="top"==this.options.orientation.item?0:this.groupIds.length-1,e=this.groupIds[t];return this.groups[e]||this.groups[x]||null},n.prototype._updateUngrouped=function(){var t,e,i=this.groups[x];if(this.groupsData){if(i){i.hide(),delete this.groups[x];for(e in this.items)if(this.items.hasOwnProperty(e)){t=this.items[e],t.parent&&t.parent.remove(t);var o=this._getGroupId(t.data),n=this.groups[o];n&&n.add(t)||t.hide()}}}else if(!i){i=new f(null,null,this),this.groups[x]=i;for(e in this.items)this.items.hasOwnProperty(e)&&(t=this.items[e],i.add(t));i.show()}},n.prototype.getLabelSet=function(){return this.dom.labelSet},n.prototype.setItems=function(t){var e,i=this,o=this.itemsData;if(t){if(!(t instanceof u||t instanceof p))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=t}else this.itemsData=null;if(o&&(l.forEach(this.itemListeners,function(t,e){o.off(e,t)}),e=o.getIds(),this._onRemove(e)),this.itemsData){var n=this.id;l.forEach(this.itemListeners,function(t,e){i.itemsData.on(e,t,n)}),e=this.itemsData.getIds(),this._onAdd(e),this._updateUngrouped()}this.body.emitter.emit("_change",{queue:!0})},n.prototype.getItems=function(){return this.itemsData},n.prototype.setGroups=function(t){var e,i=this;if(this.groupsData&&(l.forEach(this.groupListeners,function(t,e){i.groupsData.off(e,t)}),e=this.groupsData.getIds(),this.groupsData=null,this._onRemoveGroups(e)),t){if(!(t instanceof u||t instanceof p))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var o=this.groupsData;this.groupsData instanceof p&&(o=this.groupsData.getDataSet()),o.get().forEach(function(t){t.nestedGroups&&t.nestedGroups.forEach(function(e){var i=o.get(e);i.nestedInGroup=t.id,0==t.showNested&&(i.visible=!1),o.update(i)})});var n=this.id;l.forEach(this.groupListeners,function(t,e){i.groupsData.on(e,t,n)}),e=this.groupsData.getIds(),this._onAddGroups(e)}this._updateUngrouped(),this._order(),this.body.emitter.emit("_change",{queue:!0})},n.prototype.getGroups=function(){return this.groupsData},n.prototype.removeItem=function(t){var e=this.itemsData.get(t),i=this.itemsData.getDataSet();e&&this.options.onRemove(e,function(e){e&&i.remove(t)})},n.prototype._getType=function(t){return t.type||this.options.type||(t.end?"range":"box")},n.prototype._getGroupId=function(t){return"background"==this._getType(t)&&void 0==t.group?D:this.groupsData?t.group:x},n.prototype._onUpdate=function(t){var e=this;t.forEach(function(t){var i,o=e.itemsData.get(t,e.itemOptions),s=e.items[t],r=o?e._getType(o):null,a=n.types[r];if(s&&(a&&s instanceof a?e._updateItem(s,o):(i=s.selected,e._removeItem(s),s=null)),!s&&o){if(!a)throw"rangeoverflow"==r?new TypeError('Item type "rangeoverflow" is deprecated. Use css styling instead: .vis-item.vis-range .vis-item-content {overflow: visible;}'):new TypeError('Unknown item type "'+r+'"');s=new a(o,e.conversion,e.options),s.id=t,e._addItem(s),i&&(this.selection.push(t),s.select())}}.bind(this)),this._order(),this.body.emitter.emit("_change",{queue:!0})},n.prototype._onAdd=n.prototype._onUpdate,n.prototype._onRemove=function(t){var e=0,i=this;t.forEach(function(t){var o=i.items[t];o&&(e++,i._removeItem(o))}),e&&(this._order(),this.body.emitter.emit("_change",{queue:!0}))},n.prototype._order=function(){l.forEach(this.groups,function(t){t.order()})},n.prototype._onUpdateGroups=function(t){this._onAddGroups(t)},n.prototype._onAddGroups=function(t){var e=this;t.forEach(function(t){var i=e.groupsData.get(t),o=e.groups[t];if(o)o.setData(i);else{if(t==x||t==D)throw new Error("Illegal group id. "+t+" is a reserved id.");var n=(0,r.default)(e.options);l.extend(n,{height:null}),o=new f(t,i,e),e.groups[t]=o;for(var s in e.items)if(e.items.hasOwnProperty(s)){var a=e.items[s];a.data.group==t&&o.add(a)}o.order(),o.show()}}),this.body.emitter.emit("_change",{queue:!0})},n.prototype._onRemoveGroups=function(t){var e=this.groups;t.forEach(function(t){var i=e[t];i&&(i.hide(),delete e[t])}),this.markDirty(),this.body.emitter.emit("_change",{queue:!0})},n.prototype._orderGroups=function(){if(this.groupsData){var t=this.groupsData.getIds({order:this.options.groupOrder});t=this._orderNestedGroups(t);var e=!l.equalArray(t,this.groupIds);if(e){var i=this.groups;t.forEach(function(t){i[t].hide()}),t.forEach(function(t){i[t].show()}),this.groupIds=t}return e}return!1},n.prototype._orderNestedGroups=function(t){var e=[];return t.forEach(function(t){var i=this.groupsData.get(t);if(i.nestedInGroup||e.push(t),i.nestedGroups){var o=this.groupsData.get({filter:function(e){return e.nestedInGroup==t},order:this.options.groupOrder}),n=o.map(function(t){return t.id});e=e.concat(n)}},this),e},n.prototype._addItem=function(t){this.items[t.id]=t;var e=this._getGroupId(t.data),i=this.groups[e];i?i&&i.data&&i.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1,i&&i.add(t)},n.prototype._updateItem=function(t,e){t.setData(e);var i=this._getGroupId(t.data),o=this.groups[i];o?o&&o.data&&o.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1},n.prototype._removeItem=function(t){t.hide(),delete this.items[t.id];var e=this.selection.indexOf(t.id);-1!=e&&this.selection.splice(e,1),t.parent&&t.parent.remove(t)},n.prototype._constructByEndArray=function(t){for(var e=[],i=0;i<t.length;i++)t[i]instanceof b&&e.push(t[i]);return e},n.prototype._onTouch=function(t){this.touchParams.item=this.itemFromTarget(t),this.touchParams.dragLeftItem=t.target.dragLeftItem||!1,this.touchParams.dragRightItem=t.target.dragRightItem||!1,this.touchParams.itemProps=null},n.prototype._getGroupIndex=function(t){for(var e=0;e<this.groupIds.length;e++)if(t==this.groupIds[e])return e},n.prototype._onDragStart=function(t){if(!this.touchParams.itemIsDragging){var e,i=this.touchParams.item||null,o=this;if(i&&(i.selected||this.options.itemsAlwaysDraggable.item)){if(this.options.editable.overrideItems&&!this.options.editable.updateTime&&!this.options.editable.updateGroup)return;if(null!=i.editable&&!i.editable.updateTime&&!i.editable.updateGroup&&!this.options.editable.overrideItems)return;var n=this.touchParams.dragLeftItem,s=this.touchParams.dragRightItem;if(this.touchParams.itemIsDragging=!0,this.touchParams.selectedItem=i,n)e={item:n,initialX:t.center.x,dragLeft:!0,data:this._cloneItemData(i.data)},this.touchParams.itemProps=[e];else if(s)e={item:s,initialX:t.center.x,dragRight:!0,data:this._cloneItemData(i.data)},this.touchParams.itemProps=[e];else if(this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey))this._onDragStartAddItem(t);else{this.groupIds.length<1&&this.redraw();var r=this._getGroupIndex(i.data.group),a=this.options.itemsAlwaysDraggable.item&&!i.selected?[i.id]:this.getSelection();this.touchParams.itemProps=a.map(function(e){var i=o.items[e],n=o._getGroupIndex(i.data.group);return{item:i,initialX:t.center.x,groupOffset:r-n,data:this._cloneItemData(i.data)}}.bind(this))}t.stopPropagation()}else this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey)&&this._onDragStartAddItem(t)}},n.prototype._onDragStartAddItem=function(t){var e,i,o=this.options.snap||null;this.options.rtl?(e=l.getAbsoluteRight(this.dom.frame),i=e-t.center.x+10):(e=l.getAbsoluteLeft(this.dom.frame),i=t.center.x-e-10);var n=this.body.util.toTime(i),s=this.body.util.getScale(),r=this.body.util.getStep(),a=o?o(n,s,r):n,h=a,d={type:"range",start:a,end:h,content:"new item"},u=l.randomUUID();d[this.itemsData._fieldId]=u;var p=this.groupFromTarget(t);p&&(d.group=p.groupId);var c=new b(d,this.conversion,this.options);c.id=u,c.data=this._cloneItemData(d),this._addItem(c),this.touchParams.selectedItem=c;var m={item:c,initialX:t.center.x,data:c.data};this.options.rtl?m.dragLeft=!0:m.dragRight=!0,this.touchParams.itemProps=[m],t.stopPropagation()},n.prototype._onDrag=function(t){if(this.touchParams.itemProps){t.stopPropagation();var e,i=this,o=this.options.snap||null;e=this.options.rtl?this.body.dom.root.offsetLeft+this.body.domProps.right.width:this.body.dom.root.offsetLeft+this.body.domProps.left.width;var n=this.body.util.getScale(),s=this.body.util.getStep(),r=this.touchParams.selectedItem,a=(this.options.editable.overrideItems||null==r.editable)&&this.options.editable.updateGroup||!this.options.editable.overrideItems&&null!=r.editable&&r.editable.updateGroup,h=null;if(a&&r&&void 0!=r.data.group){var d=i.groupFromTarget(t);d&&(h=this._getGroupIndex(d.groupId))}this.touchParams.itemProps.forEach(function(d){var u,p,c,m,f,g=i.body.util.toTime(t.center.x-e),v=i.body.util.toTime(d.initialX-e);u=this.options.rtl?-(g-v):g-v;var y=this._cloneItemData(d.item.data);if(null==d.item.editable||d.item.editable.updateTime||d.item.editable.updateGroup||i.options.editable.overrideItems){if((this.options.editable.overrideItems||null==r.editable)&&this.options.editable.updateTime||!this.options.editable.overrideItems&&null!=r.editable&&r.editable.updateTime)if(d.dragLeft)this.options.rtl?void 0!=y.end&&(c=l.convert(d.data.end,"Date"),f=new Date(c.valueOf()+u),y.end=o?o(f,n,s):f):void 0!=y.start&&(p=l.convert(d.data.start,"Date"),m=new Date(p.valueOf()+u),y.start=o?o(m,n,s):m);else if(d.dragRight)this.options.rtl?void 0!=y.start&&(p=l.convert(d.data.start,"Date"),m=new Date(p.valueOf()+u),y.start=o?o(m,n,s):m):void 0!=y.end&&(c=l.convert(d.data.end,"Date"),f=new Date(c.valueOf()+u),y.end=o?o(f,n,s):f);else if(void 0!=y.start)if(p=l.convert(d.data.start,"Date").valueOf(),m=new Date(p+u),void 0!=y.end){c=l.convert(d.data.end,"Date");var b=c.valueOf()-p.valueOf();y.start=o?o(m,n,s):m,y.end=new Date(y.start.valueOf()+b)}else y.start=o?o(m,n,s):m;if(a&&!d.dragLeft&&!d.dragRight&&null!=h&&void 0!=y.group){var _=h-d.groupOffset;_=Math.max(0,_),_=Math.min(i.groupIds.length-1,_),y.group=i.groupIds[_]}y=this._cloneItemData(y),i.options.onMoving(y,function(t){t&&d.item.setData(this._cloneItemData(t,"Date"))}.bind(this))}}.bind(this)),this.body.emitter.emit("_change")}},n.prototype._moveToGroup=function(t,e){var i=this.groups[e];if(i&&i.groupId!=t.data.group){var o=t.parent;o.remove(t),o.order(),t.data.group=i.groupId,i.add(t),i.order()}},n.prototype._onDragEnd=function(t){if(this.touchParams.itemIsDragging=!1,this.touchParams.itemProps){t.stopPropagation();var e=this,i=this.itemsData.getDataSet(),o=this.touchParams.itemProps;this.touchParams.itemProps=null,o.forEach(function(t){var o=t.item.id;if(null!=e.itemsData.get(o,e.itemOptions)){var n=this._cloneItemData(t.item.data);e.options.onMove(n,function(n){n?(n[i._fieldId]=o,i.update(n)):(t.item.setData(t.data),e.body.emitter.emit("_change"))})}else e.options.onAdd(t.item.data,function(i){e._removeItem(t.item),i&&e.itemsData.getDataSet().add(i),e.body.emitter.emit("_change")})}.bind(this))}},n.prototype._onGroupClick=function(t){var e=this.groupFromTarget(t);if(e&&e.nestedGroups){var i=this.groupsData.getDataSet(),o=i.get(e.groupId);void 0==o.showNested&&(o.showNested=!0),o.showNested=!o.showNested;var n=i.get(e.nestedGroups).map(function(t){return t.visible=o.showNested,t});if(i.update(n.concat(o)),o.showNested)l.removeClassName(e.dom.label,"collapsed"),l.addClassName(e.dom.label,"expanded");else{l.removeClassName(e.dom.label,"expanded");var s=this.options.rtl?"collapsed-rtl":"collapsed";l.addClassName(e.dom.label,s)}}},n.prototype._onGroupDragStart=function(t){this.options.groupEditable.order&&(this.groupTouchParams.group=this.groupFromTarget(t),this.groupTouchParams.group&&(t.stopPropagation(),this.groupTouchParams.originalOrder=this.groupsData.getIds({order:this.options.groupOrder})))},n.prototype._onGroupDrag=function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this.groupsData;this.groupsData instanceof p&&(e=this.groupsData.getDataSet());var i=this.groupFromTarget(t);if(i&&i.height!=this.groupTouchParams.group.height){var o=i.top<this.groupTouchParams.group.top,n=t.center?t.center.y:t.clientY,s=l.getAbsoluteTop(i.dom.foreground),r=this.groupTouchParams.group.height;if(o){if(s+r<n)return}else{if(s+i.height-r>n)return}}if(i&&i!=this.groupTouchParams.group){var a=e.get(i.groupId),h=e.get(this.groupTouchParams.group.groupId);h&&a&&(this.options.groupOrderSwap(h,a,e),e.update(h),e.update(a));var d=e.getIds({order:this.options.groupOrder});if(!l.equalArray(d,this.groupTouchParams.originalOrder))for(var u=this.groupTouchParams.originalOrder,c=this.groupTouchParams.group.groupId,m=Math.min(u.length,d.length),f=0,g=0,v=0;f<m;){for(;f+g<m&&f+v<m&&d[f+g]==u[f+v];)f++;if(f+g>=m)break;if(d[f+g]==c)g=1;else if(u[f+v]==c)v=1;else{var y=d.indexOf(u[f+v]),b=e.get(d[f+g]),_=e.get(u[f+v]);this.options.groupOrderSwap(b,_,e),e.update(b),e.update(_);var w=d[f+g];d[f+g]=u[f+v],d[y]=w,f++}}}}},n.prototype._onGroupDragEnd=function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this,i=e.groupTouchParams.group.groupId,o=e.groupsData.getDataSet(),n=l.extend({},o.get(i));e.options.onMoveGroup(n,function(t){if(t)t[o._fieldId]=i,o.update(t);else{var n=o.getIds({order:e.options.groupOrder});if(!l.equalArray(n,e.groupTouchParams.originalOrder))for(var s=e.groupTouchParams.originalOrder,r=Math.min(s.length,n.length),a=0;a<r;){for(;a<r&&n[a]==s[a];)a++;if(a>=r)break;var h=n.indexOf(s[a]),d=o.get(n[a]),u=o.get(s[a]);e.options.groupOrderSwap(d,u,o),o.update(d),o.update(u);var p=n[a];n[a]=s[a],n[h]=p,a++}}}),e.body.emitter.emit("groupDragged",{groupId:i})}},n.prototype._onSelectItem=function(t){if(this.options.selectable){var e=t.srcEvent&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey),i=t.srcEvent&&t.srcEvent.shiftKey;if(e||i)return void this._onMultiSelectItem(t);var o=this.getSelection(),n=this.itemFromTarget(t),s=n?[n.id]:[];this.setSelection(s);var r=this.getSelection();(r.length>0||o.length>0)&&this.body.emitter.emit("select",{items:r,event:t})}},n.prototype._onMouseOver=function(t){var e=this.itemFromTarget(t);if(e){if(e!==this.itemFromRelatedTarget(t)){var i=e.getTitle();if(this.options.showTooltips&&i){null==this.popup&&(this.popup=new w(this.body.dom.root,this.options.tooltip.overflowMethod||"flip")),this.popup.setText(i);var o=this.body.dom.centerContainer;this.popup.setPosition(t.clientX-l.getAbsoluteLeft(o)+o.offsetLeft,t.clientY-l.getAbsoluteTop(o)+o.offsetTop),this.popup.show()}else null!=this.popup&&this.popup.hide();this.body.emitter.emit("itemover",{item:e.id,event:t})}}},n.prototype._onMouseOut=function(t){var e=this.itemFromTarget(t);if(e){e!==this.itemFromRelatedTarget(t)&&(null!=this.popup&&this.popup.hide(),this.body.emitter.emit("itemout",{item:e.id,event:t}))}},n.prototype._onMouseMove=function(t){if(this.itemFromTarget(t)&&this.options.showTooltips&&this.options.tooltip.followMouse&&this.popup&&!this.popup.hidden){var e=this.body.dom.centerContainer;this.popup.setPosition(t.clientX-l.getAbsoluteLeft(e)+e.offsetLeft,t.clientY-l.getAbsoluteTop(e)+e.offsetTop),this.popup.show()}},n.prototype._onMouseWheel=function(t){this.touchParams.itemIsDragging&&this._onDragEnd(t)},n.prototype._onUpdateItem=function(t){if(this.options.selectable&&this.options.editable.add){var e=this;if(t){var i=e.itemsData.get(t.id);this.options.onUpdate(i,function(t){t&&e.itemsData.getDataSet().update(t)})}}},n.prototype._onDropObjectOnItem=function(t){var e=this.itemFromTarget(t),i=JSON.parse(t.dataTransfer.getData("text"));this.options.onDropObjectOnItem(i,e)},n.prototype._onAddItem=function(t){if(this.options.selectable&&this.options.editable.add){var e,i,o=this,n=this.options.snap||null;this.options.rtl?(e=l.getAbsoluteRight(this.dom.frame),i=e-t.center.x):(e=l.getAbsoluteLeft(this.dom.frame),i=t.center.x-e);var s,r,a=this.body.util.toTime(i),h=this.body.util.getScale(),d=this.body.util.getStep();"drop"==t.type?(r=JSON.parse(t.dataTransfer.getData("text")),r.content=r.content?r.content:"new item",r.start=r.start?r.start:n?n(a,h,d):a,r.type=r.type||"box",r[this.itemsData._fieldId]=r.id||l.randomUUID(),"range"!=r.type||r.end||(s=this.body.util.toTime(i+this.props.width/5),r.end=n?n(s,h,d):s)):(r={start:n?n(a,h,d):a,content:"new item"},r[this.itemsData._fieldId]=l.randomUUID(),"range"===this.options.type&&(s=this.body.util.toTime(i+this.props.width/5),r.end=n?n(s,h,d):s));var u=this.groupFromTarget(t);u&&(r.group=u.groupId),r=this._cloneItemData(r),this.options.onAdd(r,function(e){e&&(o.itemsData.getDataSet().add(e),"drop"==t.type&&o.setSelection([e.id]))})}},n.prototype._onMultiSelectItem=function(t){if(this.options.selectable){var e=this.itemFromTarget(t);if(e){var i=this.options.multiselect?this.getSelection():[];if((t.srcEvent&&t.srcEvent.shiftKey||!1)&&this.options.multiselect){var o=this.itemsData.get(e.id).group,s=void 0;this.options.multiselectPerGroup&&i.length>0&&(s=this.itemsData.get(i[0]).group),this.options.multiselectPerGroup&&void 0!=s&&s!=o||i.push(e.id);var r=n._getItemRange(this.itemsData.get(i,this.itemOptions));if(!this.options.multiselectPerGroup||s==o){i=[];for(var a in this.items)if(this.items.hasOwnProperty(a)){var h=this.items[a],d=h.data.start,l=void 0!==h.data.end?h.data.end:d;!(d>=r.min&&l<=r.max)||this.options.multiselectPerGroup&&s!=this.itemsData.get(h.id).group||h instanceof _||i.push(h.id)}}}else{var u=i.indexOf(e.id);-1==u?i.push(e.id):i.splice(u,1)}this.setSelection(i),this.body.emitter.emit("select",{items:this.getSelection(),event:t})}}},n._getItemRange=function(t){var e=null,i=null;return t.forEach(function(t){(null==i||t.start<i)&&(i=t.start),void 0!=t.end?(null==e||t.end>e)&&(e=t.end):(null==e||t.start>e)&&(e=t.start)}),{min:i,max:e}},n.prototype.itemFromElement=function(t){for(var e=t;e;){if(e.hasOwnProperty("timeline-item"))return e["timeline-item"];e=e.parentNode}return null},n.prototype.itemFromTarget=function(t){return this.itemFromElement(t.target)},n.prototype.itemFromRelatedTarget=function(t){return this.itemFromElement(t.relatedTarget)},n.prototype.groupFromTarget=function(t){var e=t.center?t.center.y:t.clientY,i=this.groupIds;i.length<=0&&this.groupsData&&(i=this.groupsData.getIds({order:this.options.groupOrder}));for(var o=0;o<i.length;o++){var n=i[o],s=this.groups[n],r=s.dom.foreground,a=l.getAbsoluteTop(r);if(e>a&&e<a+r.offsetHeight)return s;if("top"===this.options.orientation.item){if(o===this.groupIds.length-1&&e>a)return s}else if(0===o&&e<a+r.offset)return s}return null},n.itemSetFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("timeline-itemset"))return e["timeline-itemset"];e=e.parentNode}return null},n.prototype._cloneItemData=function(t,e){var i=l.extend({},t);return e||(e=this.itemsData.getDataSet()._options.type),void 0!=i.start&&(i.start=l.convert(i.start,e&&e.start||"Date")),void 0!=i.end&&(i.end=l.convert(i.end,e&&e.end||"Date")),i},t.exports=n},function(t,e,i){e.orderByStart=function(t){t.sort(function(t,e){return t.data.start-e.data.start})},e.orderByEnd=function(t){t.sort(function(t,e){return("end"in t.data?t.data.end:t.data.start)-("end"in e.data?e.data.end:e.data.start)})},e.stack=function(t,i,o){if(o)for(var n=0;n<t.length;n++)t[n].top=null;for(var n=0;n<t.length;n++){var s=t[n];if(s.stack&&null===s.top){s.top=i.axis;do{for(var r=null,a=0,h=t.length;a<h;a++){var d=t[a];if(null!==d.top&&d!==s&&d.stack&&e.collision(s,d,i.item,d.options.rtl)){r=d;break}}null!=r&&(s.top=r.top+r.height+i.item.vertical)}while(r)}}},e.substack=function(t,i,o){for(var n=0;n<t.length;n++)t[n].top=null;var s=o.height;for(n=0;n<t.length;n++){var r=t[n];if(r.stack&&null===r.top){r.top=r.baseTop;do{for(var a=null,h=0,d=t.length;h<d;h++){var l=t[h];if(null!==l.top&&l!==r&&e.collision(r,l,i.item,l.options.rtl)){a=l;break}}null!=a&&(r.top=a.top+a.height+i.item.vertical),r.top+r.height>s&&(s=r.top+r.height)}while(a)}}o.height=s-o.top+.5*i.item.vertical},e.nostack=function(t,i,o,n){for(var s=0;s<t.length;s++)if(void 0==t[s].data.subgroup)t[s].top=i.item.vertical;else if(void 0!==t[s].data.subgroup&&n){var r=0;for(var a in o)o.hasOwnProperty(a)&&1==o[a].visible&&o[a].index<o[t[s].data.subgroup].index&&(r+=o[a].height,o[t[s].data.subgroup].top=r);t[s].top=r+.5*i.item.vertical}n||e.stackSubgroups(t,i,o)},e.stackSubgroups=function(t,i,o){for(var n in o)if(o.hasOwnProperty(n)){o[n].top=0;do{var s=null;for(var r in o)if(null!==o[r].top&&r!==n&&o[n].index>o[r].index&&e.collisionByTimes(o[n],o[r])){s=o[r];break}null!=s&&(o[n].top=s.top+s.height)}while(s)}for(var a=0;a<t.length;a++)void 0!==t[a].data.subgroup&&(t[a].top=o[t[a].data.subgroup].top+.5*i.item.vertical)},e.stackSubgroupsWithInnerStack=function(t,i,o){var n=!1,s=[];for(var r in o)o[r].hasOwnProperty("index")?s[o[r].index]=r:s.push(r);for(var a=0;a<s.length;a++)if(r=s[a],o.hasOwnProperty(r)){n=n||o[r].stack,o[r].top=0;for(var h in o)o[h].visible&&o[r].index>o[h].index&&(o[r].top+=o[h].height);for(var d=t[r],l=0;l<d.length;l++)void 0!==d[l].data.subgroup&&(d[l].top=o[d[l].data.subgroup].top+.5*i.item.vertical,o[r].stack&&(d[l].baseTop=d[l].top));n&&o[r].stack&&e.substack(t[r],i,o[r])}},e.collision=function(t,e,i,o){return o?t.right-i.horizontal+.001<e.right+e.width&&t.right+t.width+i.horizontal-.001>e.right&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top:t.left-i.horizontal+.001<e.left+e.width&&t.left+t.width+i.horizontal-.001>e.left&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top},e.collisionByTimes=function(t,e){return t.start<=e.start&&t.end>=e.start&&t.top<e.top+e.height&&t.top+t.height>e.top||e.start<=t.start&&e.end>=t.start&&e.top<t.top+t.height&&e.top+e.height>t.top}},function(t,e,i){function o(t,e,i){if(this.props={dot:{width:0,height:0},line:{width:0,height:0}},this.options=i,t&&void 0==t.start)throw new Error('Property "start" missing in item '+t);n.call(this,t,e,i)}var n=i(22);o.prototype=new n(null,null,null),o.prototype.isVisible=function(t){var e=this.options.align,i=this.width*t.getMillisecondsPerPixel();return"right"==e?this.data.start.getTime()>t.start&&this.data.start.getTime()-i<t.end:"left"==e?this.data.start.getTime()+i>t.start&&this.data.start.getTime()<t.end:this.data.start.getTime()+i/2>t.start&&this.data.start.getTime()-i/2<t.end},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.box=document.createElement("DIV"),this.dom.content=document.createElement("DIV"),this.dom.content.className="vis-item-content",this.dom.box.appendChild(this.dom.content),this.dom.line=document.createElement("DIV"),this.dom.line.className="vis-line",this.dom.dot=document.createElement("DIV"),this.dom.dot.className="vis-dot",this.dom.box["timeline-item"]=this,this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}if(!this.dom.line.parentNode){var e=this.parent.dom.background;if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(this.dom.line)}if(!this.dom.dot.parentNode){var i=this.parent.dom.axis;if(!e)throw new Error("Cannot redraw item: parent has no axis container element");i.appendChild(this.dom.dot)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className="vis-item vis-box"+e,this.dom.line.className="vis-item vis-line"+e,this.dom.dot.className="vis-item vis-dot"+e}},o.prototype._getDomComponentsSizes=function(){return{previous:{right:this.dom.box.style.right,left:this.dom.box.style.left},dot:{height:this.dom.dot.offsetHeight,width:this.dom.dot.offsetWidth},line:{width:this.dom.line.offsetWidth},box:{width:this.dom.box.offsetWidth,height:this.dom.box.offsetHeight}}},o.prototype._updateDomComponentsSizes=function(t){this.options.rtl?this.dom.box.style.right="0px":this.dom.box.style.left="0px",this.props.dot.height=t.dot.height,this.props.dot.width=t.dot.width,this.props.line.width=t.line.width,this.width=t.box.width,this.height=t.box.height,this.options.rtl?this.dom.box.style.right=t.previous.right:this.dom.box.style.left=t.previous.left,this.dirty=!1},o.prototype._repaintDomAdditionals=function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.box)},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){if(this.displayed){var t=this.dom;t.box.parentNode&&t.box.parentNode.removeChild(t.box),t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot),this.displayed=!1}},o.prototype.repositionX=function(){var t=this.conversion.toScreen(this.data.start),e=this.options.align;"right"==e?this.options.rtl?(this.right=t-this.width,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t-this.props.line.width+"px",this.dom.dot.style.right=t-this.props.line.width/2-this.props.dot.width/2+"px"):(this.left=t-this.width,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t-this.props.line.width+"px",this.dom.dot.style.left=t-this.props.line.width/2-this.props.dot.width/2+"px"):"left"==e?this.options.rtl?(this.right=t,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t+"px",this.dom.dot.style.right=t+this.props.line.width/2-this.props.dot.width/2+"px"):(this.left=t,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t+"px",this.dom.dot.style.left=t+this.props.line.width/2-this.props.dot.width/2+"px"):this.options.rtl?(this.right=t-this.width/2,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t-this.props.line.width+"px",this.dom.dot.style.right=t-this.props.dot.width/2+"px"):(this.left=t-this.width/2,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t-this.props.line.width/2+"px",this.dom.dot.style.left=t-this.props.dot.width/2+"px")},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.box,i=this.dom.line,o=this.dom.dot;if("top"==t)e.style.top=(this.top||0)+"px",i.style.top="0",i.style.height=this.parent.top+this.top+1+"px",i.style.bottom="";else{var n=this.parent.itemSet.props.height,s=n-this.parent.top-this.parent.height+this.top;e.style.top=(this.parent.height-this.top-this.height||0)+"px",i.style.top=n-s+"px",i.style.bottom="0"}o.style.top=-this.props.dot.height/2+"px"},o.prototype.getWidthLeft=function(){return this.width/2},o.prototype.getWidthRight=function(){return this.width/2},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={dot:{top:0,width:0,height:0},content:{height:0,marginLeft:0,marginRight:0}},this.options=i,t&&void 0==t.start)throw new Error('Property "start" missing in item '+t);n.call(this,t,e,i)}var n=i(22);o.prototype=new n(null,null,null),o.prototype.isVisible=function(t){var e=this.width*t.getMillisecondsPerPixel();return this.data.start.getTime()+e>t.start&&this.data.start<t.end},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.point=document.createElement("div"),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.point.appendChild(this.dom.content),this.dom.dot=document.createElement("div"),this.dom.point.appendChild(this.dom.dot),this.dom.point["timeline-item"]=this,this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.point.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.point)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.point),this._updateStyle(this.dom.point);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.point.className="vis-item vis-point"+e,this.dom.dot.className="vis-item vis-dot"+e}},o.prototype._getDomComponentsSizes=function(){return{dot:{width:this.dom.dot.offsetWidth,height:this.dom.dot.offsetHeight},content:{width:this.dom.content.offsetWidth,height:this.dom.content.offsetHeight},point:{width:this.dom.point.offsetWidth,height:this.dom.point.offsetHeight}}},o.prototype._updateDomComponentsSizes=function(t){this.props.dot.width=t.dot.width,this.props.dot.height=t.dot.height,this.props.content.height=t.content.height,this.options.rtl?this.dom.content.style.marginRight=2*this.props.dot.width+"px":this.dom.content.style.marginLeft=2*this.props.dot.width+"px",this.width=t.point.width,this.height=t.point.height,this.dom.dot.style.top=(this.height-this.props.dot.height)/2+"px",this.options.rtl?this.dom.dot.style.right=this.props.dot.width/2+"px":this.dom.dot.style.left=this.props.dot.width/2+"px",this.dirty=!1},o.prototype._repaintDomAdditionals=function(){this._repaintOnItemUpdateTimeTooltip(this.dom.point),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.point)},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){this.displayed&&(this.dom.point.parentNode&&this.dom.point.parentNode.removeChild(this.dom.point),this.displayed=!1)},o.prototype.repositionX=function(){var t=this.conversion.toScreen(this.data.start);this.options.rtl?(this.right=t-this.props.dot.width,this.dom.point.style.right=this.right+"px"):(this.left=t-this.props.dot.width,this.dom.point.style.left=this.left+"px")},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.point;e.style.top="top"==t?this.top+"px":this.parent.height-this.top-this.height+"px"},o.prototype.getWidthLeft=function(){return this.props.dot.width},o.prototype.getWidthRight=function(){return this.props.dot.width},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={content:{width:0}},this.overflow=!1,t){if(void 0==t.start)throw new Error('Property "start" missing in item '+t.id);if(void 0==t.end)throw new Error('Property "end" missing in item '+t.id)}n.call(this,t,e,i)}var n=i(22),s=i(52),r=i(53);o.prototype=new n(null,null,null),o.prototype.baseClassName="vis-item vis-background",o.prototype.stack=!1,o.prototype.isVisible=function(t){return this.data.start<t.end&&this.data.end>t.start},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.background;if(!t)throw new Error("Cannot redraw item: parent has no background container element");t.appendChild(this.dom.box)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.content),this._updateStyle(this.dom.box);var t=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"");this.dom.box.className=this.baseClassName+t}},o.prototype._getDomComponentsSizes=function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.content).overflow,{content:{width:this.dom.content.offsetWidth}}},o.prototype._updateDomComponentsSizes=function(t){this.props.content.width=t.content.width,this.height=0,this.dirty=!1},o.prototype._repaintDomAdditionals=function(){},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes.bind(this)())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=r.prototype.show,o.prototype.hide=r.prototype.hide,o.prototype.repositionX=r.prototype.repositionX,o.prototype.repositionY=function(t){var e,i=this.options.orientation.item;if(void 0!==this.data.subgroup){var o=this.data.subgroup;this.dom.box.style.height=this.parent.subgroups[o].height+"px",this.dom.box.style.top="top"==i?this.parent.top+this.parent.subgroups[o].top+"px":this.parent.top+this.parent.height-this.parent.subgroups[o].top-this.parent.subgroups[o].height+"px",this.dom.box.style.bottom=""}else this.parent instanceof s?(e=Math.max(this.parent.height,this.parent.itemSet.body.domProps.center.height,this.parent.itemSet.body.domProps.centerContainer.height),this.dom.box.style.bottom="bottom"==i?"0":"",this.dom.box.style.top="top"==i?"0":""):(e=this.parent.height,this.dom.box.style.top=this.parent.top+"px",this.dom.box.style.bottom="");this.dom.box.style.height=e+"px"},t.exports=o},function(t,e,i){Object.defineProperty(e,"__esModule",{value:!0});var o="string",n="boolean",s="number",r="object",a={configure:{enabled:{boolean:n},filter:{boolean:n,function:"function"},container:{dom:"dom"},__type__:{object:r,boolean:n,function:"function"}},align:{string:o},rtl:{boolean:n,undefined:"undefined"},rollingMode:{follow:{boolean:n},offset:{number:s,undefined:"undefined"},__type__:{object:r}},verticalScroll:{boolean:n,undefined:"undefined"},horizontalScroll:{boolean:n,undefined:"undefined"},autoResize:{boolean:n},throttleRedraw:{number:s},clickToUse:{boolean:n},dataAttributes:{string:o,array:"array"},editable:{add:{boolean:n,undefined:"undefined"},remove:{boolean:n,undefined:"undefined"},updateGroup:{boolean:n,undefined:"undefined"},updateTime:{boolean:n,undefined:"undefined"},overrideItems:{boolean:n,undefined:"undefined"},__type__:{boolean:n,object:r}},end:{number:s,date:"date",string:o,moment:"moment"},format:{minorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},week:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r,function:"function"}},majorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},week:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r,function:"function"}},__type__:{object:r}},moment:{function:"function"},groupOrder:{string:o,function:"function"},groupEditable:{add:{boolean:n,undefined:"undefined"},remove:{boolean:n,undefined:"undefined"},order:{boolean:n,undefined:"undefined"},__type__:{boolean:n,object:r}},groupOrderSwap:{function:"function"},height:{string:o,number:s},hiddenDates:{start:{date:"date",number:s,string:o,moment:"moment"},end:{date:"date",number:s,string:o,moment:"moment"},repeat:{string:o},__type__:{object:r,array:"array"}},itemsAlwaysDraggable:{item:{boolean:n,undefined:"undefined"},range:{boolean:n,undefined:"undefined"},__type__:{boolean:n,object:r}},limitSize:{boolean:n},locale:{string:o},locales:{__any__:{any:"any"},__type__:{object:r}},margin:{axis:{number:s},item:{horizontal:{number:s,undefined:"undefined"},vertical:{number:s,undefined:"undefined"},__type__:{object:r,number:s}},__type__:{object:r,number:s}},max:{date:"date",number:s,string:o,moment:"moment"},maxHeight:{number:s,string:o},maxMinorChars:{number:s},min:{date:"date",number:s,string:o,moment:"moment"},minHeight:{number:s,string:o},moveable:{boolean:n},multiselect:{boolean:n},multiselectPerGroup:{boolean:n},onAdd:{function:"function"},onDropObjectOnItem:{function:"function"},onUpdate:{function:"function"},onMove:{function:"function"},onMoving:{function:"function"},onRemove:{function:"function"},onAddGroup:{function:"function"},onMoveGroup:{function:"function"},onRemoveGroup:{function:"function"},onInitialDrawComplete:{function:"function"},order:{function:"function"},orientation:{axis:{string:o,undefined:"undefined"},item:{string:o,undefined:"undefined"},__type__:{string:o,object:r}},selectable:{boolean:n},showCurrentTime:{boolean:n},showMajorLabels:{boolean:n},showMinorLabels:{boolean:n},stack:{boolean:n},stackSubgroups:{boolean:n},snap:{function:"function",null:"null"},start:{date:"date",number:s,string:o,moment:"moment"},template:{function:"function"},groupTemplate:{function:"function"},visibleFrameTemplate:{string:o,function:"function"},showTooltips:{boolean:n},tooltip:{followMouse:{boolean:n},overflowMethod:{string:["cap","flip"]},__type__:{object:r}},tooltipOnItemUpdateTime:{template:{function:"function"},__type__:{boolean:n,object:r}},timeAxis:{scale:{string:o,undefined:"undefined"},step:{number:s,undefined:"undefined"},__type__:{object:r}},type:{string:o},width:{string:o,number:s},zoomable:{boolean:n},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:s},zoomMin:{number:s},__type__:{object:r}},h={global:{align:["center","left","right"],direction:!1,autoResize:!0,clickToUse:!1,editable:{add:!1,remove:!1,updateGroup:!1,updateTime:!1},end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}},groupsDraggable:!1,height:"",locale:"",margin:{axis:[20,0,100,1],item:{horizontal:[10,0,100,1],vertical:[10,0,100,1]}},max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!1,multiselect:!1,multiselectPerGroup:!1,orientation:{axis:["both","bottom","top"],item:["bottom","top"]},selectable:!0,showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,stack:!0,stackSubgroups:!0,start:"",showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip"},tooltipOnItemUpdateTime:!1,type:["box","point","range","background"],width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1]}};e.allOptions=a,e.configureOptions=h},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0});var n=i(16),s=o(n),r=i(1),a=o(r),h=i(30),d=o(h),l=i(31),u=o(l),p=i(0),c=i(137).default,m=function(){function t(e,i,o){var n=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1;(0,d.default)(this,t),this.parent=e,this.changedOptions=[],this.container=i,this.allowCreation=!1,this.options={},this.initialized=!1,this.popupCounter=0,this.defaultOptions={enabled:!1,filter:!0,container:void 0,showButton:!0},p.extend(this.options,this.defaultOptions),this.configureOptions=o,this.moduleOptions={},this.domElements=[],this.popupDiv={},this.popupLimit=5,this.popupHistory={},this.colorPicker=new c(n),this.wrapper=void 0}return(0,u.default)(t,[{key:"setOptions",value:function(t){if(void 0!==t){this.popupHistory={},this._removePopup();var e=!0;"string"==typeof t?this.options.filter=t:t instanceof Array?this.options.filter=t.join():"object"===(void 0===t?"undefined":(0,a.default)(t))?(void 0!==t.container&&(this.options.container=t.container),void 0!==t.filter&&(this.options.filter=t.filter),void 0!==t.showButton&&(this.options.showButton=t.showButton),void 0!==t.enabled&&(e=t.enabled)):"boolean"==typeof t?(this.options.filter=!0,e=t):"function"==typeof t&&(this.options.filter=t,e=!0),!1===this.options.filter&&(e=!1),this.options.enabled=e}this._clean()}},{key:"setModuleOptions",value:function(t){this.moduleOptions=t,!0===this.options.enabled&&(this._clean(),void 0!==this.options.container&&(this.container=this.options.container),this._create())}},{key:"_create",value:function(){var t=this;this._clean(),this.changedOptions=[];var e=this.options.filter,i=0,o=!1;for(var n in this.configureOptions)this.configureOptions.hasOwnProperty(n)&&(this.allowCreation=!1,o=!1,"function"==typeof e?(o=e(n,[]),o=o||this._handleObject(this.configureOptions[n],[n],!0)):!0!==e&&-1===e.indexOf(n)||(o=!0),!1!==o&&(this.allowCreation=!0,i>0&&this._makeItem([]),this._makeHeader(n),this._handleObject(this.configureOptions[n],[n])),i++);if(!0===this.options.showButton){var s=document.createElement("div");s.className="vis-configuration vis-config-button",s.innerHTML="generate options",s.onclick=function(){t._printOptions()},s.onmouseover=function(){s.className="vis-configuration vis-config-button hover"},s.onmouseout=function(){s.className="vis-configuration vis-config-button"},this.optionsContainer=document.createElement("div"),this.optionsContainer.className="vis-configuration vis-config-option-container",this.domElements.push(this.optionsContainer),this.domElements.push(s)}this._push()}},{key:"_push",value:function(){this.wrapper=document.createElement("div"),this.wrapper.className="vis-configuration-wrapper",this.container.appendChild(this.wrapper);for(var t=0;t<this.domElements.length;t++)this.wrapper.appendChild(this.domElements[t]);this._showPopupIfNeeded()}},{key:"_clean",value:function(){for(var t=0;t<this.domElements.length;t++)this.wrapper.removeChild(this.domElements[t]);void 0!==this.wrapper&&(this.container.removeChild(this.wrapper),this.wrapper=void 0),this.domElements=[],this._removePopup()}},{key:"_getValue",value:function(t){for(var e=this.moduleOptions,i=0;i<t.length;i++){if(void 0===e[t[i]]){e=void 0;break}e=e[t[i]]}return e}},{key:"_makeItem",value:function(t){if(!0===this.allowCreation){var e=document.createElement("div");e.className="vis-configuration vis-config-item vis-config-s"+t.length;for(var i=arguments.length,o=Array(i>1?i-1:0),n=1;n<i;n++)o[n-1]=arguments[n];return o.forEach(function(t){e.appendChild(t)}),this.domElements.push(e),this.domElements.length}return 0}},{key:"_makeHeader",value:function(t){var e=document.createElement("div");e.className="vis-configuration vis-config-header",e.innerHTML=t,this._makeItem([],e)}},{key:"_makeLabel",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],o=document.createElement("div");return o.className="vis-configuration vis-config-label vis-config-s"+e.length,o.innerHTML=!0===i?"<i><b>"+t+":</b></i>":t+":",o}},{key:"_makeDropdown",value:function(t,e,i){var o=document.createElement("select");o.className="vis-configuration vis-config-select";var n=0;void 0!==e&&-1!==t.indexOf(e)&&(n=t.indexOf(e));for(var s=0;s<t.length;s++){var r=document.createElement("option");r.value=t[s],s===n&&(r.selected="selected"),r.innerHTML=t[s],o.appendChild(r)}var a=this;o.onchange=function(){a._update(this.value,i)};var h=this._makeLabel(i[i.length-1],i);this._makeItem(i,h,o)}},{key:"_makeRange",value:function(t,e,i){var o=t[0],n=t[1],s=t[2],r=t[3],a=document.createElement("input");a.className="vis-configuration vis-config-range";try{a.type="range",a.min=n,a.max=s}catch(t){}a.step=r;var h="",d=0;if(void 0!==e){e<0&&1.2*e<n?(a.min=Math.ceil(1.2*e),d=a.min,h="range increased"):e/1.2<n&&(a.min=Math.ceil(e/1.2),d=a.min,h="range increased"),1.2*e>s&&1!==s&&(a.max=Math.ceil(1.2*e),d=a.max,h="range increased"),a.value=e}else a.value=o;var l=document.createElement("input");l.className="vis-configuration vis-config-rangeinput",l.value=a.value;var u=this;a.onchange=function(){l.value=this.value,u._update(Number(this.value),i)},a.oninput=function(){l.value=this.value};var p=this._makeLabel(i[i.length-1],i),c=this._makeItem(i,p,a,l);""!==h&&this.popupHistory[c]!==d&&(this.popupHistory[c]=d,this._setupPopup(h,c))}},{key:"_setupPopup",value:function(t,e){var i=this;if(!0===this.initialized&&!0===this.allowCreation&&this.popupCounter<this.popupLimit){var o=document.createElement("div");o.id="vis-configuration-popup",o.className="vis-configuration-popup",o.innerHTML=t,o.onclick=function(){i._removePopup()},this.popupCounter+=1,this.popupDiv={html:o,index:e}}}},{key:"_removePopup",value:function(){void 0!==this.popupDiv.html&&(this.popupDiv.html.parentNode.removeChild(this.popupDiv.html),clearTimeout(this.popupDiv.hideTimeout),clearTimeout(this.popupDiv.deleteTimeout),this.popupDiv={})}},{key:"_showPopupIfNeeded",value:function(){var t=this;if(void 0!==this.popupDiv.html){var e=this.domElements[this.popupDiv.index],i=e.getBoundingClientRect();this.popupDiv.html.style.left=i.left+"px",this.popupDiv.html.style.top=i.top-30+"px",document.body.appendChild(this.popupDiv.html),this.popupDiv.hideTimeout=setTimeout(function(){t.popupDiv.html.style.opacity=0},1500),this.popupDiv.deleteTimeout=setTimeout(function(){t._removePopup()},1800)}}},{key:"_makeCheckbox",value:function(t,e,i){var o=document.createElement("input");o.type="checkbox",o.className="vis-configuration vis-config-checkbox",o.checked=t,void 0!==e&&(o.checked=e,e!==t&&("object"===(void 0===t?"undefined":(0,a.default)(t))?e!==t.enabled&&this.changedOptions.push({path:i,value:e}):this.changedOptions.push({path:i,value:e})));var n=this;o.onchange=function(){n._update(this.checked,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,o)}},{key:"_makeTextInput",value:function(t,e,i){var o=document.createElement("input");o.type="text",o.className="vis-configuration vis-config-text",o.value=e,e!==t&&this.changedOptions.push({path:i,value:e});var n=this;o.onchange=function(){n._update(this.value,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,o)}},{key:"_makeColorField",value:function(t,e,i){var o=this,n=t[1],s=document.createElement("div");e=void 0===e?n:e,"none"!==e?(s.className="vis-configuration vis-config-colorBlock",s.style.backgroundColor=e):s.className="vis-configuration vis-config-colorBlock none",e=void 0===e?n:e,s.onclick=function(){o._showColorPicker(e,s,i)};var r=this._makeLabel(i[i.length-1],i);this._makeItem(i,r,s)}},{key:"_showColorPicker",value:function(t,e,i){var o=this;e.onclick=function(){},this.colorPicker.insertTo(e),this.colorPicker.show(),this.colorPicker.setColor(t),this.colorPicker.setUpdateCallback(function(t){var n="rgba("+t.r+","+t.g+","+t.b+","+t.a+")";e.style.backgroundColor=n,o._update(n,i)}),this.colorPicker.setCloseCallback(function(){e.onclick=function(){o._showColorPicker(t,e,i)}})}},{key:"_handleObject",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[],i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],o=!1,n=this.options.filter,s=!1;for(var r in t)if(t.hasOwnProperty(r)){o=!0;var a=t[r],h=p.copyAndExtendArray(e,r);if("function"==typeof n&&!1===(o=n(r,e))&&!(a instanceof Array)&&"string"!=typeof a&&"boolean"!=typeof a&&a instanceof Object&&(this.allowCreation=!1,o=this._handleObject(a,h,!0),this.allowCreation=!1===i),!1!==o){s=!0;var d=this._getValue(h);if(a instanceof Array)this._handleArray(a,d,h);else if("string"==typeof a)this._makeTextInput(a,d,h);else if("boolean"==typeof a)this._makeCheckbox(a,d,h);else if(a instanceof Object){var l=!0;if(-1!==e.indexOf("physics")&&this.moduleOptions.physics.solver!==r&&(l=!1),!0===l)if(void 0!==a.enabled){var u=p.copyAndExtendArray(h,"enabled"),c=this._getValue(u);if(!0===c){var m=this._makeLabel(r,h,!0);this._makeItem(h,m),s=this._handleObject(a,h)||s}else this._makeCheckbox(a,c,h)}else{var f=this._makeLabel(r,h,!0);this._makeItem(h,f),s=this._handleObject(a,h)||s}}else console.error("dont know how to handle",a,r,h)}}return s}},{key:"_handleArray",value:function(t,e,i){"string"==typeof t[0]&&"color"===t[0]?(this._makeColorField(t,e,i),t[1]!==e&&this.changedOptions.push({path:i,value:e})):"string"==typeof t[0]?(this._makeDropdown(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:e})):"number"==typeof t[0]&&(this._makeRange(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:Number(e)}))}},{key:"_update",value:function(t,e){var i=this._constructOptions(t,e);this.parent.body&&this.parent.body.emitter&&this.parent.body.emitter.emit&&this.parent.body.emitter.emit("configChange",i),this.initialized=!0,this.parent.setOptions(i)}},{key:"_constructOptions",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},o=i;t="true"===t||t,t="false"!==t&&t;for(var n=0;n<e.length;n++)"global"!==e[n]&&(void 0===o[e[n]]&&(o[e[n]]={}),n!==e.length-1?o=o[e[n]]:o[e[n]]=t);return i}},{key:"_printOptions",value:function(){var t=this.getOptions();this.optionsContainer.innerHTML="<pre>var options = "+(0,s.default)(t,null,2)+"</pre>"}},{key:"getOptions",value:function(){for(var t={},e=0;e<this.changedOptions.length;e++)this._constructOptions(this.changedOptions[e].value,this.changedOptions[e].path,t);return t}}]),t}();e.default=m},function(t,e,i){function o(t,e){this.id=r.randomUUID(),this.body=t,this.defaultOptions={yAxisOrientation:"left",defaultGroup:"default",sort:!0,sampling:!0,stack:!1,graphHeight:"400px",shaded:{enabled:!1,orientation:"bottom"},style:"line",barChart:{width:50,sideBySide:!1,align:"center"},interpolation:{enabled:!0,parametrization:"centripetal",alpha:.5},drawPoints:{enabled:!0,size:6,style:"square"},dataAxis:{},legend:{},groups:{visibility:{}}},this.options=r.extend({},this.defaultOptions),this.dom={},this.props={},this.hammer=null,this.groups={},this.abortedGraphUpdate=!1,this.updateSVGheight=!1,this.updateSVGheightOnResize=!1,this.forceGraphUpdate=!0;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,o){i._onAdd(e.items)},update:function(t,e,o){i._onUpdate(e.items)},remove:function(t,e,o){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,o){i._onAddGroups(e.items)},update:function(t,e,o){i._onUpdateGroups(e.items)},remove:function(t,e,o){i._onRemoveGroups(e.items)}},this.items={},this.selection=[],this.lastStart=this.body.range.start,this.touchParams={},this.svgElements={},this.setOptions(e),this.groupsUsingDefaultStyles=[0],this.body.emitter.on("rangechanged",function(){i.lastStart=i.body.range.start,i.svg.style.left=r.option.asSize(-i.props.width),i.forceGraphUpdate=!0,i.redraw.call(i)}),this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups}}var n=i(1),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(11),h=i(15),d=i(20),l=i(5),u=i(78),p=i(80),c=i(83),m=i(81),f=i(82),g=i(54);o.prototype=new l,o.prototype._create=function(){var t=document.createElement("div");t.className="vis-line-graph",this.dom.frame=t,this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="relative",this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",this.svg.style.display="block",t.appendChild(this.svg),this.options.dataAxis.orientation="left",this.yAxisLeft=new u(this.body,this.options.dataAxis,this.svg,this.options.groups),this.options.dataAxis.orientation="right",this.yAxisRight=new u(this.body,this.options.dataAxis,this.svg,this.options.groups),delete this.options.dataAxis.orientation,this.legendLeft=new c(this.body,this.options.legend,"left",this.options.groups),this.legendRight=new c(this.body,this.options.legend,"right",this.options.groups),this.show()},o.prototype.setOptions=function(t){if(t){var e=["sampling","defaultGroup","stack","height","graphHeight","yAxisOrientation","style","barChart","dataAxis","sort","groups"];void 0===t.graphHeight&&void 0!==t.height?(this.updateSVGheight=!0,this.updateSVGheightOnResize=!0):void 0!==this.body.domProps.centerContainer.height&&void 0!==t.graphHeight&&parseInt((t.graphHeight+"").replace("px",""))<this.body.domProps.centerContainer.height&&(this.updateSVGheight=!0),r.selectiveDeepExtend(e,this.options,t),r.mergeOptions(this.options,t,"interpolation"),r.mergeOptions(this.options,t,"drawPoints"),r.mergeOptions(this.options,t,"shaded"),r.mergeOptions(this.options,t,"legend"),t.interpolation&&"object"==(0,s.default)(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5)),this.yAxisLeft&&void 0!==t.dataAxis&&(this.yAxisLeft.setOptions(this.options.dataAxis),this.yAxisRight.setOptions(this.options.dataAxis)),this.legendLeft&&void 0!==t.legend&&(this.legendLeft.setOptions(this.options.legend),this.legendRight.setOptions(this.options.legend)),this.groups.hasOwnProperty("__ungrouped__")&&this.groups.__ungrouped__.setOptions(t)}this.dom.frame&&(this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0}))},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},o.prototype.setItems=function(t){var e,i=this,o=this.itemsData;if(t){if(!(t instanceof h||t instanceof d))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=t}else this.itemsData=null;if(o&&(r.forEach(this.itemListeners,function(t,e){o.off(e,t)}),e=o.getIds(),this._onRemove(e)),this.itemsData){var n=this.id;r.forEach(this.itemListeners,function(t,e){i.itemsData.on(e,t,n)}),e=this.itemsData.getIds(),this._onAdd(e)}},o.prototype.setGroups=function(t){var e,i=this;if(this.groupsData){r.forEach(this.groupListeners,function(t,e){i.groupsData.off(e,t)}),e=this.groupsData.getIds(),this.groupsData=null;for(var o=0;o<e.length;o++)this._removeGroup(e[o])}if(t){if(!(t instanceof h||t instanceof d))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var n=this.id;r.forEach(this.groupListeners,function(t,e){i.groupsData.on(e,t,n)}),e=this.groupsData.getIds(),this._onAddGroups(e)}},o.prototype._onUpdate=function(t){this._updateAllGroupData(t)},o.prototype._onAdd=function(t){this._onUpdate(t)},o.prototype._onRemove=function(t){this._onUpdate(t)},o.prototype._onUpdateGroups=function(t){this._updateAllGroupData(null,t)},o.prototype._onAddGroups=function(t){this._onUpdateGroups(t)},o.prototype._onRemoveGroups=function(t){for(var e=0;e<t.length;e++)this._removeGroup(t[e]);this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})},o.prototype._removeGroup=function(t){this.groups.hasOwnProperty(t)&&("right"==this.groups[t].options.yAxisOrientation?(this.yAxisRight.removeGroup(t),this.legendRight.removeGroup(t),this.legendRight.redraw()):(this.yAxisLeft.removeGroup(t),this.legendLeft.removeGroup(t),this.legendLeft.redraw()),delete this.groups[t])},o.prototype._updateGroup=function(t,e){this.groups.hasOwnProperty(e)?(this.groups[e].update(t),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.updateGroup(e,this.groups[e]),this.legendRight.updateGroup(e,this.groups[e]),this.yAxisLeft.removeGroup(e),this.legendLeft.removeGroup(e)):(this.yAxisLeft.updateGroup(e,this.groups[e]),this.legendLeft.updateGroup(e,this.groups[e]),this.yAxisRight.removeGroup(e),this.legendRight.removeGroup(e))):(this.groups[e]=new p(t,e,this.options,this.groupsUsingDefaultStyles),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.addGroup(e,this.groups[e]),this.legendRight.addGroup(e,this.groups[e])):(this.yAxisLeft.addGroup(e,this.groups[e]),this.legendLeft.addGroup(e,this.groups[e]))),this.legendLeft.redraw(),this.legendRight.redraw()},o.prototype._updateAllGroupData=function(t,e){if(null!=this.itemsData){var i={},o=this.itemsData.get(),n=this.itemsData._fieldId,s={};t&&t.map(function(t){s[t]=t});for(var a={},h=0;h<o.length;h++){var d=o[h],l=d.group;null!==l&&void 0!==l||(l="__ungrouped__"),a.hasOwnProperty(l)?a[l]++:a[l]=1}var u={};if(!e&&t)for(l in this.groups)if(this.groups.hasOwnProperty(l)){g=this.groups[l];var p=g.getItems();i[l]=p.filter(function(t){return u[t[n]]=t[n],t[n]!==s[t[n]]});var c=a[l];a[l]-=i[l].length,i[l].length<c&&(i[l][c-1]={})}for(h=0;h<o.length;h++)if(d=o[h],l=d.group,null!==l&&void 0!==l||(l="__ungrouped__"),e||!t||d[n]===s[d[n]]||!u.hasOwnProperty(d[n])){i.hasOwnProperty(l)||(i[l]=new Array(a[l]));var m=r.bridgeObject(d);m.x=r.convert(d.x,"Date"),m.end=r.convert(d.end,"Date"),m.orginalY=d.y,m.y=Number(d.y),m[n]=d[n];var f=i[l].length-a[l]--;i[l][f]=m}for(l in this.groups)this.groups.hasOwnProperty(l)&&(i.hasOwnProperty(l)||(i[l]=new Array(0)));for(l in i)if(i.hasOwnProperty(l))if(0==i[l].length)this.groups.hasOwnProperty(l)&&this._removeGroup(l);else{var g=void 0;void 0!=this.groupsData&&(g=this.groupsData.get(l)),void 0==g&&(g={id:l,content:this.options.defaultGroup+l}),this._updateGroup(g,l),this.groups[l].setItems(i[l])}this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})}},o.prototype.redraw=function(){var t=!1;this.props.width=this.dom.frame.offsetWidth,this.props.height=this.body.domProps.centerContainer.height-this.body.domProps.border.top-this.body.domProps.border.bottom,t=this._isResized()||t;var e=this.body.range.end-this.body.range.start,i=e!=this.lastVisibleInterval;if(this.lastVisibleInterval=e,1==t&&(this.svg.style.width=r.option.asSize(3*this.props.width),this.svg.style.left=r.option.asSize(-this.props.width),-1==(this.options.height+"").indexOf("%")&&1!=this.updateSVGheightOnResize||(this.updateSVGheight=!0)),1==this.updateSVGheight?(this.options.graphHeight!=this.props.height+"px"&&(this.options.graphHeight=this.props.height+"px",this.svg.style.height=this.props.height+"px"),this.updateSVGheight=!1):this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",1==t||1==i||1==this.abortedGraphUpdate||1==this.forceGraphUpdate)t=this._updateGraph()||t,this.forceGraphUpdate=!1;else if(0!=this.lastStart){var o=this.body.range.start-this.lastStart,n=this.body.range.end-this.body.range.start;if(0!=this.props.width){var s=this.props.width/n,a=o*s;this.svg.style.left=-this.props.width-a+"px"}}return this.legendLeft.redraw(),this.legendRight.redraw(),t},o.prototype._getSortedGroupIds=function(){var t=[];for(var e in this.groups)if(this.groups.hasOwnProperty(e)){var i=this.groups[e];1!=i.visible||void 0!==this.options.groups.visibility[e]&&1!=this.options.groups.visibility[e]||t.push({id:e,zIndex:i.options.zIndex})}r.insertSort(t,function(t,e){var i=t.zIndex,o=e.zIndex;return void 0===i&&(i=0),void 0===o&&(o=0),i==o?0:i<o?-1:1});for(var o=new Array(t.length),n=0;n<t.length;n++)o[n]=t[n].id;return o},o.prototype._updateGraph=function(){if(a.prepareElements(this.svgElements),0!=this.props.width&&null!=this.itemsData){var t,e,i={},o=this.body.util.toGlobalTime(-this.body.domProps.root.width),n=this.body.util.toGlobalTime(2*this.body.domProps.root.width),s=this._getSortedGroupIds();if(s.length>0){var r={};for(this._getRelevantData(s,r,o,n),this._applySampling(s,r),e=0;e<s.length;e++)this._convertXcoordinates(r[s[e]]);if(this._getYRanges(s,r,i),1==this._updateYAxis(s,i))return a.cleanupElements(this.svgElements),this.abortedGraphUpdate=!0,!0;this.abortedGraphUpdate=!1;var h=void 0;for(e=0;e<s.length;e++)t=this.groups[s[e]],!0===this.options.stack&&"line"===this.options.style&&(void 0!=t.options.excludeFromStacking&&t.options.excludeFromStacking||(void 0!=h&&(this._stack(r[t.id],r[h.id]),1==t.options.shaded.enabled&&"group"!==t.options.shaded.orientation&&("top"==t.options.shaded.orientation&&"group"!==h.options.shaded.orientation?(h.options.shaded.orientation="group",h.options.shaded.groupId=t.id):(t.options.shaded.orientation="group",t.options.shaded.groupId=h.id))),h=t)),this._convertYcoordinates(r[s[e]],t);var d={};for(e=0;e<s.length;e++)if(t=this.groups[s[e]],"line"===t.options.style&&1==t.options.shaded.enabled){var l=r[s[e]];if(null==l||0==l.length)continue;if(d.hasOwnProperty(s[e])||(d[s[e]]=f.calcPath(l,t)),"group"===t.options.shaded.orientation){var u=t.options.shaded.groupId;if(-1===s.indexOf(u)){console.log(t.id+": Unknown shading group target given:"+u);continue}d.hasOwnProperty(u)||(d[u]=f.calcPath(r[u],this.groups[u])),f.drawShading(d[s[e]],t,d[u],this.framework)}else f.drawShading(d[s[e]],t,void 0,this.framework)}for(m.draw(s,r,this.framework),e=0;e<s.length;e++)if(t=this.groups[s[e]],r[s[e]].length>0)switch(t.options.style){case"line":d.hasOwnProperty(s[e])||(d[s[e]]=f.calcPath(r[s[e]],t)),f.draw(d[s[e]],t,this.framework);case"point":case"points":"point"!=t.options.style&&"points"!=t.options.style&&1!=t.options.drawPoints.enabled||g.draw(r[s[e]],t,this.framework)}}}return a.cleanupElements(this.svgElements),!1},o.prototype._stack=function(t,e){var i,o,n,s,r;i=0;for(var a=0;a<t.length;a++){s=void 0,r=void 0;for(var h=i;h<e.length;h++){if(e[h].x===t[a].x){s=e[h],r=e[h],i=h;break}if(e[h].x>t[a].x){r=e[h],s=0==h?r:e[h-1],i=h;break}}void 0===r&&(s=e[e.length-1],r=e[e.length-1]),o=r.x-s.x,n=r.y-s.y,t[a].y=0==o?t[a].orginalY+r.y:t[a].orginalY+n/o*(t[a].x-s.x)+s.y}},o.prototype._getRelevantData=function(t,e,i,o){var n,s,a,h;if(t.length>0)for(s=0;s<t.length;s++){n=this.groups[t[s]];var d=n.getItems();if(1==n.options.sort){var l=function(t,e){return t.getTime()==e.getTime()?0:t<e?-1:1},u=Math.max(0,r.binarySearchValue(d,i,"x","before",l)),p=Math.min(d.length,r.binarySearchValue(d,o,"x","after",l)+1);p<=0&&(p=d.length);var c=new Array(p-u);for(a=u;a<p;a++)h=n.itemsData[a],c[a-u]=h;e[t[s]]=c}else e[t[s]]=n.itemsData}},o.prototype._applySampling=function(t,e){var i;if(t.length>0)for(var o=0;o<t.length;o++)if(i=this.groups[t[o]],1==i.options.sampling){var n=e[t[o]];if(n.length>0){var s=1,r=n.length,a=this.body.util.toGlobalScreen(n[n.length-1].x)-this.body.util.toGlobalScreen(n[0].x),h=r/a;s=Math.min(Math.ceil(.2*r),Math.max(1,Math.round(h)));for(var d=new Array(r),l=0;l<r;l+=s){var u=Math.round(l/s);d[u]=n[l]}e[t[o]]=d.splice(0,Math.round(r/s))}}},o.prototype._getYRanges=function(t,e,i){var o,n,s,r,a=[],h=[];if(t.length>0){for(s=0;s<t.length;s++)o=e[t[s]],r=this.groups[t[s]].options,o.length>0&&(n=this.groups[t[s]],!0===r.stack&&"bar"===r.style?"left"===r.yAxisOrientation?a=a.concat(o):h=h.concat(o):i[t[s]]=n.getYRange(o,t[s]));m.getStackedYRange(a,i,t,"__barStackLeft","left"),m.getStackedYRange(h,i,t,"__barStackRight","right")}},o.prototype._updateYAxis=function(t,e){var i,o,n=!1,s=!1,r=!1,a=1e9,h=1e9,d=-1e9,l=-1e9;if(t.length>0){for(var u=0;u<t.length;u++){var p=this.groups[t[u]];p&&"right"!=p.options.yAxisOrientation?(s=!0,a=1e9,d=-1e9):p&&p.options.yAxisOrientation&&(r=!0,h=1e9,l=-1e9)}for(u=0;u<t.length;u++)e.hasOwnProperty(t[u])&&!0!==e[t[u]].ignore&&(i=e[t[u]].min,o=e[t[u]].max,"right"!=e[t[u]].yAxisOrientation?(s=!0,a=a>i?i:a,d=d<o?o:d):(r=!0,h=h>i?i:h,l=l<o?o:l));1==s&&this.yAxisLeft.setRange(a,d),1==r&&this.yAxisRight.setRange(h,l)}n=this._toggleAxisVisiblity(s,this.yAxisLeft)||n,n=this._toggleAxisVisiblity(r,this.yAxisRight)||n,1==r&&1==s?(this.yAxisLeft.drawIcons=!0,this.yAxisRight.drawIcons=!0):(this.yAxisLeft.drawIcons=!1,this.yAxisRight.drawIcons=!1),this.yAxisRight.master=!s,this.yAxisRight.masterAxis=this.yAxisLeft,0==this.yAxisRight.master?(this.yAxisLeft.lineOffset=1==r?this.yAxisRight.width:0,n=this.yAxisLeft.redraw()||n,n=this.yAxisRight.redraw()||n):n=this.yAxisRight.redraw()||n;var c=["__barStackLeft","__barStackRight","__lineStackLeft","__lineStackRight"];for(u=0;u<c.length;u++)-1!=t.indexOf(c[u])&&t.splice(t.indexOf(c[u]),1);return n},o.prototype._toggleAxisVisiblity=function(t,e){var i=!1;return 0==t?e.dom.frame.parentNode&&0==e.hidden&&(e.hide(),i=!0):e.dom.frame.parentNode||1!=e.hidden||(e.show(),i=!0),i},o.prototype._convertXcoordinates=function(t){for(var e=this.body.util.toScreen,i=0;i<t.length;i++)t[i].screen_x=e(t[i].x)+this.props.width,t[i].screen_y=t[i].y,void 0!=t[i].end?t[i].screen_end=e(t[i].end)+this.props.width:t[i].screen_end=void 0},o.prototype._convertYcoordinates=function(t,e){var i=this.yAxisLeft,o=Number(this.svg.style.height.replace("px",""));"right"==e.options.yAxisOrientation&&(i=this.yAxisRight);for(var n=0;n<t.length;n++)t[n].screen_y=Math.round(i.convertValue(t[n].y));e.setZeroPosition(Math.min(o,i.convertValue(0)))},t.exports=o},function(t,e,i){function o(t,e,i,o){this.id=r.randomUUID(),this.body=t,this.defaultOptions={orientation:"left",showMinorLabels:!0,showMajorLabels:!0,icons:!1,majorLinesOffset:7,minorLinesOffset:4,labelOffsetX:10,labelOffsetY:2,iconWidth:20,width:"40px",visible:!0,alignZeros:!0,left:{range:{min:void 0,max:void 0},format:function(t){return""+parseFloat(t.toPrecision(3))},title:{text:void 0,style:void 0}},right:{range:{min:void 0,max:void 0},format:function(t){return""+parseFloat(t.toPrecision(3))},title:{text:void 0,style:void 0}}},this.linegraphOptions=o,this.linegraphSVG=i,this.props={},this.DOMelements={lines:{},labels:{},title:{}},this.dom={},this.scale=void 0,this.range={start:0,end:0},this.options=r.extend({},this.defaultOptions),this.conversionFactor=1,this.setOptions(e),this.width=Number((""+this.options.width).replace("px","")),this.minWidth=this.width,this.height=this.linegraphSVG.getBoundingClientRect().height,this.hidden=!1,this.stepPixels=25,this.zeroCrossing=-1,this.amountOfSteps=-1,this.lineOffset=0,this.master=!0,this.masterAxis=null,this.svgElements={},this.iconsRemoved=!1,this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups};var n=this;this.body.emitter.on("verticalDrag",function(){n.dom.lineContainer.style.top=n.body.domProps.scrollTop+"px"})}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(11),h=i(5),d=i(79);o.prototype=new h,o.prototype.addGroup=function(t,e){this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1},o.prototype.updateGroup=function(t,e){this.groups.hasOwnProperty(t)||(this.amountOfGroups+=1),this.groups[t]=e},o.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},o.prototype.setOptions=function(t){if(t){var e=!1;this.options.orientation!=t.orientation&&void 0!==t.orientation&&(e=!0);var i=["orientation","showMinorLabels","showMajorLabels","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","width","visible","left","right","alignZeros"];r.selectiveDeepExtend(i,this.options,t),this.minWidth=Number((""+this.options.width).replace("px","")),!0===e&&this.dom.frame&&(this.hide(),this.show())}},o.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.style.width=this.options.width,this.dom.frame.style.height=this.height,this.dom.lineContainer=document.createElement("div"),this.dom.lineContainer.style.width="100%",this.dom.lineContainer.style.height=this.height,this.dom.lineContainer.style.position="relative",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.height="100%",this.svg.style.width="100%",this.svg.style.display="block",this.dom.frame.appendChild(this.svg)},o.prototype._redrawGroupIcons=function(){a.prepareElements(this.svgElements);var t,e=this.options.iconWidth,i=11.5;t="left"===this.options.orientation?4:this.width-e-4;var o=(0,s.default)(this.groups);o.sort(function(t,e){return t<e?-1:1});for(var n=0;n<o.length;n++){var r=o[n];!0!==this.groups[r].visible||void 0!==this.linegraphOptions.visibility[r]&&!0!==this.linegraphOptions.visibility[r]||(this.groups[r].getLegend(e,15,this.framework,t,i),i+=19)}a.cleanupElements(this.svgElements),this.iconsRemoved=!1},o.prototype._cleanupIcons=function(){!1===this.iconsRemoved&&(a.prepareElements(this.svgElements),a.cleanupElements(this.svgElements),this.iconsRemoved=!0)},o.prototype.show=function(){this.hidden=!1,this.dom.frame.parentNode||("left"===this.options.orientation?this.body.dom.left.appendChild(this.dom.frame):this.body.dom.right.appendChild(this.dom.frame)),this.dom.lineContainer.parentNode||this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer)},o.prototype.hide=function(){this.hidden=!0,this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.lineContainer.parentNode&&this.dom.lineContainer.parentNode.removeChild(this.dom.lineContainer)},o.prototype.setRange=function(t,e){this.range.start=t,this.range.end=e},o.prototype.redraw=function(){var t=!1,e=0;this.dom.lineContainer.style.top=this.body.domProps.scrollTop+"px";for(var i in this.groups)this.groups.hasOwnProperty(i)&&(!0!==this.groups[i].visible||void 0!==this.linegraphOptions.visibility[i]&&!0!==this.linegraphOptions.visibility[i]||e++);if(0===this.amountOfGroups||0===e)this.hide();else{this.show(),this.height=Number(this.linegraphSVG.style.height.replace("px","")),this.dom.lineContainer.style.height=this.height+"px",this.width=!0===this.options.visible?Number((""+this.options.width).replace("px","")):0;var o=this.props,n=this.dom.frame;n.className="vis-data-axis",this._calculateCharSize();var s=this.options.orientation,r=this.options.showMinorLabels,a=this.options.showMajorLabels;o.minorLabelHeight=r?o.minorCharHeight:0,o.majorLabelHeight=a?o.majorCharHeight:0,o.minorLineWidth=this.body.dom.backgroundHorizontal.offsetWidth-this.lineOffset-this.width+2*this.options.minorLinesOffset,o.minorLineHeight=1,o.majorLineWidth=this.body.dom.backgroundHorizontal.offsetWidth-this.lineOffset-this.width+2*this.options.majorLinesOffset,o.majorLineHeight=1,"left"===s?(n.style.top="0",n.style.left="0",n.style.bottom="",n.style.width=this.width+"px",n.style.height=this.height+"px",this.props.width=this.body.domProps.left.width,this.props.height=this.body.domProps.left.height):(n.style.top="",n.style.bottom="0",n.style.left="0",n.style.width=this.width+"px",n.style.height=this.height+"px",this.props.width=this.body.domProps.right.width,this.props.height=this.body.domProps.right.height),t=this._redrawLabels(),t=this._isResized()||t,!0===this.options.icons?this._redrawGroupIcons():this._cleanupIcons(),this._redrawTitle(s)}return t},o.prototype._redrawLabels=function(){var t=this,e=!1;a.prepareElements(this.DOMelements.lines),a.prepareElements(this.DOMelements.labels);var i=this.options.orientation,o=void 0!=this.options[i].range?this.options[i].range:{},n=!0;void 0!=o.max&&(this.range.end=o.max,n=!1);var s=!0;void 0!=o.min&&(this.range.start=o.min,s=!1),this.scale=new d(this.range.start,this.range.end,s,n,this.dom.frame.offsetHeight,this.props.majorCharHeight,this.options.alignZeros,this.options[i].format),!1===this.master&&void 0!=this.masterAxis&&this.scale.followScale(this.masterAxis.scale),this.maxLabelSize=0,this.scale.getLines().forEach(function(e){var o=e.y,n=e.major;t.options.showMinorLabels&&!1===n&&t._redrawLabel(o-2,e.val,i,"vis-y-axis vis-minor",t.props.minorCharHeight),n&&o>=0&&t._redrawLabel(o-2,e.val,i,"vis-y-axis vis-major",t.props.majorCharHeight),!0===t.master&&(n?t._redrawLine(o,i,"vis-grid vis-horizontal vis-major",t.options.majorLinesOffset,t.props.majorLineWidth):t._redrawLine(o,i,"vis-grid vis-horizontal vis-minor",t.options.minorLinesOffset,t.props.minorLineWidth))});var r=0;void 0!==this.options[i].title&&void 0!==this.options[i].title.text&&(r=this.props.titleCharHeight);var h=!0===this.options.icons?Math.max(this.options.iconWidth,r)+this.options.labelOffsetX+15:r+this.options.labelOffsetX+15;return this.maxLabelSize>this.width-h&&!0===this.options.visible?(this.width=this.maxLabelSize+h,this.options.width=this.width+"px",a.cleanupElements(this.DOMelements.lines),a.cleanupElements(this.DOMelements.labels),this.redraw(),e=!0):this.maxLabelSize<this.width-h&&!0===this.options.visible&&this.width>this.minWidth?(this.width=Math.max(this.minWidth,this.maxLabelSize+h),this.options.width=this.width+"px",a.cleanupElements(this.DOMelements.lines),a.cleanupElements(this.DOMelements.labels),this.redraw(),e=!0):(a.cleanupElements(this.DOMelements.lines),a.cleanupElements(this.DOMelements.labels),e=!1),e},o.prototype.convertValue=function(t){return this.scale.convertValue(t)},o.prototype.screenToValue=function(t){return this.scale.screenToValue(t)},o.prototype._redrawLabel=function(t,e,i,o,n){var s=a.getDOMElement("div",this.DOMelements.labels,this.dom.frame);s.className=o,s.innerHTML=e,"left"===i?(s.style.left="-"+this.options.labelOffsetX+"px",s.style.textAlign="right"):(s.style.right="-"+this.options.labelOffsetX+"px",s.style.textAlign="left"),s.style.top=t-.5*n+this.options.labelOffsetY+"px",e+="";var r=Math.max(this.props.majorCharWidth,this.props.minorCharWidth);this.maxLabelSize<e.length*r&&(this.maxLabelSize=e.length*r)},o.prototype._redrawLine=function(t,e,i,o,n){if(!0===this.master){var s=a.getDOMElement("div",this.DOMelements.lines,this.dom.lineContainer);s.className=i,s.innerHTML="","left"===e?s.style.left=this.width-o+"px":s.style.right=this.width-o+"px",s.style.width=n+"px",s.style.top=t+"px"}},o.prototype._redrawTitle=function(t){if(a.prepareElements(this.DOMelements.title),void 0!==this.options[t].title&&void 0!==this.options[t].title.text){var e=a.getDOMElement("div",this.DOMelements.title,this.dom.frame);e.className="vis-y-axis vis-title vis-"+t,e.innerHTML=this.options[t].title.text,void 0!==this.options[t].title.style&&r.addCssText(e,this.options[t].title.style),"left"===t?e.style.left=this.props.titleCharHeight+"px":e.style.right=this.props.titleCharHeight+"px",e.style.width=this.height+"px"}a.cleanupElements(this.DOMelements.title)},o.prototype._calculateCharSize=function(){if(!("minorCharHeight"in this.props)){var t=document.createTextNode("0"),e=document.createElement("div");e.className="vis-y-axis vis-minor vis-measure",e.appendChild(t),this.dom.frame.appendChild(e),this.props.minorCharHeight=e.clientHeight,this.props.minorCharWidth=e.clientWidth,this.dom.frame.removeChild(e)}if(!("majorCharHeight"in this.props)){var i=document.createTextNode("0"),o=document.createElement("div");o.className="vis-y-axis vis-major vis-measure",o.appendChild(i),this.dom.frame.appendChild(o),this.props.majorCharHeight=o.clientHeight,this.props.majorCharWidth=o.clientWidth,this.dom.frame.removeChild(o)}if(!("titleCharHeight"in this.props)){var n=document.createTextNode("0"),s=document.createElement("div");s.className="vis-y-axis vis-title vis-measure",s.appendChild(n),this.dom.frame.appendChild(s),this.props.titleCharHeight=s.clientHeight,this.props.titleCharWidth=s.clientWidth,this.dom.frame.removeChild(s)}},t.exports=o},function(t,e,i){function o(t,e,i,o,n,s){var r=arguments.length>6&&void 0!==arguments[6]&&arguments[6],a=arguments.length>7&&void 0!==arguments[7]&&arguments[7];if(this.majorSteps=[1,2,5,10],this.minorSteps=[.25,.5,1,2],this.customLines=null,this.containerHeight=n,this.majorCharHeight=s,this._start=t,this._end=e,this.scale=1,this.minorStepIdx=-1,this.magnitudefactor=1,this.determineScale(),this.zeroAlign=r,this.autoScaleStart=i,this.autoScaleEnd=o,this.formattingFunction=a,i||o){var h=this,d=function(t){var e=t-t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx]);return t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx])>h.magnitudefactor*h.minorSteps[h.minorStepIdx]*.5?e+h.magnitudefactor*h.minorSteps[h.minorStepIdx]:e};i&&(this._start-=2*this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._start=d(this._start)),o&&(this._end+=this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._end=d(this._end)),this.determineScale()}}o.prototype.setCharHeight=function(t){this.majorCharHeight=t},o.prototype.setHeight=function(t){this.containerHeight=t},o.prototype.determineScale=function(){var t=this._end-this._start;this.scale=this.containerHeight/t;var e=this.majorCharHeight/this.scale,i=t>0?Math.round(Math.log(t)/Math.LN10):0;this.minorStepIdx=-1,this.magnitudefactor=Math.pow(10,i);var o=0;i<0&&(o=i);for(var n=!1,s=o;Math.abs(s)<=Math.abs(i);s++){this.magnitudefactor=Math.pow(10,s);for(var r=0;r<this.minorSteps.length;r++){if(this.magnitudefactor*this.minorSteps[r]>=e){n=!0,this.minorStepIdx=r;break}}if(!0===n)break}},o.prototype.is_major=function(t){return t%(this.magnitudefactor*this.majorSteps[this.minorStepIdx])==0},o.prototype.getStep=function(){return this.magnitudefactor*this.minorSteps[this.minorStepIdx]},o.prototype.getFirstMajor=function(){var t=this.magnitudefactor*this.majorSteps[this.minorStepIdx];return this.convertValue(this._start+(t-this._start%t)%t)},o.prototype.formatValue=function(t){var e=t.toPrecision(5);return"function"==typeof this.formattingFunction&&(e=this.formattingFunction(t)),"number"==typeof e?""+e:"string"==typeof e?e:t.toPrecision(5)},o.prototype.getLines=function(){for(var t=[],e=this.getStep(),i=(e-this._start%e)%e,o=this._start+i;this._end-o>1e-5;o+=e)o!=this._start&&t.push({major:this.is_major(o),y:this.convertValue(o),val:this.formatValue(o)});return t},o.prototype.followScale=function(t){var e=this.minorStepIdx,i=this._start,o=this._end,n=this,s=function(){n.magnitudefactor*=2},r=function(){n.magnitudefactor/=2};t.minorStepIdx<=1&&this.minorStepIdx<=1||t.minorStepIdx>1&&this.minorStepIdx>1||(t.minorStepIdx<this.minorStepIdx?(this.minorStepIdx=1,2==e?s():(s(),s())):(this.minorStepIdx=2,1==e?r():(r(),r())));for(var a=t.convertValue(0),h=t.getStep()*t.scale,d=!1,l=0;!d&&l++<5;){this.scale=h/(this.minorSteps[this.minorStepIdx]*this.magnitudefactor);var u=this.containerHeight/this.scale;this._start=i,this._end=this._start+u;var p=this._end*this.scale,c=this.magnitudefactor*this.majorSteps[this.minorStepIdx],m=this.getFirstMajor()-t.getFirstMajor();if(this.zeroAlign){var f=a-p;this._end+=f/this.scale,this._start=this._end-u}else this.autoScaleStart?(this._start-=m/this.scale,this._end=this._start+u):(this._start+=c-m/this.scale,this._end=this._start+u);if(!this.autoScaleEnd&&this._end>o+1e-5)r(),d=!1;else{if(!this.autoScaleStart&&this._start<i-1e-5){if(!(this.zeroAlign&&i>=0)){r(),d=!1;continue}console.warn("Can't adhere to given 'min' range, due to zeroalign")}this.autoScaleStart&&this.autoScaleEnd&&u<o-i?(s(),d=!1):d=!0}}},o.prototype.convertValue=function(t){return this.containerHeight-(t-this._start)*this.scale},o.prototype.screenToValue=function(t){return(this.containerHeight-t)/this.scale+this._start},t.exports=o},function(t,e,i){function o(t,e,i,o){this.id=e;var n=["sampling","style","sort","yAxisOrientation","barChart","drawPoints","shaded","interpolation","zIndex","excludeFromStacking","excludeFromLegend"];this.options=r.selectiveBridgeObject(n,i),this.usingDefaultStyle=void 0===t.className,this.groupsUsingDefaultStyles=o,this.zeroPosition=0,this.update(t),1==this.usingDefaultStyle&&(this.groupsUsingDefaultStyles[0]+=1),this.itemsData=[],this.visible=void 0===t.visible||t.visible}var n=i(1),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(81),h=i(82),d=i(54);o.prototype.setItems=function(t){null!=t?(this.itemsData=t,1==this.options.sort&&r.insertSort(this.itemsData,function(t,e){return t.x>e.x?1:-1})):this.itemsData=[]},o.prototype.getItems=function(){return this.itemsData},o.prototype.setZeroPosition=function(t){this.zeroPosition=t},o.prototype.setOptions=function(t){if(void 0!==t){var e=["sampling","style","sort","yAxisOrientation","barChart","zIndex","excludeFromStacking","excludeFromLegend"];r.selectiveDeepExtend(e,this.options,t),"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),r.mergeOptions(this.options,t,"interpolation"),r.mergeOptions(this.options,t,"drawPoints"),r.mergeOptions(this.options,t,"shaded"),t.interpolation&&"object"==(0,s.default)(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5))}},o.prototype.update=function(t){this.group=t,this.content=t.content||"graph",this.className=t.className||this.className||"vis-graph-group"+this.groupsUsingDefaultStyles[0]%10,this.visible=void 0===t.visible||t.visible,this.style=t.style,this.setOptions(t.options)},o.prototype.getLegend=function(t,e,i,o,n){if(void 0==i||null==i){i={svg:document.createElementNS("http://www.w3.org/2000/svg","svg"),svgElements:{},options:this.options,groups:[this]}}switch(void 0!=o&&null!=o||(o=0),void 0!=n&&null!=n||(n=.5*e),this.options.style){case"line":h.drawIcon(this,o,n,t,e,i);break;case"points":case"point":d.drawIcon(this,o,n,t,e,i);break;case"bar":a.drawIcon(this,o,n,t,e,i)}return{icon:i.svg,label:this.content,orientation:this.options.yAxisOrientation}},o.prototype.getYRange=function(t){for(var e=t[0].y,i=t[0].y,o=0;o<t.length;o++)e=e>t[o].y?t[o].y:e,i=i<t[o].y?t[o].y:i;return{min:e,max:i,yAxisOrientation:this.options.yAxisOrientation}},t.exports=o},function(t,e,i){function o(t,e){}var n=i(11),s=i(54);o.drawIcon=function(t,e,i,o,s,r){var a=.5*s,h=n.getSVGElement("rect",r.svgElements,r.svg);h.setAttributeNS(null,"x",e),h.setAttributeNS(null,"y",i-a),h.setAttributeNS(null,"width",o),h.setAttributeNS(null,"height",2*a),h.setAttributeNS(null,"class","vis-outline");var d=Math.round(.3*o),l=t.options.barChart.width,u=l/d,p=Math.round(.4*s),c=Math.round(.75*s),m=Math.round((o-2*d)/3);if(n.drawBar(e+.5*d+m,i+a-p-1,d,p,t.className+" vis-bar",r.svgElements,r.svg,t.style),n.drawBar(e+1.5*d+m+2,i+a-c-1,d,c,t.className+" vis-bar",r.svgElements,r.svg,t.style),1==t.options.drawPoints.enabled){var f={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size/u,className:t.className};n.drawPoint(e+.5*d+m,i+a-p-1,f,r.svgElements,r.svg),n.drawPoint(e+1.5*d+m+2,i+a-c-1,f,r.svgElements,r.svg)}},o.draw=function(t,e,i){var r,a,h,d,l,u,p=[],c={},m=0;for(l=0;l<t.length;l++)if(d=i.groups[t[l]],"bar"===d.options.style&&!0===d.visible&&(void 0===i.options.groups.visibility[t[l]]||!0===i.options.groups.visibility[t[l]]))for(u=0;u<e[t[l]].length;u++)p.push({screen_x:e[t[l]][u].screen_x,screen_end:e[t[l]][u].screen_end,screen_y:e[t[l]][u].screen_y,x:e[t[l]][u].x,end:e[t[l]][u].end,y:e[t[l]][u].y,groupId:t[l],label:e[t[l]][u].label}),m+=1;if(0!==m)for(p.sort(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x}),o._getDataIntersections(c,p),l=0;l<p.length;l++){d=i.groups[p[l].groupId];var f=void 0!=d.options.barChart.minWidth?d.options.barChart.minWidth:.1*d.options.barChart.width;a=p[l].screen_x;var g=0;if(void 0===c[a])l+1<p.length&&(r=Math.abs(p[l+1].screen_x-a)),h=o._getSafeDrawData(r,d,f);else{var v=l+(c[a].amount-c[a].resolved);v<p.length&&(r=Math.abs(p[v].screen_x-a)),h=o._getSafeDrawData(r,d,f),c[a].resolved+=1,!0===d.options.stack&&!0!==d.options.excludeFromStacking?p[l].screen_y<d.zeroPosition?(g=c[a].accumulatedNegative,c[a].accumulatedNegative+=d.zeroPosition-p[l].screen_y):(g=c[a].accumulatedPositive,c[a].accumulatedPositive+=d.zeroPosition-p[l].screen_y):!0===d.options.barChart.sideBySide&&(h.width=h.width/c[a].amount,h.offset+=c[a].resolved*h.width-.5*h.width*(c[a].amount+1))}var y=h.width,b=p[l].screen_x;if(void 0!=p[l].screen_end?(y=p[l].screen_end-p[l].screen_x,b+=.5*y):b+=h.offset,n.drawBar(b,p[l].screen_y-g,y,d.zeroPosition-p[l].screen_y,d.className+" vis-bar",i.svgElements,i.svg,d.style),!0===d.options.drawPoints.enabled){var _={screen_x:p[l].screen_x,screen_y:p[l].screen_y-g,x:p[l].x,y:p[l].y,groupId:p[l].groupId,label:p[l].label};s.draw([_],d,i,h.offset)}}},o._getDataIntersections=function(t,e){for(var i,o=0;o<e.length;o++)o+1<e.length&&(i=Math.abs(e[o+1].screen_x-e[o].screen_x)),o>0&&(i=Math.min(i,Math.abs(e[o-1].screen_x-e[o].screen_x))),0===i&&(void 0===t[e[o].screen_x]&&(t[e[o].screen_x]={amount:0,resolved:0,accumulatedPositive:0,accumulatedNegative:0}),t[e[o].screen_x].amount+=1)},o._getSafeDrawData=function(t,e,i){var o,n;return t<e.options.barChart.width&&t>0?(o=t<i?i:t,n=0,"left"===e.options.barChart.align?n-=.5*t:"right"===e.options.barChart.align&&(n+=.5*t)):(o=e.options.barChart.width,n=0,"left"===e.options.barChart.align?n-=.5*e.options.barChart.width:"right"===e.options.barChart.align&&(n+=.5*e.options.barChart.width)),{width:o,offset:n}},o.getStackedYRange=function(t,e,i,n,s){if(t.length>0){t.sort(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x});var r={};o._getDataIntersections(r,t),e[n]=o._getStackedYRange(r,t),e[n].yAxisOrientation=s,i.push(n)}},o._getStackedYRange=function(t,e){for(var i,o=e[0].screen_y,n=e[0].screen_y,s=0;s<e.length;s++)i=e[s].screen_x,void 0===t[i]?(o=o>e[s].screen_y?e[s].screen_y:o,n=n<e[s].screen_y?e[s].screen_y:n):e[s].screen_y<0?t[i].accumulatedNegative+=e[s].screen_y:t[i].accumulatedPositive+=e[s].screen_y;for(var r in t)t.hasOwnProperty(r)&&(o=o>t[r].accumulatedNegative?t[r].accumulatedNegative:o,o=o>t[r].accumulatedPositive?t[r].accumulatedPositive:o,n=n<t[r].accumulatedNegative?t[r].accumulatedNegative:n,n=n<t[r].accumulatedPositive?t[r].accumulatedPositive:n);return{min:o,max:n}},t.exports=o},function(t,e,i){function o(t,e){}var n=i(11);o.calcPath=function(t,e){if(null!=t&&t.length>0){return 1==e.options.interpolation.enabled?o._catmullRom(t,e):o._linear(t)}},o.drawIcon=function(t,e,i,o,s,r){var a,h,d=.5*s,l=n.getSVGElement("rect",r.svgElements,r.svg);if(l.setAttributeNS(null,"x",e),l.setAttributeNS(null,"y",i-d),l.setAttributeNS(null,"width",o),l.setAttributeNS(null,"height",2*d),l.setAttributeNS(null,"class","vis-outline"),a=n.getSVGElement("path",r.svgElements,r.svg),a.setAttributeNS(null,"class",t.className),void 0!==t.style&&a.setAttributeNS(null,"style",t.style),a.setAttributeNS(null,"d","M"+e+","+i+" L"+(e+o)+","+i),1==t.options.shaded.enabled&&(h=n.getSVGElement("path",r.svgElements,r.svg),"top"==t.options.shaded.orientation?h.setAttributeNS(null,"d","M"+e+", "+(i-d)+"L"+e+","+i+" L"+(e+o)+","+i+" L"+(e+o)+","+(i-d)):h.setAttributeNS(null,"d","M"+e+","+i+" L"+e+","+(i+d)+" L"+(e+o)+","+(i+d)+"L"+(e+o)+","+i),h.setAttributeNS(null,"class",t.className+" vis-icon-fill"),void 0!==t.options.shaded.style&&""!==t.options.shaded.style&&h.setAttributeNS(null,"style",t.options.shaded.style)),1==t.options.drawPoints.enabled){var u={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size,className:t.className};n.drawPoint(e+.5*o,i,u,r.svgElements,r.svg)}},o.drawShading=function(t,e,i,o){if(1==e.options.shaded.enabled){var s=Number(o.svg.style.height.replace("px","")),r=n.getSVGElement("path",o.svgElements,o.svg),a="L";1==e.options.interpolation.enabled&&(a="C");var h,d=0;d="top"==e.options.shaded.orientation?0:"bottom"==e.options.shaded.orientation?s:Math.min(Math.max(0,e.zeroPosition),s),h="group"==e.options.shaded.orientation&&null!=i&&void 0!=i?"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" L"+i[i.length-1][0]+","+i[i.length-1][1]+" "+this.serializePath(i,a,!0)+i[0][0]+","+i[0][1]+" Z":"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" V"+d+" H"+t[0][0]+" Z",r.setAttributeNS(null,"class",e.className+" vis-fill"),void 0!==e.options.shaded.style&&r.setAttributeNS(null,"style",e.options.shaded.style),r.setAttributeNS(null,"d",h)}},o.draw=function(t,e,i){if(null!=t&&void 0!=t){var o=n.getSVGElement("path",i.svgElements,i.svg);o.setAttributeNS(null,"class",e.className),void 0!==e.style&&o.setAttributeNS(null,"style",e.style);var s="L";1==e.options.interpolation.enabled&&(s="C"),o.setAttributeNS(null,"d","M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,s,!1))}},o.serializePath=function(t,e,i){if(t.length<2)return"";var o,n=e;if(i)for(o=t.length-2;o>0;o--)n+=t[o][0]+","+t[o][1]+" ";else for(o=1;o<t.length;o++)n+=t[o][0]+","+t[o][1]+" ";return n},o._catmullRomUniform=function(t){var e,i,o,n,s,r,a=[];a.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var h=t.length,d=0;d<h-1;d++)e=0==d?t[0]:t[d-1],i=t[d],o=t[d+1],n=d+2<h?t[d+2]:o,s={screen_x:(-e.screen_x+6*i.screen_x+o.screen_x)*(1/6),screen_y:(-e.screen_y+6*i.screen_y+o.screen_y)*(1/6)},r={screen_x:(i.screen_x+6*o.screen_x-n.screen_x)*(1/6),screen_y:(i.screen_y+6*o.screen_y-n.screen_y)*(1/6)},a.push([s.screen_x,s.screen_y]),a.push([r.screen_x,r.screen_y]),a.push([o.screen_x,o.screen_y]);return a},o._catmullRom=function(t,e){var i=e.options.interpolation.alpha;if(0==i||void 0===i)return this._catmullRomUniform(t);var o,n,s,r,a,h,d,l,u,p,c,m,f,g,v,y,b,_,w,x=[];x.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var D=t.length,S=0;S<D-1;S++)o=0==S?t[0]:t[S-1],n=t[S],s=t[S+1],r=S+2<D?t[S+2]:s,d=Math.sqrt(Math.pow(o.screen_x-n.screen_x,2)+Math.pow(o.screen_y-n.screen_y,2)),l=Math.sqrt(Math.pow(n.screen_x-s.screen_x,2)+Math.pow(n.screen_y-s.screen_y,2)),u=Math.sqrt(Math.pow(s.screen_x-r.screen_x,2)+Math.pow(s.screen_y-r.screen_y,2)),g=Math.pow(u,i),y=Math.pow(u,2*i),v=Math.pow(l,i),b=Math.pow(l,2*i),w=Math.pow(d,i),_=Math.pow(d,2*i),p=2*_+3*w*v+b,c=2*y+3*g*v+b,m=3*w*(w+v),m>0&&(m=1/m),f=3*g*(g+v),f>0&&(f=1/f),a={screen_x:(-b*o.screen_x+p*n.screen_x+_*s.screen_x)*m,screen_y:(-b*o.screen_y+p*n.screen_y+_*s.screen_y)*m},h={screen_x:(y*n.screen_x+c*s.screen_x-b*r.screen_x)*f,screen_y:(y*n.screen_y+c*s.screen_y-b*r.screen_y)*f},0==a.screen_x&&0==a.screen_y&&(a=n),0==h.screen_x&&0==h.screen_y&&(h=s),x.push([a.screen_x,a.screen_y]),x.push([h.screen_x,h.screen_y]),x.push([s.screen_x,s.screen_y]);return x},o._linear=function(t){for(var e=[],i=0;i<t.length;i++)e.push([t[i].screen_x,t[i].screen_y]);return e},t.exports=o},function(t,e,i){function o(t,e,i,o){this.body=t,this.defaultOptions={enabled:!1,icons:!0,iconSize:20,iconSpacing:6,left:{visible:!0,position:"top-left"},right:{visible:!0,position:"top-right"}},this.side=i,this.options=r.extend({},this.defaultOptions),this.linegraphOptions=o,this.svgElements={},this.dom={},this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups},this.setOptions(e)}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(11),h=i(5);o.prototype=new h,o.prototype.clear=function(){this.groups={},this.amountOfGroups=0},o.prototype.addGroup=function(t,e){1!=e.options.excludeFromLegend&&(this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1)},o.prototype.updateGroup=function(t,e){this.groups[t]=e},o.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},o.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-legend",this.dom.frame.style.position="absolute",this.dom.frame.style.top="10px",this.dom.frame.style.display="block",this.dom.textArea=document.createElement("div"),this.dom.textArea.className="vis-legend-text",this.dom.textArea.style.position="relative",this.dom.textArea.style.top="0px",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.width=this.options.iconSize+5+"px",this.svg.style.height="100%",this.dom.frame.appendChild(this.svg),this.dom.frame.appendChild(this.dom.textArea)},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},o.prototype.setOptions=function(t){var e=["enabled","orientation","icons","left","right"];r.selectiveDeepExtend(e,this.options,t)},o.prototype.redraw=function(){var t=0,e=(0,s.default)(this.groups);e.sort(function(t,e){return t<e?-1:1});for(var i=0;i<e.length;i++){var o=e[i];1!=this.groups[o].visible||void 0!==this.linegraphOptions.visibility[o]&&1!=this.linegraphOptions.visibility[o]||t++}if(0==this.options[this.side].visible||0==this.amountOfGroups||0==this.options.enabled||0==t)this.hide();else{if(this.show(),"top-left"==this.options[this.side].position||"bottom-left"==this.options[this.side].position?(this.dom.frame.style.left="4px",this.dom.frame.style.textAlign="left",this.dom.textArea.style.textAlign="left",this.dom.textArea.style.left=this.options.iconSize+15+"px",this.dom.textArea.style.right="",this.svg.style.left="0px",this.svg.style.right=""):(this.dom.frame.style.right="4px",this.dom.frame.style.textAlign="right",this.dom.textArea.style.textAlign="right",this.dom.textArea.style.right=this.options.iconSize+15+"px",this.dom.textArea.style.left="",this.svg.style.right="0px",this.svg.style.left=""),"top-left"==this.options[this.side].position||"top-right"==this.options[this.side].position)this.dom.frame.style.top=4-Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.bottom="";else{var n=this.body.domProps.center.height-this.body.domProps.centerContainer.height;this.dom.frame.style.bottom=4+n+Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.top=""}0==this.options.icons?(this.dom.frame.style.width=this.dom.textArea.offsetWidth+10+"px",this.dom.textArea.style.right="",this.dom.textArea.style.left="",this.svg.style.width="0px"):(this.dom.frame.style.width=this.options.iconSize+15+this.dom.textArea.offsetWidth+10+"px",this.drawLegendIcons());var r="";for(i=0;i<e.length;i++)o=e[i],1!=this.groups[o].visible||void 0!==this.linegraphOptions.visibility[o]&&1!=this.linegraphOptions.visibility[o]||(r+=this.groups[o].content+"<br />");this.dom.textArea.innerHTML=r,this.dom.textArea.style.lineHeight=.75*this.options.iconSize+this.options.iconSpacing+"px"}},o.prototype.drawLegendIcons=function(){if(this.dom.frame.parentNode){var t=(0,s.default)(this.groups);t.sort(function(t,e){return t<e?-1:1}),a.resetElements(this.svgElements);var e=window.getComputedStyle(this.dom.frame).paddingTop,i=Number(e.replace("px","")),o=i,n=this.options.iconSize,r=.75*this.options.iconSize,h=i+.5*r+3;this.svg.style.width=n+5+i+"px";for(var d=0;d<t.length;d++){var l=t[d];1!=this.groups[l].visible||void 0!==this.linegraphOptions.visibility[l]&&1!=this.linegraphOptions.visibility[l]||(this.groups[l].getLegend(n,r,this.framework,o,h),h+=r+this.options.iconSpacing)}}},t.exports=o},function(t,e,i){Object.defineProperty(e,"__esModule",{value:!0});var o="string",n="boolean",s="number",r="object",a={configure:{enabled:{boolean:n},filter:{boolean:n,function:"function"},container:{dom:"dom"},__type__:{object:r,boolean:n,function:"function"}},yAxisOrientation:{string:["left","right"]},defaultGroup:{string:o},sort:{boolean:n},sampling:{boolean:n},stack:{boolean:n},graphHeight:{string:o,number:s},shaded:{enabled:{boolean:n},orientation:{string:["bottom","top","zero","group"]},groupId:{object:r},__type__:{boolean:n,object:r}},style:{string:["line","bar","points"]},barChart:{width:{number:s},minWidth:{number:s},sideBySide:{boolean:n},align:{string:["left","center","right"]},__type__:{object:r}},interpolation:{enabled:{boolean:n},parametrization:{string:["centripetal","chordal","uniform"]},alpha:{number:s},__type__:{object:r,boolean:n}},drawPoints:{enabled:{boolean:n},onRender:{function:"function"},size:{number:s},style:{string:["square","circle"]},__type__:{object:r,boolean:n,function:"function"}},dataAxis:{showMinorLabels:{boolean:n},showMajorLabels:{boolean:n},icons:{boolean:n},width:{string:o,number:s},visible:{boolean:n},alignZeros:{boolean:n},left:{range:{min:{number:s,undefined:"undefined"},max:{number:s,undefined:"undefined"},__type__:{object:r}},format:{function:"function"},title:{text:{string:o,number:s,undefined:"undefined"},style:{string:o,undefined:"undefined"},__type__:{object:r}},__type__:{object:r}},right:{range:{min:{number:s,undefined:"undefined"},max:{number:s,undefined:"undefined"},__type__:{object:r}},format:{function:"function"},title:{text:{string:o,number:s,undefined:"undefined"},style:{string:o,undefined:"undefined"},__type__:{object:r}},__type__:{object:r}},__type__:{object:r}},legend:{enabled:{boolean:n},icons:{boolean:n},left:{visible:{boolean:n},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:r}},right:{visible:{boolean:n},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:r}},__type__:{object:r,boolean:n}},groups:{visibility:{any:"any"},__type__:{object:r}},autoResize:{boolean:n},throttleRedraw:{number:s},clickToUse:{boolean:n},end:{number:s,date:"date",string:o,moment:"moment"},format:{minorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r}},majorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r}},__type__:{object:r}},moment:{function:"function"},height:{string:o,number:s},hiddenDates:{start:{date:"date",number:s,string:o,moment:"moment"},end:{date:"date",number:s,string:o,moment:"moment"},repeat:{string:o},__type__:{object:r,array:"array"}},locale:{string:o},locales:{__any__:{any:"any"},__type__:{object:r}},max:{date:"date",number:s,string:o,moment:"moment"},maxHeight:{number:s,string:o},maxMinorChars:{number:s},min:{date:"date",number:s,string:o,moment:"moment"},minHeight:{number:s,string:o},moveable:{boolean:n},multiselect:{boolean:n},orientation:{string:o},showCurrentTime:{boolean:n},showMajorLabels:{boolean:n},showMinorLabels:{boolean:n},start:{date:"date",number:s,string:o,moment:"moment"},timeAxis:{scale:{string:o,undefined:"undefined"},step:{number:s,undefined:"undefined"},__type__:{object:r}},width:{string:o,number:s},zoomable:{boolean:n},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:s},zoomMin:{number:s},zIndex:{number:s},__type__:{object:r}},h={global:{sort:!0,sampling:!0,stack:!1,shaded:{enabled:!1,orientation:["zero","top","bottom","group"]},style:["line","bar","points"],barChart:{width:[50,5,100,5],minWidth:[50,5,100,5],sideBySide:!1,align:["left","center","right"]},interpolation:{enabled:!0,parametrization:["centripetal","chordal","uniform"]},drawPoints:{enabled:!0,size:[6,2,30,1],style:["square","circle"]},dataAxis:{showMinorLabels:!0,showMajorLabels:!0,icons:!1,width:[40,0,200,1],visible:!0,alignZeros:!0,left:{title:{text:"",style:""}},right:{title:{text:"",style:""}}},legend:{enabled:!1,icons:!0,left:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]},right:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]}},autoResize:!0,clickToUse:!1,end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",month:"YYYY",year:""}},height:"",locale:"",max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!0,orientation:["both","bottom","top"],showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,start:"",width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1],zIndex:0}};e.allOptions=a,e.configureOptions=h},function(t,e,i){e.util=i(0),e.DOMutil=i(11),e.DataSet=i(15),e.DataView=i(20),e.Queue=i(65),e.Timeline=i(129),e.Graph2d=i(138),e.timeline={Core:i(48),DateUtil:i(21),Range:i(47),stack:i(71),TimeStep:i(49),components:{items:{Item:i(22),BackgroundItem:i(74),BoxItem:i(72),PointItem:i(73),RangeItem:i(53)},BackgroundGroup:i(52),Component:i(5),CurrentTime:i(50),CustomTime:i(29),DataAxis:i(78),DataScale:i(79),GraphGroup:i(80),Group:i(51),ItemSet:i(70),Legend:i(83),LineGraph:i(77),TimeAxis:i(28)}},e.moment=i(3),e.Hammer=i(12),e.keycharm=i(68)},function(t,e,i){t.exports={default:i(87),__esModule:!0}},function(t,e,i){i(55),i(62),t.exports=i(102)},function(t,e,i){var o=i(89),n=i(90),s=i(23),r=i(13);t.exports=i(56)(Array,"Array",function(t,e){this._t=r(t),this._i=0,this._k=e},function(){var t=this._t,e=this._k,i=this._i++;return!t||i>=t.length?(this._t=void 0,n(1)):"keys"==e?n(0,i):"values"==e?n(0,t[i]):n(0,[i,t[i]])},"values"),s.Arguments=s.Array,o("keys"),o("values"),o("entries")},function(t,e){t.exports=function(){}},function(t,e){t.exports=function(t,e){return{value:e,done:!!t}}},function(t,e,i){var o=i(33);t.exports=Object("z").propertyIsEnumerable(0)?Object:function(t){return"String"==o(t)?t.split(""):Object(t)}},function(t,e,i){var o=i(93);t.exports=function(t,e,i){if(o(t),void 0===e)return t;switch(i){case 1:return function(i){return t.call(e,i)};case 2:return function(i,o){return t.call(e,i,o)};case 3:return function(i,o,n){return t.call(e,i,o,n)}}return function(){return t.apply(e,arguments)}}},function(t,e){t.exports=function(t){if("function"!=typeof t)throw TypeError(t+" is not a function!");return t}},function(t,e,i){var o=i(37),n=i(25),s=i(42),r={};i(14)(r,i(7)("iterator"),function(){return this}),t.exports=function(t,e,i){t.prototype=o(r,{next:n(1,i)}),s(t,e+" Iterator")}},function(t,e,i){var o=i(8),n=i(18),s=i(26);t.exports=i(9)?Object.defineProperties:function(t,e){n(t);for(var i,r=s(e),a=r.length,h=0;a>h;)o.f(t,i=r[h++],e[i]);return t}},function(t,e,i){var o=i(13),n=i(97),s=i(98);t.exports=function(t){return function(e,i,r){var a,h=o(e),d=n(h.length),l=s(r,d);if(t&&i!=i){for(;d>l;)if((a=h[l++])!=a)return!0}else for(;d>l;l++)if((t||l in h)&&h[l]===i)return t||l||0;return!t&&-1}}},function(t,e,i){var o=i(38),n=Math.min;t.exports=function(t){return t>0?n(o(t),9007199254740991):0}},function(t,e,i){var o=i(38),n=Math.max,s=Math.min;t.exports=function(t,e){return t=o(t),t<0?n(t+e,0):s(t,e)}},function(t,e,i){var o=i(6).document;t.exports=o&&o.documentElement},function(t,e,i){var o=i(10),n=i(61),s=i(39)("IE_PROTO"),r=Object.prototype;t.exports=Object.getPrototypeOf||function(t){return t=n(t),o(t,s)?t[s]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?r:null}},function(t,e,i){var o=i(38),n=i(34);t.exports=function(t){return function(e,i){var s,r,a=String(n(e)),h=o(i),d=a.length;return h<0||h>=d?t?"":void 0:(s=a.charCodeAt(h),s<55296||s>56319||h+1===d||(r=a.charCodeAt(h+1))<56320||r>57343?t?a.charAt(h):s:t?a.slice(h,h+2):r-56320+(s-55296<<10)+65536)}}},function(t,e,i){var o=i(18),n=i(103);t.exports=i(2).getIterator=function(t){var e=n(t);if("function"!=typeof e)throw TypeError(t+" is not iterable!");return o(e.call(t))}},function(t,e,i){var o=i(104),n=i(7)("iterator"),s=i(23);t.exports=i(2).getIteratorMethod=function(t){if(void 0!=t)return t[n]||t["@@iterator"]||s[o(t)]}},function(t,e,i){var o=i(33),n=i(7)("toStringTag"),s="Arguments"==o(function(){return arguments}()),r=function(t,e){try{return t[e]}catch(t){}};t.exports=function(t){var e,i,a;return void 0===t?"Undefined":null===t?"Null":"string"==typeof(i=r(e=Object(t),n))?i:s?o(e):"Object"==(a=o(e))&&"function"==typeof e.callee?"Arguments":a}},function(t,e,i){i(106);var o=i(2).Object;t.exports=function(t,e){return o.create(t,e)}},function(t,e,i){var o=i(17);o(o.S,"Object",{create:i(37)})},function(t,e,i){i(108),t.exports=i(2).Object.keys},function(t,e,i){var o=i(61),n=i(26);i(109)("keys",function(){return function(t){return n(o(t))}})},function(t,e,i){var o=i(17),n=i(2),s=i(19);t.exports=function(t,e){var i=(n.Object||{})[t]||Object[t],r={};r[t]=e(i),o(o.S+o.F*s(function(){i(1)}),"Object",r)}},function(t,e,i){t.exports={default:i(111),__esModule:!0}},function(t,e,i){i(62),i(55),t.exports=i(44).f("iterator")},function(t,e,i){t.exports={default:i(113),__esModule:!0}},function(t,e,i){i(114),i(120),i(121),i(122),t.exports=i(2).Symbol},function(t,e,i){var o=i(6),n=i(10),s=i(9),r=i(17),a=i(59),h=i(115).KEY,d=i(19),l=i(40),u=i(42),p=i(27),c=i(7),m=i(44),f=i(45),g=i(116),v=i(117),y=i(18),b=i(13),_=i(36),w=i(25),x=i(37),D=i(118),S=i(119),k=i(8),C=i(26),T=S.f,M=k.f,O=D.f,E=o.Symbol,P=o.JSON,I=P&&P.stringify,N=c("_hidden"),A=c("toPrimitive"),R={}.propertyIsEnumerable,L=l("symbol-registry"),F=l("symbols"),H=l("op-symbols"),j=Object.prototype,Y="function"==typeof E,G=o.QObject,z=!G||!G.prototype||!G.prototype.findChild,W=s&&d(function(){return 7!=x(M({},"a",{get:function(){return M(this,"a",{value:7}).a}})).a})?function(t,e,i){var o=T(j,e);o&&delete j[e],M(t,e,i),o&&t!==j&&M(j,e,o)}:M,V=function(t){var e=F[t]=x(E.prototype);return e._k=t,e},B=Y&&"symbol"==typeof E.iterator?function(t){return"symbol"==typeof t}:function(t){return t instanceof E},U=function(t,e,i){return t===j&&U(H,e,i),y(t),e=_(e,!0),y(i),n(F,e)?(i.enumerable?(n(t,N)&&t[N][e]&&(t[N][e]=!1),i=x(i,{enumerable:w(0,!1)})):(n(t,N)||M(t,N,w(1,{})),t[N][e]=!0),W(t,e,i)):M(t,e,i)},q=function(t,e){y(t);for(var i,o=g(e=b(e)),n=0,s=o.length;s>n;)U(t,i=o[n++],e[i]);return t},X=function(t,e){return void 0===e?x(t):q(x(t),e)},Z=function(t){var e=R.call(this,t=_(t,!0));return!(this===j&&n(F,t)&&!n(H,t))&&(!(e||!n(this,t)||!n(F,t)||n(this,N)&&this[N][t])||e)},K=function(t,e){if(t=b(t),e=_(e,!0),t!==j||!n(F,e)||n(H,e)){var i=T(t,e);return!i||!n(F,e)||n(t,N)&&t[N][e]||(i.enumerable=!0),i}},J=function(t){for(var e,i=O(b(t)),o=[],s=0;i.length>s;)n(F,e=i[s++])||e==N||e==h||o.push(e);return o},$=function(t){for(var e,i=t===j,o=O(i?H:b(t)),s=[],r=0;o.length>r;)!n(F,e=o[r++])||i&&!n(j,e)||s.push(F[e]);return s};Y||(E=function(){if(this instanceof E)throw TypeError("Symbol is not a constructor!");var t=p(arguments.length>0?arguments[0]:void 0),e=function(i){this===j&&e.call(H,i),n(this,N)&&n(this[N],t)&&(this[N][t]=!1),W(this,t,w(1,i))};return s&&z&&W(j,t,{configurable:!0,set:e}),V(t)},a(E.prototype,"toString",function(){return this._k}),S.f=K,k.f=U,i(64).f=D.f=J,i(46).f=Z,i(63).f=$,s&&!i(35)&&a(j,"propertyIsEnumerable",Z,!0),m.f=function(t){return V(c(t))}),r(r.G+r.W+r.F*!Y,{Symbol:E});for(var Q="hasInstance,isConcatSpreadable,iterator,match,replace,search,species,split,toPrimitive,toStringTag,unscopables".split(","),tt=0;Q.length>tt;)c(Q[tt++]);for(var et=C(c.store),it=0;et.length>it;)f(et[it++]);r(r.S+r.F*!Y,"Symbol",{for:function(t){return n(L,t+="")?L[t]:L[t]=E(t)},keyFor:function(t){if(!B(t))throw TypeError(t+" is not a symbol!");for(var e in L)if(L[e]===t)return e},useSetter:function(){z=!0},useSimple:function(){z=!1}}),r(r.S+r.F*!Y,"Object",{create:X,defineProperty:U,defineProperties:q,getOwnPropertyDescriptor:K,getOwnPropertyNames:J,getOwnPropertySymbols:$}),P&&r(r.S+r.F*(!Y||d(function(){var t=E();return"[null]"!=I([t])||"{}"!=I({a:t})||"{}"!=I(Object(t))})),"JSON",{stringify:function(t){if(void 0!==t&&!B(t)){for(var e,i,o=[t],n=1;arguments.length>n;)o.push(arguments[n++]);return e=o[1],"function"==typeof e&&(i=e),!i&&v(e)||(e=function(t,e){if(i&&(e=i.call(this,t,e)),!B(e))return e}),o[1]=e,I.apply(P,o)}}}),E.prototype[A]||i(14)(E.prototype,A,E.prototype.valueOf),u(E,"Symbol"),u(Math,"Math",!0),u(o.JSON,"JSON",!0)},function(t,e,i){var o=i(27)("meta"),n=i(24),s=i(10),r=i(8).f,a=0,h=Object.isExtensible||function(){return!0},d=!i(19)(function(){return h(Object.preventExtensions({}))}),l=function(t){r(t,o,{value:{i:"O"+ ++a,w:{}}})},u=function(t,e){if(!n(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!s(t,o)){if(!h(t))return"F";if(!e)return"E";l(t)}return t[o].i},p=function(t,e){if(!s(t,o)){if(!h(t))return!0;if(!e)return!1;l(t)}return t[o].w},c=function(t){return d&&m.NEED&&h(t)&&!s(t,o)&&l(t),t},m=t.exports={KEY:o,NEED:!1,fastKey:u,getWeak:p,onFreeze:c}},function(t,e,i){var o=i(26),n=i(63),s=i(46);t.exports=function(t){var e=o(t),i=n.f;if(i)for(var r,a=i(t),h=s.f,d=0;a.length>d;)h.call(t,r=a[d++])&&e.push(r);return e}},function(t,e,i){var o=i(33);t.exports=Array.isArray||function(t){return"Array"==o(t)}},function(t,e,i){var o=i(13),n=i(64).f,s={}.toString,r="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],a=function(t){try{return n(t)}catch(t){return r.slice()}};t.exports.f=function(t){return r&&"[object Window]"==s.call(t)?a(t):n(o(t))}},function(t,e,i){var o=i(46),n=i(25),s=i(13),r=i(36),a=i(10),h=i(57),d=Object.getOwnPropertyDescriptor;e.f=i(9)?d:function(t,e){if(t=s(t),e=r(e,!0),h)try{return d(t,e)}catch(t){}if(a(t,e))return n(!o.f.call(t,e),t[e])}},function(t,e){},function(t,e,i){i(45)("asyncIterator")},function(t,e,i){i(45)("observable")},function(t,e,i){(function(t){!function(e,i){t.exports=i()}(0,function(){function e(){return To.apply(null,arguments)}function i(t){return t instanceof Array||"[object Array]"===Object.prototype.toString.call(t)}function o(t){return null!=t&&"[object Object]"===Object.prototype.toString.call(t)}function n(t){if(Object.getOwnPropertyNames)return 0===Object.getOwnPropertyNames(t).length;var e;for(e in t)if(t.hasOwnProperty(e))return!1;return!0}function s(t){return void 0===t}function r(t){return"number"==typeof t||"[object Number]"===Object.prototype.toString.call(t)}function a(t){return t instanceof Date||"[object Date]"===Object.prototype.toString.call(t)}function h(t,e){var i,o=[];for(i=0;i<t.length;++i)o.push(e(t[i],i));return o}function d(t,e){return Object.prototype.hasOwnProperty.call(t,e)}function l(t,e){for(var i in e)d(e,i)&&(t[i]=e[i]);return d(e,"toString")&&(t.toString=e.toString),d(e,"valueOf")&&(t.valueOf=e.valueOf),t}function u(t,e,i,o){return Se(t,e,i,o,!0).utc()}function p(){return{empty:!1,unusedTokens:[],unusedInput:[],overflow:-2,charsLeftOver:0,nullInput:!1,invalidMonth:null,invalidFormat:!1,userInvalidated:!1,iso:!1,parsedDateParts:[],meridiem:null,rfc2822:!1,weekdayMismatch:!1}}function c(t){return null==t._pf&&(t._pf=p()),t._pf}function m(t){if(null==t._isValid){var e=c(t),i=Mo.call(e.parsedDateParts,function(t){return null!=t}),o=!isNaN(t._d.getTime())&&e.overflow<0&&!e.empty&&!e.invalidMonth&&!e.invalidWeekday&&!e.weekdayMismatch&&!e.nullInput&&!e.invalidFormat&&!e.userInvalidated&&(!e.meridiem||e.meridiem&&i);if(t._strict&&(o=o&&0===e.charsLeftOver&&0===e.unusedTokens.length&&void 0===e.bigHour),null!=Object.isFrozen&&Object.isFrozen(t))return o;t._isValid=o}return t._isValid}function f(t){var e=u(NaN);return null!=t?l(c(e),t):c(e).userInvalidated=!0,e}function g(t,e){var i,o,n;if(s(e._isAMomentObject)||(t._isAMomentObject=e._isAMomentObject),s(e._i)||(t._i=e._i),s(e._f)||(t._f=e._f),s(e._l)||(t._l=e._l),s(e._strict)||(t._strict=e._strict),s(e._tzm)||(t._tzm=e._tzm),s(e._isUTC)||(t._isUTC=e._isUTC),s(e._offset)||(t._offset=e._offset),s(e._pf)||(t._pf=c(e)),s(e._locale)||(t._locale=e._locale),Oo.length>0)for(i=0;i<Oo.length;i++)o=Oo[i],n=e[o],s(n)||(t[o]=n);return t}function v(t){g(this,t),this._d=new Date(null!=t._d?t._d.getTime():NaN),this.isValid()||(this._d=new Date(NaN)),!1===Eo&&(Eo=!0,e.updateOffset(this),Eo=!1)}function y(t){return t instanceof v||null!=t&&null!=t._isAMomentObject}function b(t){return t<0?Math.ceil(t)||0:Math.floor(t)}function _(t){var e=+t,i=0;return 0!==e&&isFinite(e)&&(i=b(e)),i}function w(t,e,i){var o,n=Math.min(t.length,e.length),s=Math.abs(t.length-e.length),r=0;for(o=0;o<n;o++)(i&&t[o]!==e[o]||!i&&_(t[o])!==_(e[o]))&&r++;return r+s}function x(t){!1===e.suppressDeprecationWarnings&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+t)}function D(t,i){var o=!0;return l(function(){if(null!=e.deprecationHandler&&e.deprecationHandler(null,t),o){for(var n,s=[],r=0;r<arguments.length;r++){if(n="","object"==typeof arguments[r]){n+="\n["+r+"] ";for(var a in arguments[0])n+=a+": "+arguments[0][a]+", ";n=n.slice(0,-2)}else n=arguments[r];s.push(n)}x(t+"\nArguments: "+Array.prototype.slice.call(s).join("")+"\n"+(new Error).stack),o=!1}return i.apply(this,arguments)},i)}function S(t,i){null!=e.deprecationHandler&&e.deprecationHandler(t,i),Po[t]||(x(i),Po[t]=!0)}function k(t){return t instanceof Function||"[object Function]"===Object.prototype.toString.call(t)}function C(t){var e,i;for(i in t)e=t[i],k(e)?this[i]=e:this["_"+i]=e;this._config=t,this._dayOfMonthOrdinalParseLenient=new RegExp((this._dayOfMonthOrdinalParse.source||this._ordinalParse.source)+"|"+/\d{1,2}/.source)}function T(t,e){var i,n=l({},t);for(i in e)d(e,i)&&(o(t[i])&&o(e[i])?(n[i]={},l(n[i],t[i]),l(n[i],e[i])):null!=e[i]?n[i]=e[i]:delete n[i]);for(i in t)d(t,i)&&!d(e,i)&&o(t[i])&&(n[i]=l({},n[i]));return n}function M(t){null!=t&&this.set(t)}function O(t,e,i){var o=this._calendar[t]||this._calendar.sameElse;return k(o)?o.call(e,i):o}function E(t){var e=this._longDateFormat[t],i=this._longDateFormat[t.toUpperCase()];return e||!i?e:(this._longDateFormat[t]=i.replace(/MMMM|MM|DD|dddd/g,function(t){return t.slice(1)}),this._longDateFormat[t])}function P(){return this._invalidDate}function I(t){return this._ordinal.replace("%d",t)}function N(t,e,i,o){var n=this._relativeTime[i];return k(n)?n(t,e,i,o):n.replace(/%d/i,t)}function A(t,e){var i=this._relativeTime[t>0?"future":"past"];return k(i)?i(e):i.replace(/%s/i,e)}function R(t,e){var i=t.toLowerCase();Fo[i]=Fo[i+"s"]=Fo[e]=t}function L(t){return"string"==typeof t?Fo[t]||Fo[t.toLowerCase()]:void 0}function F(t){var e,i,o={};for(i in t)d(t,i)&&(e=L(i))&&(o[e]=t[i]);return o}function H(t,e){Ho[t]=e}function j(t){var e=[];for(var i in t)e.push({unit:i,priority:Ho[i]});return e.sort(function(t,e){return t.priority-e.priority}),e}function Y(t,e,i){var o=""+Math.abs(t),n=e-o.length;return(t>=0?i?"+":"":"-")+Math.pow(10,Math.max(0,n)).toString().substr(1)+o}function G(t,e,i,o){var n=o;"string"==typeof o&&(n=function(){return this[o]()}),t&&(zo[t]=n),e&&(zo[e[0]]=function(){return Y(n.apply(this,arguments),e[1],e[2])}),i&&(zo[i]=function(){return this.localeData().ordinal(n.apply(this,arguments),t)})}function z(t){return t.match(/\[[\s\S]/)?t.replace(/^\[|\]$/g,""):t.replace(/\\/g,"")}function W(t){var e,i,o=t.match(jo);for(e=0,i=o.length;e<i;e++)zo[o[e]]?o[e]=zo[o[e]]:o[e]=z(o[e]);return function(e){var n,s="";for(n=0;n<i;n++)s+=k(o[n])?o[n].call(e,t):o[n];return s}}function V(t,e){return t.isValid()?(e=B(e,t.localeData()),Go[e]=Go[e]||W(e),Go[e](t)):t.localeData().invalidDate()}function B(t,e){function i(t){return e.longDateFormat(t)||t}var o=5;for(Yo.lastIndex=0;o>=0&&Yo.test(t);)t=t.replace(Yo,i),Yo.lastIndex=0,o-=1;return t}function U(t,e,i){an[t]=k(e)?e:function(t,o){return t&&i?i:e}}function q(t,e){return d(an,t)?an[t](e._strict,e._locale):new RegExp(X(t))}function X(t){return Z(t.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,function(t,e,i,o,n){return e||i||o||n}))}function Z(t){return t.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}function K(t,e){var i,o=e;for("string"==typeof t&&(t=[t]),r(e)&&(o=function(t,i){i[e]=_(t)}),i=0;i<t.length;i++)hn[t[i]]=o}function J(t,e){K(t,function(t,i,o,n){o._w=o._w||{},e(t,o._w,o,n)})}function $(t,e,i){null!=e&&d(hn,t)&&hn[t](e,i._a,i,t)}function Q(t){return tt(t)?366:365}function tt(t){return t%4==0&&t%100!=0||t%400==0}function et(){return tt(this.year())}function it(t,i){return function(o){return null!=o?(nt(this,t,o),e.updateOffset(this,i),this):ot(this,t)}}function ot(t,e){return t.isValid()?t._d["get"+(t._isUTC?"UTC":"")+e]():NaN}function nt(t,e,i){t.isValid()&&!isNaN(i)&&("FullYear"===e&&tt(t.year())?t._d["set"+(t._isUTC?"UTC":"")+e](i,t.month(),ht(i,t.month())):t._d["set"+(t._isUTC?"UTC":"")+e](i))}function st(t){return t=L(t),k(this[t])?this[t]():this}function rt(t,e){if("object"==typeof t){t=F(t);for(var i=j(t),o=0;o<i.length;o++)this[i[o].unit](t[i[o].unit])}else if(t=L(t),k(this[t]))return this[t](e);return this}function at(t,e){return(t%e+e)%e}function ht(t,e){if(isNaN(t)||isNaN(e))return NaN;var i=at(e,12);return t+=(e-i)/12,1===i?tt(t)?29:28:31-i%7%2}function dt(t,e){return t?i(this._months)?this._months[t.month()]:this._months[(this._months.isFormat||_n).test(e)?"format":"standalone"][t.month()]:i(this._months)?this._months:this._months.standalone}function lt(t,e){return t?i(this._monthsShort)?this._monthsShort[t.month()]:this._monthsShort[_n.test(e)?"format":"standalone"][t.month()]:i(this._monthsShort)?this._monthsShort:this._monthsShort.standalone}function ut(t,e,i){var o,n,s,r=t.toLocaleLowerCase();if(!this._monthsParse)for(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[],o=0;o<12;++o)s=u([2e3,o]),this._shortMonthsParse[o]=this.monthsShort(s,"").toLocaleLowerCase(),this._longMonthsParse[o]=this.months(s,"").toLocaleLowerCase();return i?"MMM"===e?(n=yn.call(this._shortMonthsParse,r),-1!==n?n:null):(n=yn.call(this._longMonthsParse,r),-1!==n?n:null):"MMM"===e?-1!==(n=yn.call(this._shortMonthsParse,r))?n:(n=yn.call(this._longMonthsParse,r),-1!==n?n:null):-1!==(n=yn.call(this._longMonthsParse,r))?n:(n=yn.call(this._shortMonthsParse,r),-1!==n?n:null)}function pt(t,e,i){var o,n,s;if(this._monthsParseExact)return ut.call(this,t,e,i);for(this._monthsParse||(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[]),o=0;o<12;o++){if(n=u([2e3,o]),i&&!this._longMonthsParse[o]&&(this._longMonthsParse[o]=new RegExp("^"+this.months(n,"").replace(".","")+"$","i"),this._shortMonthsParse[o]=new RegExp("^"+this.monthsShort(n,"").replace(".","")+"$","i")),i||this._monthsParse[o]||(s="^"+this.months(n,"")+"|^"+this.monthsShort(n,""),this._monthsParse[o]=new RegExp(s.replace(".",""),"i")),i&&"MMMM"===e&&this._longMonthsParse[o].test(t))return o;if(i&&"MMM"===e&&this._shortMonthsParse[o].test(t))return o;if(!i&&this._monthsParse[o].test(t))return o}}function ct(t,e){var i;if(!t.isValid())return t;if("string"==typeof e)if(/^\d+$/.test(e))e=_(e);else if(e=t.localeData().monthsParse(e),!r(e))return t;return i=Math.min(t.date(),ht(t.year(),e)),t._d["set"+(t._isUTC?"UTC":"")+"Month"](e,i),t}function mt(t){return null!=t?(ct(this,t),e.updateOffset(this,!0),this):ot(this,"Month")}function ft(){return ht(this.year(),this.month())}function gt(t){return this._monthsParseExact?(d(this,"_monthsRegex")||yt.call(this),t?this._monthsShortStrictRegex:this._monthsShortRegex):(d(this,"_monthsShortRegex")||(this._monthsShortRegex=Dn),this._monthsShortStrictRegex&&t?this._monthsShortStrictRegex:this._monthsShortRegex)}function vt(t){return this._monthsParseExact?(d(this,"_monthsRegex")||yt.call(this),t?this._monthsStrictRegex:this._monthsRegex):(d(this,"_monthsRegex")||(this._monthsRegex=Sn),this._monthsStrictRegex&&t?this._monthsStrictRegex:this._monthsRegex)}function yt(){function t(t,e){return e.length-t.length}var e,i,o=[],n=[],s=[];for(e=0;e<12;e++)i=u([2e3,e]),o.push(this.monthsShort(i,"")),n.push(this.months(i,"")),s.push(this.months(i,"")),s.push(this.monthsShort(i,""));for(o.sort(t),n.sort(t),s.sort(t),e=0;e<12;e++)o[e]=Z(o[e]),n[e]=Z(n[e]);for(e=0;e<24;e++)s[e]=Z(s[e]);this._monthsRegex=new RegExp("^("+s.join("|")+")","i"),this._monthsShortRegex=this._monthsRegex,this._monthsStrictRegex=new RegExp("^("+n.join("|")+")","i"),this._monthsShortStrictRegex=new RegExp("^("+o.join("|")+")","i")}function bt(t,e,i,o,n,s,r){var a=new Date(t,e,i,o,n,s,r);return t<100&&t>=0&&isFinite(a.getFullYear())&&a.setFullYear(t),a}function _t(t){var e=new Date(Date.UTC.apply(null,arguments));return t<100&&t>=0&&isFinite(e.getUTCFullYear())&&e.setUTCFullYear(t),e}function wt(t,e,i){var o=7+e-i;return-(7+_t(t,0,o).getUTCDay()-e)%7+o-1}function xt(t,e,i,o,n){var s,r,a=(7+i-o)%7,h=wt(t,o,n),d=1+7*(e-1)+a+h;return d<=0?(s=t-1,r=Q(s)+d):d>Q(t)?(s=t+1,r=d-Q(t)):(s=t,r=d),{year:s,dayOfYear:r}}function Dt(t,e,i){var o,n,s=wt(t.year(),e,i),r=Math.floor((t.dayOfYear()-s-1)/7)+1;return r<1?(n=t.year()-1,o=r+St(n,e,i)):r>St(t.year(),e,i)?(o=r-St(t.year(),e,i),n=t.year()+1):(n=t.year(),o=r),{week:o,year:n}}function St(t,e,i){var o=wt(t,e,i),n=wt(t+1,e,i);return(Q(t)-o+n)/7}function kt(t){return Dt(t,this._week.dow,this._week.doy).week}function Ct(){return this._week.dow}function Tt(){return this._week.doy}function Mt(t){var e=this.localeData().week(this);return null==t?e:this.add(7*(t-e),"d")}function Ot(t){var e=Dt(this,1,4).week;return null==t?e:this.add(7*(t-e),"d")}function Et(t,e){return"string"!=typeof t?t:isNaN(t)?(t=e.weekdaysParse(t),"number"==typeof t?t:null):parseInt(t,10)}function Pt(t,e){return"string"==typeof t?e.weekdaysParse(t)%7||7:isNaN(t)?null:t}function It(t,e){return t?i(this._weekdays)?this._weekdays[t.day()]:this._weekdays[this._weekdays.isFormat.test(e)?"format":"standalone"][t.day()]:i(this._weekdays)?this._weekdays:this._weekdays.standalone}function Nt(t){return t?this._weekdaysShort[t.day()]:this._weekdaysShort}function At(t){return t?this._weekdaysMin[t.day()]:this._weekdaysMin}function Rt(t,e,i){var o,n,s,r=t.toLocaleLowerCase();if(!this._weekdaysParse)for(this._weekdaysParse=[],this._shortWeekdaysParse=[],this._minWeekdaysParse=[],o=0;o<7;++o)s=u([2e3,1]).day(o),this._minWeekdaysParse[o]=this.weekdaysMin(s,"").toLocaleLowerCase(),this._shortWeekdaysParse[o]=this.weekdaysShort(s,"").toLocaleLowerCase(),this._weekdaysParse[o]=this.weekdays(s,"").toLocaleLowerCase();return i?"dddd"===e?(n=yn.call(this._weekdaysParse,r),-1!==n?n:null):"ddd"===e?(n=yn.call(this._shortWeekdaysParse,r),-1!==n?n:null):(n=yn.call(this._minWeekdaysParse,r),-1!==n?n:null):"dddd"===e?-1!==(n=yn.call(this._weekdaysParse,r))?n:-1!==(n=yn.call(this._shortWeekdaysParse,r))?n:(n=yn.call(this._minWeekdaysParse,r),-1!==n?n:null):"ddd"===e?-1!==(n=yn.call(this._shortWeekdaysParse,r))?n:-1!==(n=yn.call(this._weekdaysParse,r))?n:(n=yn.call(this._minWeekdaysParse,r),-1!==n?n:null):-1!==(n=yn.call(this._minWeekdaysParse,r))?n:-1!==(n=yn.call(this._weekdaysParse,r))?n:(n=yn.call(this._shortWeekdaysParse,r),-1!==n?n:null)}function Lt(t,e,i){var o,n,s;if(this._weekdaysParseExact)return Rt.call(this,t,e,i);for(this._weekdaysParse||(this._weekdaysParse=[],this._minWeekdaysParse=[],this._shortWeekdaysParse=[],this._fullWeekdaysParse=[]),o=0;o<7;o++){if(n=u([2e3,1]).day(o),i&&!this._fullWeekdaysParse[o]&&(this._fullWeekdaysParse[o]=new RegExp("^"+this.weekdays(n,"").replace(".",".?")+"$","i"),this._shortWeekdaysParse[o]=new RegExp("^"+this.weekdaysShort(n,"").replace(".",".?")+"$","i"),this._minWeekdaysParse[o]=new RegExp("^"+this.weekdaysMin(n,"").replace(".",".?")+"$","i")),this._weekdaysParse[o]||(s="^"+this.weekdays(n,"")+"|^"+this.weekdaysShort(n,"")+"|^"+this.weekdaysMin(n,""),this._weekdaysParse[o]=new RegExp(s.replace(".",""),"i")),i&&"dddd"===e&&this._fullWeekdaysParse[o].test(t))return o;if(i&&"ddd"===e&&this._shortWeekdaysParse[o].test(t))return o;if(i&&"dd"===e&&this._minWeekdaysParse[o].test(t))return o;if(!i&&this._weekdaysParse[o].test(t))return o}}function Ft(t){if(!this.isValid())return null!=t?this:NaN;var e=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=t?(t=Et(t,this.localeData()),this.add(t-e,"d")):e}function Ht(t){if(!this.isValid())return null!=t?this:NaN;var e=(this.day()+7-this.localeData()._week.dow)%7;return null==t?e:this.add(t-e,"d")}function jt(t){if(!this.isValid())return null!=t?this:NaN;if(null!=t){var e=Pt(t,this.localeData());return this.day(this.day()%7?e:e-7)}return this.day()||7}function Yt(t){return this._weekdaysParseExact?(d(this,"_weekdaysRegex")||Wt.call(this),t?this._weekdaysStrictRegex:this._weekdaysRegex):(d(this,"_weekdaysRegex")||(this._weekdaysRegex=On),this._weekdaysStrictRegex&&t?this._weekdaysStrictRegex:this._weekdaysRegex)}function Gt(t){return this._weekdaysParseExact?(d(this,"_weekdaysRegex")||Wt.call(this),t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex):(d(this,"_weekdaysShortRegex")||(this._weekdaysShortRegex=En),this._weekdaysShortStrictRegex&&t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex)}function zt(t){return this._weekdaysParseExact?(d(this,"_weekdaysRegex")||Wt.call(this),t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex):(d(this,"_weekdaysMinRegex")||(this._weekdaysMinRegex=Pn),this._weekdaysMinStrictRegex&&t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex)}function Wt(){function t(t,e){return e.length-t.length}var e,i,o,n,s,r=[],a=[],h=[],d=[];for(e=0;e<7;e++)i=u([2e3,1]).day(e),o=this.weekdaysMin(i,""),n=this.weekdaysShort(i,""),s=this.weekdays(i,""),r.push(o),a.push(n),h.push(s),d.push(o),d.push(n),d.push(s);for(r.sort(t),a.sort(t),h.sort(t),d.sort(t),e=0;e<7;e++)a[e]=Z(a[e]),h[e]=Z(h[e]),d[e]=Z(d[e]);this._weekdaysRegex=new RegExp("^("+d.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+h.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+a.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+r.join("|")+")","i")}function Vt(){return this.hours()%12||12}function Bt(){return this.hours()||24}function Ut(t,e){G(t,0,0,function(){return this.localeData().meridiem(this.hours(),this.minutes(),e)})}function qt(t,e){return e._meridiemParse}function Xt(t){return"p"===(t+"").toLowerCase().charAt(0)}function Zt(t,e,i){return t>11?i?"pm":"PM":i?"am":"AM"}function Kt(t){return t?t.toLowerCase().replace("_","-"):t}function Jt(t){for(var e,i,o,n,s=0;s<t.length;){for(n=Kt(t[s]).split("-"),e=n.length,i=Kt(t[s+1]),i=i?i.split("-"):null;e>0;){if(o=$t(n.slice(0,e).join("-")))return o;if(i&&i.length>=e&&w(n,i,!0)>=e-1)break;e--}s++}return null}function $t(e){var i=null;if(!Ln[e]&&void 0!==t&&t&&t.exports)try{i=In._abbr;!function(){var t=new Error('Cannot find module "./locale"');throw t.code="MODULE_NOT_FOUND",t}(),Qt(i)}catch(t){}return Ln[e]}function Qt(t,e){var i;return t&&(i=s(e)?ie(t):te(t,e))&&(In=i),In._abbr}function te(t,e){if(null!==e){var i=Rn;if(e.abbr=t,null!=Ln[t])S("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale See http://momentjs.com/guides/#/warnings/define-locale/ for more info."),i=Ln[t]._config;else if(null!=e.parentLocale){if(null==Ln[e.parentLocale])return Fn[e.parentLocale]||(Fn[e.parentLocale]=[]),Fn[e.parentLocale].push({name:t,config:e}),null;i=Ln[e.parentLocale]._config}return Ln[t]=new M(T(i,e)),Fn[t]&&Fn[t].forEach(function(t){te(t.name,t.config)}),Qt(t),Ln[t]}return delete Ln[t],null}function ee(t,e){if(null!=e){var i,o=Rn;null!=Ln[t]&&(o=Ln[t]._config),e=T(o,e),i=new M(e),i.parentLocale=Ln[t],Ln[t]=i,Qt(t)}else null!=Ln[t]&&(null!=Ln[t].parentLocale?Ln[t]=Ln[t].parentLocale:null!=Ln[t]&&delete Ln[t]);return Ln[t]}function ie(t){var e;if(t&&t._locale&&t._locale._abbr&&(t=t._locale._abbr),!t)return In;if(!i(t)){if(e=$t(t))return e;t=[t]}return Jt(t)}function oe(){return Io(Ln)}function ne(t){var e,i=t._a;return i&&-2===c(t).overflow&&(e=i[ln]<0||i[ln]>11?ln:i[un]<1||i[un]>ht(i[dn],i[ln])?un:i[pn]<0||i[pn]>24||24===i[pn]&&(0!==i[cn]||0!==i[mn]||0!==i[fn])?pn:i[cn]<0||i[cn]>59?cn:i[mn]<0||i[mn]>59?mn:i[fn]<0||i[fn]>999?fn:-1,c(t)._overflowDayOfYear&&(e<dn||e>un)&&(e=un),c(t)._overflowWeeks&&-1===e&&(e=gn),c(t)._overflowWeekday&&-1===e&&(e=vn),c(t).overflow=e),t}function se(t,e,i){return null!=t?t:null!=e?e:i}function re(t){var i=new Date(e.now());return t._useUTC?[i.getUTCFullYear(),i.getUTCMonth(),i.getUTCDate()]:[i.getFullYear(),i.getMonth(),i.getDate()]}function ae(t){var e,i,o,n,s=[];if(!t._d){for(o=re(t),t._w&&null==t._a[un]&&null==t._a[ln]&&he(t),null!=t._dayOfYear&&(n=se(t._a[dn],o[dn]),(t._dayOfYear>Q(n)||0===t._dayOfYear)&&(c(t)._overflowDayOfYear=!0),i=_t(n,0,t._dayOfYear),t._a[ln]=i.getUTCMonth(),t._a[un]=i.getUTCDate()),e=0;e<3&&null==t._a[e];++e)t._a[e]=s[e]=o[e];for(;e<7;e++)t._a[e]=s[e]=null==t._a[e]?2===e?1:0:t._a[e];24===t._a[pn]&&0===t._a[cn]&&0===t._a[mn]&&0===t._a[fn]&&(t._nextDay=!0,t._a[pn]=0),t._d=(t._useUTC?_t:bt).apply(null,s),null!=t._tzm&&t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),t._nextDay&&(t._a[pn]=24),t._w&&void 0!==t._w.d&&t._w.d!==t._d.getDay()&&(c(t).weekdayMismatch=!0)}}function he(t){var e,i,o,n,s,r,a,h;if(e=t._w,null!=e.GG||null!=e.W||null!=e.E)s=1,r=4,i=se(e.GG,t._a[dn],Dt(ke(),1,4).year),o=se(e.W,1),((n=se(e.E,1))<1||n>7)&&(h=!0);else{s=t._locale._week.dow,r=t._locale._week.doy;var d=Dt(ke(),s,r);i=se(e.gg,t._a[dn],d.year),o=se(e.w,d.week),null!=e.d?((n=e.d)<0||n>6)&&(h=!0):null!=e.e?(n=e.e+s,(e.e<0||e.e>6)&&(h=!0)):n=s}o<1||o>St(i,s,r)?c(t)._overflowWeeks=!0:null!=h?c(t)._overflowWeekday=!0:(a=xt(i,o,n,s,r),t._a[dn]=a.year,t._dayOfYear=a.dayOfYear)}function de(t){var e,i,o,n,s,r,a=t._i,h=Hn.exec(a)||jn.exec(a);if(h){for(c(t).iso=!0,e=0,i=Gn.length;e<i;e++)if(Gn[e][1].exec(h[1])){n=Gn[e][0],o=!1!==Gn[e][2];break}if(null==n)return void(t._isValid=!1);if(h[3]){for(e=0,i=zn.length;e<i;e++)if(zn[e][1].exec(h[3])){s=(h[2]||" ")+zn[e][0];break}if(null==s)return void(t._isValid=!1)}if(!o&&null!=s)return void(t._isValid=!1);if(h[4]){if(!Yn.exec(h[4]))return void(t._isValid=!1);r="Z"}t._f=n+(s||"")+(r||""),ve(t)}else t._isValid=!1}function le(t,e,i,o,n,s){var r=[ue(t),xn.indexOf(e),parseInt(i,10),parseInt(o,10),parseInt(n,10)];return s&&r.push(parseInt(s,10)),r}function ue(t){var e=parseInt(t,10);return e<=49?2e3+e:e<=999?1900+e:e}function pe(t){return t.replace(/\([^)]*\)|[\n\t]/g," ").replace(/(\s\s+)/g," ").trim()}function ce(t,e,i){if(t){if(Tn.indexOf(t)!==new Date(e[0],e[1],e[2]).getDay())return c(i).weekdayMismatch=!0,i._isValid=!1,!1}return!0}function me(t,e,i){if(t)return Bn[t];if(e)return 0;var o=parseInt(i,10),n=o%100;return(o-n)/100*60+n}function fe(t){var e=Vn.exec(pe(t._i));if(e){var i=le(e[4],e[3],e[2],e[5],e[6],e[7]);if(!ce(e[1],i,t))return;t._a=i,t._tzm=me(e[8],e[9],e[10]),t._d=_t.apply(null,t._a),t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),c(t).rfc2822=!0}else t._isValid=!1}function ge(t){var i=Wn.exec(t._i);if(null!==i)return void(t._d=new Date(+i[1]));de(t),!1===t._isValid&&(delete t._isValid,fe(t),!1===t._isValid&&(delete t._isValid,e.createFromInputFallback(t)))}function ve(t){if(t._f===e.ISO_8601)return void de(t);if(t._f===e.RFC_2822)return void fe(t);t._a=[],c(t).empty=!0;var i,o,n,s,r,a=""+t._i,h=a.length,d=0;for(n=B(t._f,t._locale).match(jo)||[],i=0;i<n.length;i++)s=n[i],o=(a.match(q(s,t))||[])[0],o&&(r=a.substr(0,a.indexOf(o)),r.length>0&&c(t).unusedInput.push(r),a=a.slice(a.indexOf(o)+o.length),d+=o.length),zo[s]?(o?c(t).empty=!1:c(t).unusedTokens.push(s),$(s,o,t)):t._strict&&!o&&c(t).unusedTokens.push(s);c(t).charsLeftOver=h-d,a.length>0&&c(t).unusedInput.push(a),t._a[pn]<=12&&!0===c(t).bigHour&&t._a[pn]>0&&(c(t).bigHour=void 0),c(t).parsedDateParts=t._a.slice(0),c(t).meridiem=t._meridiem,t._a[pn]=ye(t._locale,t._a[pn],t._meridiem),ae(t),ne(t)}function ye(t,e,i){var o;return null==i?e:null!=t.meridiemHour?t.meridiemHour(e,i):null!=t.isPM?(o=t.isPM(i),o&&e<12&&(e+=12),o||12!==e||(e=0),e):e}function be(t){var e,i,o,n,s;if(0===t._f.length)return c(t).invalidFormat=!0,void(t._d=new Date(NaN));for(n=0;n<t._f.length;n++)s=0,e=g({},t),null!=t._useUTC&&(e._useUTC=t._useUTC),e._f=t._f[n],ve(e),m(e)&&(s+=c(e).charsLeftOver,s+=10*c(e).unusedTokens.length,c(e).score=s,(null==o||s<o)&&(o=s,i=e));l(t,i||e)}function _e(t){if(!t._d){var e=F(t._i);t._a=h([e.year,e.month,e.day||e.date,e.hour,e.minute,e.second,e.millisecond],function(t){return t&&parseInt(t,10)}),ae(t)}}function we(t){var e=new v(ne(xe(t)));return e._nextDay&&(e.add(1,"d"),e._nextDay=void 0),e}function xe(t){var e=t._i,o=t._f;return t._locale=t._locale||ie(t._l),null===e||void 0===o&&""===e?f({nullInput:!0}):("string"==typeof e&&(t._i=e=t._locale.preparse(e)),y(e)?new v(ne(e)):(a(e)?t._d=e:i(o)?be(t):o?ve(t):De(t),m(t)||(t._d=null),t))}function De(t){var n=t._i;s(n)?t._d=new Date(e.now()):a(n)?t._d=new Date(n.valueOf()):"string"==typeof n?ge(t):i(n)?(t._a=h(n.slice(0),function(t){return parseInt(t,10)}),ae(t)):o(n)?_e(t):r(n)?t._d=new Date(n):e.createFromInputFallback(t)}function Se(t,e,s,r,a){var h={};return!0!==s&&!1!==s||(r=s,s=void 0),(o(t)&&n(t)||i(t)&&0===t.length)&&(t=void 0),h._isAMomentObject=!0,h._useUTC=h._isUTC=a,h._l=s,h._i=t,h._f=e,h._strict=r,we(h)}function ke(t,e,i,o){return Se(t,e,i,o,!1)}function Ce(t,e){var o,n;if(1===e.length&&i(e[0])&&(e=e[0]),!e.length)return ke();for(o=e[0],n=1;n<e.length;++n)e[n].isValid()&&!e[n][t](o)||(o=e[n]);return o}function Te(){return Ce("isBefore",[].slice.call(arguments,0))}function Me(){return Ce("isAfter",[].slice.call(arguments,0))}function Oe(t){for(var e in t)if(-1===yn.call(Zn,e)||null!=t[e]&&isNaN(t[e]))return!1;for(var i=!1,o=0;o<Zn.length;++o)if(t[Zn[o]]){if(i)return!1;parseFloat(t[Zn[o]])!==_(t[Zn[o]])&&(i=!0)}return!0}function Ee(){return this._isValid}function Pe(){return Ke(NaN)}function Ie(t){var e=F(t),i=e.year||0,o=e.quarter||0,n=e.month||0,s=e.week||0,r=e.day||0,a=e.hour||0,h=e.minute||0,d=e.second||0,l=e.millisecond||0;this._isValid=Oe(e),this._milliseconds=+l+1e3*d+6e4*h+1e3*a*60*60,this._days=+r+7*s,this._months=+n+3*o+12*i,this._data={},this._locale=ie(),this._bubble()}function Ne(t){return t instanceof Ie}function Ae(t){return t<0?-1*Math.round(-1*t):Math.round(t)}function Re(t,e){G(t,0,0,function(){var t=this.utcOffset(),i="+";return t<0&&(t=-t,i="-"),i+Y(~~(t/60),2)+e+Y(~~t%60,2)})}function Le(t,e){var i=(e||"").match(t);if(null===i)return null;var o=i[i.length-1]||[],n=(o+"").match(Kn)||["-",0,0],s=60*n[1]+_(n[2]);return 0===s?0:"+"===n[0]?s:-s}function Fe(t,i){var o,n;return i._isUTC?(o=i.clone(),n=(y(t)||a(t)?t.valueOf():ke(t).valueOf())-o.valueOf(),o._d.setTime(o._d.valueOf()+n),e.updateOffset(o,!1),o):ke(t).local()}function He(t){return 15*-Math.round(t._d.getTimezoneOffset()/15)}function je(t,i,o){var n,s=this._offset||0;if(!this.isValid())return null!=t?this:NaN;if(null!=t){if("string"==typeof t){if(null===(t=Le(nn,t)))return this}else Math.abs(t)<16&&!o&&(t*=60);return!this._isUTC&&i&&(n=He(this)),this._offset=t,this._isUTC=!0,null!=n&&this.add(n,"m"),s!==t&&(!i||this._changeInProgress?ei(this,Ke(t-s,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,e.updateOffset(this,!0),this._changeInProgress=null)),this}return this._isUTC?s:He(this)}function Ye(t,e){return null!=t?("string"!=typeof t&&(t=-t),this.utcOffset(t,e),this):-this.utcOffset()}function Ge(t){return this.utcOffset(0,t)}function ze(t){return this._isUTC&&(this.utcOffset(0,t),this._isUTC=!1,t&&this.subtract(He(this),"m")),this}function We(){if(null!=this._tzm)this.utcOffset(this._tzm,!1,!0);else if("string"==typeof this._i){var t=Le(on,this._i);null!=t?this.utcOffset(t):this.utcOffset(0,!0)}return this}function Ve(t){return!!this.isValid()&&(t=t?ke(t).utcOffset():0,(this.utcOffset()-t)%60==0)}function Be(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()}function Ue(){if(!s(this._isDSTShifted))return this._isDSTShifted;var t={};if(g(t,this),t=xe(t),t._a){var e=t._isUTC?u(t._a):ke(t._a);this._isDSTShifted=this.isValid()&&w(t._a,e.toArray())>0}else this._isDSTShifted=!1;return this._isDSTShifted}function qe(){return!!this.isValid()&&!this._isUTC}function Xe(){return!!this.isValid()&&this._isUTC}function Ze(){return!!this.isValid()&&(this._isUTC&&0===this._offset)}function Ke(t,e){var i,o,n,s=t,a=null;return Ne(t)?s={ms:t._milliseconds,d:t._days,M:t._months}:r(t)?(s={},e?s[e]=t:s.milliseconds=t):(a=Jn.exec(t))?(i="-"===a[1]?-1:1,s={y:0,d:_(a[un])*i,h:_(a[pn])*i,m:_(a[cn])*i,s:_(a[mn])*i,ms:_(Ae(1e3*a[fn]))*i}):(a=$n.exec(t))?(i="-"===a[1]?-1:(a[1],1),s={y:Je(a[2],i),M:Je(a[3],i),w:Je(a[4],i),d:Je(a[5],i),h:Je(a[6],i),m:Je(a[7],i),s:Je(a[8],i)}):null==s?s={}:"object"==typeof s&&("from"in s||"to"in s)&&(n=Qe(ke(s.from),ke(s.to)),s={},s.ms=n.milliseconds,s.M=n.months),o=new Ie(s),Ne(t)&&d(t,"_locale")&&(o._locale=t._locale),o}function Je(t,e){var i=t&&parseFloat(t.replace(",","."));return(isNaN(i)?0:i)*e}function $e(t,e){var i={milliseconds:0,months:0};return i.months=e.month()-t.month()+12*(e.year()-t.year()),t.clone().add(i.months,"M").isAfter(e)&&--i.months,i.milliseconds=+e-+t.clone().add(i.months,"M"),i}function Qe(t,e){var i;return t.isValid()&&e.isValid()?(e=Fe(e,t),t.isBefore(e)?i=$e(t,e):(i=$e(e,t),i.milliseconds=-i.milliseconds,i.months=-i.months),i):{milliseconds:0,months:0}}function ti(t,e){return function(i,o){var n,s;return null===o||isNaN(+o)||(S(e,"moment()."+e+"(period, number) is deprecated. Please use moment()."+e+"(number, period). See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info."),s=i,i=o,o=s),i="string"==typeof i?+i:i,n=Ke(i,o),ei(this,n,t),this}}function ei(t,i,o,n){var s=i._milliseconds,r=Ae(i._days),a=Ae(i._months);t.isValid()&&(n=null==n||n,a&&ct(t,ot(t,"Month")+a*o),r&&nt(t,"Date",ot(t,"Date")+r*o),s&&t._d.setTime(t._d.valueOf()+s*o),n&&e.updateOffset(t,r||a))}function ii(t,e){var i=t.diff(e,"days",!0);return i<-6?"sameElse":i<-1?"lastWeek":i<0?"lastDay":i<1?"sameDay":i<2?"nextDay":i<7?"nextWeek":"sameElse"}function oi(t,i){var o=t||ke(),n=Fe(o,this).startOf("day"),s=e.calendarFormat(this,n)||"sameElse",r=i&&(k(i[s])?i[s].call(this,o):i[s]);return this.format(r||this.localeData().calendar(s,this,ke(o)))}function ni(){return new v(this)}function si(t,e){var i=y(t)?t:ke(t);return!(!this.isValid()||!i.isValid())&&(e=L(s(e)?"millisecond":e),"millisecond"===e?this.valueOf()>i.valueOf():i.valueOf()<this.clone().startOf(e).valueOf())}function ri(t,e){var i=y(t)?t:ke(t);return!(!this.isValid()||!i.isValid())&&(e=L(s(e)?"millisecond":e),"millisecond"===e?this.valueOf()<i.valueOf():this.clone().endOf(e).valueOf()<i.valueOf())}function ai(t,e,i,o){return o=o||"()",("("===o[0]?this.isAfter(t,i):!this.isBefore(t,i))&&(")"===o[1]?this.isBefore(e,i):!this.isAfter(e,i))}function hi(t,e){var i,o=y(t)?t:ke(t);return!(!this.isValid()||!o.isValid())&&(e=L(e||"millisecond"),"millisecond"===e?this.valueOf()===o.valueOf():(i=o.valueOf(),this.clone().startOf(e).valueOf()<=i&&i<=this.clone().endOf(e).valueOf()))}function di(t,e){return this.isSame(t,e)||this.isAfter(t,e)}function li(t,e){return this.isSame(t,e)||this.isBefore(t,e)}function ui(t,e,i){var o,n,s;if(!this.isValid())return NaN;if(o=Fe(t,this),!o.isValid())return NaN;switch(n=6e4*(o.utcOffset()-this.utcOffset()),e=L(e)){case"year":s=pi(this,o)/12;break;case"month":s=pi(this,o);break;case"quarter":s=pi(this,o)/3;break;case"second":s=(this-o)/1e3;break;case"minute":s=(this-o)/6e4;break;case"hour":s=(this-o)/36e5;break;case"day":s=(this-o-n)/864e5;break;case"week":s=(this-o-n)/6048e5;break;default:s=this-o}return i?s:b(s)}function pi(t,e){var i,o,n=12*(e.year()-t.year())+(e.month()-t.month()),s=t.clone().add(n,"months");return e-s<0?(i=t.clone().add(n-1,"months"),o=(e-s)/(s-i)):(i=t.clone().add(n+1,"months"),o=(e-s)/(i-s)),-(n+o)||0}function ci(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")}function mi(){if(!this.isValid())return null;var t=this.clone().utc();return t.year()<0||t.year()>9999?V(t,"YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]"):k(Date.prototype.toISOString)?this.toDate().toISOString():V(t,"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]")}function fi(){if(!this.isValid())return"moment.invalid(/* "+this._i+" */)";var t="moment",e="";this.isLocal()||(t=0===this.utcOffset()?"moment.utc":"moment.parseZone",e="Z");var i="["+t+'("]',o=0<=this.year()&&this.year()<=9999?"YYYY":"YYYYYY",n=e+'[")]';return this.format(i+o+"-MM-DD[T]HH:mm:ss.SSS"+n)}function gi(t){t||(t=this.isUtc()?e.defaultFormatUtc:e.defaultFormat);var i=V(this,t);return this.localeData().postformat(i)}function vi(t,e){return this.isValid()&&(y(t)&&t.isValid()||ke(t).isValid())?Ke({to:this,from:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()}function yi(t){return this.from(ke(),t)}function bi(t,e){return this.isValid()&&(y(t)&&t.isValid()||ke(t).isValid())?Ke({from:this,to:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()}function _i(t){return this.to(ke(),t)}function wi(t){var e;return void 0===t?this._locale._abbr:(e=ie(t),null!=e&&(this._locale=e),this)}function xi(){return this._locale}function Di(t){switch(t=L(t)){case"year":this.month(0);case"quarter":case"month":this.date(1);case"week":case"isoWeek":case"day":case"date":this.hours(0);case"hour":this.minutes(0);case"minute":this.seconds(0);case"second":this.milliseconds(0)}return"week"===t&&this.weekday(0),"isoWeek"===t&&this.isoWeekday(1),"quarter"===t&&this.month(3*Math.floor(this.month()/3)),this}function Si(t){return void 0===(t=L(t))||"millisecond"===t?this:("date"===t&&(t="day"),this.startOf(t).add(1,"isoWeek"===t?"week":t).subtract(1,"ms"))}function ki(){return this._d.valueOf()-6e4*(this._offset||0)}function Ci(){return Math.floor(this.valueOf()/1e3)}function Ti(){return new Date(this.valueOf())}function Mi(){var t=this;return[t.year(),t.month(),t.date(),t.hour(),t.minute(),t.second(),t.millisecond()]}function Oi(){var t=this;return{years:t.year(),months:t.month(),date:t.date(),hours:t.hours(),minutes:t.minutes(),seconds:t.seconds(),milliseconds:t.milliseconds()}}function Ei(){return this.isValid()?this.toISOString():null}function Pi(){return m(this)}function Ii(){return l({},c(this))}function Ni(){return c(this).overflow}function Ai(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}}function Ri(t,e){G(0,[t,t.length],0,e)}function Li(t){return Yi.call(this,t,this.week(),this.weekday(),this.localeData()._week.dow,this.localeData()._week.doy)}function Fi(t){return Yi.call(this,t,this.isoWeek(),this.isoWeekday(),1,4)}function Hi(){return St(this.year(),1,4)}function ji(){var t=this.localeData()._week;return St(this.year(),t.dow,t.doy)}function Yi(t,e,i,o,n){var s;return null==t?Dt(this,o,n).year:(s=St(t,o,n),e>s&&(e=s),Gi.call(this,t,e,i,o,n))}function Gi(t,e,i,o,n){var s=xt(t,e,i,o,n),r=_t(s.year,0,s.dayOfYear);return this.year(r.getUTCFullYear()),this.month(r.getUTCMonth()),this.date(r.getUTCDate()),this}function zi(t){return null==t?Math.ceil((this.month()+1)/3):this.month(3*(t-1)+this.month()%3)}function Wi(t){var e=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==t?e:this.add(t-e,"d")}function Vi(t,e){e[fn]=_(1e3*("0."+t))}function Bi(){return this._isUTC?"UTC":""}function Ui(){return this._isUTC?"Coordinated Universal Time":""}function qi(t){return ke(1e3*t)}function Xi(){return ke.apply(null,arguments).parseZone()}function Zi(t){return t}function Ki(t,e,i,o){var n=ie(),s=u().set(o,e);return n[i](s,t)}function Ji(t,e,i){if(r(t)&&(e=t,t=void 0),t=t||"",null!=e)return Ki(t,e,i,"month");var o,n=[];for(o=0;o<12;o++)n[o]=Ki(t,o,i,"month");return n}function $i(t,e,i,o){"boolean"==typeof t?(r(e)&&(i=e,e=void 0),e=e||""):(e=t,i=e,t=!1,r(e)&&(i=e,e=void 0),e=e||"");var n=ie(),s=t?n._week.dow:0;if(null!=i)return Ki(e,(i+s)%7,o,"day");var a,h=[];for(a=0;a<7;a++)h[a]=Ki(e,(a+s)%7,o,"day");return h}function Qi(t,e){return Ji(t,e,"months")}function to(t,e){return Ji(t,e,"monthsShort")}function eo(t,e,i){return $i(t,e,i,"weekdays")}function io(t,e,i){return $i(t,e,i,"weekdaysShort")}function oo(t,e,i){return $i(t,e,i,"weekdaysMin")}function no(){var t=this._data;return this._milliseconds=ds(this._milliseconds),this._days=ds(this._days),this._months=ds(this._months),t.milliseconds=ds(t.milliseconds),t.seconds=ds(t.seconds),t.minutes=ds(t.minutes),t.hours=ds(t.hours),t.months=ds(t.months),t.years=ds(t.years),this}function so(t,e,i,o){var n=Ke(e,i);return t._milliseconds+=o*n._milliseconds,t._days+=o*n._days,t._months+=o*n._months,t._bubble()}function ro(t,e){return so(this,t,e,1)}function ao(t,e){return so(this,t,e,-1)}function ho(t){return t<0?Math.floor(t):Math.ceil(t)}function lo(){var t,e,i,o,n,s=this._milliseconds,r=this._days,a=this._months,h=this._data;return s>=0&&r>=0&&a>=0||s<=0&&r<=0&&a<=0||(s+=864e5*ho(po(a)+r),r=0,a=0),h.milliseconds=s%1e3,t=b(s/1e3),h.seconds=t%60,e=b(t/60),h.minutes=e%60,i=b(e/60),h.hours=i%24,r+=b(i/24),n=b(uo(r)),a+=n,r-=ho(po(n)),o=b(a/12),a%=12,h.days=r,h.months=a,h.years=o,this}function uo(t){return 4800*t/146097}function po(t){return 146097*t/4800}function co(t){if(!this.isValid())return NaN;var e,i,o=this._milliseconds;if("month"===(t=L(t))||"year"===t)return e=this._days+o/864e5,i=this._months+uo(e),"month"===t?i:i/12;switch(e=this._days+Math.round(po(this._months)),t){case"week":return e/7+o/6048e5;case"day":return e+o/864e5;case"hour":return 24*e+o/36e5;case"minute":return 1440*e+o/6e4;case"second":return 86400*e+o/1e3;case"millisecond":return Math.floor(864e5*e)+o;default:throw new Error("Unknown unit "+t)}}function mo(){return this.isValid()?this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*_(this._months/12):NaN}function fo(t){return function(){return this.as(t)}}function go(){return Ke(this)}function vo(t){return t=L(t),this.isValid()?this[t+"s"]():NaN}function yo(t){return function(){return this.isValid()?this._data[t]:NaN}}function bo(){return b(this.days()/7)}function _o(t,e,i,o,n){return n.relativeTime(e||1,!!i,t,o)}function wo(t,e,i){var o=Ke(t).abs(),n=ks(o.as("s")),s=ks(o.as("m")),r=ks(o.as("h")),a=ks(o.as("d")),h=ks(o.as("M")),d=ks(o.as("y")),l=n<=Cs.ss&&["s",n]||n<Cs.s&&["ss",n]||s<=1&&["m"]||s<Cs.m&&["mm",s]||r<=1&&["h"]||r<Cs.h&&["hh",r]||a<=1&&["d"]||a<Cs.d&&["dd",a]||h<=1&&["M"]||h<Cs.M&&["MM",h]||d<=1&&["y"]||["yy",d];return l[2]=e,l[3]=+t>0,l[4]=i,_o.apply(null,l)}function xo(t){return void 0===t?ks:"function"==typeof t&&(ks=t,!0)}function Do(t,e){return void 0!==Cs[t]&&(void 0===e?Cs[t]:(Cs[t]=e,"s"===t&&(Cs.ss=e-1),!0))}function So(t){if(!this.isValid())return this.localeData().invalidDate();var e=this.localeData(),i=wo(this,!t,e);return t&&(i=e.pastFuture(+this,i)),e.postformat(i)}function ko(t){return(t>0)-(t<0)||+t}function Co(){if(!this.isValid())return this.localeData().invalidDate();var t,e,i,o=Ts(this._milliseconds)/1e3,n=Ts(this._days),s=Ts(this._months);t=b(o/60),e=b(t/60),o%=60,t%=60,i=b(s/12),s%=12;var r=i,a=s,h=n,d=e,l=t,u=o?o.toFixed(3).replace(/\.?0+$/,""):"",p=this.asSeconds();if(!p)return"P0D";var c=p<0?"-":"",m=ko(this._months)!==ko(p)?"-":"",f=ko(this._days)!==ko(p)?"-":"",g=ko(this._milliseconds)!==ko(p)?"-":"";return c+"P"+(r?m+r+"Y":"")+(a?m+a+"M":"")+(h?f+h+"D":"")+(d||l||u?"T":"")+(d?g+d+"H":"")+(l?g+l+"M":"")+(u?g+u+"S":"")}var To,Mo;Mo=Array.prototype.some?Array.prototype.some:function(t){for(var e=Object(this),i=e.length>>>0,o=0;o<i;o++)if(o in e&&t.call(this,e[o],o,e))return!0;return!1};var Oo=e.momentProperties=[],Eo=!1,Po={};e.suppressDeprecationWarnings=!1,e.deprecationHandler=null;var Io;Io=Object.keys?Object.keys:function(t){var e,i=[];for(e in t)d(t,e)&&i.push(e);return i};var No={sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},Ao={LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},Ro=/\d{1,2}/,Lo={future:"in %s",past:"%s ago",s:"a few seconds",ss:"%d seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},Fo={},Ho={},jo=/(\[[^\[]*\])|(\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,Yo=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,Go={},zo={},Wo=/\d/,Vo=/\d\d/,Bo=/\d{3}/,Uo=/\d{4}/,qo=/[+-]?\d{6}/,Xo=/\d\d?/,Zo=/\d\d\d\d?/,Ko=/\d\d\d\d\d\d?/,Jo=/\d{1,3}/,$o=/\d{1,4}/,Qo=/[+-]?\d{1,6}/,tn=/\d+/,en=/[+-]?\d+/,on=/Z|[+-]\d\d:?\d\d/gi,nn=/Z|[+-]\d\d(?::?\d\d)?/gi,sn=/[+-]?\d+(\.\d{1,3})?/,rn=/[0-9]*['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]+|[\u0600-\u06FF\/]+(\s*?[\u0600-\u06FF]+){1,2}/i,an={},hn={},dn=0,ln=1,un=2,pn=3,cn=4,mn=5,fn=6,gn=7,vn=8;G("Y",0,0,function(){var t=this.year();return t<=9999?""+t:"+"+t}),G(0,["YY",2],0,function(){return this.year()%100}),G(0,["YYYY",4],0,"year"),G(0,["YYYYY",5],0,"year"),G(0,["YYYYYY",6,!0],0,"year"),R("year","y"),H("year",1),U("Y",en),U("YY",Xo,Vo),U("YYYY",$o,Uo),U("YYYYY",Qo,qo),U("YYYYYY",Qo,qo),K(["YYYYY","YYYYYY"],dn),K("YYYY",function(t,i){i[dn]=2===t.length?e.parseTwoDigitYear(t):_(t)}),K("YY",function(t,i){i[dn]=e.parseTwoDigitYear(t)}),K("Y",function(t,e){e[dn]=parseInt(t,10)}),e.parseTwoDigitYear=function(t){return _(t)+(_(t)>68?1900:2e3)};var yn,bn=it("FullYear",!0);yn=Array.prototype.indexOf?Array.prototype.indexOf:function(t){var e;for(e=0;e<this.length;++e)if(this[e]===t)return e;return-1},G("M",["MM",2],"Mo",function(){return this.month()+1}),G("MMM",0,0,function(t){return this.localeData().monthsShort(this,t)}),G("MMMM",0,0,function(t){return this.localeData().months(this,t)}),R("month","M"),H("month",8),U("M",Xo),U("MM",Xo,Vo),U("MMM",function(t,e){return e.monthsShortRegex(t)}),U("MMMM",function(t,e){return e.monthsRegex(t)}),K(["M","MM"],function(t,e){e[ln]=_(t)-1}),K(["MMM","MMMM"],function(t,e,i,o){var n=i._locale.monthsParse(t,o,i._strict);null!=n?e[ln]=n:c(i).invalidMonth=t});var _n=/D[oD]?(\[[^\[\]]*\]|\s)+MMMM?/,wn="January_February_March_April_May_June_July_August_September_October_November_December".split("_"),xn="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_"),Dn=rn,Sn=rn;G("w",["ww",2],"wo","week"),G("W",["WW",2],"Wo","isoWeek"),R("week","w"),R("isoWeek","W"),H("week",5),H("isoWeek",5),U("w",Xo),U("ww",Xo,Vo),U("W",Xo),U("WW",Xo,Vo),J(["w","ww","W","WW"],function(t,e,i,o){e[o.substr(0,1)]=_(t)});var kn={dow:0,doy:6};G("d",0,"do","day"),G("dd",0,0,function(t){return this.localeData().weekdaysMin(this,t)}),G("ddd",0,0,function(t){return this.localeData().weekdaysShort(this,t)}),G("dddd",0,0,function(t){return this.localeData().weekdays(this,t)}),G("e",0,0,"weekday"),G("E",0,0,"isoWeekday"),R("day","d"),R("weekday","e"),R("isoWeekday","E"),H("day",11),H("weekday",11),H("isoWeekday",11),U("d",Xo),U("e",Xo),U("E",Xo),U("dd",function(t,e){return e.weekdaysMinRegex(t)}),U("ddd",function(t,e){return e.weekdaysShortRegex(t)}),U("dddd",function(t,e){return e.weekdaysRegex(t)}),J(["dd","ddd","dddd"],function(t,e,i,o){var n=i._locale.weekdaysParse(t,o,i._strict);null!=n?e.d=n:c(i).invalidWeekday=t}),J(["d","e","E"],function(t,e,i,o){e[o]=_(t)});var Cn="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),Tn="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),Mn="Su_Mo_Tu_We_Th_Fr_Sa".split("_"),On=rn,En=rn,Pn=rn;G("H",["HH",2],0,"hour"),G("h",["hh",2],0,Vt),G("k",["kk",2],0,Bt),G("hmm",0,0,function(){return""+Vt.apply(this)+Y(this.minutes(),2)}),G("hmmss",0,0,function(){return""+Vt.apply(this)+Y(this.minutes(),2)+Y(this.seconds(),2)}),G("Hmm",0,0,function(){return""+this.hours()+Y(this.minutes(),2)}),G("Hmmss",0,0,function(){return""+this.hours()+Y(this.minutes(),2)+Y(this.seconds(),2)}),Ut("a",!0),Ut("A",!1),R("hour","h"),H("hour",13),U("a",qt),U("A",qt),U("H",Xo),U("h",Xo),U("k",Xo),U("HH",Xo,Vo),U("hh",Xo,Vo),U("kk",Xo,Vo),U("hmm",Zo),U("hmmss",Ko),U("Hmm",Zo),U("Hmmss",Ko),K(["H","HH"],pn),K(["k","kk"],function(t,e,i){var o=_(t);e[pn]=24===o?0:o}),K(["a","A"],function(t,e,i){i._isPm=i._locale.isPM(t),i._meridiem=t}),K(["h","hh"],function(t,e,i){e[pn]=_(t),c(i).bigHour=!0}),K("hmm",function(t,e,i){var o=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o)),c(i).bigHour=!0}),K("hmmss",function(t,e,i){var o=t.length-4,n=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o,2)),e[mn]=_(t.substr(n)),c(i).bigHour=!0}),K("Hmm",function(t,e,i){var o=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o))}),K("Hmmss",function(t,e,i){var o=t.length-4,n=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o,2)),e[mn]=_(t.substr(n))});var In,Nn=/[ap]\.?m?\.?/i,An=it("Hours",!0),Rn={calendar:No,longDateFormat:Ao,invalidDate:"Invalid date",ordinal:"%d",dayOfMonthOrdinalParse:Ro,relativeTime:Lo,months:wn,monthsShort:xn,week:kn,weekdays:Cn,weekdaysMin:Mn,weekdaysShort:Tn,meridiemParse:Nn},Ln={},Fn={},Hn=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,jn=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,Yn=/Z|[+-]\d\d(?::?\d\d)?/,Gn=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/]],zn=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],Wn=/^\/?Date\((\-?\d+)/i,Vn=/^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\s)?(\d{1,2})\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s(\d{2,4})\s(\d\d):(\d\d)(?::(\d\d))?\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\d{4}))$/,Bn={UT:0,GMT:0,EDT:-240,EST:-300,CDT:-300,CST:-360,MDT:-360,MST:-420,PDT:-420,PST:-480};e.createFromInputFallback=D("value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are discouraged and will be removed in an upcoming major release. Please refer to http://momentjs.com/guides/#/warnings/js-date/ for more info.",function(t){t._d=new Date(t._i+(t._useUTC?" UTC":""))}),e.ISO_8601=function(){},e.RFC_2822=function(){};var Un=D("moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/",function(){var t=ke.apply(null,arguments);return this.isValid()&&t.isValid()?t<this?this:t:f()}),qn=D("moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/",function(){var t=ke.apply(null,arguments);return this.isValid()&&t.isValid()?t>this?this:t:f()}),Xn=function(){return Date.now?Date.now():+new Date},Zn=["year","quarter","month","week","day","hour","minute","second","millisecond"];Re("Z",":"),Re("ZZ",""),U("Z",nn),U("ZZ",nn),K(["Z","ZZ"],function(t,e,i){i._useUTC=!0,i._tzm=Le(nn,t)});var Kn=/([\+\-]|\d\d)/gi;e.updateOffset=function(){};var Jn=/^(\-|\+)?(?:(\d*)[. ])?(\d+)\:(\d+)(?:\:(\d+)(\.\d*)?)?$/,$n=/^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;Ke.fn=Ie.prototype,Ke.invalid=Pe;var Qn=ti(1,"add"),ts=ti(-1,"subtract");e.defaultFormat="YYYY-MM-DDTHH:mm:ssZ",e.defaultFormatUtc="YYYY-MM-DDTHH:mm:ss[Z]";var es=D("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",function(t){return void 0===t?this.localeData():this.locale(t)});G(0,["gg",2],0,function(){return this.weekYear()%100}),G(0,["GG",2],0,function(){return this.isoWeekYear()%100}),Ri("gggg","weekYear"),Ri("ggggg","weekYear"),Ri("GGGG","isoWeekYear"),Ri("GGGGG","isoWeekYear"),R("weekYear","gg"),R("isoWeekYear","GG"),H("weekYear",1),H("isoWeekYear",1),U("G",en),U("g",en),U("GG",Xo,Vo),U("gg",Xo,Vo),U("GGGG",$o,Uo),U("gggg",$o,Uo),U("GGGGG",Qo,qo),U("ggggg",Qo,qo),J(["gggg","ggggg","GGGG","GGGGG"],function(t,e,i,o){e[o.substr(0,2)]=_(t)}),J(["gg","GG"],function(t,i,o,n){i[n]=e.parseTwoDigitYear(t)}),G("Q",0,"Qo","quarter"),R("quarter","Q"),H("quarter",7),U("Q",Wo),K("Q",function(t,e){e[ln]=3*(_(t)-1)}),G("D",["DD",2],"Do","date"),R("date","D"),H("date",9),U("D",Xo),U("DD",Xo,Vo),U("Do",function(t,e){return t?e._dayOfMonthOrdinalParse||e._ordinalParse:e._dayOfMonthOrdinalParseLenient}),K(["D","DD"],un),K("Do",function(t,e){e[un]=_(t.match(Xo)[0],10)});var is=it("Date",!0);G("DDD",["DDDD",3],"DDDo","dayOfYear"),R("dayOfYear","DDD"),H("dayOfYear",4),U("DDD",Jo),U("DDDD",Bo),K(["DDD","DDDD"],function(t,e,i){i._dayOfYear=_(t)}),G("m",["mm",2],0,"minute"),R("minute","m"),H("minute",14),U("m",Xo),U("mm",Xo,Vo),K(["m","mm"],cn);var os=it("Minutes",!1);G("s",["ss",2],0,"second"),R("second","s"),H("second",15),U("s",Xo),U("ss",Xo,Vo),K(["s","ss"],mn);var ns=it("Seconds",!1);G("S",0,0,function(){return~~(this.millisecond()/100)}),G(0,["SS",2],0,function(){return~~(this.millisecond()/10)}),G(0,["SSS",3],0,"millisecond"),G(0,["SSSS",4],0,function(){return 10*this.millisecond()}),G(0,["SSSSS",5],0,function(){return 100*this.millisecond()}),G(0,["SSSSSS",6],0,function(){return 1e3*this.millisecond()}),G(0,["SSSSSSS",7],0,function(){return 1e4*this.millisecond()}),G(0,["SSSSSSSS",8],0,function(){return 1e5*this.millisecond()}),G(0,["SSSSSSSSS",9],0,function(){return 1e6*this.millisecond()}),R("millisecond","ms"),H("millisecond",16),U("S",Jo,Wo),U("SS",Jo,Vo),U("SSS",Jo,Bo);var ss;for(ss="SSSS";ss.length<=9;ss+="S")U(ss,tn);for(ss="S";ss.length<=9;ss+="S")K(ss,Vi);var rs=it("Milliseconds",!1);G("z",0,0,"zoneAbbr"),G("zz",0,0,"zoneName");var as=v.prototype;as.add=Qn,as.calendar=oi,as.clone=ni,as.diff=ui,as.endOf=Si,as.format=gi,as.from=vi,as.fromNow=yi,as.to=bi,as.toNow=_i,as.get=st,as.invalidAt=Ni,as.isAfter=si,as.isBefore=ri,as.isBetween=ai,as.isSame=hi,as.isSameOrAfter=di,as.isSameOrBefore=li,as.isValid=Pi,as.lang=es,as.locale=wi,as.localeData=xi,as.max=qn,as.min=Un,as.parsingFlags=Ii,as.set=rt,as.startOf=Di,as.subtract=ts,as.toArray=Mi,as.toObject=Oi,as.toDate=Ti,as.toISOString=mi,as.inspect=fi,as.toJSON=Ei,as.toString=ci,as.unix=Ci,as.valueOf=ki,as.creationData=Ai,as.year=bn,as.isLeapYear=et,as.weekYear=Li,as.isoWeekYear=Fi,as.quarter=as.quarters=zi,as.month=mt,as.daysInMonth=ft,as.week=as.weeks=Mt,as.isoWeek=as.isoWeeks=Ot,as.weeksInYear=ji,as.isoWeeksInYear=Hi,as.date=is,as.day=as.days=Ft,as.weekday=Ht,as.isoWeekday=jt,as.dayOfYear=Wi,as.hour=as.hours=An,as.minute=as.minutes=os,as.second=as.seconds=ns,as.millisecond=as.milliseconds=rs,as.utcOffset=je,as.utc=Ge,as.local=ze,as.parseZone=We,as.hasAlignedHourOffset=Ve,as.isDST=Be,as.isLocal=qe,as.isUtcOffset=Xe,as.isUtc=Ze,as.isUTC=Ze,as.zoneAbbr=Bi,as.zoneName=Ui,as.dates=D("dates accessor is deprecated. Use date instead.",is),as.months=D("months accessor is deprecated. Use month instead",mt),as.years=D("years accessor is deprecated. Use year instead",bn),as.zone=D("moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/",Ye),as.isDSTShifted=D("isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information",Ue);var hs=M.prototype;hs.calendar=O,hs.longDateFormat=E,hs.invalidDate=P,hs.ordinal=I,hs.preparse=Zi,hs.postformat=Zi,hs.relativeTime=N,hs.pastFuture=A,hs.set=C,hs.months=dt,hs.monthsShort=lt,hs.monthsParse=pt,hs.monthsRegex=vt,hs.monthsShortRegex=gt,hs.week=kt,hs.firstDayOfYear=Tt,hs.firstDayOfWeek=Ct,hs.weekdays=It,hs.weekdaysMin=At,hs.weekdaysShort=Nt,hs.weekdaysParse=Lt,hs.weekdaysRegex=Yt,hs.weekdaysShortRegex=Gt,hs.weekdaysMinRegex=zt,hs.isPM=Xt,hs.meridiem=Zt,Qt("en",{dayOfMonthOrdinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(t){var e=t%10;return t+(1===_(t%100/10)?"th":1===e?"st":2===e?"nd":3===e?"rd":"th")}}),e.lang=D("moment.lang is deprecated. Use moment.locale instead.",Qt),e.langData=D("moment.langData is deprecated. Use moment.localeData instead.",ie);var ds=Math.abs,ls=fo("ms"),us=fo("s"),ps=fo("m"),cs=fo("h"),ms=fo("d"),fs=fo("w"),gs=fo("M"),vs=fo("y"),ys=yo("milliseconds"),bs=yo("seconds"),_s=yo("minutes"),ws=yo("hours"),xs=yo("days"),Ds=yo("months"),Ss=yo("years"),ks=Math.round,Cs={ss:44,s:45,m:45,h:22,d:26,M:11},Ts=Math.abs,Ms=Ie.prototype;return Ms.isValid=Ee,Ms.abs=no,Ms.add=ro,Ms.subtract=ao,Ms.as=co,Ms.asMilliseconds=ls,Ms.asSeconds=us,Ms.asMinutes=ps,Ms.asHours=cs,Ms.asDays=ms,Ms.asWeeks=fs,Ms.asMonths=gs,Ms.asYears=vs,Ms.valueOf=mo,Ms._bubble=lo,Ms.clone=go,Ms.get=vo,Ms.milliseconds=ys,Ms.seconds=bs,Ms.minutes=_s,Ms.hours=ws,Ms.days=xs,Ms.weeks=bo,Ms.months=Ds,Ms.years=Ss,Ms.humanize=So,Ms.toISOString=Co,Ms.toString=Co,Ms.toJSON=Co,Ms.locale=wi,Ms.localeData=xi,Ms.toIsoString=D("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",Co),Ms.lang=es,G("X",0,0,"unix"),G("x",0,0,"valueOf"),U("x",en),U("X",sn),K("X",function(t,e,i){i._d=new Date(1e3*parseFloat(t,10))}),K("x",function(t,e,i){i._d=new Date(_(t))}),e.version="2.19.1",function(t){To=t}(ke),e.fn=as,e.min=Te,e.max=Me,e.now=Xn,e.utc=u,e.unix=qi,e.months=Qi,e.isDate=a,e.locale=Qt,e.invalid=f,e.duration=Ke,e.isMoment=y,e.weekdays=eo,e.parseZone=Xi,e.localeData=ie,e.isDuration=Ne,e.monthsShort=to,e.weekdaysMin=oo,e.defineLocale=te,e.updateLocale=ee,e.locales=oe,e.weekdaysShort=io,e.normalizeUnits=L,e.relativeTimeRounding=xo,e.relativeTimeThreshold=Do,e.calendarFormat=ii,e.prototype=as,e})}).call(e,i(124)(t))},function(t,e){t.exports=function(t){return t.webpackPolyfill||(t.deprecate=function(){},t.paths=[],t.children||(t.children=[]),Object.defineProperty(t,"loaded",{enumerable:!0,get:function(){return t.l}}),Object.defineProperty(t,"id",{enumerable:!0,get:function(){return t.i}}),t.webpackPolyfill=1),t}},function(t,e){function i(t){throw new Error("Cannot find module '"+t+"'.")}i.keys=function(){return[]},i.resolve=i,t.exports=i,i.id=125},function(t,e,i){(function(e){function i(t,e,i){var o=e&&i||0,n=0;for(e=e||[],t.toLowerCase().replace(/[0-9a-f]{2}/g,function(t){n<16&&(e[o+n++]=u[t])});n<16;)e[o+n++]=0;return e}function o(t,e){var i=e||0,o=l;return o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]}function n(t,e,i){var n=e&&i||0,s=e||[];t=t||{};var r=void 0!==t.clockseq?t.clockseq:f,a=void 0!==t.msecs?t.msecs:(new Date).getTime(),h=void 0!==t.nsecs?t.nsecs:v+1,d=a-g+(h-v)/1e4;if(d<0&&void 0===t.clockseq&&(r=r+1&16383),(d<0||a>g)&&void 0===t.nsecs&&(h=0),h>=1e4)throw new Error("uuid.v1(): Can't create more than 10M uuids/sec");g=a,v=h,f=r,a+=122192928e5;var l=(1e4*(268435455&a)+h)%4294967296;s[n++]=l>>>24&255,s[n++]=l>>>16&255,s[n++]=l>>>8&255,s[n++]=255&l;var u=a/4294967296*1e4&268435455;s[n++]=u>>>8&255,s[n++]=255&u,s[n++]=u>>>24&15|16,s[n++]=u>>>16&255,s[n++]=r>>>8|128,s[n++]=255&r;for(var p=t.node||m,c=0;c<6;c++)s[n+c]=p[c];return e||o(s)}function s(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"==t?new Array(16):null,t=null),t=t||{};var s=t.random||(t.rng||r)();if(s[6]=15&s[6]|64,s[8]=63&s[8]|128,e)for(var a=0;a<16;a++)e[n+a]=s[a];return e||o(s)}var r,a="undefined"!=typeof window?window:void 0!==e?e:null;if(a&&a.crypto&&crypto.getRandomValues){var h=new Uint8Array(16);r=function(){return crypto.getRandomValues(h),h}}if(!r){var d=new Array(16);r=function(){for(var t,e=0;e<16;e++)0==(3&e)&&(t=4294967296*Math.random()),d[e]=t>>>((3&e)<<3)&255;return d}}for(var l=[],u={},p=0;p<256;p++)l[p]=(p+256).toString(16).substr(1),u[l[p]]=p;var c=r(),m=[1|c[0],c[1],c[2],c[3],c[4],c[5]],f=16383&(c[6]<<8|c[7]),g=0,v=0,y=s;y.v1=n,y.v4=s,y.parse=i,y.unparse=o,t.exports=y}).call(e,i(127))},function(t,e){var i;i=function(){return this}();try{i=i||Function("return this")()||(0,eval)("this")}catch(t){"object"==typeof window&&(i=window)}t.exports=i},function(t,e,i){var o=i(2),n=o.JSON||(o.JSON={stringify:JSON.stringify});t.exports=function(t){return n.stringify.apply(n,arguments)}},function(t,e,i){function o(t,e,i,n){if(!(this instanceof o))throw new SyntaxError("Constructor must be called with the new operator");if(!(Array.isArray(i)||i instanceof d||i instanceof l)&&i instanceof Object){var s=n;n=i,i=s}n&&n.throttleRedraw&&console.warn('Timeline option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var r=this;if(this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:a,width:null,height:null,maxHeight:null,minHeight:null},this.options=h.deepExtend({},this.defaultOptions),this._create(t),!n||n&&void 0===n.rtl){this.dom.root.style.visibility="hidden";for(var p,f=this.dom.root;!p&&f;)p=window.getComputedStyle(f,null).direction,f=f.parentElement;this.options.rtl=p&&"rtl"==p.toLowerCase()}else this.options.rtl=n.rtl;this.options.rollingMode=n&&n.rollingMode,this.options.onInitialDrawComplete=n&&n.onInitialDrawComplete,this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this)},hiddenDates:[],util:{getScale:function(){return r.timeAxis.step.scale},getStep:function(){return r.timeAxis.step.step},toScreen:r._toScreen.bind(r),toGlobalScreen:r._toGlobalScreen.bind(r),toTime:r._toTime.bind(r),toGlobalTime:r._toGlobalTime.bind(r)}},this.range=new u(this.body,this.options),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new c(this.body,this.options),this.timeAxis2=null,this.components.push(this.timeAxis),this.currentTime=new m(this.body,this.options),this.components.push(this.currentTime),this.itemSet=new g(this.body,this.options),this.components.push(this.itemSet),this.itemsData=null,this.groupsData=null,this.dom.root.onclick=function(t){r.emit("click",r.getEventProperties(t))},this.dom.root.ondblclick=function(t){r.emit("doubleClick",r.getEventProperties(t))},this.dom.root.oncontextmenu=function(t){r.emit("contextmenu",r.getEventProperties(t))},this.dom.root.onmouseover=function(t){r.emit("mouseOver",r.getEventProperties(t))},window.PointerEvent?(this.dom.root.onpointerdown=function(t){r.emit("mouseDown",r.getEventProperties(t))},this.dom.root.onpointermove=function(t){r.emit("mouseMove",r.getEventProperties(t))},this.dom.root.onpointerup=function(t){r.emit("mouseUp",r.getEventProperties(t))}):(this.dom.root.onmousemove=function(t){r.emit("mouseMove",r.getEventProperties(t))},this.dom.root.onmousedown=function(t){r.emit("mouseDown",r.getEventProperties(t))},this.dom.root.onmouseup=function(t){r.emit("mouseUp",r.getEventProperties(t))}),this.initialFitDone=!1,this.on("changed",function(){if(null!=this.itemsData&&!this.options.rollingMode){if(!r.initialFitDone)if(r.initialFitDone=!0,void 0!=r.options.start||void 0!=r.options.end){if(void 0==r.options.start||void 0==r.options.end)var t=r.getItemRange();var e=void 0!=r.options.start?r.options.start:t.min,i=void 0!=r.options.end?r.options.end:t.max;r.setWindow(e,i,{animation:!1})}else r.fit({animation:!1});!r.initialDrawDone&&r.initialRangeChangeDone&&(r.initialDrawDone=!0,r.dom.root.style.visibility="visible",r.options.onInitialDrawComplete&&setTimeout(function(){return r.options.onInitialDrawComplete()},0))}}),n&&this.setOptions(n),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}function n(t){return h.convert(t.data.start,"Date").valueOf()}function s(t){var e=void 0!=t.data.end?t.data.end:t.data.start;return h.convert(e,"Date").valueOf()}function r(t,e){var i=t.props.leftContainer.height,o=t.props.left.height,n=e.parent,s=n.top,r=!0,a=t.timeAxis.options.orientation.axis,h=function(){return"bottom"==a?n.height-e.top-e.height:e.top},d=-1*t._getScrollTop(),l=s+h(),u=e.height;return l<d?s+i<=s+h()+u&&(s+=h()-t.itemSet.options.margin.item.vertical):l+u>d+i?s+=h()+u-i+t.itemSet.options.margin.item.vertical:r=!1,s=Math.min(s,o-i),{shouldScroll:r,scrollOffset:s,itemTop:l}}var a=i(3),h=i(0),d=i(15),l=i(20),u=i(47),p=i(48),c=i(28),m=i(50),f=i(29),g=i(70),v=i(32).printStyle,y=i(75).allOptions,b=i(75).configureOptions,_=i(76).default,w=i(32).default;o.prototype=new p,o.prototype._createConfigurator=function(){return new _(this,this.dom.container,b)},o.prototype.redraw=function(){this.itemSet&&this.itemSet.markDirty({refreshItems:!0}),this._redraw()},o.prototype.setOptions=function(t){if(!0===w.validate(t,y)&&console.log("%cErrors have been found in the supplied options object.",v),p.prototype.setOptions.call(this,t),"type"in t&&t.type!==this.options.type){this.options.type=t.type;var e=this.itemsData;if(e){var i=this.getSelection();this.setItems(null),this.setItems(e),this.setSelection(i)}}},o.prototype.setItems=function(t){var e;e=t?t instanceof d||t instanceof l?t:new d(t,{type:{start:"Date",end:"Date"}}):null,this.itemsData=e,this.itemSet&&this.itemSet.setItems(e)},o.prototype.setGroups=function(t){var e;if(t){var i=function(t){return!1!==t.visible};e=t instanceof d||t instanceof l?new l(t,{filter:i}):new d(t.filter(i))}else e=null;this.groupsData=e,this.itemSet.setGroups(e)},o.prototype.setData=function(t){t&&t.groups&&this.setGroups(t.groups),t&&t.items&&this.setItems(t.items)},o.prototype.setSelection=function(t,e){this.itemSet&&this.itemSet.setSelection(t),e&&e.focus&&this.focus(t,e)},o.prototype.getSelection=function(){return this.itemSet&&this.itemSet.getSelection()||[]},o.prototype.focus=function(t,e){if(this.itemsData&&void 0!=t){var i=Array.isArray(t)?t:[t],o=this.itemsData.getDataSet().get(i,{type:{start:"Date",end:"Date"}}),n=null,s=null;if(o.forEach(function(t){var e=t.start.valueOf(),i="end"in t?t.end.valueOf():t.start.valueOf();(null===n||e<n)&&(n=e),(null===s||i>s)&&(s=i)}),null!==n&&null!==s){var a=this,h=this.itemSet.items[i[0]],d=-1*this._getScrollTop(),l=null,u=function(t,e,i){var o=r(a,h);if(l||(l=o),l.itemTop!=o.itemTop||l.shouldScroll){l.itemTop!=o.itemTop&&o.shouldScroll&&(l=o,d=-1*a._getScrollTop());var n=d,s=l.scrollOffset,u=i?s:n+(s-n)*t;a._setScrollTop(-u),e||a._redraw()}},p=function(){var t=r(a,h);t.shouldScroll&&t.itemTop!=l.itemTop&&(a._setScrollTop(-t.scrollOffset),a._redraw())},c=function(){p(),setTimeout(p,100)},m=(n+s)/2,f=Math.max(this.range.end-this.range.start,1.1*(s-n)),g=!e||void 0===e.animation||e.animation;g||(l={shouldScroll:!1,scrollOffset:-1,itemTop:-1}),this.range.setRange(m-f/2,m+f/2,{animation:g},c,u)}}},o.prototype.fit=function(t,e){var i,o=!t||void 0===t.animation||t.animation,n=this.itemsData&&this.itemsData.getDataSet();1===n.length&&void 0===n.get()[0].end?(i=this.getDataRange(),this.moveTo(i.min.valueOf(),{animation:o},e)):(i=this.getItemRange(),this.range.setRange(i.min,i.max,{animation:o},e))},o.prototype.getItemRange=function(){var t=this.getDataRange(),e=null!==t.min?t.min.valueOf():null,i=null!==t.max?t.max.valueOf():null,o=null,r=null;if(null!=e&&null!=i){var a=i-e;a<=0&&(a=10);var d=a/this.props.center.width,l={},u=0;h.forEach(this.itemSet.items,function(t,e){if(t.groupShowing){l[e]=t.redraw(!0),u=l[e].length}});if(u>0)for(var p=0;p<u;p++)h.forEach(l,function(t){t[p]()});if(h.forEach(this.itemSet.items,function(t){var a,h,l=n(t),u=s(t);this.options.rtl?(a=l-(t.getWidthRight()+10)*d,h=u+(t.getWidthLeft()+10)*d):(a=l-(t.getWidthLeft()+10)*d,h=u+(t.getWidthRight()+10)*d),a<e&&(e=a,o=t),h>i&&(i=h,r=t)}.bind(this)),o&&r){var c=o.getWidthLeft()+10,m=r.getWidthRight()+10,f=this.props.center.width-c-m;f>0&&(this.options.rtl?(e=n(o)-m*a/f,i=s(r)+c*a/f):(e=n(o)-c*a/f,i=s(r)+m*a/f))}}return{min:null!=e?new Date(e):null,max:null!=i?new Date(i):null}},o.prototype.getDataRange=function(){var t=null,e=null,i=this.itemsData&&this.itemsData.getDataSet();return i&&i.forEach(function(i){var o=h.convert(i.start,"Date").valueOf(),n=h.convert(void 0!=i.end?i.end:i.start,"Date").valueOf();(null===t||o<t)&&(t=o),(null===e||n>e)&&(e=n)}),{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},o.prototype.getEventProperties=function(t){var e,i=t.center?t.center.x:t.clientX,o=t.center?t.center.y:t.clientY;e=this.options.rtl?h.getAbsoluteRight(this.dom.centerContainer)-i:i-h.getAbsoluteLeft(this.dom.centerContainer);var n=o-h.getAbsoluteTop(this.dom.centerContainer),s=this.itemSet.itemFromTarget(t),r=this.itemSet.groupFromTarget(t),a=f.customTimeFromTarget(t),d=this.itemSet.options.snap||null,l=this.body.util.getScale(),u=this.body.util.getStep(),p=this._toTime(e),c=d?d(p,l,u):p,m=h.getTarget(t),g=null;return null!=s?g="item":null!=a?g="custom-time":h.hasParent(m,this.timeAxis.dom.foreground)?g="axis":this.timeAxis2&&h.hasParent(m,this.timeAxis2.dom.foreground)?g="axis":h.hasParent(m,this.itemSet.dom.labelSet)?g="group-label":h.hasParent(m,this.currentTime.bar)?g="current-time":h.hasParent(m,this.dom.center)&&(g="background"),{event:t,item:s?s.id:null,group:r?r.groupId:null,what:g,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:e,y:n,time:p,snappedTime:c}},o.prototype.toggleRollingMode=function(){this.range.rolling?this.range.stopRolling():(void 0==this.options.rollingMode&&this.setOptions(this.options),this.range.startRolling())},t.exports=o},function(t,e,i){var o,n,s;!function(i){n=[],o=i,void 0!==(s="function"==typeof o?o.apply(e,n):o)&&(t.exports=s)}(function(){var t=null;return function e(i,o){function n(t){return t.match(/[^ ]+/g)}function s(e){if("hammer.input"!==e.type){if(e.srcEvent._handled||(e.srcEvent._handled={}),e.srcEvent._handled[e.type])return;e.srcEvent._handled[e.type]=!0}var i=!1;e.stopPropagation=function(){i=!0};var o=e.srcEvent.stopPropagation.bind(e.srcEvent);"function"==typeof o&&(e.srcEvent.stopPropagation=function(){o(),e.stopPropagation()}),e.firstTarget=t;for(var n=t;n&&!i;){var s=n.hammer;if(s)for(var r,a=0;a<s.length;a++)if(r=s[a]._handlers[e.type])for(var h=0;h<r.length&&!i;h++)r[h](e);n=n.parentNode}}var r=o||{preventDefault:!1};if(i.Manager){var a=i,h=function(t,i){var o=Object.create(r);return i&&a.assign(o,i),e(new a(t,o),o)};return a.assign(h,a),h.Manager=function(t,i){var o=Object.create(r);return i&&a.assign(o,i),e(new a.Manager(t,o),o)},h}var d=Object.create(i),l=i.element;return l.hammer||(l.hammer=[]),l.hammer.push(d),i.on("hammer.input",function(e){!0!==r.preventDefault&&r.preventDefault!==e.pointerType||e.preventDefault(),e.isFirst&&(t=e.target)}),d._handlers={},d.on=function(t,e){return n(t).forEach(function(t){var o=d._handlers[t];o||(d._handlers[t]=o=[],i.on(t,s)),o.push(e)}),d},d.off=function(t,e){return n(t).forEach(function(t){var o=d._handlers[t];o&&(o=e?o.filter(function(t){return t!==e}):[],o.length>0?d._handlers[t]=o:(i.off(t,s),delete d._handlers[t]))}),d},d.emit=function(e,o){t=o.target,i.emit(e,o)},d.destroy=function(){var t=i.element.hammer,e=t.indexOf(d);-1!==e&&t.splice(e,1),t.length||delete i.element.hammer,d._handlers={},i.destroy()},d}})},function(t,e,i){var o;/*! Hammer.JS - v2.0.7 - 2016-04-22
+ * http://hammerjs.github.io/
+ *
+ * Copyright (c) 2016 Jorik Tangelder;
+ * Licensed under the MIT license */
+!function(n,s,r,a){function h(t,e,i){return setTimeout(c(t,i),e)}function d(t,e,i){return!!Array.isArray(t)&&(l(t,i[e],i),!0)}function l(t,e,i){var o;if(t)if(t.forEach)t.forEach(e,i);else if(t.length!==a)for(o=0;o<t.length;)e.call(i,t[o],o,t),o++;else for(o in t)t.hasOwnProperty(o)&&e.call(i,t[o],o,t)}function u(t,e,i){var o="DEPRECATED METHOD: "+e+"\n"+i+" AT \n";return function(){var e=new Error("get-stack-trace"),i=e&&e.stack?e.stack.replace(/^[^\(]+?[\n$]/gm,"").replace(/^\s+at\s+/gm,"").replace(/^Object.<anonymous>\s*\(/gm,"{anonymous}()@"):"Unknown Stack Trace",s=n.console&&(n.console.warn||n.console.log);return s&&s.call(n.console,o,i),t.apply(this,arguments)}}function p(t,e,i){var o,n=e.prototype;o=t.prototype=Object.create(n),o.constructor=t,o._super=n,i&&mt(o,i)}function c(t,e){return function(){return t.apply(e,arguments)}}function m(t,e){return typeof t==vt?t.apply(e?e[0]||a:a,e):t}function f(t,e){return t===a?e:t}function g(t,e,i){l(_(e),function(e){t.addEventListener(e,i,!1)})}function v(t,e,i){l(_(e),function(e){t.removeEventListener(e,i,!1)})}function y(t,e){for(;t;){if(t==e)return!0;t=t.parentNode}return!1}function b(t,e){return t.indexOf(e)>-1}function _(t){return t.trim().split(/\s+/g)}function w(t,e,i){if(t.indexOf&&!i)return t.indexOf(e);for(var o=0;o<t.length;){if(i&&t[o][i]==e||!i&&t[o]===e)return o;o++}return-1}function x(t){return Array.prototype.slice.call(t,0)}function D(t,e,i){for(var o=[],n=[],s=0;s<t.length;){var r=e?t[s][e]:t[s];w(n,r)<0&&o.push(t[s]),n[s]=r,s++}return i&&(o=e?o.sort(function(t,i){return t[e]>i[e]}):o.sort()),o}function S(t,e){for(var i,o,n=e[0].toUpperCase()+e.slice(1),s=0;s<ft.length;){if(i=ft[s],(o=i?i+n:e)in t)return o;s++}return a}function k(){return Dt++}function C(t){var e=t.ownerDocument||t;return e.defaultView||e.parentWindow||n}function T(t,e){var i=this;this.manager=t,this.callback=e,this.element=t.element,this.target=t.options.inputTarget,this.domHandler=function(e){m(t.options.enable,[t])&&i.handler(e)},this.init()}function M(t){var e=t.options.inputClass;return new(e||(Ct?z:Tt?B:kt?q:G))(t,O)}function O(t,e,i){var o=i.pointers.length,n=i.changedPointers.length,s=e&Ot&&o-n==0,r=e&(Pt|It)&&o-n==0;i.isFirst=!!s,i.isFinal=!!r,s&&(t.session={}),i.eventType=e,E(t,i),t.emit("hammer.input",i),t.recognize(i),t.session.prevInput=i}function E(t,e){var i=t.session,o=e.pointers,n=o.length;i.firstInput||(i.firstInput=N(e)),n>1&&!i.firstMultiple?i.firstMultiple=N(e):1===n&&(i.firstMultiple=!1);var s=i.firstInput,r=i.firstMultiple,a=r?r.center:s.center,h=e.center=A(o);e.timeStamp=_t(),e.deltaTime=e.timeStamp-s.timeStamp,e.angle=H(a,h),e.distance=F(a,h),P(i,e),e.offsetDirection=L(e.deltaX,e.deltaY);var d=R(e.deltaTime,e.deltaX,e.deltaY);e.overallVelocityX=d.x,e.overallVelocityY=d.y,e.overallVelocity=bt(d.x)>bt(d.y)?d.x:d.y,e.scale=r?Y(r.pointers,o):1,e.rotation=r?j(r.pointers,o):0,e.maxPointers=i.prevInput?e.pointers.length>i.prevInput.maxPointers?e.pointers.length:i.prevInput.maxPointers:e.pointers.length,I(i,e);var l=t.element;y(e.srcEvent.target,l)&&(l=e.srcEvent.target),e.target=l}function P(t,e){var i=e.center,o=t.offsetDelta||{},n=t.prevDelta||{},s=t.prevInput||{};e.eventType!==Ot&&s.eventType!==Pt||(n=t.prevDelta={x:s.deltaX||0,y:s.deltaY||0},o=t.offsetDelta={x:i.x,y:i.y}),e.deltaX=n.x+(i.x-o.x),e.deltaY=n.y+(i.y-o.y)}function I(t,e){var i,o,n,s,r=t.lastInterval||e,h=e.timeStamp-r.timeStamp;if(e.eventType!=It&&(h>Mt||r.velocity===a)){var d=e.deltaX-r.deltaX,l=e.deltaY-r.deltaY,u=R(h,d,l);o=u.x,n=u.y,i=bt(u.x)>bt(u.y)?u.x:u.y,s=L(d,l),t.lastInterval=e}else i=r.velocity,o=r.velocityX,n=r.velocityY,s=r.direction;e.velocity=i,e.velocityX=o,e.velocityY=n,e.direction=s}function N(t){for(var e=[],i=0;i<t.pointers.length;)e[i]={clientX:yt(t.pointers[i].clientX),clientY:yt(t.pointers[i].clientY)},i++;return{timeStamp:_t(),pointers:e,center:A(e),deltaX:t.deltaX,deltaY:t.deltaY}}function A(t){var e=t.length;if(1===e)return{x:yt(t[0].clientX),y:yt(t[0].clientY)};for(var i=0,o=0,n=0;n<e;)i+=t[n].clientX,o+=t[n].clientY,n++;return{x:yt(i/e),y:yt(o/e)}}function R(t,e,i){return{x:e/t||0,y:i/t||0}}function L(t,e){return t===e?Nt:bt(t)>=bt(e)?t<0?At:Rt:e<0?Lt:Ft}function F(t,e,i){i||(i=Gt);var o=e[i[0]]-t[i[0]],n=e[i[1]]-t[i[1]];return Math.sqrt(o*o+n*n)}function H(t,e,i){i||(i=Gt);var o=e[i[0]]-t[i[0]],n=e[i[1]]-t[i[1]];return 180*Math.atan2(n,o)/Math.PI}function j(t,e){return H(e[1],e[0],zt)+H(t[1],t[0],zt)}function Y(t,e){return F(e[0],e[1],zt)/F(t[0],t[1],zt)}function G(){this.evEl=Vt,this.evWin=Bt,this.pressed=!1,T.apply(this,arguments)}function z(){this.evEl=Xt,this.evWin=Zt,T.apply(this,arguments),this.store=this.manager.session.pointerEvents=[]}function W(){this.evTarget=Jt,this.evWin=$t,this.started=!1,T.apply(this,arguments)}function V(t,e){var i=x(t.touches),o=x(t.changedTouches);return e&(Pt|It)&&(i=D(i.concat(o),"identifier",!0)),[i,o]}function B(){this.evTarget=te,this.targetIds={},T.apply(this,arguments)}function U(t,e){var i=x(t.touches),o=this.targetIds;if(e&(Ot|Et)&&1===i.length)return o[i[0].identifier]=!0,[i,i];var n,s,r=x(t.changedTouches),a=[],h=this.target;if(s=i.filter(function(t){return y(t.target,h)}),e===Ot)for(n=0;n<s.length;)o[s[n].identifier]=!0,n++;for(n=0;n<r.length;)o[r[n].identifier]&&a.push(r[n]),e&(Pt|It)&&delete o[r[n].identifier],n++;return a.length?[D(s.concat(a),"identifier",!0),a]:void 0}function q(){T.apply(this,arguments);var t=c(this.handler,this);this.touch=new B(this.manager,t),this.mouse=new G(this.manager,t),this.primaryTouch=null,this.lastTouches=[]}function X(t,e){t&Ot?(this.primaryTouch=e.changedPointers[0].identifier,Z.call(this,e)):t&(Pt|It)&&Z.call(this,e)}function Z(t){var e=t.changedPointers[0];if(e.identifier===this.primaryTouch){var i={x:e.clientX,y:e.clientY};this.lastTouches.push(i);var o=this.lastTouches,n=function(){var t=o.indexOf(i);t>-1&&o.splice(t,1)};setTimeout(n,ee)}}function K(t){for(var e=t.srcEvent.clientX,i=t.srcEvent.clientY,o=0;o<this.lastTouches.length;o++){var n=this.lastTouches[o],s=Math.abs(e-n.x),r=Math.abs(i-n.y);if(s<=ie&&r<=ie)return!0}return!1}function J(t,e){this.manager=t,this.set(e)}function $(t){if(b(t,ae))return ae;var e=b(t,he),i=b(t,de);return e&&i?ae:e||i?e?he:de:b(t,re)?re:se}function Q(t){this.options=mt({},this.defaults,t||{}),this.id=k(),this.manager=null,this.options.enable=f(this.options.enable,!0),this.state=ue,this.simultaneous={},this.requireFail=[]}function tt(t){return t&ge?"cancel":t&me?"end":t&ce?"move":t&pe?"start":""}function et(t){return t==Ft?"down":t==Lt?"up":t==At?"left":t==Rt?"right":""}function it(t,e){var i=e.manager;return i?i.get(t):t}function ot(){Q.apply(this,arguments)}function nt(){ot.apply(this,arguments),this.pX=null,this.pY=null}function st(){ot.apply(this,arguments)}function rt(){Q.apply(this,arguments),this._timer=null,this._input=null}function at(){ot.apply(this,arguments)}function ht(){ot.apply(this,arguments)}function dt(){Q.apply(this,arguments),this.pTime=!1,this.pCenter=!1,this._timer=null,this._input=null,this.count=0}function lt(t,e){return e=e||{},e.recognizers=f(e.recognizers,lt.defaults.preset),new ut(t,e)}function ut(t,e){this.options=mt({},lt.defaults,e||{}),this.options.inputTarget=this.options.inputTarget||t,this.handlers={},this.session={},this.recognizers=[],this.oldCssProps={},this.element=t,this.input=M(this),this.touchAction=new J(this,this.options.touchAction),pt(this,!0),l(this.options.recognizers,function(t){var e=this.add(new t[0](t[1]));t[2]&&e.recognizeWith(t[2]),t[3]&&e.requireFailure(t[3])},this)}function pt(t,e){var i=t.element;if(i.style){var o;l(t.options.cssProps,function(n,s){o=S(i.style,s),e?(t.oldCssProps[o]=i.style[o],i.style[o]=n):i.style[o]=t.oldCssProps[o]||""}),e||(t.oldCssProps={})}}function ct(t,e){var i=s.createEvent("Event");i.initEvent(t,!0,!0),i.gesture=e,e.target.dispatchEvent(i)}var mt,ft=["","webkit","Moz","MS","ms","o"],gt=s.createElement("div"),vt="function",yt=Math.round,bt=Math.abs,_t=Date.now;mt="function"!=typeof Object.assign?function(t){if(t===a||null===t)throw new TypeError("Cannot convert undefined or null to object");for(var e=Object(t),i=1;i<arguments.length;i++){var o=arguments[i];if(o!==a&&null!==o)for(var n in o)o.hasOwnProperty(n)&&(e[n]=o[n])}return e}:Object.assign;var wt=u(function(t,e,i){for(var o=Object.keys(e),n=0;n<o.length;)(!i||i&&t[o[n]]===a)&&(t[o[n]]=e[o[n]]),n++;return t},"extend","Use `assign`."),xt=u(function(t,e){return wt(t,e,!0)},"merge","Use `assign`."),Dt=1,St=/mobile|tablet|ip(ad|hone|od)|android/i,kt="ontouchstart"in n,Ct=S(n,"PointerEvent")!==a,Tt=kt&&St.test(navigator.userAgent),Mt=25,Ot=1,Et=2,Pt=4,It=8,Nt=1,At=2,Rt=4,Lt=8,Ft=16,Ht=At|Rt,jt=Lt|Ft,Yt=Ht|jt,Gt=["x","y"],zt=["clientX","clientY"];T.prototype={handler:function(){},init:function(){this.evEl&&g(this.element,this.evEl,this.domHandler),this.evTarget&&g(this.target,this.evTarget,this.domHandler),this.evWin&&g(C(this.element),this.evWin,this.domHandler)},destroy:function(){this.evEl&&v(this.element,this.evEl,this.domHandler),this.evTarget&&v(this.target,this.evTarget,this.domHandler),this.evWin&&v(C(this.element),this.evWin,this.domHandler)}};var Wt={mousedown:Ot,mousemove:Et,mouseup:Pt},Vt="mousedown",Bt="mousemove mouseup";p(G,T,{handler:function(t){var e=Wt[t.type];e&Ot&&0===t.button&&(this.pressed=!0),e&Et&&1!==t.which&&(e=Pt),this.pressed&&(e&Pt&&(this.pressed=!1),this.callback(this.manager,e,{pointers:[t],changedPointers:[t],pointerType:"mouse",srcEvent:t}))}});var Ut={pointerdown:Ot,pointermove:Et,pointerup:Pt,pointercancel:It,pointerout:It},qt={2:"touch",3:"pen",4:"mouse",5:"kinect"},Xt="pointerdown",Zt="pointermove pointerup pointercancel";n.MSPointerEvent&&!n.PointerEvent&&(Xt="MSPointerDown",Zt="MSPointerMove MSPointerUp MSPointerCancel"),p(z,T,{handler:function(t){var e=this.store,i=!1,o=t.type.toLowerCase().replace("ms",""),n=Ut[o],s=qt[t.pointerType]||t.pointerType,r="touch"==s,a=w(e,t.pointerId,"pointerId");n&Ot&&(0===t.button||r)?a<0&&(e.push(t),a=e.length-1):n&(Pt|It)&&(i=!0),a<0||(e[a]=t,this.callback(this.manager,n,{pointers:e,changedPointers:[t],pointerType:s,srcEvent:t}),i&&e.splice(a,1))}});var Kt={touchstart:Ot,touchmove:Et,touchend:Pt,touchcancel:It},Jt="touchstart",$t="touchstart touchmove touchend touchcancel";p(W,T,{handler:function(t){var e=Kt[t.type];if(e===Ot&&(this.started=!0),this.started){var i=V.call(this,t,e);e&(Pt|It)&&i[0].length-i[1].length==0&&(this.started=!1),this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})}}});var Qt={touchstart:Ot,touchmove:Et,touchend:Pt,touchcancel:It},te="touchstart touchmove touchend touchcancel";p(B,T,{handler:function(t){var e=Qt[t.type],i=U.call(this,t,e);i&&this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})}});var ee=2500,ie=25;p(q,T,{handler:function(t,e,i){var o="touch"==i.pointerType,n="mouse"==i.pointerType;if(!(n&&i.sourceCapabilities&&i.sourceCapabilities.firesTouchEvents)){if(o)X.call(this,e,i);else if(n&&K.call(this,i))return;this.callback(t,e,i)}},destroy:function(){this.touch.destroy(),this.mouse.destroy()}});var oe=S(gt.style,"touchAction"),ne=oe!==a,se="auto",re="manipulation",ae="none",he="pan-x",de="pan-y",le=function(){if(!ne)return!1;var t={},e=n.CSS&&n.CSS.supports;return["auto","manipulation","pan-y","pan-x","pan-x pan-y","none"].forEach(function(i){t[i]=!e||n.CSS.supports("touch-action",i)}),t}();J.prototype={set:function(t){"compute"==t&&(t=this.compute()),ne&&this.manager.element.style&&le[t]&&(this.manager.element.style[oe]=t),this.actions=t.toLowerCase().trim()},update:function(){this.set(this.manager.options.touchAction)},compute:function(){var t=[];return l(this.manager.recognizers,function(e){m(e.options.enable,[e])&&(t=t.concat(e.getTouchAction()))}),$(t.join(" "))},preventDefaults:function(t){var e=t.srcEvent,i=t.offsetDirection;if(this.manager.session.prevented)return void e.preventDefault();var o=this.actions,n=b(o,ae)&&!le[ae],s=b(o,de)&&!le[de],r=b(o,he)&&!le[he];if(n){var a=1===t.pointers.length,h=t.distance<2,d=t.deltaTime<250;if(a&&h&&d)return}return r&&s?void 0:n||s&&i&Ht||r&&i&jt?this.preventSrc(e):void 0},preventSrc:function(t){this.manager.session.prevented=!0,t.preventDefault()}};var ue=1,pe=2,ce=4,me=8,fe=me,ge=16;Q.prototype={defaults:{},set:function(t){return mt(this.options,t),this.manager&&this.manager.touchAction.update(),this},recognizeWith:function(t){if(d(t,"recognizeWith",this))return this;var e=this.simultaneous;return t=it(t,this),e[t.id]||(e[t.id]=t,t.recognizeWith(this)),this},dropRecognizeWith:function(t){return d(t,"dropRecognizeWith",this)?this:(t=it(t,this),delete this.simultaneous[t.id],this)},requireFailure:function(t){if(d(t,"requireFailure",this))return this;var e=this.requireFail;return t=it(t,this),-1===w(e,t)&&(e.push(t),t.requireFailure(this)),this},dropRequireFailure:function(t){if(d(t,"dropRequireFailure",this))return this;t=it(t,this);var e=w(this.requireFail,t);return e>-1&&this.requireFail.splice(e,1),this},hasRequireFailures:function(){return this.requireFail.length>0},canRecognizeWith:function(t){return!!this.simultaneous[t.id]},emit:function(t){function e(e){i.manager.emit(e,t)}var i=this,o=this.state;o<me&&e(i.options.event+tt(o)),e(i.options.event),t.additionalEvent&&e(t.additionalEvent),o>=me&&e(i.options.event+tt(o))},tryEmit:function(t){if(this.canEmit())return this.emit(t);this.state=32},canEmit:function(){for(var t=0;t<this.requireFail.length;){if(!(this.requireFail[t].state&(32|ue)))return!1;t++}return!0},recognize:function(t){var e=mt({},t);if(!m(this.options.enable,[this,e]))return this.reset(),void(this.state=32);this.state&(fe|ge|32)&&(this.state=ue),this.state=this.process(e),this.state&(pe|ce|me|ge)&&this.tryEmit(e)},process:function(t){},getTouchAction:function(){},reset:function(){}},p(ot,Q,{defaults:{pointers:1},attrTest:function(t){var e=this.options.pointers;return 0===e||t.pointers.length===e},process:function(t){var e=this.state,i=t.eventType,o=e&(pe|ce),n=this.attrTest(t);return o&&(i&It||!n)?e|ge:o||n?i&Pt?e|me:e&pe?e|ce:pe:32}}),p(nt,ot,{defaults:{event:"pan",threshold:10,pointers:1,direction:Yt},getTouchAction:function(){var t=this.options.direction,e=[];return t&Ht&&e.push(de),t&jt&&e.push(he),e},directionTest:function(t){var e=this.options,i=!0,o=t.distance,n=t.direction,s=t.deltaX,r=t.deltaY;return n&e.direction||(e.direction&Ht?(n=0===s?Nt:s<0?At:Rt,i=s!=this.pX,o=Math.abs(t.deltaX)):(n=0===r?Nt:r<0?Lt:Ft,i=r!=this.pY,o=Math.abs(t.deltaY))),t.direction=n,i&&o>e.threshold&&n&e.direction},attrTest:function(t){return ot.prototype.attrTest.call(this,t)&&(this.state&pe||!(this.state&pe)&&this.directionTest(t))},emit:function(t){this.pX=t.deltaX,this.pY=t.deltaY;var e=et(t.direction);e&&(t.additionalEvent=this.options.event+e),this._super.emit.call(this,t)}}),p(st,ot,{defaults:{event:"pinch",threshold:0,pointers:2},getTouchAction:function(){return[ae]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.scale-1)>this.options.threshold||this.state&pe)},emit:function(t){if(1!==t.scale){var e=t.scale<1?"in":"out";t.additionalEvent=this.options.event+e}this._super.emit.call(this,t)}}),p(rt,Q,{defaults:{event:"press",pointers:1,time:251,threshold:9},getTouchAction:function(){return[se]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,o=t.distance<e.threshold,n=t.deltaTime>e.time;if(this._input=t,!o||!i||t.eventType&(Pt|It)&&!n)this.reset();else if(t.eventType&Ot)this.reset(),this._timer=h(function(){this.state=fe,this.tryEmit()},e.time,this);else if(t.eventType&Pt)return fe;return 32},reset:function(){clearTimeout(this._timer)},emit:function(t){this.state===fe&&(t&&t.eventType&Pt?this.manager.emit(this.options.event+"up",t):(this._input.timeStamp=_t(),this.manager.emit(this.options.event,this._input)))}}),p(at,ot,{defaults:{event:"rotate",threshold:0,pointers:2},getTouchAction:function(){return[ae]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.rotation)>this.options.threshold||this.state&pe)}}),p(ht,ot,{defaults:{event:"swipe",threshold:10,velocity:.3,direction:Ht|jt,pointers:1},getTouchAction:function(){return nt.prototype.getTouchAction.call(this)},attrTest:function(t){var e,i=this.options.direction;return i&(Ht|jt)?e=t.overallVelocity:i&Ht?e=t.overallVelocityX:i&jt&&(e=t.overallVelocityY),this._super.attrTest.call(this,t)&&i&t.offsetDirection&&t.distance>this.options.threshold&&t.maxPointers==this.options.pointers&&bt(e)>this.options.velocity&&t.eventType&Pt},emit:function(t){var e=et(t.offsetDirection);e&&this.manager.emit(this.options.event+e,t),this.manager.emit(this.options.event,t)}}),p(dt,Q,{defaults:{event:"tap",pointers:1,taps:1,interval:300,time:250,threshold:9,posThreshold:10},getTouchAction:function(){return[re]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,o=t.distance<e.threshold,n=t.deltaTime<e.time;if(this.reset(),t.eventType&Ot&&0===this.count)return this.failTimeout();if(o&&n&&i){if(t.eventType!=Pt)return this.failTimeout();var s=!this.pTime||t.timeStamp-this.pTime<e.interval,r=!this.pCenter||F(this.pCenter,t.center)<e.posThreshold;this.pTime=t.timeStamp,this.pCenter=t.center,r&&s?this.count+=1:this.count=1,this._input=t;if(0===this.count%e.taps)return this.hasRequireFailures()?(this._timer=h(function(){this.state=fe,this.tryEmit()},e.interval,this),pe):fe}return 32},failTimeout:function(){return this._timer=h(function(){this.state=32},this.options.interval,this),32},reset:function(){clearTimeout(this._timer)},emit:function(){this.state==fe&&(this._input.tapCount=this.count,this.manager.emit(this.options.event,this._input))}}),lt.VERSION="2.0.7",lt.defaults={domEvents:!1,touchAction:"compute",enable:!0,inputTarget:null,inputClass:null,preset:[[at,{enable:!1}],[st,{enable:!1},["rotate"]],[ht,{direction:Ht}],[nt,{direction:Ht},["swipe"]],[dt],[dt,{event:"doubletap",taps:2},["tap"]],[rt]],cssProps:{userSelect:"none",touchSelect:"none",touchCallout:"none",contentZooming:"none",userDrag:"none",tapHighlightColor:"rgba(0,0,0,0)"}};ut.prototype={set:function(t){return mt(this.options,t),t.touchAction&&this.touchAction.update(),t.inputTarget&&(this.input.destroy(),this.input.target=t.inputTarget,this.input.init()),this},stop:function(t){this.session.stopped=t?2:1},recognize:function(t){var e=this.session;if(!e.stopped){this.touchAction.preventDefaults(t);var i,o=this.recognizers,n=e.curRecognizer;(!n||n&&n.state&fe)&&(n=e.curRecognizer=null);for(var s=0;s<o.length;)i=o[s],2===e.stopped||n&&i!=n&&!i.canRecognizeWith(n)?i.reset():i.recognize(t),!n&&i.state&(pe|ce|me)&&(n=e.curRecognizer=i),s++}},get:function(t){if(t instanceof Q)return t;for(var e=this.recognizers,i=0;i<e.length;i++)if(e[i].options.event==t)return e[i];return null},add:function(t){if(d(t,"add",this))return this;var e=this.get(t.options.event);return e&&this.remove(e),this.recognizers.push(t),t.manager=this,this.touchAction.update(),t},remove:function(t){if(d(t,"remove",this))return this;if(t=this.get(t)){var e=this.recognizers,i=w(e,t);-1!==i&&(e.splice(i,1),this.touchAction.update())}return this},on:function(t,e){if(t!==a&&e!==a){var i=this.handlers;return l(_(t),function(t){i[t]=i[t]||[],i[t].push(e)}),this}},off:function(t,e){if(t!==a){var i=this.handlers;return l(_(t),function(t){e?i[t]&&i[t].splice(w(i[t],e),1):delete i[t]}),this}},emit:function(t,e){this.options.domEvents&&ct(t,e);var i=this.handlers[t]&&this.handlers[t].slice();if(i&&i.length){e.type=t,e.preventDefault=function(){e.srcEvent.preventDefault()};for(var o=0;o<i.length;)i[o](e),o++}},destroy:function(){this.element&&pt(this,!1),this.handlers={},this.session={},this.input.destroy(),this.element=null}},mt(lt,{INPUT_START:Ot,INPUT_MOVE:Et,INPUT_END:Pt,INPUT_CANCEL:It,STATE_POSSIBLE:ue,STATE_BEGAN:pe,STATE_CHANGED:ce,STATE_ENDED:me,STATE_RECOGNIZED:fe,STATE_CANCELLED:ge,STATE_FAILED:32,DIRECTION_NONE:Nt,DIRECTION_LEFT:At,DIRECTION_RIGHT:Rt,DIRECTION_UP:Lt,DIRECTION_DOWN:Ft,DIRECTION_HORIZONTAL:Ht,DIRECTION_VERTICAL:jt,DIRECTION_ALL:Yt,Manager:ut,Input:T,TouchAction:J,TouchInput:B,MouseInput:G,PointerEventInput:z,TouchMouseInput:q,SingleTouchInput:W,Recognizer:Q,AttrRecognizer:ot,Tap:dt,Pan:nt,Swipe:ht,Pinch:st,Rotate:at,Press:rt,on:g,off:v,each:l,merge:xt,extend:wt,assign:mt,inherit:p,bindFn:c,prefixed:S}),(void 0!==n?n:"undefined"!=typeof self?self:{}).Hammer=lt,(o=function(){return lt}.call(e,i,e,t))!==a&&(t.exports=o)}(window,document)},function(t,e,i){function o(t){this.active=!1,this.dom={container:t},this.dom.overlay=document.createElement("div"),this.dom.overlay.className="vis-overlay",this.dom.container.appendChild(this.dom.overlay),this.hammer=a(this.dom.overlay),this.hammer.on("tap",this._onTapOverlay.bind(this));var e=this;["tap","doubletap","press","pinch","pan","panstart","panmove","panend"].forEach(function(t){e.hammer.on(t,function(t){t.stopPropagation()})}),document&&document.body&&(this.onClick=function(i){n(i.target,t)||e.deactivate()},document.body.addEventListener("click",this.onClick)),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=s(),this.escListener=this.deactivate.bind(this)}function n(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1}var s=i(68),r=i(66),a=i(12),h=i(0);r(o.prototype),o.current=null,o.prototype.destroy=function(){this.deactivate(),this.dom.overlay.parentNode.removeChild(this.dom.overlay),this.onClick&&document.body.removeEventListener("click",this.onClick),this.hammer.destroy(),this.hammer=null},o.prototype.activate=function(){o.current&&o.current.deactivate(),o.current=this,this.active=!0,this.dom.overlay.style.display="none",h.addClassName(this.dom.container,"vis-active"),this.emit("change"),this.emit("activate"),this.keycharm.bind("esc",this.escListener)},o.prototype.deactivate=function(){this.active=!1,this.dom.overlay.style.display="",h.removeClassName(this.dom.container,"vis-active"),this.keycharm.unbind("esc",this.escListener),this.emit("change"),this.emit("deactivate")},o.prototype._onTapOverlay=function(t){this.activate(),t.stopPropagation()},t.exports=o},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0});var n=i(30),s=o(n),r=i(31),a=o(r),h=function(){function t(e,i){(0,s.default)(this,t),this.container=e,this.overflowMethod=i||"cap",this.x=0,this.y=0,this.padding=5,this.hidden=!1,this.frame=document.createElement("div"),this.frame.className="vis-tooltip",this.container.appendChild(this.frame)}return(0,a.default)(t,[{key:"setPosition",value:function(t,e){this.x=parseInt(t),this.y=parseInt(e)}},{key:"setText",value:function(t){t instanceof Element?(this.frame.innerHTML="",this.frame.appendChild(t)):this.frame.innerHTML=t}},{key:"show",value:function(t){if(void 0===t&&(t=!0),!0===t){var e=this.frame.clientHeight,i=this.frame.clientWidth,o=this.frame.parentNode.clientHeight,n=this.frame.parentNode.clientWidth,s=0,r=0;if("flip"==this.overflowMethod){var a=!1,h=!0;this.y-e<this.padding&&(h=!1),this.x+i>n-this.padding&&(a=!0),s=a?this.x-i:this.x,r=h?this.y-e:this.y}else r=this.y-e,r+e+this.padding>o&&(r=o-e-this.padding),r<this.padding&&(r=this.padding),s=this.x,s+i+this.padding>n&&(s=n-i-this.padding),s<this.padding&&(s=this.padding);this.frame.style.left=s+"px",this.frame.style.top=r+"px",this.frame.style.visibility="visible",this.hidden=!1}else this.hide()}},{key:"hide",value:function(){this.hidden=!0,this.frame.style.left="0",this.frame.style.top="0",this.frame.style.visibility="hidden"}},{key:"destroy",value:function(){this.frame.parentNode.removeChild(this.frame)}}]),t}();e.default=h},function(t,e,i){t.exports={default:i(135),__esModule:!0}},function(t,e,i){i(136);var o=i(2).Object;t.exports=function(t,e,i){return o.defineProperty(t,e,i)}},function(t,e,i){var o=i(17);o(o.S+o.F*!i(9),"Object",{defineProperty:i(8).f})},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0});var n=i(16),s=o(n),r=i(30),a=o(r),h=i(31),d=o(h),l=i(12),u=i(67),p=i(0),c=function(){function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;(0,a.default)(this,t),this.pixelRatio=e,this.generated=!1,this.centerCoordinates={x:144.5,y:144.5},this.r=289*.49,this.color={r:255,g:255,b:255,a:1},this.hueCircle=void 0,this.initialColor={r:255,g:255,b:255,a:1},this.previousColor=void 0,this.applied=!1,this.updateCallback=function(){},this.closeCallback=function(){},this._create()}return(0,d.default)(t,[{key:"insertTo",value:function(t){void 0!==this.hammer&&(this.hammer.destroy(),this.hammer=void 0),this.container=t,this.container.appendChild(this.frame),this._bindHammer(),this._setSize()}},{key:"setUpdateCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker update callback is not a function.");this.updateCallback=t}},{key:"setCloseCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker closing callback is not a function.");this.closeCallback=t}},{key:"_isColorString",value:function(t){var e={black:"#000000",navy:"#000080",darkblue:"#00008B",mediumblue:"#0000CD",blue:"#0000FF",darkgreen:"#006400",green:"#008000",teal:"#008080",darkcyan:"#008B8B",deepskyblue:"#00BFFF",darkturquoise:"#00CED1",mediumspringgreen:"#00FA9A",lime:"#00FF00",springgreen:"#00FF7F",aqua:"#00FFFF",cyan:"#00FFFF",midnightblue:"#191970",dodgerblue:"#1E90FF",lightseagreen:"#20B2AA",forestgreen:"#228B22",seagreen:"#2E8B57",darkslategray:"#2F4F4F",limegreen:"#32CD32",mediumseagreen:"#3CB371",turquoise:"#40E0D0",royalblue:"#4169E1",steelblue:"#4682B4",darkslateblue:"#483D8B",mediumturquoise:"#48D1CC",indigo:"#4B0082",darkolivegreen:"#556B2F",cadetblue:"#5F9EA0",cornflowerblue:"#6495ED",mediumaquamarine:"#66CDAA",dimgray:"#696969",slateblue:"#6A5ACD",olivedrab:"#6B8E23",slategray:"#708090",lightslategray:"#778899",mediumslateblue:"#7B68EE",lawngreen:"#7CFC00",chartreuse:"#7FFF00",aquamarine:"#7FFFD4",maroon:"#800000",purple:"#800080",olive:"#808000",gray:"#808080",skyblue:"#87CEEB",lightskyblue:"#87CEFA",blueviolet:"#8A2BE2",darkred:"#8B0000",darkmagenta:"#8B008B",saddlebrown:"#8B4513",darkseagreen:"#8FBC8F",lightgreen:"#90EE90",mediumpurple:"#9370D8",darkviolet:"#9400D3",palegreen:"#98FB98",darkorchid:"#9932CC",yellowgreen:"#9ACD32",sienna:"#A0522D",brown:"#A52A2A",darkgray:"#A9A9A9",lightblue:"#ADD8E6",greenyellow:"#ADFF2F",paleturquoise:"#AFEEEE",lightsteelblue:"#B0C4DE",powderblue:"#B0E0E6",firebrick:"#B22222",darkgoldenrod:"#B8860B",mediumorchid:"#BA55D3",rosybrown:"#BC8F8F",darkkhaki:"#BDB76B",silver:"#C0C0C0",mediumvioletred:"#C71585",indianred:"#CD5C5C",peru:"#CD853F",chocolate:"#D2691E",tan:"#D2B48C",lightgrey:"#D3D3D3",palevioletred:"#D87093",thistle:"#D8BFD8",orchid:"#DA70D6",goldenrod:"#DAA520",crimson:"#DC143C",gainsboro:"#DCDCDC",plum:"#DDA0DD",burlywood:"#DEB887",lightcyan:"#E0FFFF",lavender:"#E6E6FA",darksalmon:"#E9967A",violet:"#EE82EE",palegoldenrod:"#EEE8AA",lightcoral:"#F08080",khaki:"#F0E68C",aliceblue:"#F0F8FF",honeydew:"#F0FFF0",azure:"#F0FFFF",sandybrown:"#F4A460",wheat:"#F5DEB3",beige:"#F5F5DC",whitesmoke:"#F5F5F5",mintcream:"#F5FFFA",ghostwhite:"#F8F8FF",salmon:"#FA8072",antiquewhite:"#FAEBD7",linen:"#FAF0E6",lightgoldenrodyellow:"#FAFAD2",oldlace:"#FDF5E6",red:"#FF0000",fuchsia:"#FF00FF",magenta:"#FF00FF",deeppink:"#FF1493",orangered:"#FF4500",tomato:"#FF6347",hotpink:"#FF69B4",coral:"#FF7F50",darkorange:"#FF8C00",lightsalmon:"#FFA07A",orange:"#FFA500",lightpink:"#FFB6C1",pink:"#FFC0CB",gold:"#FFD700",peachpuff:"#FFDAB9",navajowhite:"#FFDEAD",moccasin:"#FFE4B5",bisque:"#FFE4C4",mistyrose:"#FFE4E1",blanchedalmond:"#FFEBCD",papayawhip:"#FFEFD5",lavenderblush:"#FFF0F5",seashell:"#FFF5EE",cornsilk:"#FFF8DC",lemonchiffon:"#FFFACD",floralwhite:"#FFFAF0",snow:"#FFFAFA",yellow:"#FFFF00",lightyellow:"#FFFFE0",ivory:"#FFFFF0",white:"#FFFFFF"};if("string"==typeof t)return e[t]}},{key:"setColor",value:function(t){var e=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];if("none"!==t){var i=void 0,o=this._isColorString(t);if(void 0!==o&&(t=o),!0===p.isString(t)){if(!0===p.isValidRGB(t)){var n=t.substr(4).substr(0,t.length-5).split(",");i={r:n[0],g:n[1],b:n[2],a:1}}else if(!0===p.isValidRGBA(t)){var r=t.substr(5).substr(0,t.length-6).split(",");i={r:r[0],g:r[1],b:r[2],a:r[3]}}else if(!0===p.isValidHex(t)){var a=p.hexToRGB(t);i={r:a.r,g:a.g,b:a.b,a:1}}}else if(t instanceof Object&&void 0!==t.r&&void 0!==t.g&&void 0!==t.b){var h=void 0!==t.a?t.a:"1.0";i={r:t.r,g:t.g,b:t.b,a:h}}if(void 0===i)throw new Error("Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: "+(0,s.default)(t));this._setColor(i,e)}}},{key:"show",value:function(){void 0!==this.closeCallback&&(this.closeCallback(),this.closeCallback=void 0),this.applied=!1,this.frame.style.display="block",this._generateHueCircle()}},{key:"_hide",value:function(){var t=this;!0===(!(arguments.length>0&&void 0!==arguments[0])||arguments[0])&&(this.previousColor=p.extend({},this.color)),!0===this.applied&&this.updateCallback(this.initialColor),this.frame.style.display="none",setTimeout(function(){void 0!==t.closeCallback&&(t.closeCallback(),t.closeCallback=void 0)},0)}},{key:"_save",value:function(){this.updateCallback(this.color),this.applied=!1,this._hide()}},{key:"_apply",value:function(){this.applied=!0,this.updateCallback(this.color),this._updatePicker(this.color)}},{key:"_loadLast",value:function(){void 0!==this.previousColor?this.setColor(this.previousColor,!1):alert("There is no last color to load...")}},{key:"_setColor",value:function(t){!0===(!(arguments.length>1&&void 0!==arguments[1])||arguments[1])&&(this.initialColor=p.extend({},t)),this.color=t;var e=p.RGBToHSV(t.r,t.g,t.b),i=2*Math.PI,o=this.r*e.s,n=this.centerCoordinates.x+o*Math.sin(i*e.h),s=this.centerCoordinates.y+o*Math.cos(i*e.h);this.colorPickerSelector.style.left=n-.5*this.colorPickerSelector.clientWidth+"px",this.colorPickerSelector.style.top=s-.5*this.colorPickerSelector.clientHeight+"px",this._updatePicker(t)}},{key:"_setOpacity",value:function(t){this.color.a=t/100,this._updatePicker(this.color)}},{key:"_setBrightness",value:function(t){var e=p.RGBToHSV(this.color.r,this.color.g,this.color.b);e.v=t/100;var i=p.HSVToRGB(e.h,e.s,e.v);i.a=this.color.a,this.color=i,this._updatePicker()}},{key:"_updatePicker",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:this.color,e=p.RGBToHSV(t.r,t.g,t.b),i=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(i.webkitBackingStorePixelRatio||i.mozBackingStorePixelRatio||i.msBackingStorePixelRatio||i.oBackingStorePixelRatio||i.backingStorePixelRatio||1)),i.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var o=this.colorPickerCanvas.clientWidth,n=this.colorPickerCanvas.clientHeight;i.clearRect(0,0,o,n),i.putImageData(this.hueCircle,0,0),i.fillStyle="rgba(0,0,0,"+(1-e.v)+")",i.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),i.fill(),this.brightnessRange.value=100*e.v,this.opacityRange.value=100*t.a,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}},{key:"_setSize",value:function(){this.colorPickerCanvas.style.width="100%",this.colorPickerCanvas.style.height="100%",this.colorPickerCanvas.width=289*this.pixelRatio,this.colorPickerCanvas.height=289*this.pixelRatio}},{key:"_create",value:function(){if(this.frame=document.createElement("div"),this.frame.className="vis-color-picker",this.colorPickerDiv=document.createElement("div"),this.colorPickerSelector=document.createElement("div"),this.colorPickerSelector.className="vis-selector",this.colorPickerDiv.appendChild(this.colorPickerSelector),this.colorPickerCanvas=document.createElement("canvas"),this.colorPickerDiv.appendChild(this.colorPickerCanvas),this.colorPickerCanvas.getContext){var t=this.colorPickerCanvas.getContext("2d");this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1),this.colorPickerCanvas.getContext("2d").setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0)}else{var e=document.createElement("DIV");e.style.color="red",e.style.fontWeight="bold",e.style.padding="10px",e.innerHTML="Error: your browser does not support HTML canvas",this.colorPickerCanvas.appendChild(e)}this.colorPickerDiv.className="vis-color",this.opacityDiv=document.createElement("div"),this.opacityDiv.className="vis-opacity",this.brightnessDiv=document.createElement("div"),this.brightnessDiv.className="vis-brightness",this.arrowDiv=document.createElement("div"),this.arrowDiv.className="vis-arrow",this.opacityRange=document.createElement("input");try{this.opacityRange.type="range",this.opacityRange.min="0",this.opacityRange.max="100"}catch(t){}this.opacityRange.value="100",this.opacityRange.className="vis-range",this.brightnessRange=document.createElement("input");try{this.brightnessRange.type="range",this.brightnessRange.min="0",this.brightnessRange.max="100"}catch(t){}this.brightnessRange.value="100",this.brightnessRange.className="vis-range",this.opacityDiv.appendChild(this.opacityRange),this.brightnessDiv.appendChild(this.brightnessRange);var i=this;this.opacityRange.onchange=function(){i._setOpacity(this.value)},this.opacityRange.oninput=function(){i._setOpacity(this.value)},this.brightnessRange.onchange=function(){i._setBrightness(this.value)},this.brightnessRange.oninput=function(){i._setBrightness(this.value)},this.brightnessLabel=document.createElement("div"),this.brightnessLabel.className="vis-label vis-brightness",this.brightnessLabel.innerHTML="brightness:",this.opacityLabel=document.createElement("div"),this.opacityLabel.className="vis-label vis-opacity",this.opacityLabel.innerHTML="opacity:",this.newColorDiv=document.createElement("div"),this.newColorDiv.className="vis-new-color",this.newColorDiv.innerHTML="new",this.initialColorDiv=document.createElement("div"),this.initialColorDiv.className="vis-initial-color",this.initialColorDiv.innerHTML="initial",this.cancelButton=document.createElement("div"),this.cancelButton.className="vis-button vis-cancel",this.cancelButton.innerHTML="cancel",this.cancelButton.onclick=this._hide.bind(this,!1),this.applyButton=document.createElement("div"),this.applyButton.className="vis-button vis-apply",this.applyButton.innerHTML="apply",this.applyButton.onclick=this._apply.bind(this),this.saveButton=document.createElement("div"),this.saveButton.className="vis-button vis-save",this.saveButton.innerHTML="save",this.saveButton.onclick=this._save.bind(this),this.loadButton=document.createElement("div"),this.loadButton.className="vis-button vis-load",this.loadButton.innerHTML="load last",this.loadButton.onclick=this._loadLast.bind(this),this.frame.appendChild(this.colorPickerDiv),this.frame.appendChild(this.arrowDiv),this.frame.appendChild(this.brightnessLabel),this.frame.appendChild(this.brightnessDiv),this.frame.appendChild(this.opacityLabel),this.frame.appendChild(this.opacityDiv),this.frame.appendChild(this.newColorDiv),this.frame.appendChild(this.initialColorDiv),this.frame.appendChild(this.cancelButton),this.frame.appendChild(this.applyButton),this.frame.appendChild(this.saveButton),this.frame.appendChild(this.loadButton)}},{key:"_bindHammer",value:function(){var t=this;this.drag={},this.pinch={},this.hammer=new l(this.colorPickerCanvas),this.hammer.get("pinch").set({enable:!0}),u.onTouch(this.hammer,function(e){t._moveSelector(e)}),this.hammer.on("tap",function(e){t._moveSelector(e)}),this.hammer.on("panstart",function(e){t._moveSelector(e)}),this.hammer.on("panmove",function(e){t._moveSelector(e)}),this.hammer.on("panend",function(e){t._moveSelector(e)})}},{key:"_generateHueCircle",value:function(){if(!1===this.generated){var t=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1)),t.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var e=this.colorPickerCanvas.clientWidth,i=this.colorPickerCanvas.clientHeight;t.clearRect(0,0,e,i);var o=void 0,n=void 0,s=void 0,r=void 0;this.centerCoordinates={x:.5*e,y:.5*i},this.r=.49*e;var a=2*Math.PI/360,h=1/this.r,d=void 0;for(s=0;s<360;s++)for(r=0;r<this.r;r++)o=this.centerCoordinates.x+r*Math.sin(a*s),n=this.centerCoordinates.y+r*Math.cos(a*s),d=p.HSVToRGB(s*(1/360),r*h,1),t.fillStyle="rgb("+d.r+","+d.g+","+d.b+")",t.fillRect(o-.5,n-.5,2,2);t.strokeStyle="rgba(0,0,0,1)",t.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),t.stroke(),this.hueCircle=t.getImageData(0,0,e,i)}this.generated=!0}},{key:"_moveSelector",value:function(t){var e=this.colorPickerDiv.getBoundingClientRect(),i=t.center.x-e.left,o=t.center.y-e.top,n=.5*this.colorPickerDiv.clientHeight,s=.5*this.colorPickerDiv.clientWidth,r=i-s,a=o-n,h=Math.atan2(r,a),d=.98*Math.min(Math.sqrt(r*r+a*a),s),l=Math.cos(h)*d+n,u=Math.sin(h)*d+s;this.colorPickerSelector.style.top=l-.5*this.colorPickerSelector.clientHeight+"px",this.colorPickerSelector.style.left=u-.5*this.colorPickerSelector.clientWidth+"px";var c=h/(2*Math.PI);c=c<0?c+1:c;var m=d/this.r,f=p.RGBToHSV(this.color.r,this.color.g,this.color.b);f.h=c,f.s=m;var g=p.HSVToRGB(f.h,f.s,f.v);g.a=this.color.a,this.color=g,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}}]),t}();e.default=c},function(t,e,i){function o(t,e,i,o){if(!(Array.isArray(i)||i instanceof r||i instanceof a)&&i instanceof Object){var d=o;o=i,i=d}o&&o.throttleRedraw&&console.warn('Graph2d option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var p=this;this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:n,width:null,height:null,maxHeight:null,minHeight:null},this.options=s.deepExtend({},this.defaultOptions),this._create(t),this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this)},hiddenDates:[],util:{toScreen:p._toScreen.bind(p),toGlobalScreen:p._toGlobalScreen.bind(p),toTime:p._toTime.bind(p),toGlobalTime:p._toGlobalTime.bind(p)}},this.range=new h(this.body),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new l(this.body),this.components.push(this.timeAxis),this.currentTime=new u(this.body),this.components.push(this.currentTime),this.linegraph=new c(this.body),this.components.push(this.linegraph),this.itemsData=null,this.groupsData=null,this.on("tap",function(t){p.emit("click",p.getEventProperties(t))}),this.on("doubletap",function(t){p.emit("doubleClick",p.getEventProperties(t))}),this.dom.root.oncontextmenu=function(t){p.emit("contextmenu",p.getEventProperties(t))},o&&this.setOptions(o),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}var n=i(3),s=i(0),r=i(15),a=i(20),h=i(47),d=i(48),l=i(28),u=i(50),p=i(29),c=i(77),m=i(32).printStyle,f=i(84).allOptions,g=i(84).configureOptions,v=i(76).default,y=i(32).default;o.prototype=new d,o.prototype.setOptions=function(t){!0===y.validate(t,f)&&console.log("%cErrors have been found in the supplied options object.",m),d.prototype.setOptions.call(this,t)},o.prototype.setItems=function(t){var e,i=null==this.itemsData;if(e=t?t instanceof r||t instanceof a?t:new r(t,{type:{start:"Date",end:"Date"}}):null,this.itemsData=e,this.linegraph&&this.linegraph.setItems(e),i)if(void 0!=this.options.start||void 0!=this.options.end){var o=void 0!=this.options.start?this.options.start:null,n=void 0!=this.options.end?this.options.end:null;this.setWindow(o,n,{animation:!1})}else this.fit({animation:!1})},o.prototype.setGroups=function(t){var e;e=t?t instanceof r||t instanceof a?t:new r(t):null,this.groupsData=e,this.linegraph.setGroups(e)},o.prototype.getLegend=function(t,e,i){return void 0===e&&(e=15),void 0===i&&(i=15),void 0!==this.linegraph.groups[t]?this.linegraph.groups[t].getLegend(e,i):"cannot find group:'"+t+"'"},o.prototype.isGroupVisible=function(t){return void 0!==this.linegraph.groups[t]&&(this.linegraph.groups[t].visible&&(void 0===this.linegraph.options.groups.visibility[t]||1==this.linegraph.options.groups.visibility[t]))},o.prototype.getDataRange=function(){var t=null,e=null;for(var i in this.linegraph.groups)if(this.linegraph.groups.hasOwnProperty(i)&&1==this.linegraph.groups[i].visible)for(var o=0;o<this.linegraph.groups[i].itemsData.length;o++){var n=this.linegraph.groups[i].itemsData[o],r=s.convert(n.x,"Date").valueOf();t=null==t?r:t>r?r:t,e=null==e?r:e<r?r:e}return{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},o.prototype.getEventProperties=function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,o=e-s.getAbsoluteLeft(this.dom.centerContainer),n=i-s.getAbsoluteTop(this.dom.centerContainer),r=this._toTime(o),a=p.customTimeFromTarget(t),h=s.getTarget(t),d=null;s.hasParent(h,this.timeAxis.dom.foreground)?d="axis":this.timeAxis2&&s.hasParent(h,this.timeAxis2.dom.foreground)?d="axis":s.hasParent(h,this.linegraph.yAxisLeft.dom.frame)?d="data-axis":s.hasParent(h,this.linegraph.yAxisRight.dom.frame)?d="data-axis":s.hasParent(h,this.linegraph.legendLeft.dom.frame)?d="legend":s.hasParent(h,this.linegraph.legendRight.dom.frame)?d="legend":null!=a?d="custom-time":s.hasParent(h,this.currentTime.bar)?d="current-time":s.hasParent(h,this.dom.center)&&(d="background");var l=[],u=this.linegraph.yAxisLeft,c=this.linegraph.yAxisRight;return!u.hidden&&this.itemsData.length>0&&l.push(u.screenToValue(n)),!c.hidden&&this.itemsData.length>0&&l.push(c.screenToValue(n)),{event:t,what:d,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:o,y:n,time:r,value:l}},o.prototype._createConfigurator=function(){return new v(this,this.dom.container,g)},t.exports=o}])});
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis.min.css b/core/src/main/resources/org/apache/spark/ui/static/vis.min.css
deleted file mode 100644
index 40d182cfde231..0000000000000
--- a/core/src/main/resources/org/apache/spark/ui/static/vis.min.css
+++ /dev/null
@@ -1 +0,0 @@
-.vis-background,.vis-labelset,.vis-timeline{overflow:hidden}.vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}.vis [class*=span]{min-height:0;width:auto}div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper::after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}.vis-panel,.vis-timeline{padding:0;box-sizing:border-box}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:rgba(0,0,0,0);width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(to bottom,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#dedede', endColorstr='#c8c8c8', GradientType=0 );border:1px solid #999;box-shadow:#aaa 0 0 3px 0;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2 0,#385380 100%);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(to bottom,#3876c2 0,#385380 100%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#3876c2', endColorstr='#385380', GradientType=0 );box-shadow:#111927 0 0 1px 0;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:0}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(to bottom,#9d9d9d 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#9d9d9d', endColorstr='#c8c8c8', GradientType=0 )}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(to bottom,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#dedede', endColorstr='#c8c8c8', GradientType=0 );border:1px solid #999;box-shadow:#aaa 0 0 3px 0;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:#fff solid 1px;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:0 0;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0);border-left-color:rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0);border-left-color:#f2faff;border-width:12px;margin-top:-12px}.vis-timeline{position:relative;border:1px solid #bfbfbf;margin:0}.vis-panel{position:absolute;margin:0}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-itemset,.vis-labelset,.vis-labelset .vis-label{position:relative;box-sizing:border-box}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}.vis-labelset .vis-label{left:0;top:0;width:100%;color:#4d4d4d;border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}.vis-itemset{padding:0;margin:0}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-item{position:absolute;color:#1A1A1A;border-color:#97B0F8;border-width:1px;background-color:#D5DDF6;display:inline-block}.vis-item.vis-point.vis-selected,.vis-item.vis-selected{background-color:#FFF785}.vis-item.vis-selected{border-color:#FFC200;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:0 0}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item .vis-delete,.vis-item .vis-delete-rtl{background:url(img/timeline/delete.png) center no-repeat;height:24px;top:-4px;cursor:pointer}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-delete{position:absolute;width:24px;right:-24px}.vis-item .vis-delete-rtl{position:absolute;width:24px;left:-24px}.vis-item.vis-range .vis-drag-left{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0;left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0;right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}.vis-current-time{background-color:#FF7F6E;width:2px;z-index:1}.vis-custom-time{background-color:#6E94FF;width:2px;cursor:move;z-index:1}div.vis-network div.vis-close,div.vis-network div.vis-edit-mode div.vis-button,div.vis-network div.vis-manipulation div.vis-button{cursor:pointer;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;-webkit-touch-callout:none;-khtml-user-select:none}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px rgba(154,154,154,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}.vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}div.vis-network div.vis-manipulation{border-width:0;border-bottom:1px;border-style:solid;border-color:#d6d9d8;background:#fff;background:-moz-linear-gradient(top,#fff 0,#fcfcfc 48%,#fafafa 50%,#fcfcfc 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#fff),color-stop(48%,#fcfcfc),color-stop(50%,#fafafa),color-stop(100%,#fcfcfc));background:-webkit-linear-gradient(top,#fff 0,#fcfcfc 48%,#fafafa 50%,#fcfcfc 100%);background:-o-linear-gradient(top,#fff 0,#fcfcfc 48%,#fafafa 50%,#fcfcfc 100%);background:-ms-linear-gradient(top,#fff 0,#fcfcfc 48%,#fafafa 50%,#fcfcfc 100%);background:linear-gradient(to bottom,#fff 0,#fcfcfc 48%,#fafafa 50%,#fcfcfc 100%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#ffffff', endColorstr='#fcfcfc', GradientType=0 );padding-top:4px;position:absolute;left:0;top:0;width:100%;height:28px}div.vis-network div.vis-edit-mode{position:absolute;left:0;top:5px;height:30px}div.vis-network div.vis-close{position:absolute;right:0;top:0;width:30px;height:30px;background-position:20px 3px;background-repeat:no-repeat;background-image:url(img/network/cross.png);user-select:none}div.vis-network div.vis-close:hover{opacity:.6}div.vis-network div.vis-edit-mode div.vis-button,div.vis-network div.vis-manipulation div.vis-button{float:left;font-family:verdana;font-size:12px;-moz-border-radius:15px;border-radius:15px;display:inline-block;background-position:0 0;background-repeat:no-repeat;height:24px;margin-left:10px;padding:0 8px;user-select:none}div.vis-network div.vis-manipulation div.vis-button:hover{box-shadow:1px 1px 8px rgba(0,0,0,.2)}div.vis-network div.vis-manipulation div.vis-button:active{box-shadow:1px 1px 8px rgba(0,0,0,.5)}div.vis-network div.vis-manipulation div.vis-button.vis-back{background-image:url(img/network/backIcon.png)}div.vis-network div.vis-manipulation div.vis-button.vis-none:hover{box-shadow:1px 1px 8px transparent;cursor:default}div.vis-network div.vis-manipulation div.vis-button.vis-none:active{box-shadow:1px 1px 8px transparent}div.vis-network div.vis-manipulation div.vis-button.vis-none{padding:0}div.vis-network div.vis-manipulation div.notification{margin:2px;font-weight:700}div.vis-network div.vis-manipulation div.vis-button.vis-add{background-image:url(img/network/addNodeIcon.png)}div.vis-network div.vis-edit-mode div.vis-button.vis-edit,div.vis-network div.vis-manipulation div.vis-button.vis-edit{background-image:url(img/network/editIcon.png)}div.vis-network div.vis-edit-mode div.vis-button.vis-edit.vis-edit-mode{background-color:#fcfcfc;border:1px solid #ccc}div.vis-network div.vis-manipulation div.vis-button.vis-connect{background-image:url(img/network/connectIcon.png)}div.vis-network div.vis-manipulation div.vis-button.vis-delete{background-image:url(img/network/deleteIcon.png)}div.vis-network div.vis-edit-mode div.vis-label,div.vis-network div.vis-manipulation div.vis-label{margin:0 0 0 23px;line-height:25px}div.vis-network div.vis-manipulation div.vis-separator-line{float:left;display:inline-block;width:1px;height:21px;background-color:#bdbdbd;margin:0 7px 0 15px}div.vis-network-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none}div.vis-network div.vis-navigation div.vis-button{width:34px;height:34px;-moz-border-radius:17px;border-radius:17px;position:absolute;display:inline-block;background-position:2px 2px;background-repeat:no-repeat;cursor:pointer;-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}div.vis-network div.vis-navigation div.vis-button:hover{box-shadow:0 0 3px 3px rgba(56,207,21,.3)}div.vis-network div.vis-navigation div.vis-button:active{box-shadow:0 0 1px 3px rgba(56,207,21,.95)}div.vis-network div.vis-navigation div.vis-button.vis-up{background-image:url(img/network/upArrow.png);bottom:50px;left:55px}div.vis-network div.vis-navigation div.vis-button.vis-down{background-image:url(img/network/downArrow.png);bottom:10px;left:55px}div.vis-network div.vis-navigation div.vis-button.vis-left{background-image:url(img/network/leftArrow.png);bottom:10px;left:15px}div.vis-network div.vis-navigation div.vis-button.vis-right{background-image:url(img/network/rightArrow.png);bottom:10px;left:95px}div.vis-network div.vis-navigation div.vis-button.vis-zoomIn{background-image:url(img/network/plus.png);bottom:10px;right:15px}div.vis-network div.vis-navigation div.vis-button.vis-zoomOut{background-image:url(img/network/minus.png);bottom:10px;right:55px}div.vis-network div.vis-navigation div.vis-button.vis-zoomExtends{background-image:url(img/network/zoomExtends.png);bottom:50px;right:15px}div.vis-color-picker{position:absolute;top:0;left:30px;margin-top:-140px;margin-left:30px;width:310px;height:444px;z-index:1;padding:10px;border-radius:15px;background-color:#fff;display:none;box-shadow:rgba(0,0,0,.5) 0 0 10px 0}div.vis-color-picker div.vis-arrow{position:absolute;top:147px;left:5px}div.vis-color-picker div.vis-arrow::after,div.vis-color-picker div.vis-arrow::before{right:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}div.vis-color-picker div.vis-arrow:after{border-color:rgba(255,255,255,0);border-right-color:#fff;border-width:30px;margin-top:-30px}div.vis-color-picker div.vis-color{position:absolute;width:289px;height:289px;cursor:pointer}div.vis-color-picker div.vis-brightness{position:absolute;top:313px}div.vis-color-picker div.vis-opacity{position:absolute;top:350px}div.vis-color-picker div.vis-selector{position:absolute;top:137px;left:137px;width:15px;height:15px;border-radius:15px;border:1px solid #fff;background:#4c4c4c;background:-moz-linear-gradient(top,#4c4c4c 0,#595959 12%,#666 25%,#474747 39%,#2c2c2c 50%,#000 51%,#111 60%,#2b2b2b 76%,#1c1c1c 91%,#131313 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#4c4c4c),color-stop(12%,#595959),color-stop(25%,#666),color-stop(39%,#474747),color-stop(50%,#2c2c2c),color-stop(51%,#000),color-stop(60%,#111),color-stop(76%,#2b2b2b),color-stop(91%,#1c1c1c),color-stop(100%,#131313));background:-webkit-linear-gradient(top,#4c4c4c 0,#595959 12%,#666 25%,#474747 39%,#2c2c2c 50%,#000 51%,#111 60%,#2b2b2b 76%,#1c1c1c 91%,#131313 100%);background:-o-linear-gradient(top,#4c4c4c 0,#595959 12%,#666 25%,#474747 39%,#2c2c2c 50%,#000 51%,#111 60%,#2b2b2b 76%,#1c1c1c 91%,#131313 100%);background:-ms-linear-gradient(top,#4c4c4c 0,#595959 12%,#666 25%,#474747 39%,#2c2c2c 50%,#000 51%,#111 60%,#2b2b2b 76%,#1c1c1c 91%,#131313 100%);background:linear-gradient(to bottom,#4c4c4c 0,#595959 12%,#666 25%,#474747 39%,#2c2c2c 50%,#000 51%,#111 60%,#2b2b2b 76%,#1c1c1c 91%,#131313 100%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#4c4c4c', endColorstr='#131313', GradientType=0 )}div.vis-color-picker div.vis-initial-color,div.vis-color-picker div.vis-new-color{width:140px;height:20px;top:380px;font-size:10px;color:rgba(0,0,0,.4);line-height:20px;position:absolute;vertical-align:middle}div.vis-color-picker div.vis-new-color{border:1px solid rgba(0,0,0,.1);border-radius:5px;left:159px;text-align:right;padding-right:2px}div.vis-color-picker div.vis-initial-color{border:1px solid rgba(0,0,0,.1);border-radius:5px;left:10px;text-align:left;padding-left:2px}div.vis-color-picker div.vis-label{position:absolute;width:300px;left:10px}div.vis-color-picker div.vis-label.vis-brightness{top:300px}div.vis-color-picker div.vis-label.vis-opacity{top:338px}div.vis-color-picker div.vis-button{position:absolute;width:68px;height:25px;border-radius:10px;vertical-align:middle;text-align:center;line-height:25px;top:410px;border:2px solid #d9d9d9;background-color:#f7f7f7;cursor:pointer}div.vis-color-picker div.vis-button.vis-cancel{left:5px}div.vis-color-picker div.vis-button.vis-load{left:82px}div.vis-color-picker div.vis-button.vis-apply{left:159px}div.vis-color-picker div.vis-button.vis-save{left:236px}div.vis-color-picker input.vis-range{width:290px;height:20px}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis.min.js b/core/src/main/resources/org/apache/spark/ui/static/vis.min.js
deleted file mode 100644
index 92b8ed75d85fc..0000000000000
--- a/core/src/main/resources/org/apache/spark/ui/static/vis.min.js
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * vis.js
- * https://github.com/almende/vis
- *
- * A dynamic, browser-based visualization library.
- *
- * @version 4.16.1
- * @date    2016-04-18
- *
- * @license
- * Copyright (C) 2011-2016 Almende B.V, http://almende.com
- *
- * Vis.js is dual licensed under both
- *
- * * The Apache 2.0 License
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * and
- *
- * * The MIT License
- *   http://opensource.org/licenses/MIT
- *
- * Vis.js may be distributed under either license.
- */
-"use strict";!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.vis=e():t.vis=e()}(this,function(){return function(t){function e(o){if(i[o])return i[o].exports;var n=i[o]={exports:{},id:o,loaded:!1};return t[o].call(n.exports,n,n.exports,e),n.loaded=!0,n.exports}var i={};return e.m=t,e.c=i,e.p="",e(0)}([function(t,e,i){var o=i(1);o.extend(e,i(7)),o.extend(e,i(24)),o.extend(e,i(60))},function(t,e,i){var o="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},n=i(2),s=i(6);e.isNumber=function(t){return t instanceof Number||"number"==typeof t},e.recursiveDOMDelete=function(t){if(t)for(;t.hasChildNodes()===!0;)e.recursiveDOMDelete(t.firstChild),t.removeChild(t.firstChild)},e.giveRange=function(t,e,i,o){if(e==t)return.5;var n=1/(e-t);return Math.max(0,(o-t)*n)},e.isString=function(t){return t instanceof String||"string"==typeof t},e.isDate=function(t){if(t instanceof Date)return!0;if(e.isString(t)){var i=r.exec(t);if(i)return!0;if(!isNaN(Date.parse(t)))return!0}return!1},e.randomUUID=function(){return s.v4()},e.assignAllKeys=function(t,e){for(var i in t)t.hasOwnProperty(i)&&"object"!==o(t[i])&&(t[i]=e)},e.fillIfDefined=function(t,i){var n=arguments.length<=2||void 0===arguments[2]?!1:arguments[2];for(var s in t)void 0!==i[s]&&("object"!==o(i[s])?void 0!==i[s]&&null!==i[s]||void 0===t[s]||n!==!0?t[s]=i[s]:delete t[s]:"object"===o(t[s])&&e.fillIfDefined(t[s],i[s],n))},e.protoExtend=function(t,e){for(var i=1;i<arguments.length;i++){var o=arguments[i];for(var n in o)t[n]=o[n]}return t},e.extend=function(t,e){for(var i=1;i<arguments.length;i++){var o=arguments[i];for(var n in o)o.hasOwnProperty(n)&&(t[n]=o[n])}return t},e.selectiveExtend=function(t,e,i){if(!Array.isArray(t))throw new Error("Array with property names expected as first argument");for(var o=2;o<arguments.length;o++)for(var n=arguments[o],s=0;s<t.length;s++){var r=t[s];n.hasOwnProperty(r)&&(e[r]=n[r])}return e},e.selectiveDeepExtend=function(t,i,o){var n=arguments.length<=3||void 0===arguments[3]?!1:arguments[3];if(Array.isArray(o))throw new TypeError("Arrays are not supported by deepExtend");for(var s=2;s<arguments.length;s++)for(var r=arguments[s],a=0;a<t.length;a++){var h=t[a];if(r.hasOwnProperty(h))if(o[h]&&o[h].constructor===Object)void 0===i[h]&&(i[h]={}),i[h].constructor===Object?e.deepExtend(i[h],o[h],!1,n):null===o[h]&&void 0!==i[h]&&n===!0?delete i[h]:i[h]=o[h];else{if(Array.isArray(o[h]))throw new TypeError("Arrays are not supported by deepExtend");null===o[h]&&void 0!==i[h]&&n===!0?delete i[h]:i[h]=o[h]}}return i},e.selectiveNotDeepExtend=function(t,i,o){var n=arguments.length<=3||void 0===arguments[3]?!1:arguments[3];if(Array.isArray(o))throw new TypeError("Arrays are not supported by deepExtend");for(var s in o)if(o.hasOwnProperty(s)&&-1==t.indexOf(s))if(o[s]&&o[s].constructor===Object)void 0===i[s]&&(i[s]={}),i[s].constructor===Object?e.deepExtend(i[s],o[s]):null===o[s]&&void 0!==i[s]&&n===!0?delete i[s]:i[s]=o[s];else if(Array.isArray(o[s])){i[s]=[];for(var r=0;r<o[s].length;r++)i[s].push(o[s][r])}else null===o[s]&&void 0!==i[s]&&n===!0?delete i[s]:i[s]=o[s];return i},e.deepExtend=function(t,i,o,n){for(var s in i)if(i.hasOwnProperty(s)||o===!0)if(i[s]&&i[s].constructor===Object)void 0===t[s]&&(t[s]={}),t[s].constructor===Object?e.deepExtend(t[s],i[s],o):null===i[s]&&void 0!==t[s]&&n===!0?delete t[s]:t[s]=i[s];else if(Array.isArray(i[s])){t[s]=[];for(var r=0;r<i[s].length;r++)t[s].push(i[s][r])}else null===i[s]&&void 0!==t[s]&&n===!0?delete t[s]:t[s]=i[s];return t},e.equalArray=function(t,e){if(t.length!=e.length)return!1;for(var i=0,o=t.length;o>i;i++)if(t[i]!=e[i])return!1;return!0},e.convert=function(t,i){var o;if(void 0!==t){if(null===t)return null;if(!i)return t;if("string"!=typeof i&&!(i instanceof String))throw new Error("Type must be a string");switch(i){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(e.isNumber(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(n.isMoment(t))return new Date(t.valueOf());if(e.isString(t))return o=r.exec(t),o?new Date(Number(o[1])):n(t).toDate();throw new Error("Cannot convert object of type "+e.getType(t)+" to type Date");case"Moment":if(e.isNumber(t))return n(t);if(t instanceof Date)return n(t.valueOf());if(n.isMoment(t))return n(t);if(e.isString(t))return o=r.exec(t),n(o?Number(o[1]):t);throw new Error("Cannot convert object of type "+e.getType(t)+" to type Date");case"ISODate":if(e.isNumber(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(n.isMoment(t))return t.toDate().toISOString();if(e.isString(t))return o=r.exec(t),o?new Date(Number(o[1])).toISOString():new Date(t).toISOString();throw new Error("Cannot convert object of type "+e.getType(t)+" to type ISODate");case"ASPDate":if(e.isNumber(t))return"/Date("+t+")/";if(t instanceof Date)return"/Date("+t.valueOf()+")/";if(e.isString(t)){o=r.exec(t);var s;return s=o?new Date(Number(o[1])).valueOf():new Date(t).valueOf(),"/Date("+s+")/"}throw new Error("Cannot convert object of type "+e.getType(t)+" to type ASPDate");default:throw new Error('Unknown type "'+i+'"')}}};var r=/^\/?Date\((\-?\d+)/i;e.getType=function(t){var e="undefined"==typeof t?"undefined":o(t);return"object"==e?null===t?"null":t instanceof Boolean?"Boolean":t instanceof Number?"Number":t instanceof String?"String":Array.isArray(t)?"Array":t instanceof Date?"Date":"Object":"number"==e?"Number":"boolean"==e?"Boolean":"string"==e?"String":void 0===e?"undefined":e},e.copyAndExtendArray=function(t,e){for(var i=[],o=0;o<t.length;o++)i.push(t[o]);return i.push(e),i},e.copyArray=function(t){for(var e=[],i=0;i<t.length;i++)e.push(t[i]);return e},e.getAbsoluteLeft=function(t){return t.getBoundingClientRect().left},e.getAbsoluteRight=function(t){return t.getBoundingClientRect().right},e.getAbsoluteTop=function(t){return t.getBoundingClientRect().top},e.addClassName=function(t,e){var i=t.className.split(" ");-1==i.indexOf(e)&&(i.push(e),t.className=i.join(" "))},e.removeClassName=function(t,e){var i=t.className.split(" "),o=i.indexOf(e);-1!=o&&(i.splice(o,1),t.className=i.join(" "))},e.forEach=function(t,e){var i,o;if(Array.isArray(t))for(i=0,o=t.length;o>i;i++)e(t[i],i,t);else for(i in t)t.hasOwnProperty(i)&&e(t[i],i,t)},e.toArray=function(t){var e=[];for(var i in t)t.hasOwnProperty(i)&&e.push(t[i]);return e},e.updateProperty=function(t,e,i){return t[e]!==i?(t[e]=i,!0):!1},e.throttle=function(t,e){var i=null,o=!1;return function n(){i?o=!0:(o=!1,t(),i=setTimeout(function(){i=null,o&&n()},e))}},e.addEventListener=function(t,e,i,o){t.addEventListener?(void 0===o&&(o=!1),"mousewheel"===e&&navigator.userAgent.indexOf("Firefox")>=0&&(e="DOMMouseScroll"),t.addEventListener(e,i,o)):t.attachEvent("on"+e,i)},e.removeEventListener=function(t,e,i,o){t.removeEventListener?(void 0===o&&(o=!1),"mousewheel"===e&&navigator.userAgent.indexOf("Firefox")>=0&&(e="DOMMouseScroll"),t.removeEventListener(e,i,o)):t.detachEvent("on"+e,i)},e.preventDefault=function(t){t||(t=window.event),t.preventDefault?t.preventDefault():t.returnValue=!1},e.getTarget=function(t){t||(t=window.event);var e;return t.target?e=t.target:t.srcElement&&(e=t.srcElement),void 0!=e.nodeType&&3==e.nodeType&&(e=e.parentNode),e},e.hasParent=function(t,e){for(var i=t;i;){if(i===e)return!0;i=i.parentNode}return!1},e.option={},e.option.asBoolean=function(t,e){return"function"==typeof t&&(t=t()),null!=t?0!=t:e||null},e.option.asNumber=function(t,e){return"function"==typeof t&&(t=t()),null!=t?Number(t)||e||null:e||null},e.option.asString=function(t,e){return"function"==typeof t&&(t=t()),null!=t?String(t):e||null},e.option.asSize=function(t,i){return"function"==typeof t&&(t=t()),e.isString(t)?t:e.isNumber(t)?t+"px":i||null},e.option.asElement=function(t,e){return"function"==typeof t&&(t=t()),t||e||null},e.hexToRGB=function(t){var e=/^#?([a-f\d])([a-f\d])([a-f\d])$/i;t=t.replace(e,function(t,e,i,o){return e+e+i+i+o+o});var i=/^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(t);return i?{r:parseInt(i[1],16),g:parseInt(i[2],16),b:parseInt(i[3],16)}:null},e.overrideOpacity=function(t,i){if(-1!=t.indexOf("rgba"))return t;if(-1!=t.indexOf("rgb")){var o=t.substr(t.indexOf("(")+1).replace(")","").split(",");return"rgba("+o[0]+","+o[1]+","+o[2]+","+i+")"}var o=e.hexToRGB(t);return null==o?t:"rgba("+o.r+","+o.g+","+o.b+","+i+")"},e.RGBToHex=function(t,e,i){return"#"+((1<<24)+(t<<16)+(e<<8)+i).toString(16).slice(1)},e.parseColor=function(t){var i;if(e.isString(t)===!0){if(e.isValidRGB(t)===!0){var o=t.substr(4).substr(0,t.length-5).split(",").map(function(t){return parseInt(t)});t=e.RGBToHex(o[0],o[1],o[2])}if(e.isValidHex(t)===!0){var n=e.hexToHSV(t),s={h:n.h,s:.8*n.s,v:Math.min(1,1.02*n.v)},r={h:n.h,s:Math.min(1,1.25*n.s),v:.8*n.v},a=e.HSVToHex(r.h,r.s,r.v),h=e.HSVToHex(s.h,s.s,s.v);i={background:t,border:a,highlight:{background:h,border:a},hover:{background:h,border:a}}}else i={background:t,border:t,highlight:{background:t,border:t},hover:{background:t,border:t}}}else i={},i.background=t.background||void 0,i.border=t.border||void 0,e.isString(t.highlight)?i.highlight={border:t.highlight,background:t.highlight}:(i.highlight={},i.highlight.background=t.highlight&&t.highlight.background||void 0,i.highlight.border=t.highlight&&t.highlight.border||void 0),e.isString(t.hover)?i.hover={border:t.hover,background:t.hover}:(i.hover={},i.hover.background=t.hover&&t.hover.background||void 0,i.hover.border=t.hover&&t.hover.border||void 0);return i},e.RGBToHSV=function(t,e,i){t/=255,e/=255,i/=255;var o=Math.min(t,Math.min(e,i)),n=Math.max(t,Math.max(e,i));if(o==n)return{h:0,s:0,v:o};var s=t==o?e-i:i==o?t-e:i-t,r=t==o?3:i==o?1:5,a=60*(r-s/(n-o))/360,h=(n-o)/n,d=n;return{h:a,s:h,v:d}};var a={split:function(t){var e={};return t.split(";").forEach(function(t){if(""!=t.trim()){var i=t.split(":"),o=i[0].trim(),n=i[1].trim();e[o]=n}}),e},join:function(t){return Object.keys(t).map(function(e){return e+": "+t[e]}).join("; ")}};e.addCssText=function(t,i){var o=a.split(t.style.cssText),n=a.split(i),s=e.extend(o,n);t.style.cssText=a.join(s)},e.removeCssText=function(t,e){var i=a.split(t.style.cssText),o=a.split(e);for(var n in o)o.hasOwnProperty(n)&&delete i[n];t.style.cssText=a.join(i)},e.HSVToRGB=function(t,e,i){var o,n,s,r=Math.floor(6*t),a=6*t-r,h=i*(1-e),d=i*(1-a*e),l=i*(1-(1-a)*e);switch(r%6){case 0:o=i,n=l,s=h;break;case 1:o=d,n=i,s=h;break;case 2:o=h,n=i,s=l;break;case 3:o=h,n=d,s=i;break;case 4:o=l,n=h,s=i;break;case 5:o=i,n=h,s=d}return{r:Math.floor(255*o),g:Math.floor(255*n),b:Math.floor(255*s)}},e.HSVToHex=function(t,i,o){var n=e.HSVToRGB(t,i,o);return e.RGBToHex(n.r,n.g,n.b)},e.hexToHSV=function(t){var i=e.hexToRGB(t);return e.RGBToHSV(i.r,i.g,i.b)},e.isValidHex=function(t){var e=/(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(t);return e},e.isValidRGB=function(t){t=t.replace(" ","");var e=/rgb\((\d{1,3}),(\d{1,3}),(\d{1,3})\)/i.test(t);return e},e.isValidRGBA=function(t){t=t.replace(" ","");var e=/rgba\((\d{1,3}),(\d{1,3}),(\d{1,3}),(.{1,3})\)/i.test(t);return e},e.selectiveBridgeObject=function(t,i){if("object"==("undefined"==typeof i?"undefined":o(i))){for(var n=Object.create(i),s=0;s<t.length;s++)i.hasOwnProperty(t[s])&&"object"==o(i[t[s]])&&(n[t[s]]=e.bridgeObject(i[t[s]]));return n}return null},e.bridgeObject=function(t){if("object"==("undefined"==typeof t?"undefined":o(t))){var i=Object.create(t);for(var n in t)t.hasOwnProperty(n)&&"object"==o(t[n])&&(i[n]=e.bridgeObject(t[n]));return i}return null},e.insertSort=function(t,e){for(var i=0;i<t.length;i++){for(var o=t[i],n=i;n>0&&e(o,t[n-1])<0;n--)t[n]=t[n-1];t[n]=o}return t},e.mergeOptions=function(t,e,i){var o=(arguments.length<=3||void 0===arguments[3]?!1:arguments[3],arguments.length<=4||void 0===arguments[4]?{}:arguments[4]);if(null===e[i])t[i]=Object.create(o[i]);else if(void 0!==e[i])if("boolean"==typeof e[i])t[i].enabled=e[i];else{void 0===e[i].enabled&&(t[i].enabled=!0);for(var n in e[i])e[i].hasOwnProperty(n)&&(t[i][n]=e[i][n])}},e.binarySearchCustom=function(t,e,i,o){for(var n=1e4,s=0,r=0,a=t.length-1;a>=r&&n>s;){var h=Math.floor((r+a)/2),d=t[h],l=void 0===o?d[i]:d[i][o],c=e(l);if(0==c)return h;-1==c?r=h+1:a=h-1,s++}return-1},e.binarySearchValue=function(t,e,i,o,n){for(var s,r,a,h,d=1e4,l=0,c=0,u=t.length-1,n=void 0!=n?n:function(t,e){return t==e?0:e>t?-1:1};u>=c&&d>l;){if(h=Math.floor(.5*(u+c)),s=t[Math.max(0,h-1)][i],r=t[h][i],a=t[Math.min(t.length-1,h+1)][i],0==n(r,e))return h;if(n(s,e)<0&&n(r,e)>0)return"before"==o?Math.max(0,h-1):h;if(n(r,e)<0&&n(a,e)>0)return"before"==o?h:Math.min(t.length-1,h+1);n(r,e)<0?c=h+1:u=h-1,l++}return-1},e.easingFunctions={linear:function(t){return t},easeInQuad:function(t){return t*t},easeOutQuad:function(t){return t*(2-t)},easeInOutQuad:function(t){return.5>t?2*t*t:-1+(4-2*t)*t},easeInCubic:function(t){return t*t*t},easeOutCubic:function(t){return--t*t*t+1},easeInOutCubic:function(t){return.5>t?4*t*t*t:(t-1)*(2*t-2)*(2*t-2)+1},easeInQuart:function(t){return t*t*t*t},easeOutQuart:function(t){return 1- --t*t*t*t},easeInOutQuart:function(t){return.5>t?8*t*t*t*t:1-8*--t*t*t*t},easeInQuint:function(t){return t*t*t*t*t},easeOutQuint:function(t){return 1+--t*t*t*t*t},easeInOutQuint:function(t){return.5>t?16*t*t*t*t*t:1+16*--t*t*t*t*t}}},function(t,e,i){t.exports="undefined"!=typeof window&&window.moment||i(3)},function(t,e,i){(function(t){!function(e,i){t.exports=i()}(this,function(){function e(){return ro.apply(null,arguments)}function i(t){ro=t}function o(t){return t instanceof Array||"[object Array]"===Object.prototype.toString.call(t)}function n(t){return t instanceof Date||"[object Date]"===Object.prototype.toString.call(t)}function s(t,e){var i,o=[];for(i=0;i<t.length;++i)o.push(e(t[i],i));return o}function r(t,e){return Object.prototype.hasOwnProperty.call(t,e)}function a(t,e){for(var i in e)r(e,i)&&(t[i]=e[i]);return r(e,"toString")&&(t.toString=e.toString),r(e,"valueOf")&&(t.valueOf=e.valueOf),t}function h(t,e,i,o){return Lt(t,e,i,o,!0).utc()}function d(){return{empty:!1,unusedTokens:[],unusedInput:[],overflow:-2,charsLeftOver:0,nullInput:!1,invalidMonth:null,invalidFormat:!1,userInvalidated:!1,iso:!1,parsedDateParts:[],meridiem:null}}function l(t){return null==t._pf&&(t._pf=d()),t._pf}function c(t){if(null==t._isValid){var e=l(t),i=ao.call(e.parsedDateParts,function(t){return null!=t});t._isValid=!isNaN(t._d.getTime())&&e.overflow<0&&!e.empty&&!e.invalidMonth&&!e.invalidWeekday&&!e.nullInput&&!e.invalidFormat&&!e.userInvalidated&&(!e.meridiem||e.meridiem&&i),t._strict&&(t._isValid=t._isValid&&0===e.charsLeftOver&&0===e.unusedTokens.length&&void 0===e.bigHour)}return t._isValid}function u(t){var e=h(NaN);return null!=t?a(l(e),t):l(e).userInvalidated=!0,e}function p(t){return void 0===t}function f(t,e){var i,o,n;if(p(e._isAMomentObject)||(t._isAMomentObject=e._isAMomentObject),p(e._i)||(t._i=e._i),p(e._f)||(t._f=e._f),p(e._l)||(t._l=e._l),p(e._strict)||(t._strict=e._strict),p(e._tzm)||(t._tzm=e._tzm),p(e._isUTC)||(t._isUTC=e._isUTC),p(e._offset)||(t._offset=e._offset),p(e._pf)||(t._pf=l(e)),p(e._locale)||(t._locale=e._locale),ho.length>0)for(i in ho)o=ho[i],n=e[o],p(n)||(t[o]=n);return t}function m(t){f(this,t),this._d=new Date(null!=t._d?t._d.getTime():NaN),lo===!1&&(lo=!0,e.updateOffset(this),lo=!1)}function v(t){return t instanceof m||null!=t&&null!=t._isAMomentObject}function g(t){return 0>t?Math.ceil(t):Math.floor(t)}function y(t){var e=+t,i=0;return 0!==e&&isFinite(e)&&(i=g(e)),i}function b(t,e,i){var o,n=Math.min(t.length,e.length),s=Math.abs(t.length-e.length),r=0;for(o=0;n>o;o++)(i&&t[o]!==e[o]||!i&&y(t[o])!==y(e[o]))&&r++;return r+s}function w(t){e.suppressDeprecationWarnings===!1&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+t)}function _(t,i){var o=!0;return a(function(){return null!=e.deprecationHandler&&e.deprecationHandler(null,t),o&&(w(t+"\nArguments: "+Array.prototype.slice.call(arguments).join(", ")+"\n"+(new Error).stack),o=!1),i.apply(this,arguments)},i)}function x(t,i){null!=e.deprecationHandler&&e.deprecationHandler(t,i),co[t]||(w(i),co[t]=!0)}function k(t){return t instanceof Function||"[object Function]"===Object.prototype.toString.call(t)}function O(t){return"[object Object]"===Object.prototype.toString.call(t)}function M(t){var e,i;for(i in t)e=t[i],k(e)?this[i]=e:this["_"+i]=e;this._config=t,this._ordinalParseLenient=new RegExp(this._ordinalParse.source+"|"+/\d{1,2}/.source)}function D(t,e){var i,o=a({},t);for(i in e)r(e,i)&&(O(t[i])&&O(e[i])?(o[i]={},a(o[i],t[i]),a(o[i],e[i])):null!=e[i]?o[i]=e[i]:delete o[i]);return o}function S(t){null!=t&&this.set(t)}function C(t){return t?t.toLowerCase().replace("_","-"):t}function T(t){for(var e,i,o,n,s=0;s<t.length;){for(n=C(t[s]).split("-"),e=n.length,i=C(t[s+1]),i=i?i.split("-"):null;e>0;){if(o=E(n.slice(0,e).join("-")))return o;if(i&&i.length>=e&&b(n,i,!0)>=e-1)break;e--}s++}return null}function E(e){var i=null;if(!mo[e]&&"undefined"!=typeof t&&t&&t.exports)try{i=po._abbr,!function(){var t=new Error('Cannot find module "./locale"');throw t.code="MODULE_NOT_FOUND",t}(),P(i)}catch(o){}return mo[e]}function P(t,e){var i;return t&&(i=p(e)?R(t):I(t,e),i&&(po=i)),po._abbr}function I(t,e){return null!==e?(e.abbr=t,null!=mo[t]?(x("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale"),e=D(mo[t]._config,e)):null!=e.parentLocale&&(null!=mo[e.parentLocale]?e=D(mo[e.parentLocale]._config,e):x("parentLocaleUndefined","specified parentLocale is not defined yet")),mo[t]=new S(e),P(t),mo[t]):(delete mo[t],null)}function N(t,e){if(null!=e){var i;null!=mo[t]&&(e=D(mo[t]._config,e)),i=new S(e),i.parentLocale=mo[t],mo[t]=i,P(t)}else null!=mo[t]&&(null!=mo[t].parentLocale?mo[t]=mo[t].parentLocale:null!=mo[t]&&delete mo[t]);return mo[t]}function R(t){var e;if(t&&t._locale&&t._locale._abbr&&(t=t._locale._abbr),!t)return po;if(!o(t)){if(e=E(t))return e;t=[t]}return T(t)}function z(){return uo(mo)}function L(t,e){var i=t.toLowerCase();vo[i]=vo[i+"s"]=vo[e]=t}function A(t){return"string"==typeof t?vo[t]||vo[t.toLowerCase()]:void 0}function B(t){var e,i,o={};for(i in t)r(t,i)&&(e=A(i),e&&(o[e]=t[i]));return o}function F(t,i){return function(o){return null!=o?(H(this,t,o),e.updateOffset(this,i),this):j(this,t)}}function j(t,e){return t.isValid()?t._d["get"+(t._isUTC?"UTC":"")+e]():NaN}function H(t,e,i){t.isValid()&&t._d["set"+(t._isUTC?"UTC":"")+e](i)}function W(t,e){var i;if("object"==typeof t)for(i in t)this.set(i,t[i]);else if(t=A(t),k(this[t]))return this[t](e);return this}function Y(t,e,i){var o=""+Math.abs(t),n=e-o.length,s=t>=0;return(s?i?"+":"":"-")+Math.pow(10,Math.max(0,n)).toString().substr(1)+o}function G(t,e,i,o){var n=o;"string"==typeof o&&(n=function(){return this[o]()}),t&&(wo[t]=n),e&&(wo[e[0]]=function(){return Y(n.apply(this,arguments),e[1],e[2])}),i&&(wo[i]=function(){return this.localeData().ordinal(n.apply(this,arguments),t)})}function V(t){return t.match(/\[[\s\S]/)?t.replace(/^\[|\]$/g,""):t.replace(/\\/g,"")}function U(t){var e,i,o=t.match(go);for(e=0,i=o.length;i>e;e++)wo[o[e]]?o[e]=wo[o[e]]:o[e]=V(o[e]);return function(e){var n,s="";for(n=0;i>n;n++)s+=o[n]instanceof Function?o[n].call(e,t):o[n];return s}}function q(t,e){return t.isValid()?(e=X(e,t.localeData()),bo[e]=bo[e]||U(e),bo[e](t)):t.localeData().invalidDate()}function X(t,e){function i(t){return e.longDateFormat(t)||t}var o=5;for(yo.lastIndex=0;o>=0&&yo.test(t);)t=t.replace(yo,i),yo.lastIndex=0,o-=1;return t}function Z(t,e,i){Bo[t]=k(e)?e:function(t,o){return t&&i?i:e}}function K(t,e){return r(Bo,t)?Bo[t](e._strict,e._locale):new RegExp(J(t))}function J(t){return Q(t.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,function(t,e,i,o,n){return e||i||o||n}))}function Q(t){return t.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}function $(t,e){var i,o=e;for("string"==typeof t&&(t=[t]),"number"==typeof e&&(o=function(t,i){i[e]=y(t)}),i=0;i<t.length;i++)Fo[t[i]]=o}function tt(t,e){$(t,function(t,i,o,n){o._w=o._w||{},e(t,o._w,o,n)})}function et(t,e,i){null!=e&&r(Fo,t)&&Fo[t](e,i._a,i,t)}function it(t,e){return new Date(Date.UTC(t,e+1,0)).getUTCDate()}function ot(t,e){return o(this._months)?this._months[t.month()]:this._months[Zo.test(e)?"format":"standalone"][t.month()]}function nt(t,e){return o(this._monthsShort)?this._monthsShort[t.month()]:this._monthsShort[Zo.test(e)?"format":"standalone"][t.month()]}function st(t,e,i){var o,n,s,r=t.toLocaleLowerCase();if(!this._monthsParse)for(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[],o=0;12>o;++o)s=h([2e3,o]),this._shortMonthsParse[o]=this.monthsShort(s,"").toLocaleLowerCase(),this._longMonthsParse[o]=this.months(s,"").toLocaleLowerCase();return i?"MMM"===e?(n=fo.call(this._shortMonthsParse,r),-1!==n?n:null):(n=fo.call(this._longMonthsParse,r),-1!==n?n:null):"MMM"===e?(n=fo.call(this._shortMonthsParse,r),-1!==n?n:(n=fo.call(this._longMonthsParse,r),-1!==n?n:null)):(n=fo.call(this._longMonthsParse,r),-1!==n?n:(n=fo.call(this._shortMonthsParse,r),-1!==n?n:null))}function rt(t,e,i){var o,n,s;if(this._monthsParseExact)return st.call(this,t,e,i);for(this._monthsParse||(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[]),o=0;12>o;o++){if(n=h([2e3,o]),i&&!this._longMonthsParse[o]&&(this._longMonthsParse[o]=new RegExp("^"+this.months(n,"").replace(".","")+"$","i"),this._shortMonthsParse[o]=new RegExp("^"+this.monthsShort(n,"").replace(".","")+"$","i")),i||this._monthsParse[o]||(s="^"+this.months(n,"")+"|^"+this.monthsShort(n,""),this._monthsParse[o]=new RegExp(s.replace(".",""),"i")),i&&"MMMM"===e&&this._longMonthsParse[o].test(t))return o;if(i&&"MMM"===e&&this._shortMonthsParse[o].test(t))return o;if(!i&&this._monthsParse[o].test(t))return o}}function at(t,e){var i;if(!t.isValid())return t;if("string"==typeof e)if(/^\d+$/.test(e))e=y(e);else if(e=t.localeData().monthsParse(e),"number"!=typeof e)return t;return i=Math.min(t.date(),it(t.year(),e)),t._d["set"+(t._isUTC?"UTC":"")+"Month"](e,i),t}function ht(t){return null!=t?(at(this,t),e.updateOffset(this,!0),this):j(this,"Month")}function dt(){return it(this.year(),this.month())}function lt(t){return this._monthsParseExact?(r(this,"_monthsRegex")||ut.call(this),t?this._monthsShortStrictRegex:this._monthsShortRegex):this._monthsShortStrictRegex&&t?this._monthsShortStrictRegex:this._monthsShortRegex}function ct(t){return this._monthsParseExact?(r(this,"_monthsRegex")||ut.call(this),t?this._monthsStrictRegex:this._monthsRegex):this._monthsStrictRegex&&t?this._monthsStrictRegex:this._monthsRegex}function ut(){function t(t,e){return e.length-t.length}var e,i,o=[],n=[],s=[];for(e=0;12>e;e++)i=h([2e3,e]),o.push(this.monthsShort(i,"")),n.push(this.months(i,"")),s.push(this.months(i,"")),s.push(this.monthsShort(i,""));for(o.sort(t),n.sort(t),s.sort(t),e=0;12>e;e++)o[e]=Q(o[e]),n[e]=Q(n[e]),s[e]=Q(s[e]);this._monthsRegex=new RegExp("^("+s.join("|")+")","i"),this._monthsShortRegex=this._monthsRegex,this._monthsStrictRegex=new RegExp("^("+n.join("|")+")","i"),this._monthsShortStrictRegex=new RegExp("^("+o.join("|")+")","i")}function pt(t){var e,i=t._a;return i&&-2===l(t).overflow&&(e=i[Ho]<0||i[Ho]>11?Ho:i[Wo]<1||i[Wo]>it(i[jo],i[Ho])?Wo:i[Yo]<0||i[Yo]>24||24===i[Yo]&&(0!==i[Go]||0!==i[Vo]||0!==i[Uo])?Yo:i[Go]<0||i[Go]>59?Go:i[Vo]<0||i[Vo]>59?Vo:i[Uo]<0||i[Uo]>999?Uo:-1,l(t)._overflowDayOfYear&&(jo>e||e>Wo)&&(e=Wo),l(t)._overflowWeeks&&-1===e&&(e=qo),l(t)._overflowWeekday&&-1===e&&(e=Xo),l(t).overflow=e),t}function ft(t){var e,i,o,n,s,r,a=t._i,h=tn.exec(a)||en.exec(a);if(h){for(l(t).iso=!0,e=0,i=nn.length;i>e;e++)if(nn[e][1].exec(h[1])){n=nn[e][0],o=nn[e][2]!==!1;break}if(null==n)return void(t._isValid=!1);if(h[3]){for(e=0,i=sn.length;i>e;e++)if(sn[e][1].exec(h[3])){s=(h[2]||" ")+sn[e][0];break}if(null==s)return void(t._isValid=!1)}if(!o&&null!=s)return void(t._isValid=!1);if(h[4]){if(!on.exec(h[4]))return void(t._isValid=!1);r="Z"}t._f=n+(s||"")+(r||""),Tt(t)}else t._isValid=!1}function mt(t){var i=rn.exec(t._i);return null!==i?void(t._d=new Date(+i[1])):(ft(t),void(t._isValid===!1&&(delete t._isValid,e.createFromInputFallback(t))))}function vt(t,e,i,o,n,s,r){var a=new Date(t,e,i,o,n,s,r);return 100>t&&t>=0&&isFinite(a.getFullYear())&&a.setFullYear(t),a}function gt(t){var e=new Date(Date.UTC.apply(null,arguments));return 100>t&&t>=0&&isFinite(e.getUTCFullYear())&&e.setUTCFullYear(t),e}function yt(t){return bt(t)?366:365}function bt(t){return t%4===0&&t%100!==0||t%400===0}function wt(){return bt(this.year())}function _t(t,e,i){var o=7+e-i,n=(7+gt(t,0,o).getUTCDay()-e)%7;return-n+o-1}function xt(t,e,i,o,n){var s,r,a=(7+i-o)%7,h=_t(t,o,n),d=1+7*(e-1)+a+h;return 0>=d?(s=t-1,r=yt(s)+d):d>yt(t)?(s=t+1,r=d-yt(t)):(s=t,r=d),{year:s,dayOfYear:r}}function kt(t,e,i){var o,n,s=_t(t.year(),e,i),r=Math.floor((t.dayOfYear()-s-1)/7)+1;return 1>r?(n=t.year()-1,o=r+Ot(n,e,i)):r>Ot(t.year(),e,i)?(o=r-Ot(t.year(),e,i),n=t.year()+1):(n=t.year(),o=r),{week:o,year:n}}function Ot(t,e,i){var o=_t(t,e,i),n=_t(t+1,e,i);return(yt(t)-o+n)/7}function Mt(t,e,i){return null!=t?t:null!=e?e:i}function Dt(t){var i=new Date(e.now());return t._useUTC?[i.getUTCFullYear(),i.getUTCMonth(),i.getUTCDate()]:[i.getFullYear(),i.getMonth(),i.getDate()]}function St(t){var e,i,o,n,s=[];if(!t._d){for(o=Dt(t),t._w&&null==t._a[Wo]&&null==t._a[Ho]&&Ct(t),t._dayOfYear&&(n=Mt(t._a[jo],o[jo]),t._dayOfYear>yt(n)&&(l(t)._overflowDayOfYear=!0),i=gt(n,0,t._dayOfYear),t._a[Ho]=i.getUTCMonth(),t._a[Wo]=i.getUTCDate()),e=0;3>e&&null==t._a[e];++e)t._a[e]=s[e]=o[e];for(;7>e;e++)t._a[e]=s[e]=null==t._a[e]?2===e?1:0:t._a[e];24===t._a[Yo]&&0===t._a[Go]&&0===t._a[Vo]&&0===t._a[Uo]&&(t._nextDay=!0,t._a[Yo]=0),t._d=(t._useUTC?gt:vt).apply(null,s),null!=t._tzm&&t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),t._nextDay&&(t._a[Yo]=24)}}function Ct(t){var e,i,o,n,s,r,a,h;e=t._w,null!=e.GG||null!=e.W||null!=e.E?(s=1,r=4,i=Mt(e.GG,t._a[jo],kt(At(),1,4).year),o=Mt(e.W,1),n=Mt(e.E,1),(1>n||n>7)&&(h=!0)):(s=t._locale._week.dow,r=t._locale._week.doy,i=Mt(e.gg,t._a[jo],kt(At(),s,r).year),o=Mt(e.w,1),null!=e.d?(n=e.d,(0>n||n>6)&&(h=!0)):null!=e.e?(n=e.e+s,(e.e<0||e.e>6)&&(h=!0)):n=s),1>o||o>Ot(i,s,r)?l(t)._overflowWeeks=!0:null!=h?l(t)._overflowWeekday=!0:(a=xt(i,o,n,s,r),t._a[jo]=a.year,t._dayOfYear=a.dayOfYear)}function Tt(t){if(t._f===e.ISO_8601)return void ft(t);t._a=[],l(t).empty=!0;var i,o,n,s,r,a=""+t._i,h=a.length,d=0;for(n=X(t._f,t._locale).match(go)||[],i=0;i<n.length;i++)s=n[i],o=(a.match(K(s,t))||[])[0],o&&(r=a.substr(0,a.indexOf(o)),r.length>0&&l(t).unusedInput.push(r),a=a.slice(a.indexOf(o)+o.length),d+=o.length),wo[s]?(o?l(t).empty=!1:l(t).unusedTokens.push(s),et(s,o,t)):t._strict&&!o&&l(t).unusedTokens.push(s);l(t).charsLeftOver=h-d,a.length>0&&l(t).unusedInput.push(a),l(t).bigHour===!0&&t._a[Yo]<=12&&t._a[Yo]>0&&(l(t).bigHour=void 0),l(t).parsedDateParts=t._a.slice(0),l(t).meridiem=t._meridiem,t._a[Yo]=Et(t._locale,t._a[Yo],t._meridiem),St(t),pt(t)}function Et(t,e,i){var o;return null==i?e:null!=t.meridiemHour?t.meridiemHour(e,i):null!=t.isPM?(o=t.isPM(i),o&&12>e&&(e+=12),o||12!==e||(e=0),e):e}function Pt(t){var e,i,o,n,s;if(0===t._f.length)return l(t).invalidFormat=!0,void(t._d=new Date(NaN));for(n=0;n<t._f.length;n++)s=0,e=f({},t),null!=t._useUTC&&(e._useUTC=t._useUTC),e._f=t._f[n],Tt(e),c(e)&&(s+=l(e).charsLeftOver,s+=10*l(e).unusedTokens.length,l(e).score=s,(null==o||o>s)&&(o=s,i=e));a(t,i||e)}function It(t){if(!t._d){var e=B(t._i);t._a=s([e.year,e.month,e.day||e.date,e.hour,e.minute,e.second,e.millisecond],function(t){return t&&parseInt(t,10)}),St(t)}}function Nt(t){var e=new m(pt(Rt(t)));return e._nextDay&&(e.add(1,"d"),e._nextDay=void 0),e}function Rt(t){var e=t._i,i=t._f;return t._locale=t._locale||R(t._l),null===e||void 0===i&&""===e?u({nullInput:!0}):("string"==typeof e&&(t._i=e=t._locale.preparse(e)),v(e)?new m(pt(e)):(o(i)?Pt(t):i?Tt(t):n(e)?t._d=e:zt(t),c(t)||(t._d=null),t))}function zt(t){var i=t._i;void 0===i?t._d=new Date(e.now()):n(i)?t._d=new Date(i.valueOf()):"string"==typeof i?mt(t):o(i)?(t._a=s(i.slice(0),function(t){return parseInt(t,10)}),St(t)):"object"==typeof i?It(t):"number"==typeof i?t._d=new Date(i):e.createFromInputFallback(t)}function Lt(t,e,i,o,n){var s={};return"boolean"==typeof i&&(o=i,i=void 0),s._isAMomentObject=!0,s._useUTC=s._isUTC=n,s._l=i,s._i=t,s._f=e,s._strict=o,Nt(s)}function At(t,e,i,o){return Lt(t,e,i,o,!1)}function Bt(t,e){var i,n;if(1===e.length&&o(e[0])&&(e=e[0]),!e.length)return At();for(i=e[0],n=1;n<e.length;++n)e[n].isValid()&&!e[n][t](i)||(i=e[n]);return i}function Ft(){var t=[].slice.call(arguments,0);return Bt("isBefore",t)}function jt(){var t=[].slice.call(arguments,0);return Bt("isAfter",t)}function Ht(t){var e=B(t),i=e.year||0,o=e.quarter||0,n=e.month||0,s=e.week||0,r=e.day||0,a=e.hour||0,h=e.minute||0,d=e.second||0,l=e.millisecond||0;this._milliseconds=+l+1e3*d+6e4*h+1e3*a*60*60,this._days=+r+7*s,this._months=+n+3*o+12*i,this._data={},this._locale=R(),this._bubble()}function Wt(t){return t instanceof Ht}function Yt(t,e){G(t,0,0,function(){var t=this.utcOffset(),i="+";return 0>t&&(t=-t,i="-"),i+Y(~~(t/60),2)+e+Y(~~t%60,2)})}function Gt(t,e){var i=(e||"").match(t)||[],o=i[i.length-1]||[],n=(o+"").match(cn)||["-",0,0],s=+(60*n[1])+y(n[2]);return"+"===n[0]?s:-s}function Vt(t,i){var o,s;return i._isUTC?(o=i.clone(),s=(v(t)||n(t)?t.valueOf():At(t).valueOf())-o.valueOf(),o._d.setTime(o._d.valueOf()+s),e.updateOffset(o,!1),o):At(t).local()}function Ut(t){return 15*-Math.round(t._d.getTimezoneOffset()/15)}function qt(t,i){var o,n=this._offset||0;return this.isValid()?null!=t?("string"==typeof t?t=Gt(zo,t):Math.abs(t)<16&&(t=60*t),!this._isUTC&&i&&(o=Ut(this)),this._offset=t,this._isUTC=!0,null!=o&&this.add(o,"m"),n!==t&&(!i||this._changeInProgress?le(this,ne(t-n,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,e.updateOffset(this,!0),this._changeInProgress=null)),this):this._isUTC?n:Ut(this):null!=t?this:NaN}function Xt(t,e){return null!=t?("string"!=typeof t&&(t=-t),this.utcOffset(t,e),this):-this.utcOffset()}function Zt(t){return this.utcOffset(0,t)}function Kt(t){return this._isUTC&&(this.utcOffset(0,t),this._isUTC=!1,t&&this.subtract(Ut(this),"m")),this}function Jt(){return this._tzm?this.utcOffset(this._tzm):"string"==typeof this._i&&this.utcOffset(Gt(Ro,this._i)),this}function Qt(t){return this.isValid()?(t=t?At(t).utcOffset():0,(this.utcOffset()-t)%60===0):!1}function $t(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()}function te(){if(!p(this._isDSTShifted))return this._isDSTShifted;var t={};if(f(t,this),t=Rt(t),t._a){var e=t._isUTC?h(t._a):At(t._a);this._isDSTShifted=this.isValid()&&b(t._a,e.toArray())>0}else this._isDSTShifted=!1;return this._isDSTShifted}function ee(){return this.isValid()?!this._isUTC:!1}function ie(){return this.isValid()?this._isUTC:!1}function oe(){return this.isValid()?this._isUTC&&0===this._offset:!1}function ne(t,e){var i,o,n,s=t,a=null;return Wt(t)?s={ms:t._milliseconds,d:t._days,M:t._months}:"number"==typeof t?(s={},e?s[e]=t:s.milliseconds=t):(a=un.exec(t))?(i="-"===a[1]?-1:1,s={y:0,d:y(a[Wo])*i,h:y(a[Yo])*i,m:y(a[Go])*i,s:y(a[Vo])*i,ms:y(a[Uo])*i}):(a=pn.exec(t))?(i="-"===a[1]?-1:1,s={y:se(a[2],i),M:se(a[3],i),w:se(a[4],i),d:se(a[5],i),h:se(a[6],i),m:se(a[7],i),s:se(a[8],i)}):null==s?s={}:"object"==typeof s&&("from"in s||"to"in s)&&(n=ae(At(s.from),At(s.to)),s={},s.ms=n.milliseconds,s.M=n.months),o=new Ht(s),Wt(t)&&r(t,"_locale")&&(o._locale=t._locale),o}function se(t,e){var i=t&&parseFloat(t.replace(",","."));return(isNaN(i)?0:i)*e}function re(t,e){var i={milliseconds:0,months:0};return i.months=e.month()-t.month()+12*(e.year()-t.year()),t.clone().add(i.months,"M").isAfter(e)&&--i.months,
-i.milliseconds=+e-+t.clone().add(i.months,"M"),i}function ae(t,e){var i;return t.isValid()&&e.isValid()?(e=Vt(e,t),t.isBefore(e)?i=re(t,e):(i=re(e,t),i.milliseconds=-i.milliseconds,i.months=-i.months),i):{milliseconds:0,months:0}}function he(t){return 0>t?-1*Math.round(-1*t):Math.round(t)}function de(t,e){return function(i,o){var n,s;return null===o||isNaN(+o)||(x(e,"moment()."+e+"(period, number) is deprecated. Please use moment()."+e+"(number, period)."),s=i,i=o,o=s),i="string"==typeof i?+i:i,n=ne(i,o),le(this,n,t),this}}function le(t,i,o,n){var s=i._milliseconds,r=he(i._days),a=he(i._months);t.isValid()&&(n=null==n?!0:n,s&&t._d.setTime(t._d.valueOf()+s*o),r&&H(t,"Date",j(t,"Date")+r*o),a&&at(t,j(t,"Month")+a*o),n&&e.updateOffset(t,r||a))}function ce(t,e){var i=t||At(),o=Vt(i,this).startOf("day"),n=this.diff(o,"days",!0),s=-6>n?"sameElse":-1>n?"lastWeek":0>n?"lastDay":1>n?"sameDay":2>n?"nextDay":7>n?"nextWeek":"sameElse",r=e&&(k(e[s])?e[s]():e[s]);return this.format(r||this.localeData().calendar(s,this,At(i)))}function ue(){return new m(this)}function pe(t,e){var i=v(t)?t:At(t);return this.isValid()&&i.isValid()?(e=A(p(e)?"millisecond":e),"millisecond"===e?this.valueOf()>i.valueOf():i.valueOf()<this.clone().startOf(e).valueOf()):!1}function fe(t,e){var i=v(t)?t:At(t);return this.isValid()&&i.isValid()?(e=A(p(e)?"millisecond":e),"millisecond"===e?this.valueOf()<i.valueOf():this.clone().endOf(e).valueOf()<i.valueOf()):!1}function me(t,e,i,o){return o=o||"()",("("===o[0]?this.isAfter(t,i):!this.isBefore(t,i))&&(")"===o[1]?this.isBefore(e,i):!this.isAfter(e,i))}function ve(t,e){var i,o=v(t)?t:At(t);return this.isValid()&&o.isValid()?(e=A(e||"millisecond"),"millisecond"===e?this.valueOf()===o.valueOf():(i=o.valueOf(),this.clone().startOf(e).valueOf()<=i&&i<=this.clone().endOf(e).valueOf())):!1}function ge(t,e){return this.isSame(t,e)||this.isAfter(t,e)}function ye(t,e){return this.isSame(t,e)||this.isBefore(t,e)}function be(t,e,i){var o,n,s,r;return this.isValid()?(o=Vt(t,this),o.isValid()?(n=6e4*(o.utcOffset()-this.utcOffset()),e=A(e),"year"===e||"month"===e||"quarter"===e?(r=we(this,o),"quarter"===e?r/=3:"year"===e&&(r/=12)):(s=this-o,r="second"===e?s/1e3:"minute"===e?s/6e4:"hour"===e?s/36e5:"day"===e?(s-n)/864e5:"week"===e?(s-n)/6048e5:s),i?r:g(r)):NaN):NaN}function we(t,e){var i,o,n=12*(e.year()-t.year())+(e.month()-t.month()),s=t.clone().add(n,"months");return 0>e-s?(i=t.clone().add(n-1,"months"),o=(e-s)/(s-i)):(i=t.clone().add(n+1,"months"),o=(e-s)/(i-s)),-(n+o)||0}function _e(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")}function xe(){var t=this.clone().utc();return 0<t.year()&&t.year()<=9999?k(Date.prototype.toISOString)?this.toDate().toISOString():q(t,"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]"):q(t,"YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]")}function ke(t){t||(t=this.isUtc()?e.defaultFormatUtc:e.defaultFormat);var i=q(this,t);return this.localeData().postformat(i)}function Oe(t,e){return this.isValid()&&(v(t)&&t.isValid()||At(t).isValid())?ne({to:this,from:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()}function Me(t){return this.from(At(),t)}function De(t,e){return this.isValid()&&(v(t)&&t.isValid()||At(t).isValid())?ne({from:this,to:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()}function Se(t){return this.to(At(),t)}function Ce(t){var e;return void 0===t?this._locale._abbr:(e=R(t),null!=e&&(this._locale=e),this)}function Te(){return this._locale}function Ee(t){switch(t=A(t)){case"year":this.month(0);case"quarter":case"month":this.date(1);case"week":case"isoWeek":case"day":case"date":this.hours(0);case"hour":this.minutes(0);case"minute":this.seconds(0);case"second":this.milliseconds(0)}return"week"===t&&this.weekday(0),"isoWeek"===t&&this.isoWeekday(1),"quarter"===t&&this.month(3*Math.floor(this.month()/3)),this}function Pe(t){return t=A(t),void 0===t||"millisecond"===t?this:("date"===t&&(t="day"),this.startOf(t).add(1,"isoWeek"===t?"week":t).subtract(1,"ms"))}function Ie(){return this._d.valueOf()-6e4*(this._offset||0)}function Ne(){return Math.floor(this.valueOf()/1e3)}function Re(){return this._offset?new Date(this.valueOf()):this._d}function ze(){var t=this;return[t.year(),t.month(),t.date(),t.hour(),t.minute(),t.second(),t.millisecond()]}function Le(){var t=this;return{years:t.year(),months:t.month(),date:t.date(),hours:t.hours(),minutes:t.minutes(),seconds:t.seconds(),milliseconds:t.milliseconds()}}function Ae(){return this.isValid()?this.toISOString():null}function Be(){return c(this)}function Fe(){return a({},l(this))}function je(){return l(this).overflow}function He(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}}function We(t,e){G(0,[t,t.length],0,e)}function Ye(t){return qe.call(this,t,this.week(),this.weekday(),this.localeData()._week.dow,this.localeData()._week.doy)}function Ge(t){return qe.call(this,t,this.isoWeek(),this.isoWeekday(),1,4)}function Ve(){return Ot(this.year(),1,4)}function Ue(){var t=this.localeData()._week;return Ot(this.year(),t.dow,t.doy)}function qe(t,e,i,o,n){var s;return null==t?kt(this,o,n).year:(s=Ot(t,o,n),e>s&&(e=s),Xe.call(this,t,e,i,o,n))}function Xe(t,e,i,o,n){var s=xt(t,e,i,o,n),r=gt(s.year,0,s.dayOfYear);return this.year(r.getUTCFullYear()),this.month(r.getUTCMonth()),this.date(r.getUTCDate()),this}function Ze(t){return null==t?Math.ceil((this.month()+1)/3):this.month(3*(t-1)+this.month()%3)}function Ke(t){return kt(t,this._week.dow,this._week.doy).week}function Je(){return this._week.dow}function Qe(){return this._week.doy}function $e(t){var e=this.localeData().week(this);return null==t?e:this.add(7*(t-e),"d")}function ti(t){var e=kt(this,1,4).week;return null==t?e:this.add(7*(t-e),"d")}function ei(t,e){return"string"!=typeof t?t:isNaN(t)?(t=e.weekdaysParse(t),"number"==typeof t?t:null):parseInt(t,10)}function ii(t,e){return o(this._weekdays)?this._weekdays[t.day()]:this._weekdays[this._weekdays.isFormat.test(e)?"format":"standalone"][t.day()]}function oi(t){return this._weekdaysShort[t.day()]}function ni(t){return this._weekdaysMin[t.day()]}function si(t,e,i){var o,n,s,r=t.toLocaleLowerCase();if(!this._weekdaysParse)for(this._weekdaysParse=[],this._shortWeekdaysParse=[],this._minWeekdaysParse=[],o=0;7>o;++o)s=h([2e3,1]).day(o),this._minWeekdaysParse[o]=this.weekdaysMin(s,"").toLocaleLowerCase(),this._shortWeekdaysParse[o]=this.weekdaysShort(s,"").toLocaleLowerCase(),this._weekdaysParse[o]=this.weekdays(s,"").toLocaleLowerCase();return i?"dddd"===e?(n=fo.call(this._weekdaysParse,r),-1!==n?n:null):"ddd"===e?(n=fo.call(this._shortWeekdaysParse,r),-1!==n?n:null):(n=fo.call(this._minWeekdaysParse,r),-1!==n?n:null):"dddd"===e?(n=fo.call(this._weekdaysParse,r),-1!==n?n:(n=fo.call(this._shortWeekdaysParse,r),-1!==n?n:(n=fo.call(this._minWeekdaysParse,r),-1!==n?n:null))):"ddd"===e?(n=fo.call(this._shortWeekdaysParse,r),-1!==n?n:(n=fo.call(this._weekdaysParse,r),-1!==n?n:(n=fo.call(this._minWeekdaysParse,r),-1!==n?n:null))):(n=fo.call(this._minWeekdaysParse,r),-1!==n?n:(n=fo.call(this._weekdaysParse,r),-1!==n?n:(n=fo.call(this._shortWeekdaysParse,r),-1!==n?n:null)))}function ri(t,e,i){var o,n,s;if(this._weekdaysParseExact)return si.call(this,t,e,i);for(this._weekdaysParse||(this._weekdaysParse=[],this._minWeekdaysParse=[],this._shortWeekdaysParse=[],this._fullWeekdaysParse=[]),o=0;7>o;o++){if(n=h([2e3,1]).day(o),i&&!this._fullWeekdaysParse[o]&&(this._fullWeekdaysParse[o]=new RegExp("^"+this.weekdays(n,"").replace(".",".?")+"$","i"),this._shortWeekdaysParse[o]=new RegExp("^"+this.weekdaysShort(n,"").replace(".",".?")+"$","i"),this._minWeekdaysParse[o]=new RegExp("^"+this.weekdaysMin(n,"").replace(".",".?")+"$","i")),this._weekdaysParse[o]||(s="^"+this.weekdays(n,"")+"|^"+this.weekdaysShort(n,"")+"|^"+this.weekdaysMin(n,""),this._weekdaysParse[o]=new RegExp(s.replace(".",""),"i")),i&&"dddd"===e&&this._fullWeekdaysParse[o].test(t))return o;if(i&&"ddd"===e&&this._shortWeekdaysParse[o].test(t))return o;if(i&&"dd"===e&&this._minWeekdaysParse[o].test(t))return o;if(!i&&this._weekdaysParse[o].test(t))return o}}function ai(t){if(!this.isValid())return null!=t?this:NaN;var e=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=t?(t=ei(t,this.localeData()),this.add(t-e,"d")):e}function hi(t){if(!this.isValid())return null!=t?this:NaN;var e=(this.day()+7-this.localeData()._week.dow)%7;return null==t?e:this.add(t-e,"d")}function di(t){return this.isValid()?null==t?this.day()||7:this.day(this.day()%7?t:t-7):null!=t?this:NaN}function li(t){return this._weekdaysParseExact?(r(this,"_weekdaysRegex")||pi.call(this),t?this._weekdaysStrictRegex:this._weekdaysRegex):this._weekdaysStrictRegex&&t?this._weekdaysStrictRegex:this._weekdaysRegex}function ci(t){return this._weekdaysParseExact?(r(this,"_weekdaysRegex")||pi.call(this),t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex):this._weekdaysShortStrictRegex&&t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex}function ui(t){return this._weekdaysParseExact?(r(this,"_weekdaysRegex")||pi.call(this),t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex):this._weekdaysMinStrictRegex&&t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex}function pi(){function t(t,e){return e.length-t.length}var e,i,o,n,s,r=[],a=[],d=[],l=[];for(e=0;7>e;e++)i=h([2e3,1]).day(e),o=this.weekdaysMin(i,""),n=this.weekdaysShort(i,""),s=this.weekdays(i,""),r.push(o),a.push(n),d.push(s),l.push(o),l.push(n),l.push(s);for(r.sort(t),a.sort(t),d.sort(t),l.sort(t),e=0;7>e;e++)a[e]=Q(a[e]),d[e]=Q(d[e]),l[e]=Q(l[e]);this._weekdaysRegex=new RegExp("^("+l.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+d.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+a.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+r.join("|")+")","i")}function fi(t){var e=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==t?e:this.add(t-e,"d")}function mi(){return this.hours()%12||12}function vi(){return this.hours()||24}function gi(t,e){G(t,0,0,function(){return this.localeData().meridiem(this.hours(),this.minutes(),e)})}function yi(t,e){return e._meridiemParse}function bi(t){return"p"===(t+"").toLowerCase().charAt(0)}function wi(t,e,i){return t>11?i?"pm":"PM":i?"am":"AM"}function _i(t,e){e[Uo]=y(1e3*("0."+t))}function xi(){return this._isUTC?"UTC":""}function ki(){return this._isUTC?"Coordinated Universal Time":""}function Oi(t){return At(1e3*t)}function Mi(){return At.apply(null,arguments).parseZone()}function Di(t,e,i){var o=this._calendar[t];return k(o)?o.call(e,i):o}function Si(t){var e=this._longDateFormat[t],i=this._longDateFormat[t.toUpperCase()];return e||!i?e:(this._longDateFormat[t]=i.replace(/MMMM|MM|DD|dddd/g,function(t){return t.slice(1)}),this._longDateFormat[t])}function Ci(){return this._invalidDate}function Ti(t){return this._ordinal.replace("%d",t)}function Ei(t){return t}function Pi(t,e,i,o){var n=this._relativeTime[i];return k(n)?n(t,e,i,o):n.replace(/%d/i,t)}function Ii(t,e){var i=this._relativeTime[t>0?"future":"past"];return k(i)?i(e):i.replace(/%s/i,e)}function Ni(t,e,i,o){var n=R(),s=h().set(o,e);return n[i](s,t)}function Ri(t,e,i){if("number"==typeof t&&(e=t,t=void 0),t=t||"",null!=e)return Ni(t,e,i,"month");var o,n=[];for(o=0;12>o;o++)n[o]=Ni(t,o,i,"month");return n}function zi(t,e,i,o){"boolean"==typeof t?("number"==typeof e&&(i=e,e=void 0),e=e||""):(e=t,i=e,t=!1,"number"==typeof e&&(i=e,e=void 0),e=e||"");var n=R(),s=t?n._week.dow:0;if(null!=i)return Ni(e,(i+s)%7,o,"day");var r,a=[];for(r=0;7>r;r++)a[r]=Ni(e,(r+s)%7,o,"day");return a}function Li(t,e){return Ri(t,e,"months")}function Ai(t,e){return Ri(t,e,"monthsShort")}function Bi(t,e,i){return zi(t,e,i,"weekdays")}function Fi(t,e,i){return zi(t,e,i,"weekdaysShort")}function ji(t,e,i){return zi(t,e,i,"weekdaysMin")}function Hi(){var t=this._data;return this._milliseconds=jn(this._milliseconds),this._days=jn(this._days),this._months=jn(this._months),t.milliseconds=jn(t.milliseconds),t.seconds=jn(t.seconds),t.minutes=jn(t.minutes),t.hours=jn(t.hours),t.months=jn(t.months),t.years=jn(t.years),this}function Wi(t,e,i,o){var n=ne(e,i);return t._milliseconds+=o*n._milliseconds,t._days+=o*n._days,t._months+=o*n._months,t._bubble()}function Yi(t,e){return Wi(this,t,e,1)}function Gi(t,e){return Wi(this,t,e,-1)}function Vi(t){return 0>t?Math.floor(t):Math.ceil(t)}function Ui(){var t,e,i,o,n,s=this._milliseconds,r=this._days,a=this._months,h=this._data;return s>=0&&r>=0&&a>=0||0>=s&&0>=r&&0>=a||(s+=864e5*Vi(Xi(a)+r),r=0,a=0),h.milliseconds=s%1e3,t=g(s/1e3),h.seconds=t%60,e=g(t/60),h.minutes=e%60,i=g(e/60),h.hours=i%24,r+=g(i/24),n=g(qi(r)),a+=n,r-=Vi(Xi(n)),o=g(a/12),a%=12,h.days=r,h.months=a,h.years=o,this}function qi(t){return 4800*t/146097}function Xi(t){return 146097*t/4800}function Zi(t){var e,i,o=this._milliseconds;if(t=A(t),"month"===t||"year"===t)return e=this._days+o/864e5,i=this._months+qi(e),"month"===t?i:i/12;switch(e=this._days+Math.round(Xi(this._months)),t){case"week":return e/7+o/6048e5;case"day":return e+o/864e5;case"hour":return 24*e+o/36e5;case"minute":return 1440*e+o/6e4;case"second":return 86400*e+o/1e3;case"millisecond":return Math.floor(864e5*e)+o;default:throw new Error("Unknown unit "+t)}}function Ki(){return this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*y(this._months/12)}function Ji(t){return function(){return this.as(t)}}function Qi(t){return t=A(t),this[t+"s"]()}function $i(t){return function(){return this._data[t]}}function to(){return g(this.days()/7)}function eo(t,e,i,o,n){return n.relativeTime(e||1,!!i,t,o)}function io(t,e,i){var o=ne(t).abs(),n=is(o.as("s")),s=is(o.as("m")),r=is(o.as("h")),a=is(o.as("d")),h=is(o.as("M")),d=is(o.as("y")),l=n<os.s&&["s",n]||1>=s&&["m"]||s<os.m&&["mm",s]||1>=r&&["h"]||r<os.h&&["hh",r]||1>=a&&["d"]||a<os.d&&["dd",a]||1>=h&&["M"]||h<os.M&&["MM",h]||1>=d&&["y"]||["yy",d];return l[2]=e,l[3]=+t>0,l[4]=i,eo.apply(null,l)}function oo(t,e){return void 0===os[t]?!1:void 0===e?os[t]:(os[t]=e,!0)}function no(t){var e=this.localeData(),i=io(this,!t,e);return t&&(i=e.pastFuture(+this,i)),e.postformat(i)}function so(){var t,e,i,o=ns(this._milliseconds)/1e3,n=ns(this._days),s=ns(this._months);t=g(o/60),e=g(t/60),o%=60,t%=60,i=g(s/12),s%=12;var r=i,a=s,h=n,d=e,l=t,c=o,u=this.asSeconds();return u?(0>u?"-":"")+"P"+(r?r+"Y":"")+(a?a+"M":"")+(h?h+"D":"")+(d||l||c?"T":"")+(d?d+"H":"")+(l?l+"M":"")+(c?c+"S":""):"P0D"}var ro,ao;ao=Array.prototype.some?Array.prototype.some:function(t){for(var e=Object(this),i=e.length>>>0,o=0;i>o;o++)if(o in e&&t.call(this,e[o],o,e))return!0;return!1};var ho=e.momentProperties=[],lo=!1,co={};e.suppressDeprecationWarnings=!1,e.deprecationHandler=null;var uo;uo=Object.keys?Object.keys:function(t){var e,i=[];for(e in t)r(t,e)&&i.push(e);return i};var po,fo,mo={},vo={},go=/(\[[^\[]*\])|(\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,yo=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,bo={},wo={},_o=/\d/,xo=/\d\d/,ko=/\d{3}/,Oo=/\d{4}/,Mo=/[+-]?\d{6}/,Do=/\d\d?/,So=/\d\d\d\d?/,Co=/\d\d\d\d\d\d?/,To=/\d{1,3}/,Eo=/\d{1,4}/,Po=/[+-]?\d{1,6}/,Io=/\d+/,No=/[+-]?\d+/,Ro=/Z|[+-]\d\d:?\d\d/gi,zo=/Z|[+-]\d\d(?::?\d\d)?/gi,Lo=/[+-]?\d+(\.\d{1,3})?/,Ao=/[0-9]*['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]+|[\u0600-\u06FF\/]+(\s*?[\u0600-\u06FF]+){1,2}/i,Bo={},Fo={},jo=0,Ho=1,Wo=2,Yo=3,Go=4,Vo=5,Uo=6,qo=7,Xo=8;fo=Array.prototype.indexOf?Array.prototype.indexOf:function(t){var e;for(e=0;e<this.length;++e)if(this[e]===t)return e;return-1},G("M",["MM",2],"Mo",function(){return this.month()+1}),G("MMM",0,0,function(t){return this.localeData().monthsShort(this,t)}),G("MMMM",0,0,function(t){return this.localeData().months(this,t)}),L("month","M"),Z("M",Do),Z("MM",Do,xo),Z("MMM",function(t,e){return e.monthsShortRegex(t)}),Z("MMMM",function(t,e){return e.monthsRegex(t)}),$(["M","MM"],function(t,e){e[Ho]=y(t)-1}),$(["MMM","MMMM"],function(t,e,i,o){var n=i._locale.monthsParse(t,o,i._strict);null!=n?e[Ho]=n:l(i).invalidMonth=t});var Zo=/D[oD]?(\[[^\[\]]*\]|\s+)+MMMM?/,Ko="January_February_March_April_May_June_July_August_September_October_November_December".split("_"),Jo="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_"),Qo=Ao,$o=Ao,tn=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?/,en=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?/,on=/Z|[+-]\d\d(?::?\d\d)?/,nn=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/]],sn=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],rn=/^\/?Date\((\-?\d+)/i;e.createFromInputFallback=_("moment construction falls back to js Date. This is discouraged and will be removed in upcoming major release. Please refer to https://github.com/moment/moment/issues/1407 for more info.",function(t){t._d=new Date(t._i+(t._useUTC?" UTC":""))}),G("Y",0,0,function(){var t=this.year();return 9999>=t?""+t:"+"+t}),G(0,["YY",2],0,function(){return this.year()%100}),G(0,["YYYY",4],0,"year"),G(0,["YYYYY",5],0,"year"),G(0,["YYYYYY",6,!0],0,"year"),L("year","y"),Z("Y",No),Z("YY",Do,xo),Z("YYYY",Eo,Oo),Z("YYYYY",Po,Mo),Z("YYYYYY",Po,Mo),$(["YYYYY","YYYYYY"],jo),$("YYYY",function(t,i){i[jo]=2===t.length?e.parseTwoDigitYear(t):y(t)}),$("YY",function(t,i){i[jo]=e.parseTwoDigitYear(t)}),$("Y",function(t,e){e[jo]=parseInt(t,10)}),e.parseTwoDigitYear=function(t){return y(t)+(y(t)>68?1900:2e3)};var an=F("FullYear",!0);e.ISO_8601=function(){};var hn=_("moment().min is deprecated, use moment.max instead. https://github.com/moment/moment/issues/1548",function(){var t=At.apply(null,arguments);return this.isValid()&&t.isValid()?this>t?this:t:u()}),dn=_("moment().max is deprecated, use moment.min instead. https://github.com/moment/moment/issues/1548",function(){var t=At.apply(null,arguments);return this.isValid()&&t.isValid()?t>this?this:t:u()}),ln=function(){return Date.now?Date.now():+new Date};Yt("Z",":"),Yt("ZZ",""),Z("Z",zo),Z("ZZ",zo),$(["Z","ZZ"],function(t,e,i){i._useUTC=!0,i._tzm=Gt(zo,t)});var cn=/([\+\-]|\d\d)/gi;e.updateOffset=function(){};var un=/^(\-)?(?:(\d*)[. ])?(\d+)\:(\d+)(?:\:(\d+)\.?(\d{3})?\d*)?$/,pn=/^(-)?P(?:(-?[0-9,.]*)Y)?(?:(-?[0-9,.]*)M)?(?:(-?[0-9,.]*)W)?(?:(-?[0-9,.]*)D)?(?:T(?:(-?[0-9,.]*)H)?(?:(-?[0-9,.]*)M)?(?:(-?[0-9,.]*)S)?)?$/;ne.fn=Ht.prototype;var fn=de(1,"add"),mn=de(-1,"subtract");e.defaultFormat="YYYY-MM-DDTHH:mm:ssZ",e.defaultFormatUtc="YYYY-MM-DDTHH:mm:ss[Z]";var vn=_("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",function(t){return void 0===t?this.localeData():this.locale(t)});G(0,["gg",2],0,function(){return this.weekYear()%100}),G(0,["GG",2],0,function(){return this.isoWeekYear()%100}),We("gggg","weekYear"),We("ggggg","weekYear"),We("GGGG","isoWeekYear"),We("GGGGG","isoWeekYear"),L("weekYear","gg"),L("isoWeekYear","GG"),Z("G",No),Z("g",No),Z("GG",Do,xo),Z("gg",Do,xo),Z("GGGG",Eo,Oo),Z("gggg",Eo,Oo),Z("GGGGG",Po,Mo),Z("ggggg",Po,Mo),tt(["gggg","ggggg","GGGG","GGGGG"],function(t,e,i,o){e[o.substr(0,2)]=y(t)}),tt(["gg","GG"],function(t,i,o,n){i[n]=e.parseTwoDigitYear(t)}),G("Q",0,"Qo","quarter"),L("quarter","Q"),Z("Q",_o),$("Q",function(t,e){e[Ho]=3*(y(t)-1)}),G("w",["ww",2],"wo","week"),G("W",["WW",2],"Wo","isoWeek"),L("week","w"),L("isoWeek","W"),Z("w",Do),Z("ww",Do,xo),Z("W",Do),Z("WW",Do,xo),tt(["w","ww","W","WW"],function(t,e,i,o){e[o.substr(0,1)]=y(t)});var gn={dow:0,doy:6};G("D",["DD",2],"Do","date"),L("date","D"),Z("D",Do),Z("DD",Do,xo),Z("Do",function(t,e){return t?e._ordinalParse:e._ordinalParseLenient}),$(["D","DD"],Wo),$("Do",function(t,e){e[Wo]=y(t.match(Do)[0],10)});var yn=F("Date",!0);G("d",0,"do","day"),G("dd",0,0,function(t){return this.localeData().weekdaysMin(this,t)}),G("ddd",0,0,function(t){return this.localeData().weekdaysShort(this,t)}),G("dddd",0,0,function(t){return this.localeData().weekdays(this,t)}),G("e",0,0,"weekday"),G("E",0,0,"isoWeekday"),L("day","d"),L("weekday","e"),L("isoWeekday","E"),Z("d",Do),Z("e",Do),Z("E",Do),Z("dd",function(t,e){return e.weekdaysMinRegex(t)}),Z("ddd",function(t,e){return e.weekdaysShortRegex(t)}),Z("dddd",function(t,e){return e.weekdaysRegex(t)}),tt(["dd","ddd","dddd"],function(t,e,i,o){var n=i._locale.weekdaysParse(t,o,i._strict);null!=n?e.d=n:l(i).invalidWeekday=t}),tt(["d","e","E"],function(t,e,i,o){e[o]=y(t)});var bn="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),wn="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),_n="Su_Mo_Tu_We_Th_Fr_Sa".split("_"),xn=Ao,kn=Ao,On=Ao;G("DDD",["DDDD",3],"DDDo","dayOfYear"),L("dayOfYear","DDD"),Z("DDD",To),Z("DDDD",ko),$(["DDD","DDDD"],function(t,e,i){i._dayOfYear=y(t)}),G("H",["HH",2],0,"hour"),G("h",["hh",2],0,mi),G("k",["kk",2],0,vi),G("hmm",0,0,function(){return""+mi.apply(this)+Y(this.minutes(),2)}),G("hmmss",0,0,function(){return""+mi.apply(this)+Y(this.minutes(),2)+Y(this.seconds(),2)}),G("Hmm",0,0,function(){return""+this.hours()+Y(this.minutes(),2)}),G("Hmmss",0,0,function(){return""+this.hours()+Y(this.minutes(),2)+Y(this.seconds(),2)}),gi("a",!0),gi("A",!1),L("hour","h"),Z("a",yi),Z("A",yi),Z("H",Do),Z("h",Do),Z("HH",Do,xo),Z("hh",Do,xo),Z("hmm",So),Z("hmmss",Co),Z("Hmm",So),Z("Hmmss",Co),$(["H","HH"],Yo),$(["a","A"],function(t,e,i){i._isPm=i._locale.isPM(t),i._meridiem=t}),$(["h","hh"],function(t,e,i){e[Yo]=y(t),l(i).bigHour=!0}),$("hmm",function(t,e,i){var o=t.length-2;e[Yo]=y(t.substr(0,o)),e[Go]=y(t.substr(o)),l(i).bigHour=!0}),$("hmmss",function(t,e,i){var o=t.length-4,n=t.length-2;e[Yo]=y(t.substr(0,o)),e[Go]=y(t.substr(o,2)),e[Vo]=y(t.substr(n)),l(i).bigHour=!0}),$("Hmm",function(t,e,i){var o=t.length-2;e[Yo]=y(t.substr(0,o)),e[Go]=y(t.substr(o))}),$("Hmmss",function(t,e,i){var o=t.length-4,n=t.length-2;e[Yo]=y(t.substr(0,o)),e[Go]=y(t.substr(o,2)),e[Vo]=y(t.substr(n))});var Mn=/[ap]\.?m?\.?/i,Dn=F("Hours",!0);G("m",["mm",2],0,"minute"),L("minute","m"),Z("m",Do),Z("mm",Do,xo),$(["m","mm"],Go);var Sn=F("Minutes",!1);G("s",["ss",2],0,"second"),L("second","s"),Z("s",Do),Z("ss",Do,xo),$(["s","ss"],Vo);var Cn=F("Seconds",!1);G("S",0,0,function(){return~~(this.millisecond()/100)}),G(0,["SS",2],0,function(){return~~(this.millisecond()/10)}),G(0,["SSS",3],0,"millisecond"),G(0,["SSSS",4],0,function(){return 10*this.millisecond()}),G(0,["SSSSS",5],0,function(){return 100*this.millisecond()}),G(0,["SSSSSS",6],0,function(){return 1e3*this.millisecond()}),G(0,["SSSSSSS",7],0,function(){return 1e4*this.millisecond()}),G(0,["SSSSSSSS",8],0,function(){return 1e5*this.millisecond()}),G(0,["SSSSSSSSS",9],0,function(){return 1e6*this.millisecond()}),L("millisecond","ms"),Z("S",To,_o),Z("SS",To,xo),Z("SSS",To,ko);var Tn;for(Tn="SSSS";Tn.length<=9;Tn+="S")Z(Tn,Io);for(Tn="S";Tn.length<=9;Tn+="S")$(Tn,_i);var En=F("Milliseconds",!1);G("z",0,0,"zoneAbbr"),G("zz",0,0,"zoneName");var Pn=m.prototype;Pn.add=fn,Pn.calendar=ce,Pn.clone=ue,Pn.diff=be,Pn.endOf=Pe,Pn.format=ke,Pn.from=Oe,Pn.fromNow=Me,Pn.to=De,Pn.toNow=Se,Pn.get=W,Pn.invalidAt=je,Pn.isAfter=pe,Pn.isBefore=fe,Pn.isBetween=me,Pn.isSame=ve,Pn.isSameOrAfter=ge,Pn.isSameOrBefore=ye,Pn.isValid=Be,Pn.lang=vn,Pn.locale=Ce,Pn.localeData=Te,Pn.max=dn,Pn.min=hn,Pn.parsingFlags=Fe,Pn.set=W,Pn.startOf=Ee,Pn.subtract=mn,Pn.toArray=ze,Pn.toObject=Le,Pn.toDate=Re,Pn.toISOString=xe,Pn.toJSON=Ae,Pn.toString=_e,Pn.unix=Ne,Pn.valueOf=Ie,Pn.creationData=He,Pn.year=an,Pn.isLeapYear=wt,Pn.weekYear=Ye,Pn.isoWeekYear=Ge,Pn.quarter=Pn.quarters=Ze,Pn.month=ht,Pn.daysInMonth=dt,Pn.week=Pn.weeks=$e,Pn.isoWeek=Pn.isoWeeks=ti,Pn.weeksInYear=Ue,Pn.isoWeeksInYear=Ve,Pn.date=yn,Pn.day=Pn.days=ai,Pn.weekday=hi,Pn.isoWeekday=di,Pn.dayOfYear=fi,Pn.hour=Pn.hours=Dn,Pn.minute=Pn.minutes=Sn,Pn.second=Pn.seconds=Cn,Pn.millisecond=Pn.milliseconds=En,Pn.utcOffset=qt,Pn.utc=Zt,Pn.local=Kt,Pn.parseZone=Jt,Pn.hasAlignedHourOffset=Qt,Pn.isDST=$t,Pn.isDSTShifted=te,Pn.isLocal=ee,Pn.isUtcOffset=ie,Pn.isUtc=oe,Pn.isUTC=oe,Pn.zoneAbbr=xi,Pn.zoneName=ki,Pn.dates=_("dates accessor is deprecated. Use date instead.",yn),Pn.months=_("months accessor is deprecated. Use month instead",ht),Pn.years=_("years accessor is deprecated. Use year instead",an),Pn.zone=_("moment().zone is deprecated, use moment().utcOffset instead. https://github.com/moment/moment/issues/1779",Xt);var In=Pn,Nn={sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},Rn={LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},zn="Invalid date",Ln="%d",An=/\d{1,2}/,Bn={future:"in %s",past:"%s ago",s:"a few seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},Fn=S.prototype;Fn._calendar=Nn,Fn.calendar=Di,Fn._longDateFormat=Rn,Fn.longDateFormat=Si,Fn._invalidDate=zn,Fn.invalidDate=Ci,Fn._ordinal=Ln,Fn.ordinal=Ti,Fn._ordinalParse=An,Fn.preparse=Ei,Fn.postformat=Ei,Fn._relativeTime=Bn,Fn.relativeTime=Pi,Fn.pastFuture=Ii,Fn.set=M,Fn.months=ot,Fn._months=Ko,Fn.monthsShort=nt,Fn._monthsShort=Jo,Fn.monthsParse=rt,Fn._monthsRegex=$o,Fn.monthsRegex=ct,Fn._monthsShortRegex=Qo,Fn.monthsShortRegex=lt,Fn.week=Ke,Fn._week=gn,Fn.firstDayOfYear=Qe,Fn.firstDayOfWeek=Je,Fn.weekdays=ii,Fn._weekdays=bn,Fn.weekdaysMin=ni,Fn._weekdaysMin=_n,Fn.weekdaysShort=oi,Fn._weekdaysShort=wn,Fn.weekdaysParse=ri,Fn._weekdaysRegex=xn,Fn.weekdaysRegex=li,Fn._weekdaysShortRegex=kn,Fn.weekdaysShortRegex=ci,Fn._weekdaysMinRegex=On,Fn.weekdaysMinRegex=ui,Fn.isPM=bi,Fn._meridiemParse=Mn,Fn.meridiem=wi,P("en",{ordinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(t){var e=t%10,i=1===y(t%100/10)?"th":1===e?"st":2===e?"nd":3===e?"rd":"th";return t+i}}),e.lang=_("moment.lang is deprecated. Use moment.locale instead.",P),e.langData=_("moment.langData is deprecated. Use moment.localeData instead.",R);var jn=Math.abs,Hn=Ji("ms"),Wn=Ji("s"),Yn=Ji("m"),Gn=Ji("h"),Vn=Ji("d"),Un=Ji("w"),qn=Ji("M"),Xn=Ji("y"),Zn=$i("milliseconds"),Kn=$i("seconds"),Jn=$i("minutes"),Qn=$i("hours"),$n=$i("days"),ts=$i("months"),es=$i("years"),is=Math.round,os={s:45,m:45,h:22,d:26,M:11},ns=Math.abs,ss=Ht.prototype;ss.abs=Hi,ss.add=Yi,ss.subtract=Gi,ss.as=Zi,ss.asMilliseconds=Hn,ss.asSeconds=Wn,ss.asMinutes=Yn,ss.asHours=Gn,ss.asDays=Vn,ss.asWeeks=Un,ss.asMonths=qn,ss.asYears=Xn,ss.valueOf=Ki,ss._bubble=Ui,ss.get=Qi,ss.milliseconds=Zn,ss.seconds=Kn,ss.minutes=Jn,ss.hours=Qn,ss.days=$n,ss.weeks=to,ss.months=ts,ss.years=es,ss.humanize=no,ss.toISOString=so,ss.toString=so,ss.toJSON=so,ss.locale=Ce,ss.localeData=Te,ss.toIsoString=_("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",so),ss.lang=vn,G("X",0,0,"unix"),G("x",0,0,"valueOf"),Z("x",No),Z("X",Lo),$("X",function(t,e,i){i._d=new Date(1e3*parseFloat(t,10))}),$("x",function(t,e,i){i._d=new Date(y(t))}),e.version="2.13.0",i(At),e.fn=In,e.min=Ft,e.max=jt,e.now=ln,e.utc=h,e.unix=Oi,e.months=Li,e.isDate=n,e.locale=P,e.invalid=u,e.duration=ne,e.isMoment=v,e.weekdays=Bi,e.parseZone=Mi,e.localeData=R,e.isDuration=Wt,e.monthsShort=Ai,e.weekdaysMin=ji,e.defineLocale=I,e.updateLocale=N,e.locales=z,e.weekdaysShort=Fi,e.normalizeUnits=A,e.relativeTimeThreshold=oo,e.prototype=In;var rs=e;return rs})}).call(e,i(4)(t))},function(t,e){t.exports=function(t){return t.webpackPolyfill||(t.deprecate=function(){},t.paths=[],t.children=[],t.webpackPolyfill=1),t}},function(t,e){function i(t){throw new Error("Cannot find module '"+t+"'.")}i.keys=function(){return[]},i.resolve=i,t.exports=i,i.id=5},function(t,e){(function(e){function i(t,e,i){var o=e&&i||0,n=0;for(e=e||[],t.toLowerCase().replace(/[0-9a-f]{2}/g,function(t){16>n&&(e[o+n++]=c[t])});16>n;)e[o+n++]=0;return e}function o(t,e){var i=e||0,o=l;return o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]}function n(t,e,i){var n=e&&i||0,s=e||[];t=t||{};var r=void 0!==t.clockseq?t.clockseq:m,a=void 0!==t.msecs?t.msecs:(new Date).getTime(),h=void 0!==t.nsecs?t.nsecs:g+1,d=a-v+(h-g)/1e4;if(0>d&&void 0===t.clockseq&&(r=r+1&16383),(0>d||a>v)&&void 0===t.nsecs&&(h=0),h>=1e4)throw new Error("uuid.v1(): Can't create more than 10M uuids/sec");v=a,g=h,m=r,a+=122192928e5;var l=(1e4*(268435455&a)+h)%4294967296;s[n++]=l>>>24&255,s[n++]=l>>>16&255,s[n++]=l>>>8&255,s[n++]=255&l;var c=a/4294967296*1e4&268435455;s[n++]=c>>>8&255,s[n++]=255&c,s[n++]=c>>>24&15|16,s[n++]=c>>>16&255,s[n++]=r>>>8|128,s[n++]=255&r;for(var u=t.node||f,p=0;6>p;p++)s[n+p]=u[p];return e?e:o(s)}function s(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"==t?new Array(16):null,t=null),t=t||{};var s=t.random||(t.rng||r)();if(s[6]=15&s[6]|64,s[8]=63&s[8]|128,e)for(var a=0;16>a;a++)e[n+a]=s[a];return e||o(s)}var r,a="undefined"!=typeof window?window:"undefined"!=typeof e?e:null;if(a&&a.crypto&&crypto.getRandomValues){var h=new Uint8Array(16);r=function(){return crypto.getRandomValues(h),h}}if(!r){var d=new Array(16);r=function(){for(var t,e=0;16>e;e++)0===(3&e)&&(t=4294967296*Math.random()),d[e]=t>>>((3&e)<<3)&255;return d}}for(var l=[],c={},u=0;256>u;u++)l[u]=(u+256).toString(16).substr(1),c[l[u]]=u;var p=r(),f=[1|p[0],p[1],p[2],p[3],p[4],p[5]],m=16383&(p[6]<<8|p[7]),v=0,g=0,y=s;y.v1=n,y.v4=s,y.parse=i,y.unparse=o,t.exports=y}).call(e,function(){return this}())},function(t,e,i){e.util=i(1),e.DOMutil=i(8),e.DataSet=i(9),e.DataView=i(11),e.Queue=i(10),e.Graph3d=i(12),e.graph3d={Camera:i(16),Filter:i(17),Point2d:i(15),Point3d:i(14),Slider:i(18),StepNumber:i(19)},e.moment=i(2),e.Hammer=i(20),e.keycharm=i(23)},function(t,e){e.prepareElements=function(t){for(var e in t)t.hasOwnProperty(e)&&(t[e].redundant=t[e].used,t[e].used=[])},e.cleanupElements=function(t){for(var e in t)if(t.hasOwnProperty(e)&&t[e].redundant){for(var i=0;i<t[e].redundant.length;i++)t[e].redundant[i].parentNode.removeChild(t[e].redundant[i]);t[e].redundant=[]}},e.resetElements=function(t){e.prepareElements(t),e.cleanupElements(t),e.prepareElements(t)},e.getSVGElement=function(t,e,i){var o;return e.hasOwnProperty(t)?e[t].redundant.length>0?(o=e[t].redundant[0],e[t].redundant.shift()):(o=document.createElementNS("http://www.w3.org/2000/svg",t),i.appendChild(o)):(o=document.createElementNS("http://www.w3.org/2000/svg",t),e[t]={used:[],redundant:[]},i.appendChild(o)),e[t].used.push(o),o},e.getDOMElement=function(t,e,i,o){var n;return e.hasOwnProperty(t)?e[t].redundant.length>0?(n=e[t].redundant[0],e[t].redundant.shift()):(n=document.createElement(t),void 0!==o?i.insertBefore(n,o):i.appendChild(n)):(n=document.createElement(t),e[t]={used:[],redundant:[]},void 0!==o?i.insertBefore(n,o):i.appendChild(n)),e[t].used.push(n),n},e.drawPoint=function(t,i,o,n,s,r){var a;if("circle"==o.style?(a=e.getSVGElement("circle",n,s),a.setAttributeNS(null,"cx",t),a.setAttributeNS(null,"cy",i),a.setAttributeNS(null,"r",.5*o.size)):(a=e.getSVGElement("rect",n,s),a.setAttributeNS(null,"x",t-.5*o.size),a.setAttributeNS(null,"y",i-.5*o.size),a.setAttributeNS(null,"width",o.size),a.setAttributeNS(null,"height",o.size)),void 0!==o.styles&&a.setAttributeNS(null,"style",o.styles),a.setAttributeNS(null,"class",o.className+" vis-point"),r){var h=e.getSVGElement("text",n,s);
-r.xOffset&&(t+=r.xOffset),r.yOffset&&(i+=r.yOffset),r.content&&(h.textContent=r.content),r.className&&h.setAttributeNS(null,"class",r.className+" vis-label"),h.setAttributeNS(null,"x",t),h.setAttributeNS(null,"y",i)}return a},e.drawBar=function(t,i,o,n,s,r,a,h){if(0!=n){0>n&&(n*=-1,i-=n);var d=e.getSVGElement("rect",r,a);d.setAttributeNS(null,"x",t-.5*o),d.setAttributeNS(null,"y",i),d.setAttributeNS(null,"width",o),d.setAttributeNS(null,"height",n),d.setAttributeNS(null,"class",s),h&&d.setAttributeNS(null,"style",h)}}},function(t,e,i){function o(t,e){if(t&&!Array.isArray(t)&&(e=t,t=null),this._options=e||{},this._data={},this.length=0,this._fieldId=this._options.fieldId||"id",this._type={},this._options.type)for(var i=Object.keys(this._options.type),o=0,n=i.length;n>o;o++){var s=i[o],r=this._options.type[s];"Date"==r||"ISODate"==r||"ASPDate"==r?this._type[s]="Date":this._type[s]=r}if(this._options.convert)throw new Error('Option "convert" is deprecated. Use "type" instead.');this._subscribers={},t&&this.add(t),this.setOptions(e)}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(1),r=i(10);o.prototype.setOptions=function(t){t&&void 0!==t.queue&&(t.queue===!1?this._queue&&(this._queue.destroy(),delete this._queue):(this._queue||(this._queue=r.extend(this,{replace:["add","update","remove"]})),"object"===n(t.queue)&&this._queue.setOptions(t.queue)))},o.prototype.on=function(t,e){var i=this._subscribers[t];i||(i=[],this._subscribers[t]=i),i.push({callback:e})},o.prototype.subscribe=function(){throw new Error("DataSet.subscribe is deprecated. Use DataSet.on instead.")},o.prototype.off=function(t,e){var i=this._subscribers[t];i&&(this._subscribers[t]=i.filter(function(t){return t.callback!=e}))},o.prototype.unsubscribe=function(){throw new Error("DataSet.unsubscribe is deprecated. Use DataSet.off instead.")},o.prototype._trigger=function(t,e,i){if("*"==t)throw new Error("Cannot trigger event *");var o=[];t in this._subscribers&&(o=o.concat(this._subscribers[t])),"*"in this._subscribers&&(o=o.concat(this._subscribers["*"]));for(var n=0,s=o.length;s>n;n++){var r=o[n];r.callback&&r.callback(t,e,i||null)}},o.prototype.add=function(t,e){var i,o=[],n=this;if(Array.isArray(t))for(var s=0,r=t.length;r>s;s++)i=n._addItem(t[s]),o.push(i);else{if(!(t instanceof Object))throw new Error("Unknown dataType");i=n._addItem(t),o.push(i)}return o.length&&this._trigger("add",{items:o},e),o},o.prototype.update=function(t,e){var i=[],o=[],n=[],r=[],a=this,h=a._fieldId,d=function(t){var e=t[h];if(a._data[e]){var d=s.extend({},a._data[e]);e=a._updateItem(t),o.push(e),r.push(t),n.push(d)}else e=a._addItem(t),i.push(e)};if(Array.isArray(t))for(var l=0,c=t.length;c>l;l++)t[l]instanceof Object?d(t[l]):console.warn("Ignoring input item, which is not an object at index "+l);else{if(!(t instanceof Object))throw new Error("Unknown dataType");d(t)}if(i.length&&this._trigger("add",{items:i},e),o.length){var u={items:o,oldData:n,data:r};this._trigger("update",u,e)}return i.concat(o)},o.prototype.get=function(t){var e,i,o,n=this,r=s.getType(arguments[0]);"String"==r||"Number"==r?(e=arguments[0],o=arguments[1]):"Array"==r?(i=arguments[0],o=arguments[1]):o=arguments[0];var a;if(o&&o.returnType){var h=["Array","Object"];a=-1==h.indexOf(o.returnType)?"Array":o.returnType}else a="Array";var d,l,c,u,p,f=o&&o.type||this._options.type,m=o&&o.filter,v=[];if(void 0!=e)d=n._getItem(e,f),d&&m&&!m(d)&&(d=null);else if(void 0!=i)for(u=0,p=i.length;p>u;u++)d=n._getItem(i[u],f),m&&!m(d)||v.push(d);else for(l=Object.keys(this._data),u=0,p=l.length;p>u;u++)c=l[u],d=n._getItem(c,f),m&&!m(d)||v.push(d);if(o&&o.order&&void 0==e&&this._sort(v,o.order),o&&o.fields){var g=o.fields;if(void 0!=e)d=this._filterFields(d,g);else for(u=0,p=v.length;p>u;u++)v[u]=this._filterFields(v[u],g)}if("Object"==a){var y,b={};for(u=0,p=v.length;p>u;u++)y=v[u],b[y.id]=y;return b}return void 0!=e?d:v},o.prototype.getIds=function(t){var e,i,o,n,s,r=this._data,a=t&&t.filter,h=t&&t.order,d=t&&t.type||this._options.type,l=Object.keys(r),c=[];if(a)if(h){for(s=[],e=0,i=l.length;i>e;e++)o=l[e],n=this._getItem(o,d),a(n)&&s.push(n);for(this._sort(s,h),e=0,i=s.length;i>e;e++)c.push(s[e][this._fieldId])}else for(e=0,i=l.length;i>e;e++)o=l[e],n=this._getItem(o,d),a(n)&&c.push(n[this._fieldId]);else if(h){for(s=[],e=0,i=l.length;i>e;e++)o=l[e],s.push(r[o]);for(this._sort(s,h),e=0,i=s.length;i>e;e++)c.push(s[e][this._fieldId])}else for(e=0,i=l.length;i>e;e++)o=l[e],n=r[o],c.push(n[this._fieldId]);return c},o.prototype.getDataSet=function(){return this},o.prototype.forEach=function(t,e){var i,o,n,s,r=e&&e.filter,a=e&&e.type||this._options.type,h=this._data,d=Object.keys(h);if(e&&e.order){var l=this.get(e);for(i=0,o=l.length;o>i;i++)n=l[i],s=n[this._fieldId],t(n,s)}else for(i=0,o=d.length;o>i;i++)s=d[i],n=this._getItem(s,a),r&&!r(n)||t(n,s)},o.prototype.map=function(t,e){var i,o,n,s,r=e&&e.filter,a=e&&e.type||this._options.type,h=[],d=this._data,l=Object.keys(d);for(i=0,o=l.length;o>i;i++)n=l[i],s=this._getItem(n,a),r&&!r(s)||h.push(t(s,n));return e&&e.order&&this._sort(h,e.order),h},o.prototype._filterFields=function(t,e){if(!t)return t;var i,o,n={},s=Object.keys(t),r=s.length;if(Array.isArray(e))for(i=0;r>i;i++)o=s[i],-1!=e.indexOf(o)&&(n[o]=t[o]);else for(i=0;r>i;i++)o=s[i],e.hasOwnProperty(o)&&(n[e[o]]=t[o]);return n},o.prototype._sort=function(t,e){if(s.isString(e)){var i=e;t.sort(function(t,e){var o=t[i],n=e[i];return o>n?1:n>o?-1:0})}else{if("function"!=typeof e)throw new TypeError("Order must be a function or a string");t.sort(e)}},o.prototype.remove=function(t,e){var i,o,n,s=[];if(Array.isArray(t))for(i=0,o=t.length;o>i;i++)n=this._remove(t[i]),null!=n&&s.push(n);else n=this._remove(t),null!=n&&s.push(n);return s.length&&this._trigger("remove",{items:s},e),s},o.prototype._remove=function(t){if(s.isNumber(t)||s.isString(t)){if(this._data[t])return delete this._data[t],this.length--,t}else if(t instanceof Object){var e=t[this._fieldId];if(void 0!==e&&this._data[e])return delete this._data[e],this.length--,e}return null},o.prototype.clear=function(t){var e=Object.keys(this._data);return this._data={},this.length=0,this._trigger("remove",{items:e},t),e},o.prototype.max=function(t){var e,i,o=this._data,n=Object.keys(o),s=null,r=null;for(e=0,i=n.length;i>e;e++){var a=n[e],h=o[a],d=h[t];null!=d&&(!s||d>r)&&(s=h,r=d)}return s},o.prototype.min=function(t){var e,i,o=this._data,n=Object.keys(o),s=null,r=null;for(e=0,i=n.length;i>e;e++){var a=n[e],h=o[a],d=h[t];null!=d&&(!s||r>d)&&(s=h,r=d)}return s},o.prototype.distinct=function(t){var e,i,o,n=this._data,r=Object.keys(n),a=[],h=this._options.type&&this._options.type[t]||null,d=0;for(e=0,o=r.length;o>e;e++){var l=r[e],c=n[l],u=c[t],p=!1;for(i=0;d>i;i++)if(a[i]==u){p=!0;break}p||void 0===u||(a[d]=u,d++)}if(h)for(e=0,o=a.length;o>e;e++)a[e]=s.convert(a[e],h);return a},o.prototype._addItem=function(t){var e=t[this._fieldId];if(void 0!=e){if(this._data[e])throw new Error("Cannot add item: item with id "+e+" already exists")}else e=s.randomUUID(),t[this._fieldId]=e;var i,o,n={},r=Object.keys(t);for(i=0,o=r.length;o>i;i++){var a=r[i],h=this._type[a];n[a]=s.convert(t[a],h)}return this._data[e]=n,this.length++,e},o.prototype._getItem=function(t,e){var i,o,n,r,a=this._data[t];if(!a)return null;var h={},d=Object.keys(a);if(e)for(n=0,r=d.length;r>n;n++)i=d[n],o=a[i],h[i]=s.convert(o,e[i]);else for(n=0,r=d.length;r>n;n++)i=d[n],o=a[i],h[i]=o;return h},o.prototype._updateItem=function(t){var e=t[this._fieldId];if(void 0==e)throw new Error("Cannot update item: item has no id (item: "+JSON.stringify(t)+")");var i=this._data[e];if(!i)throw new Error("Cannot update item: no item with id "+e+" found");for(var o=Object.keys(t),n=0,r=o.length;r>n;n++){var a=o[n],h=this._type[a];i[a]=s.convert(t[a],h)}return e},t.exports=o},function(t,e){function i(t){this.delay=null,this.max=1/0,this._queue=[],this._timeout=null,this._extended=null,this.setOptions(t)}i.prototype.setOptions=function(t){t&&"undefined"!=typeof t.delay&&(this.delay=t.delay),t&&"undefined"!=typeof t.max&&(this.max=t.max),this._flushIfNeeded()},i.extend=function(t,e){var o=new i(e);if(void 0!==t.flush)throw new Error("Target object already has a property flush");t.flush=function(){o.flush()};var n=[{name:"flush",original:void 0}];if(e&&e.replace)for(var s=0;s<e.replace.length;s++){var r=e.replace[s];n.push({name:r,original:t[r]}),o.replace(t,r)}return o._extended={object:t,methods:n},o},i.prototype.destroy=function(){if(this.flush(),this._extended){for(var t=this._extended.object,e=this._extended.methods,i=0;i<e.length;i++){var o=e[i];o.original?t[o.name]=o.original:delete t[o.name]}this._extended=null}},i.prototype.replace=function(t,e){var i=this,o=t[e];if(!o)throw new Error("Method "+e+" undefined");t[e]=function(){for(var t=[],e=0;e<arguments.length;e++)t[e]=arguments[e];i.queue({args:t,fn:o,context:this})}},i.prototype.queue=function(t){"function"==typeof t?this._queue.push({fn:t}):this._queue.push(t),this._flushIfNeeded()},i.prototype._flushIfNeeded=function(){if(this._queue.length>this.max&&this.flush(),clearTimeout(this._timeout),this.queue.length>0&&"number"==typeof this.delay){var t=this;this._timeout=setTimeout(function(){t.flush()},this.delay)}},i.prototype.flush=function(){for(;this._queue.length>0;){var t=this._queue.shift();t.fn.apply(t.context||t.fn,t.args||[])}},t.exports=i},function(t,e,i){function o(t,e){this._data=null,this._ids={},this.length=0,this._options=e||{},this._fieldId="id",this._subscribers={};var i=this;this.listener=function(){i._onEvent.apply(i,arguments)},this.setData(t)}var n=i(1),s=i(9);o.prototype.setData=function(t){var e,i,o,n;if(this._data&&(this._data.off&&this._data.off("*",this.listener),e=Object.keys(this._ids),this._ids={},this.length=0,this._trigger("remove",{items:e})),this._data=t,this._data){for(this._fieldId=this._options.fieldId||this._data&&this._data.options&&this._data.options.fieldId||"id",e=this._data.getIds({filter:this._options&&this._options.filter}),o=0,n=e.length;n>o;o++)i=e[o],this._ids[i]=!0;this.length=e.length,this._trigger("add",{items:e}),this._data.on&&this._data.on("*",this.listener)}},o.prototype.refresh=function(){var t,e,i,o=this._data.getIds({filter:this._options&&this._options.filter}),n=Object.keys(this._ids),s={},r=[],a=[];for(e=0,i=o.length;i>e;e++)t=o[e],s[t]=!0,this._ids[t]||(r.push(t),this._ids[t]=!0);for(e=0,i=n.length;i>e;e++)t=n[e],s[t]||(a.push(t),delete this._ids[t]);this.length+=r.length-a.length,r.length&&this._trigger("add",{items:r}),a.length&&this._trigger("remove",{items:a})},o.prototype.get=function(t){var e,i,o,s=this,r=n.getType(arguments[0]);"String"==r||"Number"==r||"Array"==r?(e=arguments[0],i=arguments[1],o=arguments[2]):(i=arguments[0],o=arguments[1]);var a=n.extend({},this._options,i);this._options.filter&&i&&i.filter&&(a.filter=function(t){return s._options.filter(t)&&i.filter(t)});var h=[];return void 0!=e&&h.push(e),h.push(a),h.push(o),this._data&&this._data.get.apply(this._data,h)},o.prototype.getIds=function(t){var e;if(this._data){var i,o=this._options.filter;i=t&&t.filter?o?function(e){return o(e)&&t.filter(e)}:t.filter:o,e=this._data.getIds({filter:i,order:t&&t.order})}else e=[];return e},o.prototype.map=function(t,e){var i=[];if(this._data){var o,n=this._options.filter;o=e&&e.filter?n?function(t){return n(t)&&e.filter(t)}:e.filter:n,i=this._data.map(t,{filter:o,order:e&&e.order})}else i=[];return i},o.prototype.getDataSet=function(){for(var t=this;t instanceof o;)t=t._data;return t||null},o.prototype._onEvent=function(t,e,i){var o,n,s,r,a=e&&e.items,h=this._data,d=[],l=[],c=[],u=[];if(a&&h){switch(t){case"add":for(o=0,n=a.length;n>o;o++)s=a[o],r=this.get(s),r&&(this._ids[s]=!0,l.push(s));break;case"update":for(o=0,n=a.length;n>o;o++)s=a[o],r=this.get(s),r?this._ids[s]?(c.push(s),d.push(e.data[o])):(this._ids[s]=!0,l.push(s)):this._ids[s]&&(delete this._ids[s],u.push(s));break;case"remove":for(o=0,n=a.length;n>o;o++)s=a[o],this._ids[s]&&(delete this._ids[s],u.push(s))}this.length+=l.length-u.length,l.length&&this._trigger("add",{items:l},i),c.length&&this._trigger("update",{items:c,data:d},i),u.length&&this._trigger("remove",{items:u},i)}},o.prototype.on=s.prototype.on,o.prototype.off=s.prototype.off,o.prototype._trigger=s.prototype._trigger,o.prototype.subscribe=o.prototype.on,o.prototype.unsubscribe=o.prototype.off,t.exports=o},function(t,e,i){function o(t,e,i){if(!(this instanceof o))throw new SyntaxError("Constructor must be called with the new operator");this.containerElement=t,this.width="400px",this.height="400px",this.margin=10,this.defaultXCenter="55%",this.defaultYCenter="50%",this.xLabel="x",this.yLabel="y",this.zLabel="z";var n=function(t){return t};this.xValueLabel=n,this.yValueLabel=n,this.zValueLabel=n,this.filterLabel="time",this.legendLabel="value",this.style=o.STYLE.DOT,this.showPerspective=!0,this.showGrid=!0,this.keepAspectRatio=!0,this.showShadow=!1,this.showGrayBottom=!1,this.showTooltip=!1,this.verticalRatio=.5,this.animationInterval=1e3,this.animationPreload=!1,this.camera=new p,this.camera.setArmRotation(1,.5),this.camera.setArmLength(1.7),this.eye=new c(0,0,-1),this.dataTable=null,this.dataPoints=null,this.colX=void 0,this.colY=void 0,this.colZ=void 0,this.colValue=void 0,this.colFilter=void 0,this.xMin=0,this.xStep=void 0,this.xMax=1,this.yMin=0,this.yStep=void 0,this.yMax=1,this.zMin=0,this.zStep=void 0,this.zMax=1,this.valueMin=0,this.valueMax=1,this.xBarWidth=1,this.yBarWidth=1,this.axisColor="#4D4D4D",this.gridColor="#D3D3D3",this.dataColor={fill:"#7DC1FF",stroke:"#3267D2",strokeWidth:1},this.dotSizeRatio=.02,this.create(),this.setOptions(i),e&&this.setData(e)}function n(t){return"clientX"in t?t.clientX:t.targetTouches[0]&&t.targetTouches[0].clientX||0}function s(t){return"clientY"in t?t.clientY:t.targetTouches[0]&&t.targetTouches[0].clientY||0}var r="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},a=i(13),h=i(9),d=i(11),l=i(1),c=i(14),u=i(15),p=i(16),f=i(17),m=i(18),v=i(19);a(o.prototype),o.prototype._setScale=function(){this.scale=new c(1/(this.xMax-this.xMin),1/(this.yMax-this.yMin),1/(this.zMax-this.zMin)),this.keepAspectRatio&&(this.scale.x<this.scale.y?this.scale.y=this.scale.x:this.scale.x=this.scale.y),this.scale.z*=this.verticalRatio,this.scale.value=1/(this.valueMax-this.valueMin);var t=(this.xMax+this.xMin)/2*this.scale.x,e=(this.yMax+this.yMin)/2*this.scale.y,i=(this.zMax+this.zMin)/2*this.scale.z;this.camera.setArmLocation(t,e,i)},o.prototype._convert3Dto2D=function(t){var e=this._convertPointToTranslation(t);return this._convertTranslationToScreen(e)},o.prototype._convertPointToTranslation=function(t){var e=t.x*this.scale.x,i=t.y*this.scale.y,o=t.z*this.scale.z,n=this.camera.getCameraLocation().x,s=this.camera.getCameraLocation().y,r=this.camera.getCameraLocation().z,a=Math.sin(this.camera.getCameraRotation().x),h=Math.cos(this.camera.getCameraRotation().x),d=Math.sin(this.camera.getCameraRotation().y),l=Math.cos(this.camera.getCameraRotation().y),u=Math.sin(this.camera.getCameraRotation().z),p=Math.cos(this.camera.getCameraRotation().z),f=l*(u*(i-s)+p*(e-n))-d*(o-r),m=a*(l*(o-r)+d*(u*(i-s)+p*(e-n)))+h*(p*(i-s)-u*(e-n)),v=h*(l*(o-r)+d*(u*(i-s)+p*(e-n)))-a*(p*(i-s)-u*(e-n));return new c(f,m,v)},o.prototype._convertTranslationToScreen=function(t){var e,i,o=this.eye.x,n=this.eye.y,s=this.eye.z,r=t.x,a=t.y,h=t.z;return this.showPerspective?(e=(r-o)*(s/h),i=(a-n)*(s/h)):(e=r*-(s/this.camera.getArmLength()),i=a*-(s/this.camera.getArmLength())),new u(this.xcenter+e*this.frame.canvas.clientWidth,this.ycenter-i*this.frame.canvas.clientWidth)},o.prototype._setBackgroundColor=function(t){var e="white",i="gray",o=1;if("string"==typeof t)e=t,i="none",o=0;else if("object"===("undefined"==typeof t?"undefined":r(t)))void 0!==t.fill&&(e=t.fill),void 0!==t.stroke&&(i=t.stroke),void 0!==t.strokeWidth&&(o=t.strokeWidth);else if(void 0!==t)throw"Unsupported type of backgroundColor";this.frame.style.backgroundColor=e,this.frame.style.borderColor=i,this.frame.style.borderWidth=o+"px",this.frame.style.borderStyle="solid"},o.STYLE={BAR:0,BARCOLOR:1,BARSIZE:2,DOT:3,DOTLINE:4,DOTCOLOR:5,DOTSIZE:6,GRID:7,LINE:8,SURFACE:9},o.prototype._getStyleNumber=function(t){switch(t){case"dot":return o.STYLE.DOT;case"dot-line":return o.STYLE.DOTLINE;case"dot-color":return o.STYLE.DOTCOLOR;case"dot-size":return o.STYLE.DOTSIZE;case"line":return o.STYLE.LINE;case"grid":return o.STYLE.GRID;case"surface":return o.STYLE.SURFACE;case"bar":return o.STYLE.BAR;case"bar-color":return o.STYLE.BARCOLOR;case"bar-size":return o.STYLE.BARSIZE}return-1},o.prototype._determineColumnIndexes=function(t,e){if(this.style===o.STYLE.DOT||this.style===o.STYLE.DOTLINE||this.style===o.STYLE.LINE||this.style===o.STYLE.GRID||this.style===o.STYLE.SURFACE||this.style===o.STYLE.BAR)this.colX=0,this.colY=1,this.colZ=2,this.colValue=void 0,t.getNumberOfColumns()>3&&(this.colFilter=3);else{if(this.style!==o.STYLE.DOTCOLOR&&this.style!==o.STYLE.DOTSIZE&&this.style!==o.STYLE.BARCOLOR&&this.style!==o.STYLE.BARSIZE)throw'Unknown style "'+this.style+'"';this.colX=0,this.colY=1,this.colZ=2,this.colValue=3,t.getNumberOfColumns()>4&&(this.colFilter=4)}},o.prototype.getNumberOfRows=function(t){return t.length},o.prototype.getNumberOfColumns=function(t){var e=0;for(var i in t[0])t[0].hasOwnProperty(i)&&e++;return e},o.prototype.getDistinctValues=function(t,e){for(var i=[],o=0;o<t.length;o++)-1==i.indexOf(t[o][e])&&i.push(t[o][e]);return i},o.prototype.getColumnRange=function(t,e){for(var i={min:t[0][e],max:t[0][e]},o=0;o<t.length;o++)i.min>t[o][e]&&(i.min=t[o][e]),i.max<t[o][e]&&(i.max=t[o][e]);return i},o.prototype._dataInitialize=function(t,e){var i=this;if(this.dataSet&&this.dataSet.off("*",this._onChange),void 0!==t){Array.isArray(t)&&(t=new h(t));var n;if(!(t instanceof h||t instanceof d))throw new Error("Array, DataSet, or DataView expected");if(n=t.get(),0!=n.length){this.dataSet=t,this.dataTable=n,this._onChange=function(){i.setData(i.dataSet)},this.dataSet.on("*",this._onChange),this.colX="x",this.colY="y",this.colZ="z",this.colValue="style",this.colFilter="filter",n[0].hasOwnProperty("filter")&&void 0===this.dataFilter&&(this.dataFilter=new f(t,this.colFilter,this),this.dataFilter.setOnLoadCallback(function(){i.redraw()}));var s=this.style==o.STYLE.BAR||this.style==o.STYLE.BARCOLOR||this.style==o.STYLE.BARSIZE;if(s){if(void 0!==this.defaultXBarWidth)this.xBarWidth=this.defaultXBarWidth;else{var r=this.getDistinctValues(n,this.colX);this.xBarWidth=r[1]-r[0]||1}if(void 0!==this.defaultYBarWidth)this.yBarWidth=this.defaultYBarWidth;else{var a=this.getDistinctValues(n,this.colY);this.yBarWidth=a[1]-a[0]||1}}var l=this.getColumnRange(n,this.colX);s&&(l.min-=this.xBarWidth/2,l.max+=this.xBarWidth/2),this.xMin=void 0!==this.defaultXMin?this.defaultXMin:l.min,this.xMax=void 0!==this.defaultXMax?this.defaultXMax:l.max,this.xMax<=this.xMin&&(this.xMax=this.xMin+1),this.xStep=void 0!==this.defaultXStep?this.defaultXStep:(this.xMax-this.xMin)/5;var c=this.getColumnRange(n,this.colY);s&&(c.min-=this.yBarWidth/2,c.max+=this.yBarWidth/2),this.yMin=void 0!==this.defaultYMin?this.defaultYMin:c.min,this.yMax=void 0!==this.defaultYMax?this.defaultYMax:c.max,this.yMax<=this.yMin&&(this.yMax=this.yMin+1),this.yStep=void 0!==this.defaultYStep?this.defaultYStep:(this.yMax-this.yMin)/5;var u=this.getColumnRange(n,this.colZ);if(this.zMin=void 0!==this.defaultZMin?this.defaultZMin:u.min,this.zMax=void 0!==this.defaultZMax?this.defaultZMax:u.max,this.zMax<=this.zMin&&(this.zMax=this.zMin+1),this.zStep=void 0!==this.defaultZStep?this.defaultZStep:(this.zMax-this.zMin)/5,void 0!==this.colValue){var p=this.getColumnRange(n,this.colValue);this.valueMin=void 0!==this.defaultValueMin?this.defaultValueMin:p.min,this.valueMax=void 0!==this.defaultValueMax?this.defaultValueMax:p.max,this.valueMax<=this.valueMin&&(this.valueMax=this.valueMin+1)}this._setScale()}}},o.prototype._getDataPoints=function(t){var e,i,n,s,r,a,h=[];if(this.style===o.STYLE.GRID||this.style===o.STYLE.SURFACE){var d=[],l=[];for(n=0;n<this.getNumberOfRows(t);n++)e=t[n][this.colX]||0,i=t[n][this.colY]||0,-1===d.indexOf(e)&&d.push(e),-1===l.indexOf(i)&&l.push(i);var u=function(t,e){return t-e};d.sort(u),l.sort(u);var p=[];for(n=0;n<t.length;n++){e=t[n][this.colX]||0,i=t[n][this.colY]||0,s=t[n][this.colZ]||0;var f=d.indexOf(e),m=l.indexOf(i);void 0===p[f]&&(p[f]=[]);var v=new c;v.x=e,v.y=i,v.z=s,r={},r.point=v,r.trans=void 0,r.screen=void 0,r.bottom=new c(e,i,this.zMin),p[f][m]=r,h.push(r)}for(e=0;e<p.length;e++)for(i=0;i<p[e].length;i++)p[e][i]&&(p[e][i].pointRight=e<p.length-1?p[e+1][i]:void 0,p[e][i].pointTop=i<p[e].length-1?p[e][i+1]:void 0,p[e][i].pointCross=e<p.length-1&&i<p[e].length-1?p[e+1][i+1]:void 0)}else for(n=0;n<t.length;n++)a=new c,a.x=t[n][this.colX]||0,a.y=t[n][this.colY]||0,a.z=t[n][this.colZ]||0,void 0!==this.colValue&&(a.value=t[n][this.colValue]||0),r={},r.point=a,r.bottom=new c(a.x,a.y,this.zMin),r.trans=void 0,r.screen=void 0,h.push(r);return h},o.prototype.create=function(){for(;this.containerElement.hasChildNodes();)this.containerElement.removeChild(this.containerElement.firstChild);this.frame=document.createElement("div"),this.frame.style.position="relative",this.frame.style.overflow="hidden",this.frame.canvas=document.createElement("canvas"),this.frame.canvas.style.position="relative",this.frame.appendChild(this.frame.canvas);var t=document.createElement("DIV");t.style.color="red",t.style.fontWeight="bold",t.style.padding="10px",t.innerHTML="Error: your browser does not support HTML canvas",this.frame.canvas.appendChild(t),this.frame.filter=document.createElement("div"),this.frame.filter.style.position="absolute",this.frame.filter.style.bottom="0px",this.frame.filter.style.left="0px",this.frame.filter.style.width="100%",this.frame.appendChild(this.frame.filter);var e=this,i=function(t){e._onMouseDown(t)},o=function(t){e._onTouchStart(t)},n=function(t){e._onWheel(t)},s=function(t){e._onTooltip(t)};l.addEventListener(this.frame.canvas,"keydown",onkeydown),l.addEventListener(this.frame.canvas,"mousedown",i),l.addEventListener(this.frame.canvas,"touchstart",o),l.addEventListener(this.frame.canvas,"mousewheel",n),l.addEventListener(this.frame.canvas,"mousemove",s),this.containerElement.appendChild(this.frame)},o.prototype.setSize=function(t,e){this.frame.style.width=t,this.frame.style.height=e,this._resizeCanvas()},o.prototype._resizeCanvas=function(){this.frame.canvas.style.width="100%",this.frame.canvas.style.height="100%",this.frame.canvas.width=this.frame.canvas.clientWidth,this.frame.canvas.height=this.frame.canvas.clientHeight,this.frame.filter.style.width=this.frame.canvas.clientWidth-20+"px"},o.prototype.animationStart=function(){if(!this.frame.filter||!this.frame.filter.slider)throw"No animation available";this.frame.filter.slider.play()},o.prototype.animationStop=function(){this.frame.filter&&this.frame.filter.slider&&this.frame.filter.slider.stop()},o.prototype._resizeCenter=function(){"%"===this.defaultXCenter.charAt(this.defaultXCenter.length-1)?this.xcenter=parseFloat(this.defaultXCenter)/100*this.frame.canvas.clientWidth:this.xcenter=parseFloat(this.defaultXCenter),"%"===this.defaultYCenter.charAt(this.defaultYCenter.length-1)?this.ycenter=parseFloat(this.defaultYCenter)/100*(this.frame.canvas.clientHeight-this.frame.filter.clientHeight):this.ycenter=parseFloat(this.defaultYCenter)},o.prototype.setCameraPosition=function(t){void 0!==t&&(void 0!==t.horizontal&&void 0!==t.vertical&&this.camera.setArmRotation(t.horizontal,t.vertical),void 0!==t.distance&&this.camera.setArmLength(t.distance),this.redraw())},o.prototype.getCameraPosition=function(){var t=this.camera.getArmRotation();return t.distance=this.camera.getArmLength(),t},o.prototype._readData=function(t){this._dataInitialize(t,this.style),this.dataFilter?this.dataPoints=this.dataFilter._getDataPoints():this.dataPoints=this._getDataPoints(this.dataTable),this._redrawFilter()},o.prototype.setData=function(t){this._readData(t),this.redraw(),this.animationAutoStart&&this.dataFilter&&this.animationStart()},o.prototype.setOptions=function(t){var e=void 0;if(this.animationStop(),void 0!==t){if(void 0!==t.width&&(this.width=t.width),void 0!==t.height&&(this.height=t.height),void 0!==t.xCenter&&(this.defaultXCenter=t.xCenter),void 0!==t.yCenter&&(this.defaultYCenter=t.yCenter),void 0!==t.filterLabel&&(this.filterLabel=t.filterLabel),void 0!==t.legendLabel&&(this.legendLabel=t.legendLabel),void 0!==t.xLabel&&(this.xLabel=t.xLabel),void 0!==t.yLabel&&(this.yLabel=t.yLabel),void 0!==t.zLabel&&(this.zLabel=t.zLabel),void 0!==t.xValueLabel&&(this.xValueLabel=t.xValueLabel),void 0!==t.yValueLabel&&(this.yValueLabel=t.yValueLabel),void 0!==t.zValueLabel&&(this.zValueLabel=t.zValueLabel),void 0!==t.dotSizeRatio&&(this.dotSizeRatio=t.dotSizeRatio),void 0!==t.style){var i=this._getStyleNumber(t.style);-1!==i&&(this.style=i)}void 0!==t.showGrid&&(this.showGrid=t.showGrid),void 0!==t.showPerspective&&(this.showPerspective=t.showPerspective),void 0!==t.showShadow&&(this.showShadow=t.showShadow),void 0!==t.tooltip&&(this.showTooltip=t.tooltip),void 0!==t.showAnimationControls&&(this.showAnimationControls=t.showAnimationControls),void 0!==t.keepAspectRatio&&(this.keepAspectRatio=t.keepAspectRatio),void 0!==t.verticalRatio&&(this.verticalRatio=t.verticalRatio),void 0!==t.animationInterval&&(this.animationInterval=t.animationInterval),void 0!==t.animationPreload&&(this.animationPreload=t.animationPreload),void 0!==t.animationAutoStart&&(this.animationAutoStart=t.animationAutoStart),void 0!==t.xBarWidth&&(this.defaultXBarWidth=t.xBarWidth),void 0!==t.yBarWidth&&(this.defaultYBarWidth=t.yBarWidth),void 0!==t.xMin&&(this.defaultXMin=t.xMin),void 0!==t.xStep&&(this.defaultXStep=t.xStep),void 0!==t.xMax&&(this.defaultXMax=t.xMax),void 0!==t.yMin&&(this.defaultYMin=t.yMin),void 0!==t.yStep&&(this.defaultYStep=t.yStep),void 0!==t.yMax&&(this.defaultYMax=t.yMax),void 0!==t.zMin&&(this.defaultZMin=t.zMin),void 0!==t.zStep&&(this.defaultZStep=t.zStep),void 0!==t.zMax&&(this.defaultZMax=t.zMax),void 0!==t.valueMin&&(this.defaultValueMin=t.valueMin),void 0!==t.valueMax&&(this.defaultValueMax=t.valueMax),void 0!==t.backgroundColor&&this._setBackgroundColor(t.backgroundColor),void 0!==t.cameraPosition&&(e=t.cameraPosition),void 0!==e&&(this.camera.setArmRotation(e.horizontal,e.vertical),this.camera.setArmLength(e.distance)),void 0!==t.axisColor&&(this.axisColor=t.axisColor),void 0!==t.gridColor&&(this.gridColor=t.gridColor),t.dataColor&&("string"==typeof t.dataColor?(this.dataColor.fill=t.dataColor,this.dataColor.stroke=t.dataColor):(t.dataColor.fill&&(this.dataColor.fill=t.dataColor.fill),t.dataColor.stroke&&(this.dataColor.stroke=t.dataColor.stroke),void 0!==t.dataColor.strokeWidth&&(this.dataColor.strokeWidth=t.dataColor.strokeWidth)))}this.setSize(this.width,this.height),this.dataTable&&this.setData(this.dataTable),this.animationAutoStart&&this.dataFilter&&this.animationStart()},o.prototype.redraw=function(){if(void 0===this.dataPoints)throw"Error: graph data not initialized";this._resizeCanvas(),this._resizeCenter(),this._redrawSlider(),this._redrawClear(),this._redrawAxis(),this.style===o.STYLE.GRID||this.style===o.STYLE.SURFACE?this._redrawDataGrid():this.style===o.STYLE.LINE?this._redrawDataLine():this.style===o.STYLE.BAR||this.style===o.STYLE.BARCOLOR||this.style===o.STYLE.BARSIZE?this._redrawDataBar():this._redrawDataDot(),this._redrawInfo(),this._redrawLegend()},o.prototype._redrawClear=function(){var t=this.frame.canvas,e=t.getContext("2d");e.clearRect(0,0,t.width,t.height)},o.prototype._redrawLegend=function(){var t;if(this.style===o.STYLE.DOTCOLOR||this.style===o.STYLE.DOTSIZE){var e,i,n=this.frame.clientWidth*this.dotSizeRatio;this.style===o.STYLE.DOTSIZE?(e=n/2,i=n/2+2*n):(e=20,i=20);var s=Math.max(.25*this.frame.clientHeight,100),r=this.margin,a=this.frame.clientWidth-this.margin,h=a-i,d=r+s}var l=this.frame.canvas,c=l.getContext("2d");if(c.lineWidth=1,c.font="14px arial",this.style===o.STYLE.DOTCOLOR){var u=0,p=s;for(t=u;p>t;t++){var f=(t-u)/(p-u),m=240*f,g=this._hsv2rgb(m,1,1);c.strokeStyle=g,c.beginPath(),c.moveTo(h,r+t),c.lineTo(a,r+t),c.stroke()}c.strokeStyle=this.axisColor,c.strokeRect(h,r,i,s)}if(this.style===o.STYLE.DOTSIZE&&(c.strokeStyle=this.axisColor,c.fillStyle=this.dataColor.fill,c.beginPath(),c.moveTo(h,r),c.lineTo(a,r),c.lineTo(a-i+e,d),c.lineTo(h,d),c.closePath(),c.fill(),c.stroke()),this.style===o.STYLE.DOTCOLOR||this.style===o.STYLE.DOTSIZE){var y=5,b=new v(this.valueMin,this.valueMax,(this.valueMax-this.valueMin)/5,!0);for(b.start(),b.getCurrent()<this.valueMin&&b.next();!b.end();)t=d-(b.getCurrent()-this.valueMin)/(this.valueMax-this.valueMin)*s,c.beginPath(),c.moveTo(h-y,t),c.lineTo(h,t),c.stroke(),c.textAlign="right",c.textBaseline="middle",c.fillStyle=this.axisColor,c.fillText(b.getCurrent(),h-2*y,t),b.next();c.textAlign="right",c.textBaseline="top";var w=this.legendLabel;c.fillText(w,a,d+this.margin)}},o.prototype._redrawFilter=function(){if(this.frame.filter.innerHTML="",this.dataFilter){var t={visible:this.showAnimationControls},e=new m(this.frame.filter,t);this.frame.filter.slider=e,this.frame.filter.style.padding="10px",e.setValues(this.dataFilter.values),e.setPlayInterval(this.animationInterval);var i=this,o=function(){var t=e.getIndex();i.dataFilter.selectValue(t),i.dataPoints=i.dataFilter._getDataPoints(),i.redraw()};e.setOnChangeCallback(o)}else this.frame.filter.slider=void 0},o.prototype._redrawSlider=function(){void 0!==this.frame.filter.slider&&this.frame.filter.slider.redraw()},o.prototype._redrawInfo=function(){if(this.dataFilter){var t=this.frame.canvas,e=t.getContext("2d");e.font="14px arial",e.lineStyle="gray",e.fillStyle="gray",e.textAlign="left",e.textBaseline="top";var i=this.margin,o=this.margin;e.fillText(this.dataFilter.getLabel()+": "+this.dataFilter.getSelectedValue(),i,o)}},o.prototype._redrawAxis=function(){var t,e,i,o,n,s,r,a,h,d,l,u,p,f=this.frame.canvas,m=f.getContext("2d");m.font=24/this.camera.getArmLength()+"px arial";var g=.025/this.scale.x,y=.025/this.scale.y,b=5/this.camera.getArmLength(),w=this.camera.getArmRotation().horizontal;for(m.lineWidth=1,o=void 0===this.defaultXStep,i=new v(this.xMin,this.xMax,this.xStep,o),i.start(),i.getCurrent()<this.xMin&&i.next();!i.end();){var _=i.getCurrent();this.showGrid?(t=this._convert3Dto2D(new c(_,this.yMin,this.zMin)),e=this._convert3Dto2D(new c(_,this.yMax,this.zMin)),m.strokeStyle=this.gridColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke()):(t=this._convert3Dto2D(new c(_,this.yMin,this.zMin)),e=this._convert3Dto2D(new c(_,this.yMin+g,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke(),t=this._convert3Dto2D(new c(_,this.yMax,this.zMin)),e=this._convert3Dto2D(new c(_,this.yMax-g,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke()),r=Math.cos(w)>0?this.yMin:this.yMax,n=this._convert3Dto2D(new c(_,r,this.zMin)),Math.cos(2*w)>0?(m.textAlign="center",m.textBaseline="top",n.y+=b):Math.sin(2*w)<0?(m.textAlign="right",m.textBaseline="middle"):(m.textAlign="left",m.textBaseline="middle"),m.fillStyle=this.axisColor,m.fillText("  "+this.xValueLabel(i.getCurrent())+"  ",n.x,n.y),i.next()}for(m.lineWidth=1,o=void 0===this.defaultYStep,i=new v(this.yMin,this.yMax,this.yStep,o),i.start(),i.getCurrent()<this.yMin&&i.next();!i.end();)this.showGrid?(t=this._convert3Dto2D(new c(this.xMin,i.getCurrent(),this.zMin)),e=this._convert3Dto2D(new c(this.xMax,i.getCurrent(),this.zMin)),m.strokeStyle=this.gridColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke()):(t=this._convert3Dto2D(new c(this.xMin,i.getCurrent(),this.zMin)),e=this._convert3Dto2D(new c(this.xMin+y,i.getCurrent(),this.zMin)),
-m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke(),t=this._convert3Dto2D(new c(this.xMax,i.getCurrent(),this.zMin)),e=this._convert3Dto2D(new c(this.xMax-y,i.getCurrent(),this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke()),s=Math.sin(w)>0?this.xMin:this.xMax,n=this._convert3Dto2D(new c(s,i.getCurrent(),this.zMin)),Math.cos(2*w)<0?(m.textAlign="center",m.textBaseline="top",n.y+=b):Math.sin(2*w)>0?(m.textAlign="right",m.textBaseline="middle"):(m.textAlign="left",m.textBaseline="middle"),m.fillStyle=this.axisColor,m.fillText("  "+this.yValueLabel(i.getCurrent())+"  ",n.x,n.y),i.next();for(m.lineWidth=1,o=void 0===this.defaultZStep,i=new v(this.zMin,this.zMax,this.zStep,o),i.start(),i.getCurrent()<this.zMin&&i.next(),s=Math.cos(w)>0?this.xMin:this.xMax,r=Math.sin(w)<0?this.yMin:this.yMax;!i.end();)t=this._convert3Dto2D(new c(s,r,i.getCurrent())),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(t.x-b,t.y),m.stroke(),m.textAlign="right",m.textBaseline="middle",m.fillStyle=this.axisColor,m.fillText(this.zValueLabel(i.getCurrent())+" ",t.x-5,t.y),i.next();m.lineWidth=1,t=this._convert3Dto2D(new c(s,r,this.zMin)),e=this._convert3Dto2D(new c(s,r,this.zMax)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke(),m.lineWidth=1,u=this._convert3Dto2D(new c(this.xMin,this.yMin,this.zMin)),p=this._convert3Dto2D(new c(this.xMax,this.yMin,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(u.x,u.y),m.lineTo(p.x,p.y),m.stroke(),u=this._convert3Dto2D(new c(this.xMin,this.yMax,this.zMin)),p=this._convert3Dto2D(new c(this.xMax,this.yMax,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(u.x,u.y),m.lineTo(p.x,p.y),m.stroke(),m.lineWidth=1,t=this._convert3Dto2D(new c(this.xMin,this.yMin,this.zMin)),e=this._convert3Dto2D(new c(this.xMin,this.yMax,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke(),t=this._convert3Dto2D(new c(this.xMax,this.yMin,this.zMin)),e=this._convert3Dto2D(new c(this.xMax,this.yMax,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke();var x=this.xLabel;x.length>0&&(l=.1/this.scale.y,s=(this.xMin+this.xMax)/2,r=Math.cos(w)>0?this.yMin-l:this.yMax+l,n=this._convert3Dto2D(new c(s,r,this.zMin)),Math.cos(2*w)>0?(m.textAlign="center",m.textBaseline="top"):Math.sin(2*w)<0?(m.textAlign="right",m.textBaseline="middle"):(m.textAlign="left",m.textBaseline="middle"),m.fillStyle=this.axisColor,m.fillText(x,n.x,n.y));var k=this.yLabel;k.length>0&&(d=.1/this.scale.x,s=Math.sin(w)>0?this.xMin-d:this.xMax+d,r=(this.yMin+this.yMax)/2,n=this._convert3Dto2D(new c(s,r,this.zMin)),Math.cos(2*w)<0?(m.textAlign="center",m.textBaseline="top"):Math.sin(2*w)>0?(m.textAlign="right",m.textBaseline="middle"):(m.textAlign="left",m.textBaseline="middle"),m.fillStyle=this.axisColor,m.fillText(k,n.x,n.y));var O=this.zLabel;O.length>0&&(h=30,s=Math.cos(w)>0?this.xMin:this.xMax,r=Math.sin(w)<0?this.yMin:this.yMax,a=(this.zMin+this.zMax)/2,n=this._convert3Dto2D(new c(s,r,a)),m.textAlign="right",m.textBaseline="middle",m.fillStyle=this.axisColor,m.fillText(O,n.x-h,n.y))},o.prototype._hsv2rgb=function(t,e,i){var o,n,s,r,a,h;switch(r=i*e,a=Math.floor(t/60),h=r*(1-Math.abs(t/60%2-1)),a){case 0:o=r,n=h,s=0;break;case 1:o=h,n=r,s=0;break;case 2:o=0,n=r,s=h;break;case 3:o=0,n=h,s=r;break;case 4:o=h,n=0,s=r;break;case 5:o=r,n=0,s=h;break;default:o=0,n=0,s=0}return"RGB("+parseInt(255*o)+","+parseInt(255*n)+","+parseInt(255*s)+")"},o.prototype._redrawDataGrid=function(){var t,e,i,n,s,r,a,h,d,l,u,p,f=this.frame.canvas,m=f.getContext("2d");if(m.lineJoin="round",m.lineCap="round",!(void 0===this.dataPoints||this.dataPoints.length<=0)){for(s=0;s<this.dataPoints.length;s++){var v=this._convertPointToTranslation(this.dataPoints[s].point),g=this._convertTranslationToScreen(v);this.dataPoints[s].trans=v,this.dataPoints[s].screen=g;var y=this._convertPointToTranslation(this.dataPoints[s].bottom);this.dataPoints[s].dist=this.showPerspective?y.length():-y.z}var b=function(t,e){return e.dist-t.dist};if(this.dataPoints.sort(b),this.style===o.STYLE.SURFACE){for(s=0;s<this.dataPoints.length;s++)if(t=this.dataPoints[s],e=this.dataPoints[s].pointRight,i=this.dataPoints[s].pointTop,n=this.dataPoints[s].pointCross,void 0!==t&&void 0!==e&&void 0!==i&&void 0!==n){if(this.showGrayBottom||this.showShadow){var w=c.subtract(n.trans,t.trans),_=c.subtract(i.trans,e.trans),x=c.crossProduct(w,_),k=x.length();r=x.z>0}else r=!0;r?(p=(t.point.z+e.point.z+i.point.z+n.point.z)/4,d=240*(1-(p-this.zMin)*this.scale.z/this.verticalRatio),l=1,this.showShadow?(u=Math.min(1+x.x/k/2,1),a=this._hsv2rgb(d,l,u),h=a):(u=1,a=this._hsv2rgb(d,l,u),h=this.axisColor)):(a="gray",h=this.axisColor),m.lineWidth=this._getStrokeWidth(t),m.fillStyle=a,m.strokeStyle=h,m.beginPath(),m.moveTo(t.screen.x,t.screen.y),m.lineTo(e.screen.x,e.screen.y),m.lineTo(n.screen.x,n.screen.y),m.lineTo(i.screen.x,i.screen.y),m.closePath(),m.fill(),m.stroke()}}else for(s=0;s<this.dataPoints.length;s++)t=this.dataPoints[s],e=this.dataPoints[s].pointRight,i=this.dataPoints[s].pointTop,void 0!==t&&void 0!==e&&(p=(t.point.z+e.point.z)/2,d=240*(1-(p-this.zMin)*this.scale.z/this.verticalRatio),m.lineWidth=2*this._getStrokeWidth(t),m.strokeStyle=this._hsv2rgb(d,1,1),m.beginPath(),m.moveTo(t.screen.x,t.screen.y),m.lineTo(e.screen.x,e.screen.y),m.stroke()),void 0!==t&&void 0!==i&&(p=(t.point.z+i.point.z)/2,d=240*(1-(p-this.zMin)*this.scale.z/this.verticalRatio),m.lineWidth=2*this._getStrokeWidth(t),m.strokeStyle=this._hsv2rgb(d,1,1),m.beginPath(),m.moveTo(t.screen.x,t.screen.y),m.lineTo(i.screen.x,i.screen.y),m.stroke())}},o.prototype._getStrokeWidth=function(t){return void 0!==t?this.showPerspective?1/-t.trans.z*this.dataColor.strokeWidth:-(this.eye.z/this.camera.getArmLength())*this.dataColor.strokeWidth:this.dataColor.strokeWidth},o.prototype._redrawDataDot=function(){var t,e=this.frame.canvas,i=e.getContext("2d");if(!(void 0===this.dataPoints||this.dataPoints.length<=0)){for(t=0;t<this.dataPoints.length;t++){var n=this._convertPointToTranslation(this.dataPoints[t].point),s=this._convertTranslationToScreen(n);this.dataPoints[t].trans=n,this.dataPoints[t].screen=s;var r=this._convertPointToTranslation(this.dataPoints[t].bottom);this.dataPoints[t].dist=this.showPerspective?r.length():-r.z}var a=function(t,e){return e.dist-t.dist};this.dataPoints.sort(a);var h=this.frame.clientWidth*this.dotSizeRatio;for(t=0;t<this.dataPoints.length;t++){var d=this.dataPoints[t];if(this.style===o.STYLE.DOTLINE){var l=this._convert3Dto2D(d.bottom);i.lineWidth=1,i.strokeStyle=this.gridColor,i.beginPath(),i.moveTo(l.x,l.y),i.lineTo(d.screen.x,d.screen.y),i.stroke()}var c;c=this.style===o.STYLE.DOTSIZE?h/2+2*h*(d.point.value-this.valueMin)/(this.valueMax-this.valueMin):h;var u;u=this.showPerspective?c/-d.trans.z:c*-(this.eye.z/this.camera.getArmLength()),0>u&&(u=0);var p,f,m;this.style===o.STYLE.DOTCOLOR?(p=240*(1-(d.point.value-this.valueMin)*this.scale.value),f=this._hsv2rgb(p,1,1),m=this._hsv2rgb(p,1,.8)):this.style===o.STYLE.DOTSIZE?(f=this.dataColor.fill,m=this.dataColor.stroke):(p=240*(1-(d.point.z-this.zMin)*this.scale.z/this.verticalRatio),f=this._hsv2rgb(p,1,1),m=this._hsv2rgb(p,1,.8)),i.lineWidth=this._getStrokeWidth(d),i.strokeStyle=m,i.fillStyle=f,i.beginPath(),i.arc(d.screen.x,d.screen.y,u,0,2*Math.PI,!0),i.fill(),i.stroke()}}},o.prototype._redrawDataBar=function(){var t,e,i,n,s=this.frame.canvas,r=s.getContext("2d");if(!(void 0===this.dataPoints||this.dataPoints.length<=0)){for(t=0;t<this.dataPoints.length;t++){var a=this._convertPointToTranslation(this.dataPoints[t].point),h=this._convertTranslationToScreen(a);this.dataPoints[t].trans=a,this.dataPoints[t].screen=h;var d=this._convertPointToTranslation(this.dataPoints[t].bottom);this.dataPoints[t].dist=this.showPerspective?d.length():-d.z}var l=function(t,e){return e.dist-t.dist};this.dataPoints.sort(l),r.lineJoin="round",r.lineCap="round";var u=this.xBarWidth/2,p=this.yBarWidth/2;for(t=0;t<this.dataPoints.length;t++){var f,m,v,g=this.dataPoints[t];this.style===o.STYLE.BARCOLOR?(f=240*(1-(g.point.value-this.valueMin)*this.scale.value),m=this._hsv2rgb(f,1,1),v=this._hsv2rgb(f,1,.8)):this.style===o.STYLE.BARSIZE?(m=this.dataColor.fill,v=this.dataColor.stroke):(f=240*(1-(g.point.z-this.zMin)*this.scale.z/this.verticalRatio),m=this._hsv2rgb(f,1,1),v=this._hsv2rgb(f,1,.8)),this.style===o.STYLE.BARSIZE&&(u=this.xBarWidth/2*((g.point.value-this.valueMin)/(this.valueMax-this.valueMin)*.8+.2),p=this.yBarWidth/2*((g.point.value-this.valueMin)/(this.valueMax-this.valueMin)*.8+.2));var y=this,b=g.point,w=[{point:new c(b.x-u,b.y-p,b.z)},{point:new c(b.x+u,b.y-p,b.z)},{point:new c(b.x+u,b.y+p,b.z)},{point:new c(b.x-u,b.y+p,b.z)}],_=[{point:new c(b.x-u,b.y-p,this.zMin)},{point:new c(b.x+u,b.y-p,this.zMin)},{point:new c(b.x+u,b.y+p,this.zMin)},{point:new c(b.x-u,b.y+p,this.zMin)}];w.forEach(function(t){t.screen=y._convert3Dto2D(t.point)}),_.forEach(function(t){t.screen=y._convert3Dto2D(t.point)});var x=[{corners:w,center:c.avg(_[0].point,_[2].point)},{corners:[w[0],w[1],_[1],_[0]],center:c.avg(_[1].point,_[0].point)},{corners:[w[1],w[2],_[2],_[1]],center:c.avg(_[2].point,_[1].point)},{corners:[w[2],w[3],_[3],_[2]],center:c.avg(_[3].point,_[2].point)},{corners:[w[3],w[0],_[0],_[3]],center:c.avg(_[0].point,_[3].point)}];for(g.surfaces=x,e=0;e<x.length;e++){i=x[e];var k=this._convertPointToTranslation(i.center);i.dist=this.showPerspective?k.length():-k.z}for(x.sort(function(t,e){var i=e.dist-t.dist;return i?i:t.corners===w?1:e.corners===w?-1:0}),r.lineWidth=this._getStrokeWidth(g),r.strokeStyle=v,r.fillStyle=m,e=2;e<x.length;e++)i=x[e],n=i.corners,r.beginPath(),r.moveTo(n[3].screen.x,n[3].screen.y),r.lineTo(n[0].screen.x,n[0].screen.y),r.lineTo(n[1].screen.x,n[1].screen.y),r.lineTo(n[2].screen.x,n[2].screen.y),r.lineTo(n[3].screen.x,n[3].screen.y),r.fill(),r.stroke()}}},o.prototype._redrawDataLine=function(){var t,e,i=this.frame.canvas,o=i.getContext("2d");if(!(void 0===this.dataPoints||this.dataPoints.length<=0)){for(e=0;e<this.dataPoints.length;e++){var n=this._convertPointToTranslation(this.dataPoints[e].point),s=this._convertTranslationToScreen(n);this.dataPoints[e].trans=n,this.dataPoints[e].screen=s}if(this.dataPoints.length>0){for(t=this.dataPoints[0],o.lineWidth=this._getStrokeWidth(t),o.lineJoin="round",o.lineCap="round",o.strokeStyle=this.dataColor.stroke,o.beginPath(),o.moveTo(t.screen.x,t.screen.y),e=1;e<this.dataPoints.length;e++)t=this.dataPoints[e],o.lineTo(t.screen.x,t.screen.y);o.stroke()}}},o.prototype._onMouseDown=function(t){if(t=t||window.event,this.leftButtonDown&&this._onMouseUp(t),this.leftButtonDown=t.which?1===t.which:1===t.button,this.leftButtonDown||this.touchDown){this.startMouseX=n(t),this.startMouseY=s(t),this.startStart=new Date(this.start),this.startEnd=new Date(this.end),this.startArmRotation=this.camera.getArmRotation(),this.frame.style.cursor="move";var e=this;this.onmousemove=function(t){e._onMouseMove(t)},this.onmouseup=function(t){e._onMouseUp(t)},l.addEventListener(document,"mousemove",e.onmousemove),l.addEventListener(document,"mouseup",e.onmouseup),l.preventDefault(t)}},o.prototype._onMouseMove=function(t){t=t||window.event;var e=parseFloat(n(t))-this.startMouseX,i=parseFloat(s(t))-this.startMouseY,o=this.startArmRotation.horizontal+e/200,r=this.startArmRotation.vertical+i/200,a=4,h=Math.sin(a/360*2*Math.PI);Math.abs(Math.sin(o))<h&&(o=Math.round(o/Math.PI)*Math.PI-.001),Math.abs(Math.cos(o))<h&&(o=(Math.round(o/Math.PI-.5)+.5)*Math.PI-.001),Math.abs(Math.sin(r))<h&&(r=Math.round(r/Math.PI)*Math.PI),Math.abs(Math.cos(r))<h&&(r=(Math.round(r/Math.PI-.5)+.5)*Math.PI),this.camera.setArmRotation(o,r),this.redraw();var d=this.getCameraPosition();this.emit("cameraPositionChange",d),l.preventDefault(t)},o.prototype._onMouseUp=function(t){this.frame.style.cursor="auto",this.leftButtonDown=!1,l.removeEventListener(document,"mousemove",this.onmousemove),l.removeEventListener(document,"mouseup",this.onmouseup),l.preventDefault(t)},o.prototype._onTooltip=function(t){var e=300,i=this.frame.getBoundingClientRect(),o=n(t)-i.left,r=s(t)-i.top;if(this.showTooltip){if(this.tooltipTimeout&&clearTimeout(this.tooltipTimeout),this.leftButtonDown)return void this._hideTooltip();if(this.tooltip&&this.tooltip.dataPoint){var a=this._dataPointFromXY(o,r);a!==this.tooltip.dataPoint&&(a?this._showTooltip(a):this._hideTooltip())}else{var h=this;this.tooltipTimeout=setTimeout(function(){h.tooltipTimeout=null;var t=h._dataPointFromXY(o,r);t&&h._showTooltip(t)},e)}}},o.prototype._onTouchStart=function(t){this.touchDown=!0;var e=this;this.ontouchmove=function(t){e._onTouchMove(t)},this.ontouchend=function(t){e._onTouchEnd(t)},l.addEventListener(document,"touchmove",e.ontouchmove),l.addEventListener(document,"touchend",e.ontouchend),this._onMouseDown(t)},o.prototype._onTouchMove=function(t){this._onMouseMove(t)},o.prototype._onTouchEnd=function(t){this.touchDown=!1,l.removeEventListener(document,"touchmove",this.ontouchmove),l.removeEventListener(document,"touchend",this.ontouchend),this._onMouseUp(t)},o.prototype._onWheel=function(t){t||(t=window.event);var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail&&(e=-t.detail/3),e){var i=this.camera.getArmLength(),o=i*(1-e/10);this.camera.setArmLength(o),this.redraw(),this._hideTooltip()}var n=this.getCameraPosition();this.emit("cameraPositionChange",n),l.preventDefault(t)},o.prototype._insideTriangle=function(t,e){function i(t){return t>0?1:0>t?-1:0}var o=e[0],n=e[1],s=e[2],r=i((n.x-o.x)*(t.y-o.y)-(n.y-o.y)*(t.x-o.x)),a=i((s.x-n.x)*(t.y-n.y)-(s.y-n.y)*(t.x-n.x)),h=i((o.x-s.x)*(t.y-s.y)-(o.y-s.y)*(t.x-s.x));return!(0!=r&&0!=a&&r!=a||0!=a&&0!=h&&a!=h||0!=r&&0!=h&&r!=h)},o.prototype._dataPointFromXY=function(t,e){var i,n=100,s=null,r=null,a=null,h=new u(t,e);if(this.style===o.STYLE.BAR||this.style===o.STYLE.BARCOLOR||this.style===o.STYLE.BARSIZE)for(i=this.dataPoints.length-1;i>=0;i--){s=this.dataPoints[i];var d=s.surfaces;if(d)for(var l=d.length-1;l>=0;l--){var c=d[l],p=c.corners,f=[p[0].screen,p[1].screen,p[2].screen],m=[p[2].screen,p[3].screen,p[0].screen];if(this._insideTriangle(h,f)||this._insideTriangle(h,m))return s}}else for(i=0;i<this.dataPoints.length;i++){s=this.dataPoints[i];var v=s.screen;if(v){var g=Math.abs(t-v.x),y=Math.abs(e-v.y),b=Math.sqrt(g*g+y*y);(null===a||a>b)&&n>b&&(a=b,r=s)}}return r},o.prototype._showTooltip=function(t){var e,i,o;this.tooltip?(e=this.tooltip.dom.content,i=this.tooltip.dom.line,o=this.tooltip.dom.dot):(e=document.createElement("div"),e.style.position="absolute",e.style.padding="10px",e.style.border="1px solid #4d4d4d",e.style.color="#1a1a1a",e.style.background="rgba(255,255,255,0.7)",e.style.borderRadius="2px",e.style.boxShadow="5px 5px 10px rgba(128,128,128,0.5)",i=document.createElement("div"),i.style.position="absolute",i.style.height="40px",i.style.width="0",i.style.borderLeft="1px solid #4d4d4d",o=document.createElement("div"),o.style.position="absolute",o.style.height="0",o.style.width="0",o.style.border="5px solid #4d4d4d",o.style.borderRadius="5px",this.tooltip={dataPoint:null,dom:{content:e,line:i,dot:o}}),this._hideTooltip(),this.tooltip.dataPoint=t,"function"==typeof this.showTooltip?e.innerHTML=this.showTooltip(t.point):e.innerHTML="<table><tr><td>"+this.xLabel+":</td><td>"+t.point.x+"</td></tr><tr><td>"+this.yLabel+":</td><td>"+t.point.y+"</td></tr><tr><td>"+this.zLabel+":</td><td>"+t.point.z+"</td></tr></table>",e.style.left="0",e.style.top="0",this.frame.appendChild(e),this.frame.appendChild(i),this.frame.appendChild(o);var n=e.offsetWidth,s=e.offsetHeight,r=i.offsetHeight,a=o.offsetWidth,h=o.offsetHeight,d=t.screen.x-n/2;d=Math.min(Math.max(d,10),this.frame.clientWidth-10-n),i.style.left=t.screen.x+"px",i.style.top=t.screen.y-r+"px",e.style.left=d+"px",e.style.top=t.screen.y-r-s+"px",o.style.left=t.screen.x-a/2+"px",o.style.top=t.screen.y-h/2+"px"},o.prototype._hideTooltip=function(){if(this.tooltip){this.tooltip.dataPoint=null;for(var t in this.tooltip.dom)if(this.tooltip.dom.hasOwnProperty(t)){var e=this.tooltip.dom[t];e&&e.parentNode&&e.parentNode.removeChild(e)}}},t.exports=o},function(t,e){function i(t){return t?o(t):void 0}function o(t){for(var e in i.prototype)t[e]=i.prototype[e];return t}t.exports=i,i.prototype.on=i.prototype.addEventListener=function(t,e){return this._callbacks=this._callbacks||{},(this._callbacks[t]=this._callbacks[t]||[]).push(e),this},i.prototype.once=function(t,e){function i(){o.off(t,i),e.apply(this,arguments)}var o=this;return this._callbacks=this._callbacks||{},i.fn=e,this.on(t,i),this},i.prototype.off=i.prototype.removeListener=i.prototype.removeAllListeners=i.prototype.removeEventListener=function(t,e){if(this._callbacks=this._callbacks||{},0==arguments.length)return this._callbacks={},this;var i=this._callbacks[t];if(!i)return this;if(1==arguments.length)return delete this._callbacks[t],this;for(var o,n=0;n<i.length;n++)if(o=i[n],o===e||o.fn===e){i.splice(n,1);break}return this},i.prototype.emit=function(t){this._callbacks=this._callbacks||{};var e=[].slice.call(arguments,1),i=this._callbacks[t];if(i){i=i.slice(0);for(var o=0,n=i.length;n>o;++o)i[o].apply(this,e)}return this},i.prototype.listeners=function(t){return this._callbacks=this._callbacks||{},this._callbacks[t]||[]},i.prototype.hasListeners=function(t){return!!this.listeners(t).length}},function(t,e){function i(t,e,i){this.x=void 0!==t?t:0,this.y=void 0!==e?e:0,this.z=void 0!==i?i:0}i.subtract=function(t,e){var o=new i;return o.x=t.x-e.x,o.y=t.y-e.y,o.z=t.z-e.z,o},i.add=function(t,e){var o=new i;return o.x=t.x+e.x,o.y=t.y+e.y,o.z=t.z+e.z,o},i.avg=function(t,e){return new i((t.x+e.x)/2,(t.y+e.y)/2,(t.z+e.z)/2)},i.crossProduct=function(t,e){var o=new i;return o.x=t.y*e.z-t.z*e.y,o.y=t.z*e.x-t.x*e.z,o.z=t.x*e.y-t.y*e.x,o},i.prototype.length=function(){return Math.sqrt(this.x*this.x+this.y*this.y+this.z*this.z)},t.exports=i},function(t,e){function i(t,e){this.x=void 0!==t?t:0,this.y=void 0!==e?e:0}t.exports=i},function(t,e,i){function o(){this.armLocation=new n,this.armRotation={},this.armRotation.horizontal=0,this.armRotation.vertical=0,this.armLength=1.7,this.cameraLocation=new n,this.cameraRotation=new n(.5*Math.PI,0,0),this.calculateCameraOrientation()}var n=i(14);o.prototype.setArmLocation=function(t,e,i){this.armLocation.x=t,this.armLocation.y=e,this.armLocation.z=i,this.calculateCameraOrientation()},o.prototype.setArmRotation=function(t,e){void 0!==t&&(this.armRotation.horizontal=t),void 0!==e&&(this.armRotation.vertical=e,this.armRotation.vertical<0&&(this.armRotation.vertical=0),this.armRotation.vertical>.5*Math.PI&&(this.armRotation.vertical=.5*Math.PI)),void 0===t&&void 0===e||this.calculateCameraOrientation()},o.prototype.getArmRotation=function(){var t={};return t.horizontal=this.armRotation.horizontal,t.vertical=this.armRotation.vertical,t},o.prototype.setArmLength=function(t){void 0!==t&&(this.armLength=t,this.armLength<.71&&(this.armLength=.71),this.armLength>5&&(this.armLength=5),this.calculateCameraOrientation())},o.prototype.getArmLength=function(){return this.armLength},o.prototype.getCameraLocation=function(){return this.cameraLocation},o.prototype.getCameraRotation=function(){return this.cameraRotation},o.prototype.calculateCameraOrientation=function(){this.cameraLocation.x=this.armLocation.x-this.armLength*Math.sin(this.armRotation.horizontal)*Math.cos(this.armRotation.vertical),this.cameraLocation.y=this.armLocation.y-this.armLength*Math.cos(this.armRotation.horizontal)*Math.cos(this.armRotation.vertical),this.cameraLocation.z=this.armLocation.z+this.armLength*Math.sin(this.armRotation.vertical),this.cameraRotation.x=Math.PI/2-this.armRotation.vertical,this.cameraRotation.y=0,this.cameraRotation.z=-this.armRotation.horizontal},t.exports=o},function(t,e,i){function o(t,e,i){this.data=t,this.column=e,this.graph=i,this.index=void 0,this.value=void 0,this.values=i.getDistinctValues(t.get(),this.column),this.values.sort(function(t,e){return t>e?1:e>t?-1:0}),this.values.length>0&&this.selectValue(0),this.dataPoints=[],this.loaded=!1,this.onLoadCallback=void 0,i.animationPreload?(this.loaded=!1,this.loadInBackground()):this.loaded=!0}var n=i(11);o.prototype.isLoaded=function(){return this.loaded},o.prototype.getLoadedProgress=function(){for(var t=this.values.length,e=0;this.dataPoints[e];)e++;return Math.round(e/t*100)},o.prototype.getLabel=function(){return this.graph.filterLabel},o.prototype.getColumn=function(){return this.column},o.prototype.getSelectedValue=function(){return void 0!==this.index?this.values[this.index]:void 0},o.prototype.getValues=function(){return this.values},o.prototype.getValue=function(t){if(t>=this.values.length)throw"Error: index out of range";return this.values[t]},o.prototype._getDataPoints=function(t){if(void 0===t&&(t=this.index),void 0===t)return[];var e;if(this.dataPoints[t])e=this.dataPoints[t];else{var i={};i.column=this.column,i.value=this.values[t];var o=new n(this.data,{filter:function(t){return t[i.column]==i.value}}).get();e=this.graph._getDataPoints(o),this.dataPoints[t]=e}return e},o.prototype.setOnLoadCallback=function(t){this.onLoadCallback=t},o.prototype.selectValue=function(t){if(t>=this.values.length)throw"Error: index out of range";this.index=t,this.value=this.values[t]},o.prototype.loadInBackground=function(t){void 0===t&&(t=0);var e=this.graph.frame;if(t<this.values.length){this._getDataPoints(t);void 0===e.progress&&(e.progress=document.createElement("DIV"),e.progress.style.position="absolute",e.progress.style.color="gray",e.appendChild(e.progress));var i=this.getLoadedProgress();e.progress.innerHTML="Loading animation... "+i+"%",e.progress.style.bottom="60px",e.progress.style.left="10px";var o=this;setTimeout(function(){o.loadInBackground(t+1)},10),this.loaded=!1}else this.loaded=!0,void 0!==e.progress&&(e.removeChild(e.progress),e.progress=void 0),this.onLoadCallback&&this.onLoadCallback()},t.exports=o},function(t,e,i){function o(t,e){if(void 0===t)throw"Error: No container element defined";if(this.container=t,this.visible=e&&void 0!=e.visible?e.visible:!0,this.visible){this.frame=document.createElement("DIV"),this.frame.style.width="100%",this.frame.style.position="relative",this.container.appendChild(this.frame),this.frame.prev=document.createElement("INPUT"),this.frame.prev.type="BUTTON",this.frame.prev.value="Prev",this.frame.appendChild(this.frame.prev),this.frame.play=document.createElement("INPUT"),this.frame.play.type="BUTTON",this.frame.play.value="Play",this.frame.appendChild(this.frame.play),this.frame.next=document.createElement("INPUT"),this.frame.next.type="BUTTON",this.frame.next.value="Next",this.frame.appendChild(this.frame.next),this.frame.bar=document.createElement("INPUT"),this.frame.bar.type="BUTTON",this.frame.bar.style.position="absolute",this.frame.bar.style.border="1px solid red",this.frame.bar.style.width="100px",this.frame.bar.style.height="6px",this.frame.bar.style.borderRadius="2px",this.frame.bar.style.MozBorderRadius="2px",this.frame.bar.style.border="1px solid #7F7F7F",this.frame.bar.style.backgroundColor="#E5E5E5",this.frame.appendChild(this.frame.bar),this.frame.slide=document.createElement("INPUT"),this.frame.slide.type="BUTTON",this.frame.slide.style.margin="0px",this.frame.slide.value=" ",this.frame.slide.style.position="relative",this.frame.slide.style.left="-100px",this.frame.appendChild(this.frame.slide);var i=this;this.frame.slide.onmousedown=function(t){i._onMouseDown(t)},this.frame.prev.onclick=function(t){i.prev(t)},this.frame.play.onclick=function(t){i.togglePlay(t)},this.frame.next.onclick=function(t){i.next(t)}}this.onChangeCallback=void 0,this.values=[],this.index=void 0,this.playTimeout=void 0,this.playInterval=1e3,this.playLoop=!0}var n=i(1);o.prototype.prev=function(){var t=this.getIndex();t>0&&(t--,this.setIndex(t))},o.prototype.next=function(){var t=this.getIndex();t<this.values.length-1&&(t++,this.setIndex(t))},o.prototype.playNext=function(){var t=new Date,e=this.getIndex();e<this.values.length-1?(e++,this.setIndex(e)):this.playLoop&&(e=0,this.setIndex(e));var i=new Date,o=i-t,n=Math.max(this.playInterval-o,0),s=this;this.playTimeout=setTimeout(function(){s.playNext()},n)},o.prototype.togglePlay=function(){void 0===this.playTimeout?this.play():this.stop()},o.prototype.play=function(){this.playTimeout||(this.playNext(),this.frame&&(this.frame.play.value="Stop"))},o.prototype.stop=function(){clearInterval(this.playTimeout),this.playTimeout=void 0,this.frame&&(this.frame.play.value="Play")},o.prototype.setOnChangeCallback=function(t){this.onChangeCallback=t},o.prototype.setPlayInterval=function(t){this.playInterval=t},o.prototype.getPlayInterval=function(t){return this.playInterval},o.prototype.setPlayLoop=function(t){this.playLoop=t},o.prototype.onChange=function(){void 0!==this.onChangeCallback&&this.onChangeCallback()},o.prototype.redraw=function(){if(this.frame){this.frame.bar.style.top=this.frame.clientHeight/2-this.frame.bar.offsetHeight/2+"px",this.frame.bar.style.width=this.frame.clientWidth-this.frame.prev.clientWidth-this.frame.play.clientWidth-this.frame.next.clientWidth-30+"px";var t=this.indexToLeft(this.index);this.frame.slide.style.left=t+"px"}},o.prototype.setValues=function(t){this.values=t,this.values.length>0?this.setIndex(0):this.index=void 0},o.prototype.setIndex=function(t){if(!(t<this.values.length))throw"Error: index out of range";this.index=t,this.redraw(),this.onChange()},o.prototype.getIndex=function(){return this.index},o.prototype.get=function(){return this.values[this.index]},o.prototype._onMouseDown=function(t){var e=t.which?1===t.which:1===t.button;if(e){this.startClientX=t.clientX,this.startSlideX=parseFloat(this.frame.slide.style.left),this.frame.style.cursor="move";var i=this;this.onmousemove=function(t){i._onMouseMove(t)},this.onmouseup=function(t){i._onMouseUp(t)},n.addEventListener(document,"mousemove",this.onmousemove),n.addEventListener(document,"mouseup",this.onmouseup),n.preventDefault(t)}},o.prototype.leftToIndex=function(t){var e=parseFloat(this.frame.bar.style.width)-this.frame.slide.clientWidth-10,i=t-3,o=Math.round(i/e*(this.values.length-1));return 0>o&&(o=0),o>this.values.length-1&&(o=this.values.length-1),o},o.prototype.indexToLeft=function(t){var e=parseFloat(this.frame.bar.style.width)-this.frame.slide.clientWidth-10,i=t/(this.values.length-1)*e,o=i+3;return o},o.prototype._onMouseMove=function(t){var e=t.clientX-this.startClientX,i=this.startSlideX+e,o=this.leftToIndex(i);this.setIndex(o),n.preventDefault()},o.prototype._onMouseUp=function(t){this.frame.style.cursor="auto",n.removeEventListener(document,"mousemove",this.onmousemove),n.removeEventListener(document,"mouseup",this.onmouseup),n.preventDefault()},t.exports=o},function(t,e){function i(t,e,i,o){this._start=0,this._end=0,this._step=1,this.prettyStep=!0,this.precision=5,this._current=0,this.setRange(t,e,i,o)}i.prototype.setRange=function(t,e,i,o){this._start=t?t:0,this._end=e?e:0,this.setStep(i,o)},i.prototype.setStep=function(t,e){void 0===t||0>=t||(void 0!==e&&(this.prettyStep=e),this.prettyStep===!0?this._step=i.calculatePrettyStep(t):this._step=t)},i.calculatePrettyStep=function(t){var e=function(t){return Math.log(t)/Math.LN10},i=Math.pow(10,Math.round(e(t))),o=2*Math.pow(10,Math.round(e(t/2))),n=5*Math.pow(10,Math.round(e(t/5))),s=i;return Math.abs(o-t)<=Math.abs(s-t)&&(s=o),Math.abs(n-t)<=Math.abs(s-t)&&(s=n),0>=s&&(s=1),s},i.prototype.getCurrent=function(){return parseFloat(this._current.toPrecision(this.precision))},i.prototype.getStep=function(){return this._step},i.prototype.start=function(){this._current=this._start-this._start%this._step},i.prototype.next=function(){this._current+=this._step},i.prototype.end=function(){return this._current>this._end},t.exports=i},function(t,e,i){if("undefined"!=typeof window){var o=i(21),n=window.Hammer||i(22);t.exports=o(n,{preventDefault:"mouse"})}else t.exports=function(){throw Error("hammer.js is only available in a browser, not in node.js.")}},function(t,e,i){var o,n,s;!function(i){n=[],o=i,s="function"==typeof o?o.apply(e,n):o,!(void 0!==s&&(t.exports=s))}(function(){var t=null;return function e(i,o){function n(t){return t.match(/[^ ]+/g)}function s(e){if("hammer.input"!==e.type){if(e.srcEvent._handled||(e.srcEvent._handled={}),e.srcEvent._handled[e.type])return;e.srcEvent._handled[e.type]=!0}var i=!1;e.stopPropagation=function(){i=!0};var o=e.srcEvent.stopPropagation.bind(e.srcEvent);"function"==typeof o&&(e.srcEvent.stopPropagation=function(){o(),e.stopPropagation()}),e.firstTarget=t;for(var n=t;n&&!i;){var s=n.hammer;if(s)for(var r,a=0;a<s.length;a++)if(r=s[a]._handlers[e.type])for(var h=0;h<r.length&&!i;h++)r[h](e);n=n.parentNode}}var r=o||{preventDefault:!1};if(i.Manager){var a=i,h=function(t,i){var o=Object.create(r);return i&&a.assign(o,i),e(new a(t,o),o)};return a.assign(h,a),h.Manager=function(t,i){var o=Object.create(r);return i&&a.assign(o,i),e(new a.Manager(t,o),o)},h}var d=Object.create(i),l=i.element;return l.hammer||(l.hammer=[]),l.hammer.push(d),i.on("hammer.input",function(e){r.preventDefault!==!0&&r.preventDefault!==e.pointerType||e.preventDefault(),e.isFirst&&(t=e.target)}),d._handlers={},d.on=function(t,e){return n(t).forEach(function(t){var o=d._handlers[t];o||(d._handlers[t]=o=[],i.on(t,s)),o.push(e)}),d},d.off=function(t,e){return n(t).forEach(function(t){var o=d._handlers[t];o&&(o=e?o.filter(function(t){return t!==e}):[],o.length>0?d._handlers[t]=o:(i.off(t,s),delete d._handlers[t]))}),d},d.emit=function(e,o){t=o.target,i.emit(e,o)},d.destroy=function(){var t=i.element.hammer,e=t.indexOf(d);-1!==e&&t.splice(e,1),t.length||delete i.element.hammer,d._handlers={},i.destroy()},d}})},function(t,e,i){var o;!function(n,s,r,a){function h(t,e,i){return setTimeout(p(t,i),e)}function d(t,e,i){return Array.isArray(t)?(l(t,i[e],i),!0):!1}function l(t,e,i){var o;if(t)if(t.forEach)t.forEach(e,i);else if(t.length!==a)for(o=0;o<t.length;)e.call(i,t[o],o,t),o++;else for(o in t)t.hasOwnProperty(o)&&e.call(i,t[o],o,t)}function c(t,e,i){var o="DEPRECATED METHOD: "+e+"\n"+i+" AT \n";return function(){var e=new Error("get-stack-trace"),i=e&&e.stack?e.stack.replace(/^[^\(]+?[\n$]/gm,"").replace(/^\s+at\s+/gm,"").replace(/^Object.<anonymous>\s*\(/gm,"{anonymous}()@"):"Unknown Stack Trace",s=n.console&&(n.console.warn||n.console.log);return s&&s.call(n.console,o,i),t.apply(this,arguments)}}function u(t,e,i){var o,n=e.prototype;o=t.prototype=Object.create(n),o.constructor=t,o._super=n,i&&ct(o,i)}function p(t,e){return function(){return t.apply(e,arguments)}}function f(t,e){return typeof t==ft?t.apply(e?e[0]||a:a,e):t}function m(t,e){return t===a?e:t}function v(t,e,i){l(w(e),function(e){t.addEventListener(e,i,!1)})}function g(t,e,i){l(w(e),function(e){t.removeEventListener(e,i,!1)})}function y(t,e){for(;t;){if(t==e)return!0;t=t.parentNode}return!1}function b(t,e){return t.indexOf(e)>-1}function w(t){return t.trim().split(/\s+/g)}function _(t,e,i){if(t.indexOf&&!i)return t.indexOf(e);for(var o=0;o<t.length;){if(i&&t[o][i]==e||!i&&t[o]===e)return o;o++}return-1}function x(t){return Array.prototype.slice.call(t,0)}function k(t,e,i){for(var o=[],n=[],s=0;s<t.length;){var r=e?t[s][e]:t[s];_(n,r)<0&&o.push(t[s]),n[s]=r,s++}return i&&(o=e?o.sort(function(t,i){return t[e]>i[e]}):o.sort()),o}function O(t,e){for(var i,o,n=e[0].toUpperCase()+e.slice(1),s=0;s<ut.length;){if(i=ut[s],o=i?i+n:e,o in t)return o;s++}return a}function M(){return wt++}function D(t){var e=t.ownerDocument||t;return e.defaultView||e.parentWindow||n}function S(t,e){var i=this;this.manager=t,this.callback=e,this.element=t.element,this.target=t.options.inputTarget,this.domHandler=function(e){f(t.options.enable,[t])&&i.handler(e)},this.init()}function C(t){var e,i=t.options.inputClass;return new(e=i?i:kt?W:Ot?V:xt?q:H)(t,T)}function T(t,e,i){var o=i.pointers.length,n=i.changedPointers.length,s=e&Et&&o-n===0,r=e&(It|Nt)&&o-n===0;
-i.isFirst=!!s,i.isFinal=!!r,s&&(t.session={}),i.eventType=e,E(t,i),t.emit("hammer.input",i),t.recognize(i),t.session.prevInput=i}function E(t,e){var i=t.session,o=e.pointers,n=o.length;i.firstInput||(i.firstInput=N(e)),n>1&&!i.firstMultiple?i.firstMultiple=N(e):1===n&&(i.firstMultiple=!1);var s=i.firstInput,r=i.firstMultiple,a=r?r.center:s.center,h=e.center=R(o);e.timeStamp=gt(),e.deltaTime=e.timeStamp-s.timeStamp,e.angle=B(a,h),e.distance=A(a,h),P(i,e),e.offsetDirection=L(e.deltaX,e.deltaY);var d=z(e.deltaTime,e.deltaX,e.deltaY);e.overallVelocityX=d.x,e.overallVelocityY=d.y,e.overallVelocity=vt(d.x)>vt(d.y)?d.x:d.y,e.scale=r?j(r.pointers,o):1,e.rotation=r?F(r.pointers,o):0,e.maxPointers=i.prevInput?e.pointers.length>i.prevInput.maxPointers?e.pointers.length:i.prevInput.maxPointers:e.pointers.length,I(i,e);var l=t.element;y(e.srcEvent.target,l)&&(l=e.srcEvent.target),e.target=l}function P(t,e){var i=e.center,o=t.offsetDelta||{},n=t.prevDelta||{},s=t.prevInput||{};e.eventType!==Et&&s.eventType!==It||(n=t.prevDelta={x:s.deltaX||0,y:s.deltaY||0},o=t.offsetDelta={x:i.x,y:i.y}),e.deltaX=n.x+(i.x-o.x),e.deltaY=n.y+(i.y-o.y)}function I(t,e){var i,o,n,s,r=t.lastInterval||e,h=e.timeStamp-r.timeStamp;if(e.eventType!=Nt&&(h>Tt||r.velocity===a)){var d=e.deltaX-r.deltaX,l=e.deltaY-r.deltaY,c=z(h,d,l);o=c.x,n=c.y,i=vt(c.x)>vt(c.y)?c.x:c.y,s=L(d,l),t.lastInterval=e}else i=r.velocity,o=r.velocityX,n=r.velocityY,s=r.direction;e.velocity=i,e.velocityX=o,e.velocityY=n,e.direction=s}function N(t){for(var e=[],i=0;i<t.pointers.length;)e[i]={clientX:mt(t.pointers[i].clientX),clientY:mt(t.pointers[i].clientY)},i++;return{timeStamp:gt(),pointers:e,center:R(e),deltaX:t.deltaX,deltaY:t.deltaY}}function R(t){var e=t.length;if(1===e)return{x:mt(t[0].clientX),y:mt(t[0].clientY)};for(var i=0,o=0,n=0;e>n;)i+=t[n].clientX,o+=t[n].clientY,n++;return{x:mt(i/e),y:mt(o/e)}}function z(t,e,i){return{x:e/t||0,y:i/t||0}}function L(t,e){return t===e?Rt:vt(t)>=vt(e)?0>t?zt:Lt:0>e?At:Bt}function A(t,e,i){i||(i=Wt);var o=e[i[0]]-t[i[0]],n=e[i[1]]-t[i[1]];return Math.sqrt(o*o+n*n)}function B(t,e,i){i||(i=Wt);var o=e[i[0]]-t[i[0]],n=e[i[1]]-t[i[1]];return 180*Math.atan2(n,o)/Math.PI}function F(t,e){return B(e[1],e[0],Yt)+B(t[1],t[0],Yt)}function j(t,e){return A(e[0],e[1],Yt)/A(t[0],t[1],Yt)}function H(){this.evEl=Vt,this.evWin=Ut,this.allow=!0,this.pressed=!1,S.apply(this,arguments)}function W(){this.evEl=Zt,this.evWin=Kt,S.apply(this,arguments),this.store=this.manager.session.pointerEvents=[]}function Y(){this.evTarget=Qt,this.evWin=$t,this.started=!1,S.apply(this,arguments)}function G(t,e){var i=x(t.touches),o=x(t.changedTouches);return e&(It|Nt)&&(i=k(i.concat(o),"identifier",!0)),[i,o]}function V(){this.evTarget=ee,this.targetIds={},S.apply(this,arguments)}function U(t,e){var i=x(t.touches),o=this.targetIds;if(e&(Et|Pt)&&1===i.length)return o[i[0].identifier]=!0,[i,i];var n,s,r=x(t.changedTouches),a=[],h=this.target;if(s=i.filter(function(t){return y(t.target,h)}),e===Et)for(n=0;n<s.length;)o[s[n].identifier]=!0,n++;for(n=0;n<r.length;)o[r[n].identifier]&&a.push(r[n]),e&(It|Nt)&&delete o[r[n].identifier],n++;return a.length?[k(s.concat(a),"identifier",!0),a]:void 0}function q(){S.apply(this,arguments);var t=p(this.handler,this);this.touch=new V(this.manager,t),this.mouse=new H(this.manager,t)}function X(t,e){this.manager=t,this.set(e)}function Z(t){if(b(t,ae))return ae;var e=b(t,he),i=b(t,de);return e&&i?ae:e||i?e?he:de:b(t,re)?re:se}function K(t){this.options=ct({},this.defaults,t||{}),this.id=M(),this.manager=null,this.options.enable=m(this.options.enable,!0),this.state=le,this.simultaneous={},this.requireFail=[]}function J(t){return t&me?"cancel":t&pe?"end":t&ue?"move":t&ce?"start":""}function Q(t){return t==Bt?"down":t==At?"up":t==zt?"left":t==Lt?"right":""}function $(t,e){var i=e.manager;return i?i.get(t):t}function tt(){K.apply(this,arguments)}function et(){tt.apply(this,arguments),this.pX=null,this.pY=null}function it(){tt.apply(this,arguments)}function ot(){K.apply(this,arguments),this._timer=null,this._input=null}function nt(){tt.apply(this,arguments)}function st(){tt.apply(this,arguments)}function rt(){K.apply(this,arguments),this.pTime=!1,this.pCenter=!1,this._timer=null,this._input=null,this.count=0}function at(t,e){return e=e||{},e.recognizers=m(e.recognizers,at.defaults.preset),new ht(t,e)}function ht(t,e){this.options=ct({},at.defaults,e||{}),this.options.inputTarget=this.options.inputTarget||t,this.handlers={},this.session={},this.recognizers=[],this.element=t,this.input=C(this),this.touchAction=new X(this,this.options.touchAction),dt(this,!0),l(this.options.recognizers,function(t){var e=this.add(new t[0](t[1]));t[2]&&e.recognizeWith(t[2]),t[3]&&e.requireFailure(t[3])},this)}function dt(t,e){var i=t.element;i.style&&l(t.options.cssProps,function(t,o){i.style[O(i.style,o)]=e?t:""})}function lt(t,e){var i=s.createEvent("Event");i.initEvent(t,!0,!0),i.gesture=e,e.target.dispatchEvent(i)}var ct,ut=["","webkit","Moz","MS","ms","o"],pt=s.createElement("div"),ft="function",mt=Math.round,vt=Math.abs,gt=Date.now;ct="function"!=typeof Object.assign?function(t){if(t===a||null===t)throw new TypeError("Cannot convert undefined or null to object");for(var e=Object(t),i=1;i<arguments.length;i++){var o=arguments[i];if(o!==a&&null!==o)for(var n in o)o.hasOwnProperty(n)&&(e[n]=o[n])}return e}:Object.assign;var yt=c(function(t,e,i){for(var o=Object.keys(e),n=0;n<o.length;)(!i||i&&t[o[n]]===a)&&(t[o[n]]=e[o[n]]),n++;return t},"extend","Use `assign`."),bt=c(function(t,e){return yt(t,e,!0)},"merge","Use `assign`."),wt=1,_t=/mobile|tablet|ip(ad|hone|od)|android/i,xt="ontouchstart"in n,kt=O(n,"PointerEvent")!==a,Ot=xt&&_t.test(navigator.userAgent),Mt="touch",Dt="pen",St="mouse",Ct="kinect",Tt=25,Et=1,Pt=2,It=4,Nt=8,Rt=1,zt=2,Lt=4,At=8,Bt=16,Ft=zt|Lt,jt=At|Bt,Ht=Ft|jt,Wt=["x","y"],Yt=["clientX","clientY"];S.prototype={handler:function(){},init:function(){this.evEl&&v(this.element,this.evEl,this.domHandler),this.evTarget&&v(this.target,this.evTarget,this.domHandler),this.evWin&&v(D(this.element),this.evWin,this.domHandler)},destroy:function(){this.evEl&&g(this.element,this.evEl,this.domHandler),this.evTarget&&g(this.target,this.evTarget,this.domHandler),this.evWin&&g(D(this.element),this.evWin,this.domHandler)}};var Gt={mousedown:Et,mousemove:Pt,mouseup:It},Vt="mousedown",Ut="mousemove mouseup";u(H,S,{handler:function(t){var e=Gt[t.type];e&Et&&0===t.button&&(this.pressed=!0),e&Pt&&1!==t.which&&(e=It),this.pressed&&this.allow&&(e&It&&(this.pressed=!1),this.callback(this.manager,e,{pointers:[t],changedPointers:[t],pointerType:St,srcEvent:t}))}});var qt={pointerdown:Et,pointermove:Pt,pointerup:It,pointercancel:Nt,pointerout:Nt},Xt={2:Mt,3:Dt,4:St,5:Ct},Zt="pointerdown",Kt="pointermove pointerup pointercancel";n.MSPointerEvent&&!n.PointerEvent&&(Zt="MSPointerDown",Kt="MSPointerMove MSPointerUp MSPointerCancel"),u(W,S,{handler:function(t){var e=this.store,i=!1,o=t.type.toLowerCase().replace("ms",""),n=qt[o],s=Xt[t.pointerType]||t.pointerType,r=s==Mt,a=_(e,t.pointerId,"pointerId");n&Et&&(0===t.button||r)?0>a&&(e.push(t),a=e.length-1):n&(It|Nt)&&(i=!0),0>a||(e[a]=t,this.callback(this.manager,n,{pointers:e,changedPointers:[t],pointerType:s,srcEvent:t}),i&&e.splice(a,1))}});var Jt={touchstart:Et,touchmove:Pt,touchend:It,touchcancel:Nt},Qt="touchstart",$t="touchstart touchmove touchend touchcancel";u(Y,S,{handler:function(t){var e=Jt[t.type];if(e===Et&&(this.started=!0),this.started){var i=G.call(this,t,e);e&(It|Nt)&&i[0].length-i[1].length===0&&(this.started=!1),this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:Mt,srcEvent:t})}}});var te={touchstart:Et,touchmove:Pt,touchend:It,touchcancel:Nt},ee="touchstart touchmove touchend touchcancel";u(V,S,{handler:function(t){var e=te[t.type],i=U.call(this,t,e);i&&this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:Mt,srcEvent:t})}}),u(q,S,{handler:function(t,e,i){var o=i.pointerType==Mt,n=i.pointerType==St;if(o)this.mouse.allow=!1;else if(n&&!this.mouse.allow)return;e&(It|Nt)&&(this.mouse.allow=!0),this.callback(t,e,i)},destroy:function(){this.touch.destroy(),this.mouse.destroy()}});var ie=O(pt.style,"touchAction"),oe=ie!==a,ne="compute",se="auto",re="manipulation",ae="none",he="pan-x",de="pan-y";X.prototype={set:function(t){t==ne&&(t=this.compute()),oe&&this.manager.element.style&&(this.manager.element.style[ie]=t),this.actions=t.toLowerCase().trim()},update:function(){this.set(this.manager.options.touchAction)},compute:function(){var t=[];return l(this.manager.recognizers,function(e){f(e.options.enable,[e])&&(t=t.concat(e.getTouchAction()))}),Z(t.join(" "))},preventDefaults:function(t){if(!oe){var e=t.srcEvent,i=t.offsetDirection;if(this.manager.session.prevented)return void e.preventDefault();var o=this.actions,n=b(o,ae),s=b(o,de),r=b(o,he);if(n){var a=1===t.pointers.length,h=t.distance<2,d=t.deltaTime<250;if(a&&h&&d)return}if(!r||!s)return n||s&&i&Ft||r&&i&jt?this.preventSrc(e):void 0}},preventSrc:function(t){this.manager.session.prevented=!0,t.preventDefault()}};var le=1,ce=2,ue=4,pe=8,fe=pe,me=16,ve=32;K.prototype={defaults:{},set:function(t){return ct(this.options,t),this.manager&&this.manager.touchAction.update(),this},recognizeWith:function(t){if(d(t,"recognizeWith",this))return this;var e=this.simultaneous;return t=$(t,this),e[t.id]||(e[t.id]=t,t.recognizeWith(this)),this},dropRecognizeWith:function(t){return d(t,"dropRecognizeWith",this)?this:(t=$(t,this),delete this.simultaneous[t.id],this)},requireFailure:function(t){if(d(t,"requireFailure",this))return this;var e=this.requireFail;return t=$(t,this),-1===_(e,t)&&(e.push(t),t.requireFailure(this)),this},dropRequireFailure:function(t){if(d(t,"dropRequireFailure",this))return this;t=$(t,this);var e=_(this.requireFail,t);return e>-1&&this.requireFail.splice(e,1),this},hasRequireFailures:function(){return this.requireFail.length>0},canRecognizeWith:function(t){return!!this.simultaneous[t.id]},emit:function(t){function e(e){i.manager.emit(e,t)}var i=this,o=this.state;pe>o&&e(i.options.event+J(o)),e(i.options.event),t.additionalEvent&&e(t.additionalEvent),o>=pe&&e(i.options.event+J(o))},tryEmit:function(t){return this.canEmit()?this.emit(t):void(this.state=ve)},canEmit:function(){for(var t=0;t<this.requireFail.length;){if(!(this.requireFail[t].state&(ve|le)))return!1;t++}return!0},recognize:function(t){var e=ct({},t);return f(this.options.enable,[this,e])?(this.state&(fe|me|ve)&&(this.state=le),this.state=this.process(e),void(this.state&(ce|ue|pe|me)&&this.tryEmit(e))):(this.reset(),void(this.state=ve))},process:function(t){},getTouchAction:function(){},reset:function(){}},u(tt,K,{defaults:{pointers:1},attrTest:function(t){var e=this.options.pointers;return 0===e||t.pointers.length===e},process:function(t){var e=this.state,i=t.eventType,o=e&(ce|ue),n=this.attrTest(t);return o&&(i&Nt||!n)?e|me:o||n?i&It?e|pe:e&ce?e|ue:ce:ve}}),u(et,tt,{defaults:{event:"pan",threshold:10,pointers:1,direction:Ht},getTouchAction:function(){var t=this.options.direction,e=[];return t&Ft&&e.push(de),t&jt&&e.push(he),e},directionTest:function(t){var e=this.options,i=!0,o=t.distance,n=t.direction,s=t.deltaX,r=t.deltaY;return n&e.direction||(e.direction&Ft?(n=0===s?Rt:0>s?zt:Lt,i=s!=this.pX,o=Math.abs(t.deltaX)):(n=0===r?Rt:0>r?At:Bt,i=r!=this.pY,o=Math.abs(t.deltaY))),t.direction=n,i&&o>e.threshold&&n&e.direction},attrTest:function(t){return tt.prototype.attrTest.call(this,t)&&(this.state&ce||!(this.state&ce)&&this.directionTest(t))},emit:function(t){this.pX=t.deltaX,this.pY=t.deltaY;var e=Q(t.direction);e&&(t.additionalEvent=this.options.event+e),this._super.emit.call(this,t)}}),u(it,tt,{defaults:{event:"pinch",threshold:0,pointers:2},getTouchAction:function(){return[ae]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.scale-1)>this.options.threshold||this.state&ce)},emit:function(t){if(1!==t.scale){var e=t.scale<1?"in":"out";t.additionalEvent=this.options.event+e}this._super.emit.call(this,t)}}),u(ot,K,{defaults:{event:"press",pointers:1,time:251,threshold:9},getTouchAction:function(){return[se]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,o=t.distance<e.threshold,n=t.deltaTime>e.time;if(this._input=t,!o||!i||t.eventType&(It|Nt)&&!n)this.reset();else if(t.eventType&Et)this.reset(),this._timer=h(function(){this.state=fe,this.tryEmit()},e.time,this);else if(t.eventType&It)return fe;return ve},reset:function(){clearTimeout(this._timer)},emit:function(t){this.state===fe&&(t&&t.eventType&It?this.manager.emit(this.options.event+"up",t):(this._input.timeStamp=gt(),this.manager.emit(this.options.event,this._input)))}}),u(nt,tt,{defaults:{event:"rotate",threshold:0,pointers:2},getTouchAction:function(){return[ae]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.rotation)>this.options.threshold||this.state&ce)}}),u(st,tt,{defaults:{event:"swipe",threshold:10,velocity:.3,direction:Ft|jt,pointers:1},getTouchAction:function(){return et.prototype.getTouchAction.call(this)},attrTest:function(t){var e,i=this.options.direction;return i&(Ft|jt)?e=t.overallVelocity:i&Ft?e=t.overallVelocityX:i&jt&&(e=t.overallVelocityY),this._super.attrTest.call(this,t)&&i&t.offsetDirection&&t.distance>this.options.threshold&&t.maxPointers==this.options.pointers&&vt(e)>this.options.velocity&&t.eventType&It},emit:function(t){var e=Q(t.offsetDirection);e&&this.manager.emit(this.options.event+e,t),this.manager.emit(this.options.event,t)}}),u(rt,K,{defaults:{event:"tap",pointers:1,taps:1,interval:300,time:250,threshold:9,posThreshold:10},getTouchAction:function(){return[re]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,o=t.distance<e.threshold,n=t.deltaTime<e.time;if(this.reset(),t.eventType&Et&&0===this.count)return this.failTimeout();if(o&&n&&i){if(t.eventType!=It)return this.failTimeout();var s=this.pTime?t.timeStamp-this.pTime<e.interval:!0,r=!this.pCenter||A(this.pCenter,t.center)<e.posThreshold;this.pTime=t.timeStamp,this.pCenter=t.center,r&&s?this.count+=1:this.count=1,this._input=t;var a=this.count%e.taps;if(0===a)return this.hasRequireFailures()?(this._timer=h(function(){this.state=fe,this.tryEmit()},e.interval,this),ce):fe}return ve},failTimeout:function(){return this._timer=h(function(){this.state=ve},this.options.interval,this),ve},reset:function(){clearTimeout(this._timer)},emit:function(){this.state==fe&&(this._input.tapCount=this.count,this.manager.emit(this.options.event,this._input))}}),at.VERSION="2.0.6",at.defaults={domEvents:!1,touchAction:ne,enable:!0,inputTarget:null,inputClass:null,preset:[[nt,{enable:!1}],[it,{enable:!1},["rotate"]],[st,{direction:Ft}],[et,{direction:Ft},["swipe"]],[rt],[rt,{event:"doubletap",taps:2},["tap"]],[ot]],cssProps:{userSelect:"none",touchSelect:"none",touchCallout:"none",contentZooming:"none",userDrag:"none",tapHighlightColor:"rgba(0,0,0,0)"}};var ge=1,ye=2;ht.prototype={set:function(t){return ct(this.options,t),t.touchAction&&this.touchAction.update(),t.inputTarget&&(this.input.destroy(),this.input.target=t.inputTarget,this.input.init()),this},stop:function(t){this.session.stopped=t?ye:ge},recognize:function(t){var e=this.session;if(!e.stopped){this.touchAction.preventDefaults(t);var i,o=this.recognizers,n=e.curRecognizer;(!n||n&&n.state&fe)&&(n=e.curRecognizer=null);for(var s=0;s<o.length;)i=o[s],e.stopped===ye||n&&i!=n&&!i.canRecognizeWith(n)?i.reset():i.recognize(t),!n&&i.state&(ce|ue|pe)&&(n=e.curRecognizer=i),s++}},get:function(t){if(t instanceof K)return t;for(var e=this.recognizers,i=0;i<e.length;i++)if(e[i].options.event==t)return e[i];return null},add:function(t){if(d(t,"add",this))return this;var e=this.get(t.options.event);return e&&this.remove(e),this.recognizers.push(t),t.manager=this,this.touchAction.update(),t},remove:function(t){if(d(t,"remove",this))return this;if(t=this.get(t)){var e=this.recognizers,i=_(e,t);-1!==i&&(e.splice(i,1),this.touchAction.update())}return this},on:function(t,e){var i=this.handlers;return l(w(t),function(t){i[t]=i[t]||[],i[t].push(e)}),this},off:function(t,e){var i=this.handlers;return l(w(t),function(t){e?i[t]&&i[t].splice(_(i[t],e),1):delete i[t]}),this},emit:function(t,e){this.options.domEvents&&lt(t,e);var i=this.handlers[t]&&this.handlers[t].slice();if(i&&i.length){e.type=t,e.preventDefault=function(){e.srcEvent.preventDefault()};for(var o=0;o<i.length;)i[o](e),o++}},destroy:function(){this.element&&dt(this,!1),this.handlers={},this.session={},this.input.destroy(),this.element=null}},ct(at,{INPUT_START:Et,INPUT_MOVE:Pt,INPUT_END:It,INPUT_CANCEL:Nt,STATE_POSSIBLE:le,STATE_BEGAN:ce,STATE_CHANGED:ue,STATE_ENDED:pe,STATE_RECOGNIZED:fe,STATE_CANCELLED:me,STATE_FAILED:ve,DIRECTION_NONE:Rt,DIRECTION_LEFT:zt,DIRECTION_RIGHT:Lt,DIRECTION_UP:At,DIRECTION_DOWN:Bt,DIRECTION_HORIZONTAL:Ft,DIRECTION_VERTICAL:jt,DIRECTION_ALL:Ht,Manager:ht,Input:S,TouchAction:X,TouchInput:V,MouseInput:H,PointerEventInput:W,TouchMouseInput:q,SingleTouchInput:Y,Recognizer:K,AttrRecognizer:tt,Tap:rt,Pan:et,Swipe:st,Pinch:it,Rotate:nt,Press:ot,on:v,off:g,each:l,merge:bt,extend:yt,assign:ct,inherit:u,bindFn:p,prefixed:O});var be="undefined"!=typeof n?n:"undefined"!=typeof self?self:{};be.Hammer=at,o=function(){return at}.call(e,i,e,t),!(o!==a&&(t.exports=o))}(window,document,"Hammer")},function(t,e,i){var o,n,s;!function(i,r){n=[],o=r,s="function"==typeof o?o.apply(e,n):o,!(void 0!==s&&(t.exports=s))}(this,function(){function t(t){var e,i=t&&t.preventDefault||!1,o=t&&t.container||window,n={},s={keydown:{},keyup:{}},r={};for(e=97;122>=e;e++)r[String.fromCharCode(e)]={code:65+(e-97),shift:!1};for(e=65;90>=e;e++)r[String.fromCharCode(e)]={code:e,shift:!0};for(e=0;9>=e;e++)r[""+e]={code:48+e,shift:!1};for(e=1;12>=e;e++)r["F"+e]={code:111+e,shift:!1};for(e=0;9>=e;e++)r["num"+e]={code:96+e,shift:!1};r["num*"]={code:106,shift:!1},r["num+"]={code:107,shift:!1},r["num-"]={code:109,shift:!1},r["num/"]={code:111,shift:!1},r["num."]={code:110,shift:!1},r.left={code:37,shift:!1},r.up={code:38,shift:!1},r.right={code:39,shift:!1},r.down={code:40,shift:!1},r.space={code:32,shift:!1},r.enter={code:13,shift:!1},r.shift={code:16,shift:void 0},r.esc={code:27,shift:!1},r.backspace={code:8,shift:!1},r.tab={code:9,shift:!1},r.ctrl={code:17,shift:!1},r.alt={code:18,shift:!1},r["delete"]={code:46,shift:!1},r.pageup={code:33,shift:!1},r.pagedown={code:34,shift:!1},r["="]={code:187,shift:!1},r["-"]={code:189,shift:!1},r["]"]={code:221,shift:!1},r["["]={code:219,shift:!1};var a=function(t){d(t,"keydown")},h=function(t){d(t,"keyup")},d=function(t,e){if(void 0!==s[e][t.keyCode]){for(var o=s[e][t.keyCode],n=0;n<o.length;n++)void 0===o[n].shift?o[n].fn(t):1==o[n].shift&&1==t.shiftKey?o[n].fn(t):0==o[n].shift&&0==t.shiftKey&&o[n].fn(t);1==i&&t.preventDefault()}};return n.bind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);void 0===s[i][r[t].code]&&(s[i][r[t].code]=[]),s[i][r[t].code].push({fn:e,shift:r[t].shift})},n.bindAll=function(t,e){void 0===e&&(e="keydown");for(var i in r)r.hasOwnProperty(i)&&n.bind(i,t,e)},n.getKey=function(t){for(var e in r)if(r.hasOwnProperty(e)){if(1==t.shiftKey&&1==r[e].shift&&t.keyCode==r[e].code)return e;if(0==t.shiftKey&&0==r[e].shift&&t.keyCode==r[e].code)return e;if(t.keyCode==r[e].code&&"shift"==e)return e}return"unknown key, currently not supported"},n.unbind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);if(void 0!==e){var o=[],n=s[i][r[t].code];if(void 0!==n)for(var a=0;a<n.length;a++)n[a].fn==e&&n[a].shift==r[t].shift||o.push(s[i][r[t].code][a]);s[i][r[t].code]=o}else s[i][r[t].code]=[]},n.reset=function(){s={keydown:{},keyup:{}}},n.destroy=function(){s={keydown:{},keyup:{}},o.removeEventListener("keydown",a,!0),o.removeEventListener("keyup",h,!0)},o.addEventListener("keydown",a,!0),o.addEventListener("keyup",h,!0),n}return t})},function(t,e,i){e.util=i(1),e.DOMutil=i(8),e.DataSet=i(9),e.DataView=i(11),e.Queue=i(10),e.Timeline=i(25),e.Graph2d=i(50),e.timeline={Core:i(33),DateUtil:i(32),Range:i(30),stack:i(37),TimeStep:i(35),components:{items:{Item:i(39),BackgroundItem:i(43),BoxItem:i(41),PointItem:i(42),RangeItem:i(38)},BackgroundGroup:i(40),Component:i(31),CurrentTime:i(48),CustomTime:i(46),DataAxis:i(52),DataScale:i(53),GraphGroup:i(54),Group:i(36),ItemSet:i(34),Legend:i(58),LineGraph:i(51),TimeAxis:i(44)}},e.moment=i(2),e.Hammer=i(20),e.keycharm=i(23)},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e,i,o){if(!(this instanceof n))throw new SyntaxError("Constructor must be called with the new operator");if(!(Array.isArray(i)||i instanceof c||i instanceof u)&&i instanceof Object){var s=o;o=i,i=s}var r=this;this.defaultOptions={start:null,end:null,autoResize:!0,throttleRedraw:0,orientation:{axis:"bottom",item:"bottom"},rtl:!1,moment:d,width:null,height:null,maxHeight:null,minHeight:null},this.options=l.deepExtend({},this.defaultOptions),this._create(t),this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this)},hiddenDates:[],util:{getScale:function(){return r.timeAxis.step.scale},getStep:function(){return r.timeAxis.step.step},toScreen:r._toScreen.bind(r),toGlobalScreen:r._toGlobalScreen.bind(r),toTime:r._toTime.bind(r),toGlobalTime:r._toGlobalTime.bind(r)}},this.range=new p(this.body),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new m(this.body),this.timeAxis2=null,this.components.push(this.timeAxis),this.currentTime=new v(this.body),this.components.push(this.currentTime),this.itemSet=new y(this.body,this.options),this.components.push(this.itemSet),this.itemsData=null,this.groupsData=null,this.on("tap",function(t){r.emit("click",r.getEventProperties(t))}),this.on("doubletap",function(t){r.emit("doubleClick",r.getEventProperties(t))}),this.dom.root.oncontextmenu=function(t){r.emit("contextmenu",r.getEventProperties(t))},this.fitDone=!1,this.on("changed",function(){if(null!=this.itemsData&&!r.fitDone)if(r.fitDone=!0,void 0!=r.options.start||void 0!=r.options.end){if(void 0==r.options.start||void 0==r.options.end)var t=r.getItemRange();var e=void 0!=r.options.start?r.options.start:t.min,i=void 0!=r.options.end?r.options.end:t.max;r.setWindow(e,i,{animation:!1})}else r.fit({animation:!1})}),o&&this.setOptions(o),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}var s=i(26),r=o(s),a=i(29),h=o(a),d=(i(13),i(20),i(2)),l=i(1),c=i(9),u=i(11),p=i(30),f=i(33),m=i(44),v=i(48),g=i(46),y=i(34),b=i(29).printStyle,w=i(49).allOptions,_=i(49).configureOptions;n.prototype=new f,n.prototype._createConfigurator=function(){return new r["default"](this,this.dom.container,_)},n.prototype.redraw=function(){this.itemSet&&this.itemSet.markDirty({refreshItems:!0}),this._redraw()},n.prototype.setOptions=function(t){var e=h["default"].validate(t,w);if(e===!0&&console.log("%cErrors have been found in the supplied options object.",b),f.prototype.setOptions.call(this,t),"type"in t&&t.type!==this.options.type){this.options.type=t.type;var i=this.itemsData;if(i){var o=this.getSelection();this.setItems(null),this.setItems(i),this.setSelection(o)}}},n.prototype.setItems=function(t){var e;e=t?t instanceof c||t instanceof u?t:new c(t,{type:{start:"Date",end:"Date"}}):null,this.itemsData=e,this.itemSet&&this.itemSet.setItems(e)},n.prototype.setGroups=function(t){var e;e=t?t instanceof c||t instanceof u?t:new c(t):null,this.groupsData=e,this.itemSet.setGroups(e)},n.prototype.setData=function(t){t&&t.groups&&this.setGroups(t.groups),t&&t.items&&this.setItems(t.items)},n.prototype.setSelection=function(t,e){this.itemSet&&this.itemSet.setSelection(t),e&&e.focus&&this.focus(t,e)},n.prototype.getSelection=function(){return this.itemSet&&this.itemSet.getSelection()||[]},n.prototype.focus=function(t,e){if(this.itemsData&&void 0!=t){var i=Array.isArray(t)?t:[t],o=this.itemsData.getDataSet().get(i,{type:{start:"Date",end:"Date"}}),n=null,s=null;if(o.forEach(function(t){var e=t.start.valueOf(),i="end"in t?t.end.valueOf():t.start.valueOf();(null===n||n>e)&&(n=e),(null===s||i>s)&&(s=i)}),null!==n&&null!==s){var r=(n+s)/2,a=Math.max(this.range.end-this.range.start,1.1*(s-n)),h=e&&void 0!==e.animation?e.animation:!0;this.range.setRange(r-a/2,r+a/2,h)}}},n.prototype.fit=function(t){var e,i=t&&void 0!==t.animation?t.animation:!0,o=this.itemsData&&this.itemsData.getDataSet();1===o.length&&void 0===o.get()[0].end?(e=this.getDataRange(),this.moveTo(e.min.valueOf(),{animation:i})):(e=this.getItemRange(),this.range.setRange(e.min,e.max,i))},n.prototype.getItemRange=function(){var t=this,e=this.getDataRange(),i=null!==e.min?e.min.valueOf():null,o=null!==e.max?e.max.valueOf():null,n=null,s=null;if(null!=i&&null!=o){var r,a,h,d,c;!function(){var e=function(t){return l.convert(t.data.start,"Date").valueOf()},u=function(t){var e=void 0!=t.data.end?t.data.end:t.data.start;return l.convert(e,"Date").valueOf()};r=o-i,0>=r&&(r=10),a=r/t.props.center.width,l.forEach(t.itemSet.items,function(t){t.show(),t.repositionX();var r=e(t),h=u(t);if(this.options.rtl)var d=r-(t.getWidthRight()+10)*a,l=h+(t.getWidthLeft()+10)*a;else var d=r-(t.getWidthLeft()+10)*a,l=h+(t.getWidthRight()+10)*a;i>d&&(i=d,n=t),l>o&&(o=l,s=t)}.bind(t)),n&&s&&(h=n.getWidthLeft()+10,d=s.getWidthRight()+10,c=t.props.center.width-h-d,c>0&&(t.options.rtl?(i=e(n)-d*r/c,o=u(s)+h*r/c):(i=e(n)-h*r/c,o=u(s)+d*r/c)))}()}return{min:null!=i?new Date(i):null,max:null!=o?new Date(o):null}},n.prototype.getDataRange=function(){var t=null,e=null,i=this.itemsData&&this.itemsData.getDataSet();return i&&i.forEach(function(i){var o=l.convert(i.start,"Date").valueOf(),n=l.convert(void 0!=i.end?i.end:i.start,"Date").valueOf();(null===t||t>o)&&(t=o),(null===e||n>e)&&(e=n)}),{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},n.prototype.getEventProperties=function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY;if(this.options.rtl)var o=l.getAbsoluteRight(this.dom.centerContainer)-e;else var o=e-l.getAbsoluteLeft(this.dom.centerContainer);var n=i-l.getAbsoluteTop(this.dom.centerContainer),s=this.itemSet.itemFromTarget(t),r=this.itemSet.groupFromTarget(t),a=g.customTimeFromTarget(t),h=this.itemSet.options.snap||null,d=this.body.util.getScale(),c=this.body.util.getStep(),u=this._toTime(o),p=h?h(u,d,c):u,f=l.getTarget(t),m=null;return null!=s?m="item":null!=a?m="custom-time":l.hasParent(f,this.timeAxis.dom.foreground)?m="axis":this.timeAxis2&&l.hasParent(f,this.timeAxis2.dom.foreground)?m="axis":l.hasParent(f,this.itemSet.dom.labelSet)?m="group-label":l.hasParent(f,this.currentTime.bar)?m="current-time":l.hasParent(f,this.dom.center)&&(m="background"),{event:t,item:s?s.id:null,group:r?r.groupId:null,what:m,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:o,y:n,time:u,snappedTime:p}},t.exports=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},r=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),a=i(27),h=o(a),d=i(1),l=function(){function t(e,i,o){var s=arguments.length<=3||void 0===arguments[3]?1:arguments[3];n(this,t),this.parent=e,this.changedOptions=[],this.container=i,this.allowCreation=!1,this.options={},this.initialized=!1,this.popupCounter=0,this.defaultOptions={enabled:!1,filter:!0,container:void 0,showButton:!0},d.extend(this.options,this.defaultOptions),this.configureOptions=o,this.moduleOptions={},this.domElements=[],this.popupDiv={},this.popupLimit=5,this.popupHistory={},this.colorPicker=new h["default"](s),this.wrapper=void 0}return r(t,[{key:"setOptions",value:function(t){if(void 0!==t){this.popupHistory={},this._removePopup();var e=!0;"string"==typeof t?this.options.filter=t:t instanceof Array?this.options.filter=t.join():"object"===("undefined"==typeof t?"undefined":s(t))?(void 0!==t.container&&(this.options.container=t.container),void 0!==t.filter&&(this.options.filter=t.filter),void 0!==t.showButton&&(this.options.showButton=t.showButton),void 0!==t.enabled&&(e=t.enabled)):"boolean"==typeof t?(this.options.filter=!0,e=t):"function"==typeof t&&(this.options.filter=t,e=!0),this.options.filter===!1&&(e=!1),this.options.enabled=e}this._clean()}},{key:"setModuleOptions",value:function(t){this.moduleOptions=t,this.options.enabled===!0&&(this._clean(),void 0!==this.options.container&&(this.container=this.options.container),this._create())}},{key:"_create",value:function(){var t=this;this._clean(),this.changedOptions=[];var e=this.options.filter,i=0,o=!1;for(var n in this.configureOptions)this.configureOptions.hasOwnProperty(n)&&(this.allowCreation=!1,o=!1,"function"==typeof e?(o=e(n,[]),o=o||this._handleObject(this.configureOptions[n],[n],!0)):e!==!0&&-1===e.indexOf(n)||(o=!0),o!==!1&&(this.allowCreation=!0,i>0&&this._makeItem([]),this._makeHeader(n),this._handleObject(this.configureOptions[n],[n])),i++);this.options.showButton===!0&&!function(){var e=document.createElement("div");e.className="vis-configuration vis-config-button",e.innerHTML="generate options",e.onclick=function(){t._printOptions()},e.onmouseover=function(){e.className="vis-configuration vis-config-button hover"},e.onmouseout=function(){e.className="vis-configuration vis-config-button"},t.optionsContainer=document.createElement("div"),t.optionsContainer.className="vis-configuration vis-config-option-container",t.domElements.push(t.optionsContainer),t.domElements.push(e)}(),this._push()}},{key:"_push",value:function(){this.wrapper=document.createElement("div"),this.wrapper.className="vis-configuration-wrapper",this.container.appendChild(this.wrapper);for(var t=0;t<this.domElements.length;t++)this.wrapper.appendChild(this.domElements[t]);this._showPopupIfNeeded()}},{key:"_clean",value:function(){for(var t=0;t<this.domElements.length;t++)this.wrapper.removeChild(this.domElements[t]);void 0!==this.wrapper&&(this.container.removeChild(this.wrapper),this.wrapper=void 0),this.domElements=[],this._removePopup()}},{key:"_getValue",value:function(t){for(var e=this.moduleOptions,i=0;i<t.length;i++){if(void 0===e[t[i]]){e=void 0;break}e=e[t[i]]}return e}},{key:"_makeItem",value:function(t){var e=arguments,i=this;if(this.allowCreation===!0){var o,n,r,a=function(){var s=document.createElement("div");for(s.className="vis-configuration vis-config-item vis-config-s"+t.length,o=e.length,n=Array(o>1?o-1:0),r=1;o>r;r++)n[r-1]=e[r];return n.forEach(function(t){s.appendChild(t)}),i.domElements.push(s),{v:i.domElements.length}}();if("object"===("undefined"==typeof a?"undefined":s(a)))return a.v}return 0}},{key:"_makeHeader",value:function(t){var e=document.createElement("div");e.className="vis-configuration vis-config-header",e.innerHTML=t,this._makeItem([],e)}},{key:"_makeLabel",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=document.createElement("div");return o.className="vis-configuration vis-config-label vis-config-s"+e.length,i===!0?o.innerHTML="<i><b>"+t+":</b></i>":o.innerHTML=t+":",o}},{key:"_makeDropdown",value:function(t,e,i){var o=document.createElement("select");o.className="vis-configuration vis-config-select";var n=0;void 0!==e&&-1!==t.indexOf(e)&&(n=t.indexOf(e));for(var s=0;s<t.length;s++){var r=document.createElement("option");r.value=t[s],s===n&&(r.selected="selected"),r.innerHTML=t[s],o.appendChild(r)}var a=this;o.onchange=function(){a._update(this.value,i)};var h=this._makeLabel(i[i.length-1],i);this._makeItem(i,h,o)}},{key:"_makeRange",value:function(t,e,i){
-var o=t[0],n=t[1],s=t[2],r=t[3],a=document.createElement("input");a.className="vis-configuration vis-config-range";try{a.type="range",a.min=n,a.max=s}catch(h){}a.step=r;var d="",l=0;if(void 0!==e){var c=1.2;0>e&&n>e*c?(a.min=Math.ceil(e*c),l=a.min,d="range increased"):n>e/c&&(a.min=Math.ceil(e/c),l=a.min,d="range increased"),e*c>s&&1!==s&&(a.max=Math.ceil(e*c),l=a.max,d="range increased"),a.value=e}else a.value=o;var u=document.createElement("input");u.className="vis-configuration vis-config-rangeinput",u.value=a.value;var p=this;a.onchange=function(){u.value=this.value,p._update(Number(this.value),i)},a.oninput=function(){u.value=this.value};var f=this._makeLabel(i[i.length-1],i),m=this._makeItem(i,f,a,u);""!==d&&this.popupHistory[m]!==l&&(this.popupHistory[m]=l,this._setupPopup(d,m))}},{key:"_setupPopup",value:function(t,e){var i=this;if(this.initialized===!0&&this.allowCreation===!0&&this.popupCounter<this.popupLimit){var o=document.createElement("div");o.id="vis-configuration-popup",o.className="vis-configuration-popup",o.innerHTML=t,o.onclick=function(){i._removePopup()},this.popupCounter+=1,this.popupDiv={html:o,index:e}}}},{key:"_removePopup",value:function(){void 0!==this.popupDiv.html&&(this.popupDiv.html.parentNode.removeChild(this.popupDiv.html),clearTimeout(this.popupDiv.hideTimeout),clearTimeout(this.popupDiv.deleteTimeout),this.popupDiv={})}},{key:"_showPopupIfNeeded",value:function(){var t=this;if(void 0!==this.popupDiv.html){var e=this.domElements[this.popupDiv.index],i=e.getBoundingClientRect();this.popupDiv.html.style.left=i.left+"px",this.popupDiv.html.style.top=i.top-30+"px",document.body.appendChild(this.popupDiv.html),this.popupDiv.hideTimeout=setTimeout(function(){t.popupDiv.html.style.opacity=0},1500),this.popupDiv.deleteTimeout=setTimeout(function(){t._removePopup()},1800)}}},{key:"_makeCheckbox",value:function(t,e,i){var o=document.createElement("input");o.type="checkbox",o.className="vis-configuration vis-config-checkbox",o.checked=t,void 0!==e&&(o.checked=e,e!==t&&("object"===("undefined"==typeof t?"undefined":s(t))?e!==t.enabled&&this.changedOptions.push({path:i,value:e}):this.changedOptions.push({path:i,value:e})));var n=this;o.onchange=function(){n._update(this.checked,i)};var r=this._makeLabel(i[i.length-1],i);this._makeItem(i,r,o)}},{key:"_makeTextInput",value:function(t,e,i){var o=document.createElement("input");o.type="text",o.className="vis-configuration vis-config-text",o.value=e,e!==t&&this.changedOptions.push({path:i,value:e});var n=this;o.onchange=function(){n._update(this.value,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,o)}},{key:"_makeColorField",value:function(t,e,i){var o=this,n=t[1],s=document.createElement("div");e=void 0===e?n:e,"none"!==e?(s.className="vis-configuration vis-config-colorBlock",s.style.backgroundColor=e):s.className="vis-configuration vis-config-colorBlock none",e=void 0===e?n:e,s.onclick=function(){o._showColorPicker(e,s,i)};var r=this._makeLabel(i[i.length-1],i);this._makeItem(i,r,s)}},{key:"_showColorPicker",value:function(t,e,i){var o=this;e.onclick=function(){},this.colorPicker.insertTo(e),this.colorPicker.show(),this.colorPicker.setColor(t),this.colorPicker.setUpdateCallback(function(t){var n="rgba("+t.r+","+t.g+","+t.b+","+t.a+")";e.style.backgroundColor=n,o._update(n,i)}),this.colorPicker.setCloseCallback(function(){e.onclick=function(){o._showColorPicker(t,e,i)}})}},{key:"_handleObject",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?[]:arguments[1],i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=!1,n=this.options.filter,s=!1;for(var r in t)if(t.hasOwnProperty(r)){o=!0;var a=t[r],h=d.copyAndExtendArray(e,r);if("function"==typeof n&&(o=n(r,e),o===!1&&!(a instanceof Array)&&"string"!=typeof a&&"boolean"!=typeof a&&a instanceof Object&&(this.allowCreation=!1,o=this._handleObject(a,h,!0),this.allowCreation=i===!1)),o!==!1){s=!0;var l=this._getValue(h);if(a instanceof Array)this._handleArray(a,l,h);else if("string"==typeof a)this._makeTextInput(a,l,h);else if("boolean"==typeof a)this._makeCheckbox(a,l,h);else if(a instanceof Object){var c=!0;if(-1!==e.indexOf("physics")&&this.moduleOptions.physics.solver!==r&&(c=!1),c===!0)if(void 0!==a.enabled){var u=d.copyAndExtendArray(h,"enabled"),p=this._getValue(u);if(p===!0){var f=this._makeLabel(r,h,!0);this._makeItem(h,f),s=this._handleObject(a,h)||s}else this._makeCheckbox(a,p,h)}else{var m=this._makeLabel(r,h,!0);this._makeItem(h,m),s=this._handleObject(a,h)||s}}else console.error("dont know how to handle",a,r,h)}}return s}},{key:"_handleArray",value:function(t,e,i){"string"==typeof t[0]&&"color"===t[0]?(this._makeColorField(t,e,i),t[1]!==e&&this.changedOptions.push({path:i,value:e})):"string"==typeof t[0]?(this._makeDropdown(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:e})):"number"==typeof t[0]&&(this._makeRange(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:Number(e)}))}},{key:"_update",value:function(t,e){var i=this._constructOptions(t,e);this.parent.body&&this.parent.body.emitter&&this.parent.body.emitter.emit&&this.parent.body.emitter.emit("configChange",i),this.initialized=!0,this.parent.setOptions(i)}},{key:"_constructOptions",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?{}:arguments[2],o=i;t="true"===t?!0:t,t="false"===t?!1:t;for(var n=0;n<e.length;n++)"global"!==e[n]&&(void 0===o[e[n]]&&(o[e[n]]={}),n!==e.length-1?o=o[e[n]]:o[e[n]]=t);return i}},{key:"_printOptions",value:function(){var t=this.getOptions();this.optionsContainer.innerHTML="<pre>var options = "+JSON.stringify(t,null,2)+"</pre>"}},{key:"getOptions",value:function(){for(var t={},e=0;e<this.changedOptions.length;e++)this._constructOptions(this.changedOptions[e].value,this.changedOptions[e].path,t);return t}}]),t}();e["default"]=l},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=i(20),r=i(28),a=i(1),h=function(){function t(){var e=arguments.length<=0||void 0===arguments[0]?1:arguments[0];o(this,t),this.pixelRatio=e,this.generated=!1,this.centerCoordinates={x:144.5,y:144.5},this.r=289*.49,this.color={r:255,g:255,b:255,a:1},this.hueCircle=void 0,this.initialColor={r:255,g:255,b:255,a:1},this.previousColor=void 0,this.applied=!1,this.updateCallback=function(){},this.closeCallback=function(){},this._create()}return n(t,[{key:"insertTo",value:function(t){void 0!==this.hammer&&(this.hammer.destroy(),this.hammer=void 0),this.container=t,this.container.appendChild(this.frame),this._bindHammer(),this._setSize()}},{key:"setUpdateCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker update callback is not a function.");this.updateCallback=t}},{key:"setCloseCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker closing callback is not a function.");this.closeCallback=t}},{key:"_isColorString",value:function(t){var e={black:"#000000",navy:"#000080",darkblue:"#00008B",mediumblue:"#0000CD",blue:"#0000FF",darkgreen:"#006400",green:"#008000",teal:"#008080",darkcyan:"#008B8B",deepskyblue:"#00BFFF",darkturquoise:"#00CED1",mediumspringgreen:"#00FA9A",lime:"#00FF00",springgreen:"#00FF7F",aqua:"#00FFFF",cyan:"#00FFFF",midnightblue:"#191970",dodgerblue:"#1E90FF",lightseagreen:"#20B2AA",forestgreen:"#228B22",seagreen:"#2E8B57",darkslategray:"#2F4F4F",limegreen:"#32CD32",mediumseagreen:"#3CB371",turquoise:"#40E0D0",royalblue:"#4169E1",steelblue:"#4682B4",darkslateblue:"#483D8B",mediumturquoise:"#48D1CC",indigo:"#4B0082",darkolivegreen:"#556B2F",cadetblue:"#5F9EA0",cornflowerblue:"#6495ED",mediumaquamarine:"#66CDAA",dimgray:"#696969",slateblue:"#6A5ACD",olivedrab:"#6B8E23",slategray:"#708090",lightslategray:"#778899",mediumslateblue:"#7B68EE",lawngreen:"#7CFC00",chartreuse:"#7FFF00",aquamarine:"#7FFFD4",maroon:"#800000",purple:"#800080",olive:"#808000",gray:"#808080",skyblue:"#87CEEB",lightskyblue:"#87CEFA",blueviolet:"#8A2BE2",darkred:"#8B0000",darkmagenta:"#8B008B",saddlebrown:"#8B4513",darkseagreen:"#8FBC8F",lightgreen:"#90EE90",mediumpurple:"#9370D8",darkviolet:"#9400D3",palegreen:"#98FB98",darkorchid:"#9932CC",yellowgreen:"#9ACD32",sienna:"#A0522D",brown:"#A52A2A",darkgray:"#A9A9A9",lightblue:"#ADD8E6",greenyellow:"#ADFF2F",paleturquoise:"#AFEEEE",lightsteelblue:"#B0C4DE",powderblue:"#B0E0E6",firebrick:"#B22222",darkgoldenrod:"#B8860B",mediumorchid:"#BA55D3",rosybrown:"#BC8F8F",darkkhaki:"#BDB76B",silver:"#C0C0C0",mediumvioletred:"#C71585",indianred:"#CD5C5C",peru:"#CD853F",chocolate:"#D2691E",tan:"#D2B48C",lightgrey:"#D3D3D3",palevioletred:"#D87093",thistle:"#D8BFD8",orchid:"#DA70D6",goldenrod:"#DAA520",crimson:"#DC143C",gainsboro:"#DCDCDC",plum:"#DDA0DD",burlywood:"#DEB887",lightcyan:"#E0FFFF",lavender:"#E6E6FA",darksalmon:"#E9967A",violet:"#EE82EE",palegoldenrod:"#EEE8AA",lightcoral:"#F08080",khaki:"#F0E68C",aliceblue:"#F0F8FF",honeydew:"#F0FFF0",azure:"#F0FFFF",sandybrown:"#F4A460",wheat:"#F5DEB3",beige:"#F5F5DC",whitesmoke:"#F5F5F5",mintcream:"#F5FFFA",ghostwhite:"#F8F8FF",salmon:"#FA8072",antiquewhite:"#FAEBD7",linen:"#FAF0E6",lightgoldenrodyellow:"#FAFAD2",oldlace:"#FDF5E6",red:"#FF0000",fuchsia:"#FF00FF",magenta:"#FF00FF",deeppink:"#FF1493",orangered:"#FF4500",tomato:"#FF6347",hotpink:"#FF69B4",coral:"#FF7F50",darkorange:"#FF8C00",lightsalmon:"#FFA07A",orange:"#FFA500",lightpink:"#FFB6C1",pink:"#FFC0CB",gold:"#FFD700",peachpuff:"#FFDAB9",navajowhite:"#FFDEAD",moccasin:"#FFE4B5",bisque:"#FFE4C4",mistyrose:"#FFE4E1",blanchedalmond:"#FFEBCD",papayawhip:"#FFEFD5",lavenderblush:"#FFF0F5",seashell:"#FFF5EE",cornsilk:"#FFF8DC",lemonchiffon:"#FFFACD",floralwhite:"#FFFAF0",snow:"#FFFAFA",yellow:"#FFFF00",lightyellow:"#FFFFE0",ivory:"#FFFFF0",white:"#FFFFFF"};return"string"==typeof t?e[t]:void 0}},{key:"setColor",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];if("none"!==t){var i=void 0,o=this._isColorString(t);if(void 0!==o&&(t=o),a.isString(t)===!0){if(a.isValidRGB(t)===!0){var n=t.substr(4).substr(0,t.length-5).split(",");i={r:n[0],g:n[1],b:n[2],a:1}}else if(a.isValidRGBA(t)===!0){var s=t.substr(5).substr(0,t.length-6).split(",");i={r:s[0],g:s[1],b:s[2],a:s[3]}}else if(a.isValidHex(t)===!0){var r=a.hexToRGB(t);i={r:r.r,g:r.g,b:r.b,a:1}}}else if(t instanceof Object&&void 0!==t.r&&void 0!==t.g&&void 0!==t.b){var h=void 0!==t.a?t.a:"1.0";i={r:t.r,g:t.g,b:t.b,a:h}}if(void 0===i)throw new Error("Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: "+JSON.stringify(t));this._setColor(i,e)}}},{key:"show",value:function(){void 0!==this.closeCallback&&(this.closeCallback(),this.closeCallback=void 0),this.applied=!1,this.frame.style.display="block",this._generateHueCircle()}},{key:"_hide",value:function(){var t=this,e=arguments.length<=0||void 0===arguments[0]?!0:arguments[0];e===!0&&(this.previousColor=a.extend({},this.color)),this.applied===!0&&this.updateCallback(this.initialColor),this.frame.style.display="none",setTimeout(function(){void 0!==t.closeCallback&&(t.closeCallback(),t.closeCallback=void 0)},0)}},{key:"_save",value:function(){this.updateCallback(this.color),this.applied=!1,this._hide()}},{key:"_apply",value:function(){this.applied=!0,this.updateCallback(this.color),this._updatePicker(this.color)}},{key:"_loadLast",value:function(){void 0!==this.previousColor?this.setColor(this.previousColor,!1):alert("There is no last color to load...")}},{key:"_setColor",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];e===!0&&(this.initialColor=a.extend({},t)),this.color=t;var i=a.RGBToHSV(t.r,t.g,t.b),o=2*Math.PI,n=this.r*i.s,s=this.centerCoordinates.x+n*Math.sin(o*i.h),r=this.centerCoordinates.y+n*Math.cos(o*i.h);this.colorPickerSelector.style.left=s-.5*this.colorPickerSelector.clientWidth+"px",this.colorPickerSelector.style.top=r-.5*this.colorPickerSelector.clientHeight+"px",this._updatePicker(t)}},{key:"_setOpacity",value:function(t){this.color.a=t/100,this._updatePicker(this.color)}},{key:"_setBrightness",value:function(t){var e=a.RGBToHSV(this.color.r,this.color.g,this.color.b);e.v=t/100;var i=a.HSVToRGB(e.h,e.s,e.v);i.a=this.color.a,this.color=i,this._updatePicker()}},{key:"_updatePicker",value:function(){var t=arguments.length<=0||void 0===arguments[0]?this.color:arguments[0],e=a.RGBToHSV(t.r,t.g,t.b),i=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(i.webkitBackingStorePixelRatio||i.mozBackingStorePixelRatio||i.msBackingStorePixelRatio||i.oBackingStorePixelRatio||i.backingStorePixelRatio||1)),i.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var o=this.colorPickerCanvas.clientWidth,n=this.colorPickerCanvas.clientHeight;i.clearRect(0,0,o,n),i.putImageData(this.hueCircle,0,0),i.fillStyle="rgba(0,0,0,"+(1-e.v)+")",i.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),i.fill(),this.brightnessRange.value=100*e.v,this.opacityRange.value=100*t.a,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}},{key:"_setSize",value:function(){this.colorPickerCanvas.style.width="100%",this.colorPickerCanvas.style.height="100%",this.colorPickerCanvas.width=289*this.pixelRatio,this.colorPickerCanvas.height=289*this.pixelRatio}},{key:"_create",value:function(){if(this.frame=document.createElement("div"),this.frame.className="vis-color-picker",this.colorPickerDiv=document.createElement("div"),this.colorPickerSelector=document.createElement("div"),this.colorPickerSelector.className="vis-selector",this.colorPickerDiv.appendChild(this.colorPickerSelector),this.colorPickerCanvas=document.createElement("canvas"),this.colorPickerDiv.appendChild(this.colorPickerCanvas),this.colorPickerCanvas.getContext){var t=this.colorPickerCanvas.getContext("2d");this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1),this.colorPickerCanvas.getContext("2d").setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0)}else{var e=document.createElement("DIV");e.style.color="red",e.style.fontWeight="bold",e.style.padding="10px",e.innerHTML="Error: your browser does not support HTML canvas",this.colorPickerCanvas.appendChild(e)}this.colorPickerDiv.className="vis-color",this.opacityDiv=document.createElement("div"),this.opacityDiv.className="vis-opacity",this.brightnessDiv=document.createElement("div"),this.brightnessDiv.className="vis-brightness",this.arrowDiv=document.createElement("div"),this.arrowDiv.className="vis-arrow",this.opacityRange=document.createElement("input");try{this.opacityRange.type="range",this.opacityRange.min="0",this.opacityRange.max="100"}catch(i){}this.opacityRange.value="100",this.opacityRange.className="vis-range",this.brightnessRange=document.createElement("input");try{this.brightnessRange.type="range",this.brightnessRange.min="0",this.brightnessRange.max="100"}catch(i){}this.brightnessRange.value="100",this.brightnessRange.className="vis-range",this.opacityDiv.appendChild(this.opacityRange),this.brightnessDiv.appendChild(this.brightnessRange);var o=this;this.opacityRange.onchange=function(){o._setOpacity(this.value)},this.opacityRange.oninput=function(){o._setOpacity(this.value)},this.brightnessRange.onchange=function(){o._setBrightness(this.value)},this.brightnessRange.oninput=function(){o._setBrightness(this.value)},this.brightnessLabel=document.createElement("div"),this.brightnessLabel.className="vis-label vis-brightness",this.brightnessLabel.innerHTML="brightness:",this.opacityLabel=document.createElement("div"),this.opacityLabel.className="vis-label vis-opacity",this.opacityLabel.innerHTML="opacity:",this.newColorDiv=document.createElement("div"),this.newColorDiv.className="vis-new-color",this.newColorDiv.innerHTML="new",this.initialColorDiv=document.createElement("div"),this.initialColorDiv.className="vis-initial-color",this.initialColorDiv.innerHTML="initial",this.cancelButton=document.createElement("div"),this.cancelButton.className="vis-button vis-cancel",this.cancelButton.innerHTML="cancel",this.cancelButton.onclick=this._hide.bind(this,!1),this.applyButton=document.createElement("div"),this.applyButton.className="vis-button vis-apply",this.applyButton.innerHTML="apply",this.applyButton.onclick=this._apply.bind(this),this.saveButton=document.createElement("div"),this.saveButton.className="vis-button vis-save",this.saveButton.innerHTML="save",this.saveButton.onclick=this._save.bind(this),this.loadButton=document.createElement("div"),this.loadButton.className="vis-button vis-load",this.loadButton.innerHTML="load last",this.loadButton.onclick=this._loadLast.bind(this),this.frame.appendChild(this.colorPickerDiv),this.frame.appendChild(this.arrowDiv),this.frame.appendChild(this.brightnessLabel),this.frame.appendChild(this.brightnessDiv),this.frame.appendChild(this.opacityLabel),this.frame.appendChild(this.opacityDiv),this.frame.appendChild(this.newColorDiv),this.frame.appendChild(this.initialColorDiv),this.frame.appendChild(this.cancelButton),this.frame.appendChild(this.applyButton),this.frame.appendChild(this.saveButton),this.frame.appendChild(this.loadButton)}},{key:"_bindHammer",value:function(){var t=this;this.drag={},this.pinch={},this.hammer=new s(this.colorPickerCanvas),this.hammer.get("pinch").set({enable:!0}),r.onTouch(this.hammer,function(e){t._moveSelector(e)}),this.hammer.on("tap",function(e){t._moveSelector(e)}),this.hammer.on("panstart",function(e){t._moveSelector(e)}),this.hammer.on("panmove",function(e){t._moveSelector(e)}),this.hammer.on("panend",function(e){t._moveSelector(e)})}},{key:"_generateHueCircle",value:function(){if(this.generated===!1){var t=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1)),t.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var e=this.colorPickerCanvas.clientWidth,i=this.colorPickerCanvas.clientHeight;t.clearRect(0,0,e,i);var o=void 0,n=void 0,s=void 0,r=void 0;this.centerCoordinates={x:.5*e,y:.5*i},this.r=.49*e;var h=2*Math.PI/360,d=1/360,l=1/this.r,c=void 0;for(s=0;360>s;s++)for(r=0;r<this.r;r++)o=this.centerCoordinates.x+r*Math.sin(h*s),n=this.centerCoordinates.y+r*Math.cos(h*s),c=a.HSVToRGB(s*d,r*l,1),t.fillStyle="rgb("+c.r+","+c.g+","+c.b+")",t.fillRect(o-.5,n-.5,2,2);t.strokeStyle="rgba(0,0,0,1)",t.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),t.stroke(),this.hueCircle=t.getImageData(0,0,e,i)}this.generated=!0}},{key:"_moveSelector",value:function(t){var e=this.colorPickerDiv.getBoundingClientRect(),i=t.center.x-e.left,o=t.center.y-e.top,n=.5*this.colorPickerDiv.clientHeight,s=.5*this.colorPickerDiv.clientWidth,r=i-s,h=o-n,d=Math.atan2(r,h),l=.98*Math.min(Math.sqrt(r*r+h*h),s),c=Math.cos(d)*l+n,u=Math.sin(d)*l+s;this.colorPickerSelector.style.top=c-.5*this.colorPickerSelector.clientHeight+"px",this.colorPickerSelector.style.left=u-.5*this.colorPickerSelector.clientWidth+"px";var p=d/(2*Math.PI);p=0>p?p+1:p;var f=l/this.r,m=a.RGBToHSV(this.color.r,this.color.g,this.color.b);m.h=p,m.s=f;var v=a.HSVToRGB(m.h,m.s,m.v);v.a=this.color.a,this.color=v,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}}]),t}();e["default"]=h},function(t,e,i){i(20);e.onTouch=function(t,e){e.inputHandler=function(t){t.isFirst&&e(t)},t.on("hammer.input",e.inputHandler)},e.onRelease=function(t,e){return e.inputHandler=function(t){t.isFinal&&e(t)},t.on("hammer.input",e.inputHandler)},e.offTouch=function(t,e){t.off("hammer.input",e.inputHandler)},e.offRelease=e.offTouch,e.disablePreventDefaultVertically=function(t){var e="pan-y";return t.getTouchAction=function(){return[e]},t}},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(1),a=!1,h=void 0,d="background: #FFeeee; color: #dd0000",l=function(){function t(){o(this,t)}return s(t,null,[{key:"validate",value:function(e,i,o){a=!1,h=i;var n=i;return void 0!==o&&(n=i[o]),t.parse(e,n,[]),a}},{key:"parse",value:function(e,i,o){for(var n in e)e.hasOwnProperty(n)&&t.check(n,e,i,o)}},{key:"check",value:function(e,i,o,n){void 0===o[e]&&void 0===o.__any__?t.getSuggestion(e,o,n):void 0===o[e]&&void 0!==o.__any__?"object"===t.getType(i[e])&&void 0!==o.__any__.__type__?t.checkFields(e,i,o,"__any__",o.__any__.__type__,n):t.checkFields(e,i,o,"__any__",o.__any__,n):void 0!==o[e].__type__?t.checkFields(e,i,o,e,o[e].__type__,n):t.checkFields(e,i,o,e,o[e],n)}},{key:"checkFields",value:function(e,i,o,n,s,h){var l=t.getType(i[e]),c=s[l];void 0!==c?"array"===t.getType(c)&&-1===c.indexOf(i[e])?(console.log('%cInvalid option detected in "'+e+'". Allowed values are:'+t.print(c)+' not "'+i[e]+'". '+t.printLocation(h,e),d),a=!0):"object"===l&&"__any__"!==n&&(h=r.copyAndExtendArray(h,e),t.parse(i[e],o[n],h)):void 0===s.any&&(console.log('%cInvalid type received for "'+e+'". Expected: '+t.print(Object.keys(s))+". Received ["+l+'] "'+i[e]+'"'+t.printLocation(h,e),d),a=!0)}},{key:"getType",value:function(t){var e="undefined"==typeof t?"undefined":n(t);return"object"===e?null===t?"null":t instanceof Boolean?"boolean":t instanceof Number?"number":t instanceof String?"string":Array.isArray(t)?"array":t instanceof Date?"date":void 0!==t.nodeType?"dom":t._isAMomentObject===!0?"moment":"object":"number"===e?"number":"boolean"===e?"boolean":"string"===e?"string":void 0===e?"undefined":e}},{key:"getSuggestion",value:function(e,i,o){var n=t.findInOptions(e,i,o,!1),s=t.findInOptions(e,h,[],!0),r=8,l=4;void 0!==n.indexMatch?console.log('%cUnknown option detected: "'+e+'" in '+t.printLocation(n.path,e,"")+'Perhaps it was incomplete? Did you mean: "'+n.indexMatch+'"?\n\n',d):s.distance<=l&&n.distance>s.distance?console.log('%cUnknown option detected: "'+e+'" in '+t.printLocation(n.path,e,"")+"Perhaps it was misplaced? Matching option found at: "+t.printLocation(s.path,s.closestMatch,""),d):n.distance<=r?console.log('%cUnknown option detected: "'+e+'". Did you mean "'+n.closestMatch+'"?'+t.printLocation(n.path,e),d):console.log('%cUnknown option detected: "'+e+'". Did you mean one of these: '+t.print(Object.keys(i))+t.printLocation(o,e),d),a=!0}},{key:"findInOptions",value:function(e,i,o){var n=arguments.length<=3||void 0===arguments[3]?!1:arguments[3],s=1e9,a="",h=[],d=e.toLowerCase(),l=void 0;for(var c in i){var u=void 0;if(void 0!==i[c].__type__&&n===!0){var p=t.findInOptions(e,i[c],r.copyAndExtendArray(o,c));s>p.distance&&(a=p.closestMatch,h=p.path,s=p.distance,l=p.indexMatch)}else-1!==c.toLowerCase().indexOf(d)&&(l=c),u=t.levenshteinDistance(e,c),s>u&&(a=c,h=r.copyArray(o),s=u)}return{closestMatch:a,path:h,distance:s,indexMatch:l}}},{key:"printLocation",value:function(t,e){for(var i=arguments.length<=2||void 0===arguments[2]?"Problem value found at: \n":arguments[2],o="\n\n"+i+"options = {\n",n=0;n<t.length;n++){for(var s=0;n+1>s;s++)o+="  ";o+=t[n]+": {\n"}for(var r=0;r<t.length+1;r++)o+="  ";o+=e+"\n";for(var a=0;a<t.length+1;a++){for(var h=0;h<t.length-a;h++)o+="  ";o+="}\n"}return o+"\n\n"}},{key:"print",value:function(t){return JSON.stringify(t).replace(/(\")|(\[)|(\])|(,"__type__")/g,"").replace(/(\,)/g,", ")}},{key:"levenshteinDistance",value:function(t,e){if(0===t.length)return e.length;if(0===e.length)return t.length;var i,o=[];for(i=0;i<=e.length;i++)o[i]=[i];var n;for(n=0;n<=t.length;n++)o[0][n]=n;for(i=1;i<=e.length;i++)for(n=1;n<=t.length;n++)e.charAt(i-1)==t.charAt(n-1)?o[i][n]=o[i-1][n-1]:o[i][n]=Math.min(o[i-1][n-1]+1,Math.min(o[i][n-1]+1,o[i-1][n]+1));return o[e.length][t.length]}}]),t}();e["default"]=l,e.printStyle=d},function(t,e,i){function o(t,e){var i=a().hours(0).minutes(0).seconds(0).milliseconds(0);this.start=i.clone().add(-3,"days").valueOf(),this.end=i.clone().add(4,"days").valueOf(),this.body=t,this.deltaDifference=0,this.scaleOffset=0,this.startToFront=!1,this.endToFront=!0,this.defaultOptions={rtl:!1,start:null,end:null,moment:a,direction:"horizontal",moveable:!0,zoomable:!0,min:null,max:null,zoomMin:10,zoomMax:31536e10},this.options=r.extend({},this.defaultOptions),this.props={touch:{}},this.animationTimer=null,this.body.emitter.on("panstart",this._onDragStart.bind(this)),this.body.emitter.on("panmove",this._onDrag.bind(this)),this.body.emitter.on("panend",this._onDragEnd.bind(this)),this.body.emitter.on("mousewheel",this._onMouseWheel.bind(this)),this.body.emitter.on("touch",this._onTouch.bind(this)),this.body.emitter.on("pinch",this._onPinch.bind(this)),this.setOptions(e)}function n(t){if("horizontal"!=t&&"vertical"!=t)throw new TypeError('Unknown direction "'+t+'". Choose "horizontal" or "vertical".')}var s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},r=i(1),a=(i(28),i(2)),h=i(31),d=i(32);o.prototype=new h,o.prototype.setOptions=function(t){if(t){var e=["direction","min","max","zoomMin","zoomMax","moveable","zoomable","moment","activate","hiddenDates","zoomKey","rtl"];r.selectiveExtend(e,this.options,t),("start"in t||"end"in t)&&this.setRange(t.start,t.end)}},o.prototype.setRange=function(t,e,i,o){o!==!0&&(o=!1);var n=void 0!=t?r.convert(t,"Date").valueOf():null,a=void 0!=e?r.convert(e,"Date").valueOf():null;if(this._cancelAnimation(),i){var h=this,l=this.start,c=this.end,u="object"===("undefined"==typeof i?"undefined":s(i))&&"duration"in i?i.duration:500,p="object"===("undefined"==typeof i?"undefined":s(i))&&"easingFunction"in i?i.easingFunction:"easeInOutQuad",f=r.easingFunctions[p];if(!f)throw new Error("Unknown easing function "+JSON.stringify(p)+". Choose from: "+Object.keys(r.easingFunctions).join(", "));var m=(new Date).valueOf(),v=!1,g=function w(){if(!h.props.touch.dragging){var t=(new Date).valueOf(),e=t-m,i=f(e/u),s=e>u,r=s||null===n?n:l+(n-l)*i,p=s||null===a?a:c+(a-c)*i;y=h._applyRange(r,p),d.updateHiddenDates(h.options.moment,h.body,h.options.hiddenDates),v=v||y,y&&h.body.emitter.emit("rangechange",{start:new Date(h.start),end:new Date(h.end),byUser:o}),s?v&&h.body.emitter.emit("rangechanged",{start:new Date(h.start),end:new Date(h.end),byUser:o}):h.animationTimer=setTimeout(w,20)}};return g()}var y=this._applyRange(n,a);if(d.updateHiddenDates(this.options.moment,this.body,this.options.hiddenDates),y){var b={start:new Date(this.start),end:new Date(this.end),byUser:o};this.body.emitter.emit("rangechange",b),this.body.emitter.emit("rangechanged",b)}},o.prototype._cancelAnimation=function(){this.animationTimer&&(clearTimeout(this.animationTimer),this.animationTimer=null)},o.prototype._applyRange=function(t,e){var i,o=null!=t?r.convert(t,"Date").valueOf():this.start,n=null!=e?r.convert(e,"Date").valueOf():this.end,s=null!=this.options.max?r.convert(this.options.max,"Date").valueOf():null,a=null!=this.options.min?r.convert(this.options.min,"Date").valueOf():null;if(isNaN(o)||null===o)throw new Error('Invalid start "'+t+'"');if(isNaN(n)||null===n)throw new Error('Invalid end "'+e+'"');if(o>n&&(n=o),null!==a&&a>o&&(i=a-o,o+=i,n+=i,null!=s&&n>s&&(n=s)),null!==s&&n>s&&(i=n-s,o-=i,n-=i,null!=a&&a>o&&(o=a)),null!==this.options.zoomMin){var h=parseFloat(this.options.zoomMin);0>h&&(h=0),h>n-o&&(this.end-this.start===h&&o>this.start&&n<this.end?(o=this.start,n=this.end):(i=h-(n-o),o-=i/2,n+=i/2))}if(null!==this.options.zoomMax){var d=parseFloat(this.options.zoomMax);0>d&&(d=0),n-o>d&&(this.end-this.start===d&&o<this.start&&n>this.end?(o=this.start,n=this.end):(i=n-o-d,o+=i/2,n-=i/2))}var l=this.start!=o||this.end!=n;return o>=this.start&&o<=this.end||n>=this.start&&n<=this.end||this.start>=o&&this.start<=n||this.end>=o&&this.end<=n||this.body.emitter.emit("checkRangedItems"),this.start=o,this.end=n,l},o.prototype.getRange=function(){return{start:this.start,end:this.end}},o.prototype.conversion=function(t,e){return o.conversion(this.start,this.end,t,e)},o.conversion=function(t,e,i,o){return void 0===o&&(o=0),0!=i&&e-t!=0?{offset:t,scale:i/(e-t-o)}:{offset:0,scale:1}},o.prototype._onDragStart=function(t){this.deltaDifference=0,this.previousDelta=0,this.options.moveable&&this._isInsideRange(t)&&this.props.touch.allowDragging&&(this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.dragging=!0,this.body.dom.root&&(this.body.dom.root.style.cursor="move"))},o.prototype._onDrag=function(t){if(this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging){var e=this.options.direction;n(e);var i="horizontal"==e?t.deltaX:t.deltaY;i-=this.deltaDifference;var o=this.props.touch.end-this.props.touch.start,s=d.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end);o-=s;var r="horizontal"==e?this.body.domProps.center.width:this.body.domProps.center.height;if(this.options.rtl)var a=i/r*o;else var a=-i/r*o;var h=this.props.touch.start+a,l=this.props.touch.end+a,c=d.snapAwayFromHidden(this.body.hiddenDates,h,this.previousDelta-i,!0),u=d.snapAwayFromHidden(this.body.hiddenDates,l,this.previousDelta-i,!0);if(c!=h||u!=l)return this.deltaDifference+=i,this.props.touch.start=c,this.props.touch.end=u,void this._onDrag(t);this.previousDelta=i,this._applyRange(h,l);var p=new Date(this.start),f=new Date(this.end);this.body.emitter.emit("rangechange",{start:p,end:f,byUser:!0})}},o.prototype._onDragEnd=function(t){this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging&&(this.props.touch.dragging=!1,this.body.dom.root&&(this.body.dom.root.style.cursor="auto"),this.body.emitter.emit("rangechanged",{start:new Date(this.start),end:new Date(this.end),byUser:!0}))},o.prototype._onMouseWheel=function(t){if(this.options.zoomable&&this.options.moveable&&this._isInsideRange(t)&&(!this.options.zoomKey||t[this.options.zoomKey])){var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail&&(e=-t.detail/3),e){var i;i=0>e?1-e/5:1/(1+e/5);var o=this.getPointer({x:t.clientX,y:t.clientY},this.body.dom.center),n=this._pointerToDate(o);this.zoom(i,n,e)}t.preventDefault()}},o.prototype._onTouch=function(t){this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.allowDragging=!0,this.props.touch.center=null,this.scaleOffset=0,this.deltaDifference=0},o.prototype._onPinch=function(t){if(this.options.zoomable&&this.options.moveable){this.props.touch.allowDragging=!1,this.props.touch.center||(this.props.touch.center=this.getPointer(t.center,this.body.dom.center));var e=1/(t.scale+this.scaleOffset),i=this._pointerToDate(this.props.touch.center),o=d.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end),n=d.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this,i),s=o-n,r=i-n+(this.props.touch.start-(i-n))*e,a=i+s+(this.props.touch.end-(i+s))*e;
-this.startToFront=0>=1-e,this.endToFront=0>=e-1;var h=d.snapAwayFromHidden(this.body.hiddenDates,r,1-e,!0),l=d.snapAwayFromHidden(this.body.hiddenDates,a,e-1,!0);h==r&&l==a||(this.props.touch.start=h,this.props.touch.end=l,this.scaleOffset=1-t.scale,r=h,a=l),this.setRange(r,a,!1,!0),this.startToFront=!1,this.endToFront=!0}},o.prototype._isInsideRange=function(t){var e=t.center?t.center.x:t.clientX;if(this.options.rtl)var i=e-r.getAbsoluteLeft(this.body.dom.centerContainer);else var i=r.getAbsoluteRight(this.body.dom.centerContainer)-e;var o=this.body.util.toTime(i);return o>=this.start&&o<=this.end},o.prototype._pointerToDate=function(t){var e,i=this.options.direction;if(n(i),"horizontal"==i)return this.body.util.toTime(t.x).valueOf();var o=this.body.domProps.center.height;return e=this.conversion(o),t.y/e.scale+e.offset},o.prototype.getPointer=function(t,e){return this.options.rtl?{x:r.getAbsoluteRight(e)-t.x,y:t.y-r.getAbsoluteTop(e)}:{x:t.x-r.getAbsoluteLeft(e),y:t.y-r.getAbsoluteTop(e)}},o.prototype.zoom=function(t,e,i){null==e&&(e=(this.start+this.end)/2);var o=d.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end),n=d.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this,e),s=o-n,r=e-n+(this.start-(e-n))*t,a=e+s+(this.end-(e+s))*t;this.startToFront=!(i>0),this.endToFront=!(-i>0);var h=d.snapAwayFromHidden(this.body.hiddenDates,r,i,!0),l=d.snapAwayFromHidden(this.body.hiddenDates,a,-i,!0);h==r&&l==a||(r=h,a=l),this.setRange(r,a,!1,!0),this.startToFront=!1,this.endToFront=!0},o.prototype.move=function(t){var e=this.end-this.start,i=this.start+e*t,o=this.end+e*t;this.start=i,this.end=o},o.prototype.moveTo=function(t){var e=(this.start+this.end)/2,i=e-t,o=this.start-i,n=this.end-i;this.setRange(o,n)},t.exports=o},function(t,e){function i(t,e){this.options=null,this.props=null}i.prototype.setOptions=function(t){t&&util.extend(this.options,t)},i.prototype.redraw=function(){return!1},i.prototype.destroy=function(){},i.prototype._isResized=function(){var t=this.props._previousWidth!==this.props.width||this.props._previousHeight!==this.props.height;return this.props._previousWidth=this.props.width,this.props._previousHeight=this.props.height,t},t.exports=i},function(t,e){e.convertHiddenOptions=function(t,i,o){if(o&&!Array.isArray(o))return e.convertHiddenOptions(t,i,[o]);if(i.hiddenDates=[],o&&1==Array.isArray(o)){for(var n=0;n<o.length;n++)if(void 0===o[n].repeat){var s={};s.start=t(o[n].start).toDate().valueOf(),s.end=t(o[n].end).toDate().valueOf(),i.hiddenDates.push(s)}i.hiddenDates.sort(function(t,e){return t.start-e.start})}},e.updateHiddenDates=function(t,i,o){if(o&&!Array.isArray(o))return e.updateHiddenDates(t,i,[o]);if(o&&void 0!==i.domProps.centerContainer.width){e.convertHiddenOptions(t,i,o);for(var n=t(i.range.start),s=t(i.range.end),r=i.range.end-i.range.start,a=r/i.domProps.centerContainer.width,h=0;h<o.length;h++)if(void 0!==o[h].repeat){var d=t(o[h].start),l=t(o[h].end);if("Invalid Date"==d._d)throw new Error("Supplied start date is not valid: "+o[h].start);if("Invalid Date"==l._d)throw new Error("Supplied end date is not valid: "+o[h].end);var c=l-d;if(c>=4*a){var u=0,p=s.clone();switch(o[h].repeat){case"daily":d.day()!=l.day()&&(u=1),d.dayOfYear(n.dayOfYear()),d.year(n.year()),d.subtract(7,"days"),l.dayOfYear(n.dayOfYear()),l.year(n.year()),l.subtract(7-u,"days"),p.add(1,"weeks");break;case"weekly":var f=l.diff(d,"days"),m=d.day();d.date(n.date()),d.month(n.month()),d.year(n.year()),l=d.clone(),d.day(m),l.day(m),l.add(f,"days"),d.subtract(1,"weeks"),l.subtract(1,"weeks"),p.add(1,"weeks");break;case"monthly":d.month()!=l.month()&&(u=1),d.month(n.month()),d.year(n.year()),d.subtract(1,"months"),l.month(n.month()),l.year(n.year()),l.subtract(1,"months"),l.add(u,"months"),p.add(1,"months");break;case"yearly":d.year()!=l.year()&&(u=1),d.year(n.year()),d.subtract(1,"years"),l.year(n.year()),l.subtract(1,"years"),l.add(u,"years"),p.add(1,"years");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",o[h].repeat)}for(;p>d;)switch(i.hiddenDates.push({start:d.valueOf(),end:l.valueOf()}),o[h].repeat){case"daily":d.add(1,"days"),l.add(1,"days");break;case"weekly":d.add(1,"weeks"),l.add(1,"weeks");break;case"monthly":d.add(1,"months"),l.add(1,"months");break;case"yearly":d.add(1,"y"),l.add(1,"y");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",o[h].repeat)}i.hiddenDates.push({start:d.valueOf(),end:l.valueOf()})}}e.removeDuplicates(i);var v=e.isHidden(i.range.start,i.hiddenDates),g=e.isHidden(i.range.end,i.hiddenDates),y=i.range.start,b=i.range.end;1==v.hidden&&(y=1==i.range.startToFront?v.startDate-1:v.endDate+1),1==g.hidden&&(b=1==i.range.endToFront?g.startDate-1:g.endDate+1),1!=v.hidden&&1!=g.hidden||i.range._applyRange(y,b)}},e.removeDuplicates=function(t){for(var e=t.hiddenDates,i=[],o=0;o<e.length;o++)for(var n=0;n<e.length;n++)o!=n&&1!=e[n].remove&&1!=e[o].remove&&(e[n].start>=e[o].start&&e[n].end<=e[o].end?e[n].remove=!0:e[n].start>=e[o].start&&e[n].start<=e[o].end?(e[o].end=e[n].end,e[n].remove=!0):e[n].end>=e[o].start&&e[n].end<=e[o].end&&(e[o].start=e[n].start,e[n].remove=!0));for(var o=0;o<e.length;o++)e[o].remove!==!0&&i.push(e[o]);t.hiddenDates=i,t.hiddenDates.sort(function(t,e){return t.start-e.start})},e.printDates=function(t){for(var e=0;e<t.length;e++)console.log(e,new Date(t[e].start),new Date(t[e].end),t[e].start,t[e].end,t[e].remove)},e.stepOverHiddenDates=function(t,e,i){for(var o=!1,n=e.current.valueOf(),s=0;s<e.hiddenDates.length;s++){var r=e.hiddenDates[s].start,a=e.hiddenDates[s].end;if(n>=r&&a>n){o=!0;break}}if(1==o&&n<e._end.valueOf()&&n!=i){var h=t(i),d=t(a);h.year()!=d.year()?e.switchedYear=!0:h.month()!=d.month()?e.switchedMonth=!0:h.dayOfYear()!=d.dayOfYear()&&(e.switchedDay=!0),e.current=d}},e.toScreen=function(t,i,o){if(0==t.body.hiddenDates.length){var n=t.range.conversion(o);return(i.valueOf()-n.offset)*n.scale}var s=e.isHidden(i,t.body.hiddenDates);1==s.hidden&&(i=s.startDate);var r=e.getHiddenDurationBetween(t.body.hiddenDates,t.range.start,t.range.end);i=e.correctTimeForHidden(t.options.moment,t.body.hiddenDates,t.range,i);var n=t.range.conversion(o,r);return(i.valueOf()-n.offset)*n.scale},e.toTime=function(t,i,o){if(0==t.body.hiddenDates.length){var n=t.range.conversion(o);return new Date(i/n.scale+n.offset)}var s=e.getHiddenDurationBetween(t.body.hiddenDates,t.range.start,t.range.end),r=t.range.end-t.range.start-s,a=r*i/o,h=e.getAccumulatedHiddenDuration(t.body.hiddenDates,t.range,a),d=new Date(h+a+t.range.start);return d},e.getHiddenDurationBetween=function(t,e,i){for(var o=0,n=0;n<t.length;n++){var s=t[n].start,r=t[n].end;s>=e&&i>r&&(o+=r-s)}return o},e.correctTimeForHidden=function(t,i,o,n){return n=t(n).toDate().valueOf(),n-=e.getHiddenDurationBefore(t,i,o,n)},e.getHiddenDurationBefore=function(t,e,i,o){var n=0;o=t(o).toDate().valueOf();for(var s=0;s<e.length;s++){var r=e[s].start,a=e[s].end;r>=i.start&&a<i.end&&o>=a&&(n+=a-r)}return n},e.getAccumulatedHiddenDuration=function(t,e,i){for(var o=0,n=0,s=e.start,r=0;r<t.length;r++){var a=t[r].start,h=t[r].end;if(a>=e.start&&h<e.end){if(n+=a-s,s=h,n>=i)break;o+=h-a}}return o},e.snapAwayFromHidden=function(t,i,o,n){var s=e.isHidden(i,t);return 1==s.hidden?0>o?1==n?s.startDate-(s.endDate-i)-1:s.startDate-1:1==n?s.endDate+(i-s.startDate)+1:s.endDate+1:i},e.isHidden=function(t,e){for(var i=0;i<e.length;i++){var o=e[i].start,n=e[i].end;if(t>=o&&n>t)return{hidden:!0,startDate:o,endDate:n}}return{hidden:!1,startDate:o,endDate:n}}},function(t,e,i){function o(){}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(13),r=i(20),a=i(28),h=i(1),d=(i(9),i(11),i(30),i(34),i(44)),l=i(45),c=i(32),u=i(46);s(o.prototype),o.prototype._create=function(t){function e(t){i.isActive()&&i.emit("mousewheel",t)}this.dom={},this.dom.container=t,this.dom.root=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.backgroundVertical=document.createElement("div"),this.dom.backgroundHorizontal=document.createElement("div"),this.dom.centerContainer=document.createElement("div"),this.dom.leftContainer=document.createElement("div"),this.dom.rightContainer=document.createElement("div"),this.dom.center=document.createElement("div"),this.dom.left=document.createElement("div"),this.dom.right=document.createElement("div"),this.dom.top=document.createElement("div"),this.dom.bottom=document.createElement("div"),this.dom.shadowTop=document.createElement("div"),this.dom.shadowBottom=document.createElement("div"),this.dom.shadowTopLeft=document.createElement("div"),this.dom.shadowBottomLeft=document.createElement("div"),this.dom.shadowTopRight=document.createElement("div"),this.dom.shadowBottomRight=document.createElement("div"),this.dom.root.className="vis-timeline",this.dom.background.className="vis-panel vis-background",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical",this.dom.backgroundHorizontal.className="vis-panel vis-background vis-horizontal",this.dom.centerContainer.className="vis-panel vis-center",this.dom.leftContainer.className="vis-panel vis-left",this.dom.rightContainer.className="vis-panel vis-right",this.dom.top.className="vis-panel vis-top",this.dom.bottom.className="vis-panel vis-bottom",this.dom.left.className="vis-content",this.dom.center.className="vis-content",this.dom.right.className="vis-content",this.dom.shadowTop.className="vis-shadow vis-top",this.dom.shadowBottom.className="vis-shadow vis-bottom",this.dom.shadowTopLeft.className="vis-shadow vis-top",this.dom.shadowBottomLeft.className="vis-shadow vis-bottom",this.dom.shadowTopRight.className="vis-shadow vis-top",this.dom.shadowBottomRight.className="vis-shadow vis-bottom",this.dom.root.appendChild(this.dom.background),this.dom.root.appendChild(this.dom.backgroundVertical),this.dom.root.appendChild(this.dom.backgroundHorizontal),this.dom.root.appendChild(this.dom.centerContainer),this.dom.root.appendChild(this.dom.leftContainer),this.dom.root.appendChild(this.dom.rightContainer),this.dom.root.appendChild(this.dom.top),this.dom.root.appendChild(this.dom.bottom),this.dom.centerContainer.appendChild(this.dom.center),this.dom.leftContainer.appendChild(this.dom.left),this.dom.rightContainer.appendChild(this.dom.right),this.dom.centerContainer.appendChild(this.dom.shadowTop),this.dom.centerContainer.appendChild(this.dom.shadowBottom),this.dom.leftContainer.appendChild(this.dom.shadowTopLeft),this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft),this.dom.rightContainer.appendChild(this.dom.shadowTopRight),this.dom.rightContainer.appendChild(this.dom.shadowBottomRight),this.on("rangechange",function(){this.initialDrawDone===!0&&this._redraw()}.bind(this)),this.on("touch",this._onTouch.bind(this)),this.on("pan",this._onDrag.bind(this));var i=this;this.on("_change",function(t){t&&1==t.queue?i._redrawTimer||(i._redrawTimer=setTimeout(function(){i._redrawTimer=null,i._redraw()},0)):i._redraw()}),this.hammer=new r(this.dom.root);var o=this.hammer.get("pinch").set({enable:!0});a.disablePreventDefaultVertically(o),this.hammer.get("pan").set({threshold:5,direction:r.DIRECTION_HORIZONTAL}),this.listeners={};var n=["tap","doubletap","press","pinch","pan","panstart","panmove","panend"];if(n.forEach(function(t){var e=function(e){i.isActive()&&i.emit(t,e)};i.hammer.on(t,e),i.listeners[t]=e}),a.onTouch(this.hammer,function(t){i.emit("touch",t)}.bind(this)),a.onRelease(this.hammer,function(t){i.emit("release",t)}.bind(this)),this.dom.root.addEventListener("mousewheel",e),this.dom.root.addEventListener("DOMMouseScroll",e),this.props={root:{},background:{},centerContainer:{},leftContainer:{},rightContainer:{},center:{},left:{},right:{},top:{},bottom:{},border:{},scrollTop:0,scrollTopMin:0},this.customTimes=[],this.touch={},this.redrawCount=0,this.initialDrawDone=!1,!t)throw new Error("No container provided");t.appendChild(this.dom.root)},o.prototype.setOptions=function(t){if(t){var e=["width","height","minHeight","maxHeight","autoResize","start","end","clickToUse","dataAttributes","hiddenDates","locale","locales","moment","rtl","throttleRedraw"];if(h.selectiveExtend(e,this.options,t),this.options.rtl){var i=this.dom.leftContainer;this.dom.leftContainer=this.dom.rightContainer,this.dom.rightContainer=i,this.dom.container.style.direction="rtl",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical-rtl"}if(this.options.orientation={item:void 0,axis:void 0},"orientation"in t&&("string"==typeof t.orientation?this.options.orientation={item:t.orientation,axis:t.orientation}:"object"===n(t.orientation)&&("item"in t.orientation&&(this.options.orientation.item=t.orientation.item),"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis))),"both"===this.options.orientation.axis){if(!this.timeAxis2){var o=this.timeAxis2=new d(this.body);o.setOptions=function(t){var e=t?h.extend({},t):{};e.orientation="top",d.prototype.setOptions.call(o,e)},this.components.push(o)}}else if(this.timeAxis2){var s=this.components.indexOf(this.timeAxis2);-1!==s&&this.components.splice(s,1),this.timeAxis2.destroy(),this.timeAxis2=null}if("function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),"hiddenDates"in this.options&&c.convertHiddenOptions(this.options.moment,this.body,this.options.hiddenDates),"clickToUse"in t&&(t.clickToUse?this.activator||(this.activator=new l(this.dom.root)):this.activator&&(this.activator.destroy(),delete this.activator)),"showCustomTime"in t)throw new Error("Option `showCustomTime` is deprecated. Create a custom time bar via timeline.addCustomTime(time [, id])");this._initAutoResize()}if(this.components.forEach(function(e){return e.setOptions(t)}),"configure"in t){this.configurator||(this.configurator=this._createConfigurator()),this.configurator.setOptions(t.configure);var r=h.deepExtend({},this.options);this.components.forEach(function(t){h.deepExtend(r,t.options)}),this.configurator.setModuleOptions({global:r})}this._origRedraw?this._redraw():(this._origRedraw=this._redraw.bind(this),this._redraw=h.throttle(this._origRedraw,this.options.throttleRedraw))},o.prototype.isActive=function(){return!this.activator||this.activator.active},o.prototype.destroy=function(){this.setItems(null),this.setGroups(null),this.off(),this._stopAutoResize(),this.dom.root.parentNode&&this.dom.root.parentNode.removeChild(this.dom.root),this.dom=null,this.activator&&(this.activator.destroy(),delete this.activator);for(var t in this.listeners)this.listeners.hasOwnProperty(t)&&delete this.listeners[t];this.listeners=null,this.hammer=null,this.components.forEach(function(t){return t.destroy()}),this.body=null},o.prototype.setCustomTime=function(t,e){var i=this.customTimes.filter(function(t){return e===t.options.id});if(0===i.length)throw new Error("No custom time bar found with id "+JSON.stringify(e));i.length>0&&i[0].setCustomTime(t)},o.prototype.getCustomTime=function(t){var e=this.customTimes.filter(function(e){return e.options.id===t});if(0===e.length)throw new Error("No custom time bar found with id "+JSON.stringify(t));return e[0].getCustomTime()},o.prototype.setCustomTimeTitle=function(t,e){var i=this.customTimes.filter(function(t){return t.options.id===e});if(0===i.length)throw new Error("No custom time bar found with id "+JSON.stringify(e));return i.length>0?i[0].setCustomTitle(t):void 0},o.prototype.getEventProperties=function(t){return{event:t}},o.prototype.addCustomTime=function(t,e){var i=void 0!==t?h.convert(t,"Date").valueOf():new Date,o=this.customTimes.some(function(t){return t.options.id===e});if(o)throw new Error("A custom time with id "+JSON.stringify(e)+" already exists");var n=new u(this.body,h.extend({},this.options,{time:i,id:e}));return this.customTimes.push(n),this.components.push(n),this._redraw(),e},o.prototype.removeCustomTime=function(t){var e=this.customTimes.filter(function(e){return e.options.id===t});if(0===e.length)throw new Error("No custom time bar found with id "+JSON.stringify(t));e.forEach(function(t){this.customTimes.splice(this.customTimes.indexOf(t),1),this.components.splice(this.components.indexOf(t),1),t.destroy()}.bind(this))},o.prototype.getVisibleItems=function(){return this.itemSet&&this.itemSet.getVisibleItems()||[]},o.prototype.fit=function(t){var e=this.getDataRange();if(null!==e.min||null!==e.max){var i=e.max-e.min,o=new Date(e.min.valueOf()-.01*i),n=new Date(e.max.valueOf()+.01*i),s=t&&void 0!==t.animation?t.animation:!0;this.range.setRange(o,n,s)}},o.prototype.getDataRange=function(){throw new Error("Cannot invoke abstract method getDataRange")},o.prototype.setWindow=function(t,e,i){var o;if(1==arguments.length){var n=arguments[0];o=void 0!==n.animation?n.animation:!0,this.range.setRange(n.start,n.end,o)}else o=i&&void 0!==i.animation?i.animation:!0,this.range.setRange(t,e,o)},o.prototype.moveTo=function(t,e){var i=this.range.end-this.range.start,o=h.convert(t,"Date").valueOf(),n=o-i/2,s=o+i/2,r=e&&void 0!==e.animation?e.animation:!0;this.range.setRange(n,s,r)},o.prototype.getWindow=function(){var t=this.range.getRange();return{start:new Date(t.start),end:new Date(t.end)}},o.prototype.redraw=function(){this._redraw()},o.prototype._redraw=function(){this.redrawCount++;var t=!1,e=this.options,i=this.props,o=this.dom;if(o&&o.container&&0!=o.root.offsetWidth){c.updateHiddenDates(this.options.moment,this.body,this.options.hiddenDates),"top"==e.orientation?(h.addClassName(o.root,"vis-top"),h.removeClassName(o.root,"vis-bottom")):(h.removeClassName(o.root,"vis-top"),h.addClassName(o.root,"vis-bottom")),o.root.style.maxHeight=h.option.asSize(e.maxHeight,""),o.root.style.minHeight=h.option.asSize(e.minHeight,""),o.root.style.width=h.option.asSize(e.width,""),i.border.left=(o.centerContainer.offsetWidth-o.centerContainer.clientWidth)/2,i.border.right=i.border.left,i.border.top=(o.centerContainer.offsetHeight-o.centerContainer.clientHeight)/2,i.border.bottom=i.border.top;var n=o.root.offsetHeight-o.root.clientHeight,s=o.root.offsetWidth-o.root.clientWidth;0===o.centerContainer.clientHeight&&(i.border.left=i.border.top,i.border.right=i.border.left),0===o.root.clientHeight&&(s=n),i.center.height=o.center.offsetHeight,i.left.height=o.left.offsetHeight,i.right.height=o.right.offsetHeight,i.top.height=o.top.clientHeight||-i.border.top,i.bottom.height=o.bottom.clientHeight||-i.border.bottom;var a=Math.max(i.left.height,i.center.height,i.right.height),d=i.top.height+a+i.bottom.height+n+i.border.top+i.border.bottom;o.root.style.height=h.option.asSize(e.height,d+"px"),i.root.height=o.root.offsetHeight,i.background.height=i.root.height-n;var l=i.root.height-i.top.height-i.bottom.height-n;i.centerContainer.height=l,i.leftContainer.height=l,i.rightContainer.height=i.leftContainer.height,i.root.width=o.root.offsetWidth,i.background.width=i.root.width-s,i.left.width=o.leftContainer.clientWidth||-i.border.left,i.leftContainer.width=i.left.width,i.right.width=o.rightContainer.clientWidth||-i.border.right,i.rightContainer.width=i.right.width;var u=i.root.width-i.left.width-i.right.width-s;i.center.width=u,i.centerContainer.width=u,i.top.width=u,i.bottom.width=u,o.background.style.height=i.background.height+"px",o.backgroundVertical.style.height=i.background.height+"px",o.backgroundHorizontal.style.height=i.centerContainer.height+"px",o.centerContainer.style.height=i.centerContainer.height+"px",o.leftContainer.style.height=i.leftContainer.height+"px",o.rightContainer.style.height=i.rightContainer.height+"px",o.background.style.width=i.background.width+"px",o.backgroundVertical.style.width=i.centerContainer.width+"px",o.backgroundHorizontal.style.width=i.background.width+"px",o.centerContainer.style.width=i.center.width+"px",o.top.style.width=i.top.width+"px",o.bottom.style.width=i.bottom.width+"px",o.background.style.left="0",o.background.style.top="0",o.backgroundVertical.style.left=i.left.width+i.border.left+"px",o.backgroundVertical.style.top="0",o.backgroundHorizontal.style.left="0",o.backgroundHorizontal.style.top=i.top.height+"px",o.centerContainer.style.left=i.left.width+"px",o.centerContainer.style.top=i.top.height+"px",o.leftContainer.style.left="0",o.leftContainer.style.top=i.top.height+"px",o.rightContainer.style.left=i.left.width+i.center.width+"px",o.rightContainer.style.top=i.top.height+"px",o.top.style.left=i.left.width+"px",o.top.style.top="0",o.bottom.style.left=i.left.width+"px",o.bottom.style.top=i.top.height+i.centerContainer.height+"px",this._updateScrollTop();var p=this.props.scrollTop;"top"!=e.orientation.item&&(p+=Math.max(this.props.centerContainer.height-this.props.center.height-this.props.border.top-this.props.border.bottom,0)),o.center.style.left="0",o.center.style.top=p+"px",o.left.style.left="0",o.left.style.top=p+"px",o.right.style.left="0",o.right.style.top=p+"px";var f=0==this.props.scrollTop?"hidden":"",m=this.props.scrollTop==this.props.scrollTopMin?"hidden":"";o.shadowTop.style.visibility=f,o.shadowBottom.style.visibility=m,o.shadowTopLeft.style.visibility=f,o.shadowBottomLeft.style.visibility=m,o.shadowTopRight.style.visibility=f,o.shadowBottomRight.style.visibility=m;var v=this.props.center.height>this.props.centerContainer.height;this.hammer.get("pan").set({direction:v?r.DIRECTION_ALL:r.DIRECTION_HORIZONTAL}),this.components.forEach(function(e){t=e.redraw()||t});var g=5;if(t){if(this.redrawCount<g)return void this.body.emitter.emit("_change");console.log("WARNING: infinite loop in redraw?")}else this.redrawCount=0;this.initialDrawDone=!0,this.body.emitter.emit("changed")}},o.prototype.repaint=function(){throw new Error("Function repaint is deprecated. Use redraw instead.")},o.prototype.setCurrentTime=function(t){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");this.currentTime.setCurrentTime(t)},o.prototype.getCurrentTime=function(){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");return this.currentTime.getCurrentTime()},o.prototype._toTime=function(t){return c.toTime(this,t,this.props.center.width)},o.prototype._toGlobalTime=function(t){return c.toTime(this,t,this.props.root.width)},o.prototype._toScreen=function(t){return c.toScreen(this,t,this.props.center.width)},o.prototype._toGlobalScreen=function(t){return c.toScreen(this,t,this.props.root.width)},o.prototype._initAutoResize=function(){1==this.options.autoResize?this._startAutoResize():this._stopAutoResize()},o.prototype._startAutoResize=function(){var t=this;this._stopAutoResize(),this._onResize=function(){return 1!=t.options.autoResize?void t._stopAutoResize():void(t.dom.root&&(t.dom.root.offsetWidth==t.props.lastWidth&&t.dom.root.offsetHeight==t.props.lastHeight||(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight,t.body.emitter.emit("_change"))))},h.addEventListener(window,"resize",this._onResize),t.dom.root&&(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight),this.watchTimer=setInterval(this._onResize,1e3)},o.prototype._stopAutoResize=function(){this.watchTimer&&(clearInterval(this.watchTimer),this.watchTimer=void 0),this._onResize&&(h.removeEventListener(window,"resize",this._onResize),this._onResize=null)},o.prototype._onTouch=function(t){this.touch.allowDragging=!0,this.touch.initialScrollTop=this.props.scrollTop},o.prototype._onPinch=function(t){this.touch.allowDragging=!1},o.prototype._onDrag=function(t){if(this.touch.allowDragging){var e=t.deltaY,i=this._getScrollTop(),o=this._setScrollTop(this.touch.initialScrollTop+e);o!=i&&this.emit("verticalDrag")}},o.prototype._setScrollTop=function(t){return this.props.scrollTop=t,this._updateScrollTop(),this.props.scrollTop},o.prototype._updateScrollTop=function(){var t=Math.min(this.props.centerContainer.height-this.props.center.height,0);return t!=this.props.scrollTopMin&&("top"!=this.options.orientation.item&&(this.props.scrollTop+=t-this.props.scrollTopMin),this.props.scrollTopMin=t),this.props.scrollTop>0&&(this.props.scrollTop=0),this.props.scrollTop<t&&(this.props.scrollTop=t),this.props.scrollTop},o.prototype._getScrollTop=function(){return this.props.scrollTop},o.prototype._createConfigurator=function(){throw new Error("Cannot invoke abstract method _createConfigurator")},t.exports=o},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={rtl:!1,type:null,orientation:{item:"bottom"},align:"auto",stack:!0,groupOrderSwap:function(t,e,i){var o=e.order;e.order=t.order,t.order=o},groupOrder:"order",selectable:!0,multiselect:!1,itemsAlwaysDraggable:!1,editable:{updateTime:!1,updateGroup:!1,add:!1,remove:!1},groupEditable:{order:!1,add:!1,remove:!1},snap:d.snap,onAdd:function(t,e){e(t)},onUpdate:function(t,e){e(t)},onMove:function(t,e){e(t)},onRemove:function(t,e){e(t)},onMoving:function(t,e){e(t)},onAddGroup:function(t,e){e(t)},onMoveGroup:function(t,e){e(t)},onRemoveGroup:function(t,e){e(t)},margin:{item:{horizontal:10,vertical:10},axis:20}},this.options=r.extend({},this.defaultOptions),this.itemOptions={type:{start:"Date",end:"Date"}},this.conversion={toScreen:t.util.toScreen,toTime:t.util.toTime},this.dom={},this.props={},this.hammer=null;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,o){i._onAdd(e.items)},update:function(t,e,o){i._onUpdate(e.items)},remove:function(t,e,o){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,o){i._onAddGroups(e.items)},update:function(t,e,o){i._onUpdateGroups(e.items)},remove:function(t,e,o){i._onRemoveGroups(e.items)}},this.items={},this.groups={},this.groupIds=[],this.selection=[],this.stackDirty=!0,this.touchParams={},this.groupTouchParams={},this._create(),this.setOptions(e)}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(20),r=i(1),a=i(9),h=i(11),d=i(35),l=i(31),c=i(36),u=i(40),p=i(41),f=i(42),m=i(38),v=i(43),g="__ungrouped__",y="__background__";o.prototype=new l,o.types={background:v,box:p,range:m,point:f},o.prototype._create=function(){var t=document.createElement("div");t.className="vis-itemset",t["timeline-itemset"]=this,this.dom.frame=t;var e=document.createElement("div");e.className="vis-background",t.appendChild(e),this.dom.background=e;var i=document.createElement("div");i.className="vis-foreground",t.appendChild(i),this.dom.foreground=i;var o=document.createElement("div");o.className="vis-axis",this.dom.axis=o;var n=document.createElement("div");n.className="vis-labelset",this.dom.labelSet=n,this._updateUngrouped();var r=new u(y,null,this);r.show(),this.groups[y]=r,this.hammer=new s(this.body.dom.centerContainer),this.hammer.on("hammer.input",function(t){t.isFirst&&this._onTouch(t)}.bind(this)),this.hammer.on("panstart",this._onDragStart.bind(this)),this.hammer.on("panmove",this._onDrag.bind(this)),this.hammer.on("panend",this._onDragEnd.bind(this)),this.hammer.get("pan").set({threshold:5,direction:s.DIRECTION_HORIZONTAL}),this.hammer.on("tap",this._onSelectItem.bind(this)),this.hammer.on("press",this._onMultiSelectItem.bind(this)),this.hammer.on("doubletap",this._onAddItem.bind(this)),this.groupHammer=new s(this.body.dom.leftContainer),this.groupHammer.on("panstart",this._onGroupDragStart.bind(this)),this.groupHammer.on("panmove",this._onGroupDrag.bind(this)),this.groupHammer.on("panend",this._onGroupDragEnd.bind(this)),this.groupHammer.get("pan").set({threshold:5,direction:s.DIRECTION_HORIZONTAL}),this.show()},o.prototype.setOptions=function(t){if(t){var e=["type","rtl","align","order","stack","selectable","multiselect","itemsAlwaysDraggable","multiselectPerGroup","groupOrder","dataAttributes","template","groupTemplate","hide","snap","groupOrderSwap"];r.selectiveExtend(e,this.options,t),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.item="top"===t.orientation?"top":"bottom":"object"===n(t.orientation)&&"item"in t.orientation&&(this.options.orientation.item=t.orientation.item)),"margin"in t&&("number"==typeof t.margin?(this.options.margin.axis=t.margin,this.options.margin.item.horizontal=t.margin,this.options.margin.item.vertical=t.margin):"object"===n(t.margin)&&(r.selectiveExtend(["axis"],this.options.margin,t.margin),"item"in t.margin&&("number"==typeof t.margin.item?(this.options.margin.item.horizontal=t.margin.item,this.options.margin.item.vertical=t.margin.item):"object"===n(t.margin.item)&&r.selectiveExtend(["horizontal","vertical"],this.options.margin.item,t.margin.item)))),"editable"in t&&("boolean"==typeof t.editable?(this.options.editable.updateTime=t.editable,this.options.editable.updateGroup=t.editable,this.options.editable.add=t.editable,this.options.editable.remove=t.editable):"object"===n(t.editable)&&r.selectiveExtend(["updateTime","updateGroup","add","remove"],this.options.editable,t.editable)),"groupEditable"in t&&("boolean"==typeof t.groupEditable?(this.options.groupEditable.order=t.groupEditable,this.options.groupEditable.add=t.groupEditable,this.options.groupEditable.remove=t.groupEditable):"object"===n(t.groupEditable)&&r.selectiveExtend(["order","add","remove"],this.options.groupEditable,t.groupEditable));var i=function(e){var i=t[e];if(i){if(!(i instanceof Function))throw new Error("option "+e+" must be a function "+e+"(item, callback)");this.options[e]=i}}.bind(this);["onAdd","onUpdate","onRemove","onMove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup"].forEach(i),this.markDirty()}},o.prototype.markDirty=function(t){this.groupIds=[],this.stackDirty=!0,t&&t.refreshItems&&r.forEach(this.items,function(t){t.dirty=!0,t.displayed&&t.redraw()})},o.prototype.destroy=function(){this.hide(),this.setItems(null),this.setGroups(null),this.hammer=null,this.body=null,this.conversion=null},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.axis.parentNode&&this.dom.axis.parentNode.removeChild(this.dom.axis),this.dom.labelSet.parentNode&&this.dom.labelSet.parentNode.removeChild(this.dom.labelSet)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame),this.dom.axis.parentNode||this.body.dom.backgroundVertical.appendChild(this.dom.axis),this.dom.labelSet.parentNode||this.body.dom.left.appendChild(this.dom.labelSet)},o.prototype.setSelection=function(t){var e,i,o,n;for(void 0==t&&(t=[]),Array.isArray(t)||(t=[t]),e=0,i=this.selection.length;i>e;e++)o=this.selection[e],n=this.items[o],n&&n.unselect();for(this.selection=[],e=0,i=t.length;i>e;e++)o=t[e],n=this.items[o],n&&(this.selection.push(o),n.select())},o.prototype.getSelection=function(){return this.selection.concat([])},o.prototype.getVisibleItems=function(){var t=this.body.range.getRange();if(this.options.rtl)var e=this.body.util.toScreen(t.start),i=this.body.util.toScreen(t.end);else var i=this.body.util.toScreen(t.start),e=this.body.util.toScreen(t.end);var o=[];for(var n in this.groups)if(this.groups.hasOwnProperty(n))for(var s=this.groups[n],r=s.visibleItems,a=0;a<r.length;a++){var h=r[a];this.options.rtl?h.right<i&&h.right+h.width>e&&o.push(h.id):h.left<e&&h.left+h.width>i&&o.push(h.id)}return o},o.prototype._deselect=function(t){for(var e=this.selection,i=0,o=e.length;o>i;i++)if(e[i]==t){e.splice(i,1);break}},o.prototype.redraw=function(){var t=this.options.margin,e=this.body.range,i=r.option.asSize,o=this.options,n=o.orientation.item,s=!1,a=this.dom.frame;this.props.top=this.body.domProps.top.height+this.body.domProps.border.top,this.options.rtl?this.props.right=this.body.domProps.right.width+this.body.domProps.border.right:this.props.left=this.body.domProps.left.width+this.body.domProps.border.left,a.className="vis-itemset",s=this._orderGroups()||s;var h=e.end-e.start,d=h!=this.lastVisibleInterval||this.props.width!=this.props.lastWidth;d&&(this.stackDirty=!0),
-this.lastVisibleInterval=h,this.props.lastWidth=this.props.width;var l=this.stackDirty,c=this._firstGroup(),u={item:t.item,axis:t.axis},p={item:t.item,axis:t.item.vertical/2},f=0,m=t.axis+t.item.vertical;return this.groups[y].redraw(e,p,l),r.forEach(this.groups,function(t){var i=t==c?u:p,o=t.redraw(e,i,l);s=o||s,f+=t.height}),f=Math.max(f,m),this.stackDirty=!1,a.style.height=i(f),this.props.width=a.offsetWidth,this.props.height=f,this.dom.axis.style.top=i("top"==n?this.body.domProps.top.height+this.body.domProps.border.top:this.body.domProps.top.height+this.body.domProps.centerContainer.height),this.options.rtl?this.dom.axis.style.right="0":this.dom.axis.style.left="0",s=this._isResized()||s},o.prototype._firstGroup=function(){var t="top"==this.options.orientation.item?0:this.groupIds.length-1,e=this.groupIds[t],i=this.groups[e]||this.groups[g];return i||null},o.prototype._updateUngrouped=function(){var t,e,i=this.groups[g];this.groups[y];if(this.groupsData){if(i){i.hide(),delete this.groups[g];for(e in this.items)if(this.items.hasOwnProperty(e)){t=this.items[e],t.parent&&t.parent.remove(t);var o=this._getGroupId(t.data),n=this.groups[o];n&&n.add(t)||t.hide()}}}else if(!i){var s=null,r=null;i=new c(s,r,this),this.groups[g]=i;for(e in this.items)this.items.hasOwnProperty(e)&&(t=this.items[e],i.add(t));i.show()}},o.prototype.getLabelSet=function(){return this.dom.labelSet},o.prototype.setItems=function(t){var e,i=this,o=this.itemsData;if(t){if(!(t instanceof a||t instanceof h))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=t}else this.itemsData=null;if(o&&(r.forEach(this.itemListeners,function(t,e){o.off(e,t)}),e=o.getIds(),this._onRemove(e)),this.itemsData){var n=this.id;r.forEach(this.itemListeners,function(t,e){i.itemsData.on(e,t,n)}),e=this.itemsData.getIds(),this._onAdd(e),this._updateUngrouped()}this.body.emitter.emit("_change",{queue:!0})},o.prototype.getItems=function(){return this.itemsData},o.prototype.setGroups=function(t){var e,i=this;if(this.groupsData&&(r.forEach(this.groupListeners,function(t,e){i.groupsData.off(e,t)}),e=this.groupsData.getIds(),this.groupsData=null,this._onRemoveGroups(e)),t){if(!(t instanceof a||t instanceof h))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var o=this.id;r.forEach(this.groupListeners,function(t,e){i.groupsData.on(e,t,o)}),e=this.groupsData.getIds(),this._onAddGroups(e)}this._updateUngrouped(),this._order(),this.body.emitter.emit("_change",{queue:!0})},o.prototype.getGroups=function(){return this.groupsData},o.prototype.removeItem=function(t){var e=this.itemsData.get(t),i=this.itemsData.getDataSet();e&&this.options.onRemove(e,function(e){e&&i.remove(t)})},o.prototype._getType=function(t){return t.type||this.options.type||(t.end?"range":"box")},o.prototype._getGroupId=function(t){var e=this._getType(t);return"background"==e&&void 0==t.group?y:this.groupsData?t.group:g},o.prototype._onUpdate=function(t){var e=this;t.forEach(function(t){var i,n=e.itemsData.get(t,e.itemOptions),s=e.items[t],r=e._getType(n),a=o.types[r];if(s&&(a&&s instanceof a?e._updateItem(s,n):(i=s.selected,e._removeItem(s),s=null)),!s){if(!a)throw"rangeoverflow"==r?new TypeError('Item type "rangeoverflow" is deprecated. Use css styling instead: .vis-item.vis-range .vis-item-content {overflow: visible;}'):new TypeError('Unknown item type "'+r+'"');s=new a(n,e.conversion,e.options),s.id=t,e._addItem(s),i&&(this.selection.push(t),s.select())}}.bind(this)),this._order(),this.stackDirty=!0,this.body.emitter.emit("_change",{queue:!0})},o.prototype._onAdd=o.prototype._onUpdate,o.prototype._onRemove=function(t){var e=0,i=this;t.forEach(function(t){var o=i.items[t];o&&(e++,i._removeItem(o))}),e&&(this._order(),this.stackDirty=!0,this.body.emitter.emit("_change",{queue:!0}))},o.prototype._order=function(){r.forEach(this.groups,function(t){t.order()})},o.prototype._onUpdateGroups=function(t){this._onAddGroups(t)},o.prototype._onAddGroups=function(t){var e=this;t.forEach(function(t){var i=e.groupsData.get(t),o=e.groups[t];if(o)o.setData(i);else{if(t==g||t==y)throw new Error("Illegal group id. "+t+" is a reserved id.");var n=Object.create(e.options);r.extend(n,{height:null}),o=new c(t,i,e),e.groups[t]=o;for(var s in e.items)if(e.items.hasOwnProperty(s)){var a=e.items[s];a.data.group==t&&o.add(a)}o.order(),o.show()}}),this.body.emitter.emit("_change",{queue:!0})},o.prototype._onRemoveGroups=function(t){var e=this.groups;t.forEach(function(t){var i=e[t];i&&(i.hide(),delete e[t])}),this.markDirty(),this.body.emitter.emit("_change",{queue:!0})},o.prototype._orderGroups=function(){if(this.groupsData){var t=this.groupsData.getIds({order:this.options.groupOrder}),e=!r.equalArray(t,this.groupIds);if(e){var i=this.groups;t.forEach(function(t){i[t].hide()}),t.forEach(function(t){i[t].show()}),this.groupIds=t}return e}return!1},o.prototype._addItem=function(t){this.items[t.id]=t;var e=this._getGroupId(t.data),i=this.groups[e];i&&i.add(t)},o.prototype._updateItem=function(t,e){var i=t.data.group,o=t.data.subgroup;if(t.setData(e),i!=t.data.group||o!=t.data.subgroup){var n=this.groups[i];n&&n.remove(t);var s=this._getGroupId(t.data),r=this.groups[s];r&&r.add(t)}},o.prototype._removeItem=function(t){t.hide(),delete this.items[t.id];var e=this.selection.indexOf(t.id);-1!=e&&this.selection.splice(e,1),t.parent&&t.parent.remove(t)},o.prototype._constructByEndArray=function(t){for(var e=[],i=0;i<t.length;i++)t[i]instanceof m&&e.push(t[i]);return e},o.prototype._onTouch=function(t){this.touchParams.item=this.itemFromTarget(t),this.touchParams.dragLeftItem=t.target.dragLeftItem||!1,this.touchParams.dragRightItem=t.target.dragRightItem||!1,this.touchParams.itemProps=null},o.prototype._getGroupIndex=function(t){for(var e=0;e<this.groupIds.length;e++)if(t==this.groupIds[e])return e},o.prototype._onDragStart=function(t){var e,i=this.touchParams.item||null,o=this;if(i&&(i.selected||this.options.itemsAlwaysDraggable)){if(!this.options.editable.updateTime&&!this.options.editable.updateGroup&&!i.editable)return;if(i.editable===!1)return;var n=this.touchParams.dragLeftItem,s=this.touchParams.dragRightItem;if(n)e={item:n,initialX:t.center.x,dragLeft:!0,data:this._cloneItemData(i.data)},this.touchParams.itemProps=[e];else if(s)e={item:s,initialX:t.center.x,dragRight:!0,data:this._cloneItemData(i.data)},this.touchParams.itemProps=[e];else{this.touchParams.selectedItem=i;var r=this._getGroupIndex(i.data.group),a=this.options.itemsAlwaysDraggable&&!i.selected?[i.id]:this.getSelection();this.touchParams.itemProps=a.map(function(e){var i=o.items[e],n=o._getGroupIndex(i.data.group);return{item:i,initialX:t.center.x,groupOffset:r-n,data:this._cloneItemData(i.data)}}.bind(this))}t.stopPropagation()}else this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey)&&this._onDragStartAddItem(t)},o.prototype._onDragStartAddItem=function(t){var e=this.options.snap||null;if(this.options.rtl)var i=r.getAbsoluteRight(this.dom.frame),o=i-t.center.x+10;else var i=r.getAbsoluteLeft(this.dom.frame),o=t.center.x-i-10;var n=this.body.util.toTime(o),s=this.body.util.getScale(),a=this.body.util.getStep(),h=e?e(n,s,a):n,d=h,l={type:"range",start:h,end:d,content:"new item"},c=r.randomUUID();l[this.itemsData._fieldId]=c;var u=this.groupFromTarget(t);u&&(l.group=u.groupId);var p=new m(l,this.conversion,this.options);p.id=c,p.data=this._cloneItemData(l),this._addItem(p);var f={item:p,initialX:t.center.x,data:p.data};this.options.rtl?f.dragLeft=!0:f.dragRight=!0,this.touchParams.itemProps=[f],t.stopPropagation()},o.prototype._onDrag=function(t){if(this.touchParams.itemProps){t.stopPropagation();var e=this,i=this.options.snap||null;if(this.options.rtl)var o=this.body.dom.root.offsetLeft+this.body.domProps.right.width;else var o=this.body.dom.root.offsetLeft+this.body.domProps.left.width;var n=this.body.util.getScale(),s=this.body.util.getStep(),a=this.touchParams.selectedItem,h=e.options.editable.updateGroup,d=null;if(h&&a&&void 0!=a.data.group){var l=e.groupFromTarget(t);l&&(d=this._getGroupIndex(l.groupId))}this.touchParams.itemProps.forEach(function(a){var h=e.body.util.toTime(t.center.x-o),l=e.body.util.toTime(a.initialX-o);if(this.options.rtl)var c=-(h-l);else var c=h-l;var u=this._cloneItemData(a.item.data);if(a.item.editable!==!1){var p=e.options.editable.updateTime||a.item.editable===!0;if(p)if(a.dragLeft){if(this.options.rtl){if(void 0!=u.end){var f=r.convert(a.data.end,"Date"),m=new Date(f.valueOf()+c);u.end=i?i(m,n,s):m}}else if(void 0!=u.start){var v=r.convert(a.data.start,"Date"),g=new Date(v.valueOf()+c);u.start=i?i(g,n,s):g}}else if(a.dragRight){if(this.options.rtl){if(void 0!=u.start){var v=r.convert(a.data.start,"Date"),g=new Date(v.valueOf()+c);u.start=i?i(g,n,s):g}}else if(void 0!=u.end){var f=r.convert(a.data.end,"Date"),m=new Date(f.valueOf()+c);u.end=i?i(m,n,s):m}}else if(void 0!=u.start){var v=r.convert(a.data.start,"Date").valueOf(),g=new Date(v+c);if(void 0!=u.end){var f=r.convert(a.data.end,"Date"),y=f.valueOf()-v.valueOf();u.start=i?i(g,n,s):g,u.end=new Date(u.start.valueOf()+y)}else u.start=i?i(g,n,s):g}var b=e.options.editable.updateGroup||a.item.editable===!0;if(b&&!a.dragLeft&&!a.dragRight&&null!=d&&void 0!=u.group){var w=d-a.groupOffset;w=Math.max(0,w),w=Math.min(e.groupIds.length-1,w),u.group=e.groupIds[w]}u=this._cloneItemData(u),e.options.onMoving(u,function(t){t&&a.item.setData(this._cloneItemData(t,"Date"))}.bind(this))}}.bind(this)),this.stackDirty=!0,this.body.emitter.emit("_change")}},o.prototype._moveToGroup=function(t,e){var i=this.groups[e];if(i&&i.groupId!=t.data.group){var o=t.parent;o.remove(t),o.order(),i.add(t),i.order(),t.data.group=i.groupId}},o.prototype._onDragEnd=function(t){if(this.touchParams.itemProps){t.stopPropagation();var e=this,i=this.itemsData.getDataSet(),o=this.touchParams.itemProps;this.touchParams.itemProps=null,o.forEach(function(t){var o=t.item.id,n=null!=e.itemsData.get(o,e.itemOptions);if(n){var s=this._cloneItemData(t.item.data);e.options.onMove(s,function(n){n?(n[i._fieldId]=o,i.update(n)):(t.item.setData(t.data),e.stackDirty=!0,e.body.emitter.emit("_change"))})}else e.options.onAdd(t.item.data,function(i){e._removeItem(t.item),i&&e.itemsData.getDataSet().add(i),e.stackDirty=!0,e.body.emitter.emit("_change")})}.bind(this))}},o.prototype._onGroupDragStart=function(t){this.options.groupEditable.order&&(this.groupTouchParams.group=this.groupFromTarget(t),this.groupTouchParams.group&&(t.stopPropagation(),this.groupTouchParams.originalOrder=this.groupsData.getIds({order:this.options.groupOrder})))},o.prototype._onGroupDrag=function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this.groupFromTarget(t);if(e&&e.height!=this.groupTouchParams.group.height){var i=e.top<this.groupTouchParams.group.top,o=t.center?t.center.y:t.clientY,n=r.getAbsoluteTop(e.dom.foreground),s=this.groupTouchParams.group.height;if(i){if(o>n+s)return}else{var a=e.height;if(n+a-s>o)return}}if(e&&e!=this.groupTouchParams.group){var h=this.groupsData,d=h.get(e.groupId),l=h.get(this.groupTouchParams.group.groupId);l&&d&&(this.options.groupOrderSwap(l,d,this.groupsData),this.groupsData.update(l),this.groupsData.update(d));var c=this.groupsData.getIds({order:this.options.groupOrder});if(!r.equalArray(c,this.groupTouchParams.originalOrder))for(var h=this.groupsData,u=this.groupTouchParams.originalOrder,p=this.groupTouchParams.group.groupId,f=Math.min(u.length,c.length),m=0,v=0,g=0;f>m;){for(;f>m+v&&f>m+g&&c[m+v]==u[m+g];)m++;if(m+v>=f)break;if(c[m+v]!=p)if(u[m+g]!=p){var y=c.indexOf(u[m+g]),b=h.get(c[m+v]),w=h.get(u[m+g]);this.options.groupOrderSwap(b,w,h),h.update(b),h.update(w);var _=c[m+v];c[m+v]=u[m+g],c[y]=_,m++}else g=1;else v=1}}}},o.prototype._onGroupDragEnd=function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this,i=e.groupTouchParams.group.groupId,o=e.groupsData.getDataSet(),n=r.extend({},o.get(i));e.options.onMoveGroup(n,function(t){if(t)t[o._fieldId]=i,o.update(t);else{var n=o.getIds({order:e.options.groupOrder});if(!r.equalArray(n,e.groupTouchParams.originalOrder))for(var s=e.groupTouchParams.originalOrder,a=Math.min(s.length,n.length),h=0;a>h;){for(;a>h&&n[h]==s[h];)h++;if(h>=a)break;var d=n.indexOf(s[h]),l=o.get(n[h]),c=o.get(s[h]);e.options.groupOrderSwap(l,c,o),groupsData.update(l),groupsData.update(c);var u=n[h];n[h]=s[h],n[d]=u,h++}}}),e.body.emitter.emit("groupDragged",{groupId:i})}},o.prototype._onSelectItem=function(t){if(this.options.selectable){var e=t.srcEvent&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey),i=t.srcEvent&&t.srcEvent.shiftKey;if(e||i)return void this._onMultiSelectItem(t);var o=this.getSelection(),n=this.itemFromTarget(t),s=n?[n.id]:[];this.setSelection(s);var r=this.getSelection();(r.length>0||o.length>0)&&this.body.emitter.emit("select",{items:r,event:t})}},o.prototype._onAddItem=function(t){if(this.options.selectable&&this.options.editable.add){var e=this,i=this.options.snap||null,o=this.itemFromTarget(t);if(o){var n=e.itemsData.get(o.id);this.options.onUpdate(n,function(t){t&&e.itemsData.getDataSet().update(t)})}else{if(this.options.rtl)var s=r.getAbsoluteRight(this.dom.frame),a=s-t.center.x;else var s=r.getAbsoluteLeft(this.dom.frame),a=t.center.x-s;var h=this.body.util.toTime(a),d=this.body.util.getScale(),l=this.body.util.getStep(),c={start:i?i(h,d,l):h,content:"new item"};if("range"===this.options.type){var u=this.body.util.toTime(a+this.props.width/5);c.end=i?i(u,d,l):u}c[this.itemsData._fieldId]=r.randomUUID();var p=this.groupFromTarget(t);p&&(c.group=p.groupId),c=this._cloneItemData(c),this.options.onAdd(c,function(t){t&&e.itemsData.getDataSet().add(t)})}}},o.prototype._onMultiSelectItem=function(t){if(this.options.selectable){var e=this.itemFromTarget(t);if(e){var i=this.options.multiselect?this.getSelection():[],n=t.srcEvent&&t.srcEvent.shiftKey||!1;if(n&&this.options.multiselect){var s=this.itemsData.get(e.id).group,r=void 0;this.options.multiselectPerGroup&&i.length>0&&(r=this.itemsData.get(i[0]).group),this.options.multiselectPerGroup&&void 0!=r&&r!=s||i.push(e.id);var a=o._getItemRange(this.itemsData.get(i,this.itemOptions));if(!this.options.multiselectPerGroup||r==s){i=[];for(var h in this.items)if(this.items.hasOwnProperty(h)){var d=this.items[h],l=d.data.start,c=void 0!==d.data.end?d.data.end:l;!(l>=a.min&&c<=a.max)||this.options.multiselectPerGroup&&r!=this.itemsData.get(d.id).group||d instanceof v||i.push(d.id)}}}else{var u=i.indexOf(e.id);-1==u?i.push(e.id):i.splice(u,1)}this.setSelection(i),this.body.emitter.emit("select",{items:this.getSelection(),event:t})}}},o._getItemRange=function(t){var e=null,i=null;return t.forEach(function(t){(null==i||t.start<i)&&(i=t.start),void 0!=t.end?(null==e||t.end>e)&&(e=t.end):(null==e||t.start>e)&&(e=t.start)}),{min:i,max:e}},o.prototype.itemFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("timeline-item"))return e["timeline-item"];e=e.parentNode}return null},o.prototype.groupFromTarget=function(t){for(var e=t.center?t.center.y:t.clientY,i=0;i<this.groupIds.length;i++){var o=this.groupIds[i],n=this.groups[o],s=n.dom.foreground,a=r.getAbsoluteTop(s);if(e>a&&e<a+s.offsetHeight)return n;if("top"===this.options.orientation.item){if(i===this.groupIds.length-1&&e>a)return n}else if(0===i&&e<a+s.offset)return n}return null},o.itemSetFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("timeline-itemset"))return e["timeline-itemset"];e=e.parentNode}return null},o.prototype._cloneItemData=function(t,e){var i=r.extend({},t);return e||(e=this.itemsData.getDataSet()._options.type),void 0!=i.start&&(i.start=r.convert(i.start,e&&e.start||"Date")),void 0!=i.end&&(i.end=r.convert(i.end,e&&e.end||"Date")),i},t.exports=o},function(t,e,i){function o(t,e,i,s){this.moment=n,this.current=this.moment(),this._start=this.moment(),this._end=this.moment(),this.autoScale=!0,this.scale="day",this.step=1,this.setRange(t,e,i),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,Array.isArray(s)?this.hiddenDates=s:void 0!=s?this.hiddenDates=[s]:this.hiddenDates=[],this.format=o.FORMAT}var n=i(2),s=i(32),r=i(1);o.FORMAT={minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",month:"YYYY",year:""}},o.prototype.setMoment=function(t){this.moment=t,this.current=this.moment(this.current),this._start=this.moment(this._start),this._end=this.moment(this._end)},o.prototype.setFormat=function(t){var e=r.deepExtend({},o.FORMAT);this.format=r.deepExtend(e,t)},o.prototype.setRange=function(t,e,i){if(!(t instanceof Date&&e instanceof Date))throw"No legal start or end date in method setRange";this._start=void 0!=t?this.moment(t.valueOf()):new Date,this._end=void 0!=e?this.moment(e.valueOf()):new Date,this.autoScale&&this.setMinimumStep(i)},o.prototype.start=function(){this.current=this._start.clone(),this.roundToMinor()},o.prototype.roundToMinor=function(){switch(this.scale){case"year":this.current.year(this.step*Math.floor(this.current.year()/this.step)),this.current.month(0);case"month":this.current.date(1);case"day":case"weekday":this.current.hours(0);case"hour":this.current.minutes(0);case"minute":this.current.seconds(0);case"second":this.current.milliseconds(0)}if(1!=this.step)switch(this.scale){case"millisecond":this.current.subtract(this.current.milliseconds()%this.step,"milliseconds");break;case"second":this.current.subtract(this.current.seconds()%this.step,"seconds");break;case"minute":this.current.subtract(this.current.minutes()%this.step,"minutes");break;case"hour":this.current.subtract(this.current.hours()%this.step,"hours");break;case"weekday":case"day":this.current.subtract((this.current.date()-1)%this.step,"day");break;case"month":this.current.subtract(this.current.month()%this.step,"month");break;case"year":this.current.subtract(this.current.year()%this.step,"year")}},o.prototype.hasNext=function(){return this.current.valueOf()<=this._end.valueOf()},o.prototype.next=function(){var t=this.current.valueOf();if(this.current.month()<6)switch(this.scale){case"millisecond":this.current.add(this.step,"millisecond");break;case"second":this.current.add(this.step,"second");break;case"minute":this.current.add(this.step,"minute");break;case"hour":this.current.add(this.step,"hour"),this.current.subtract(this.current.hours()%this.step,"hour");break;case"weekday":case"day":this.current.add(this.step,"day");break;case"month":this.current.add(this.step,"month");break;case"year":this.current.add(this.step,"year")}else switch(this.scale){case"millisecond":this.current.add(this.step,"millisecond");break;case"second":this.current.add(this.step,"second");break;case"minute":this.current.add(this.step,"minute");break;case"hour":this.current.add(this.step,"hour");break;case"weekday":case"day":this.current.add(this.step,"day");break;case"month":this.current.add(this.step,"month");break;case"year":this.current.add(this.step,"year")}if(1!=this.step)switch(this.scale){case"millisecond":this.current.milliseconds()<this.step&&this.current.milliseconds(0);break;case"second":this.current.seconds()<this.step&&this.current.seconds(0);break;case"minute":this.current.minutes()<this.step&&this.current.minutes(0);break;case"hour":this.current.hours()<this.step&&this.current.hours(0);break;case"weekday":case"day":this.current.date()<this.step+1&&this.current.date(1);break;case"month":this.current.month()<this.step&&this.current.month(0);break;case"year":}this.current.valueOf()==t&&(this.current=this._end.clone()),s.stepOverHiddenDates(this.moment,this,t)},o.prototype.getCurrent=function(){return this.current},o.prototype.setScale=function(t){t&&"string"==typeof t.scale&&(this.scale=t.scale,this.step=t.step>0?t.step:1,this.autoScale=!1)},o.prototype.setAutoScale=function(t){this.autoScale=t},o.prototype.setMinimumStep=function(t){if(void 0!=t){var e=31104e6,i=2592e6,o=864e5,n=36e5,s=6e4,r=1e3,a=1;1e3*e>t&&(this.scale="year",this.step=1e3),500*e>t&&(this.scale="year",this.step=500),100*e>t&&(this.scale="year",this.step=100),50*e>t&&(this.scale="year",this.step=50),10*e>t&&(this.scale="year",this.step=10),5*e>t&&(this.scale="year",this.step=5),e>t&&(this.scale="year",this.step=1),3*i>t&&(this.scale="month",this.step=3),i>t&&(this.scale="month",this.step=1),5*o>t&&(this.scale="day",this.step=5),2*o>t&&(this.scale="day",this.step=2),o>t&&(this.scale="day",this.step=1),o/2>t&&(this.scale="weekday",this.step=1),4*n>t&&(this.scale="hour",this.step=4),n>t&&(this.scale="hour",this.step=1),15*s>t&&(this.scale="minute",this.step=15),10*s>t&&(this.scale="minute",this.step=10),5*s>t&&(this.scale="minute",this.step=5),s>t&&(this.scale="minute",this.step=1),15*r>t&&(this.scale="second",this.step=15),10*r>t&&(this.scale="second",this.step=10),5*r>t&&(this.scale="second",this.step=5),r>t&&(this.scale="second",this.step=1),200*a>t&&(this.scale="millisecond",this.step=200),100*a>t&&(this.scale="millisecond",this.step=100),50*a>t&&(this.scale="millisecond",this.step=50),10*a>t&&(this.scale="millisecond",this.step=10),5*a>t&&(this.scale="millisecond",this.step=5),a>t&&(this.scale="millisecond",this.step=1)}},o.snap=function(t,e,i){var o=n(t);if("year"==e){var s=o.year()+Math.round(o.month()/12);o.year(Math.round(s/i)*i),o.month(0),o.date(0),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("month"==e)o.date()>15?(o.date(1),o.add(1,"month")):o.date(1),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0);else if("day"==e){switch(i){case 5:case 2:o.hours(24*Math.round(o.hours()/24));break;default:o.hours(12*Math.round(o.hours()/12))}o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("weekday"==e){switch(i){case 5:case 2:o.hours(12*Math.round(o.hours()/12));break;default:o.hours(6*Math.round(o.hours()/6))}o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("hour"==e){switch(i){case 4:o.minutes(60*Math.round(o.minutes()/60));break;default:o.minutes(30*Math.round(o.minutes()/30))}o.seconds(0),o.milliseconds(0)}else if("minute"==e){switch(i){case 15:case 10:o.minutes(5*Math.round(o.minutes()/5)),o.seconds(0);break;case 5:o.seconds(60*Math.round(o.seconds()/60));break;default:o.seconds(30*Math.round(o.seconds()/30))}o.milliseconds(0)}else if("second"==e)switch(i){case 15:case 10:o.seconds(5*Math.round(o.seconds()/5)),o.milliseconds(0);break;case 5:o.milliseconds(1e3*Math.round(o.milliseconds()/1e3));break;default:o.milliseconds(500*Math.round(o.milliseconds()/500))}else if("millisecond"==e){var r=i>5?i/2:1;o.milliseconds(Math.round(o.milliseconds()/r)*r)}return o},o.prototype.isMajor=function(){if(1==this.switchedYear)switch(this.switchedYear=!1,this.scale){case"year":case"month":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedMonth)switch(this.switchedMonth=!1,this.scale){case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedDay)switch(this.switchedDay=!1,this.scale){case"millisecond":case"second":case"minute":case"hour":return!0;default:return!1}var t=this.moment(this.current);switch(this.scale){case"millisecond":return 0==t.milliseconds();case"second":return 0==t.seconds();case"minute":return 0==t.hours()&&0==t.minutes();case"hour":return 0==t.hours();case"weekday":case"day":return 1==t.date();case"month":return 0==t.month();case"year":return!1;default:return!1}},o.prototype.getLabelMinor=function(t){void 0==t&&(t=this.current);var e=this.format.minorLabels[this.scale];return e&&e.length>0?this.moment(t).format(e):""},o.prototype.getLabelMajor=function(t){void 0==t&&(t=this.current);var e=this.format.majorLabels[this.scale];return e&&e.length>0?this.moment(t).format(e):""},o.prototype.getClassName=function(){function t(t){return t/h%2==0?" vis-even":" vis-odd"}function e(t){return t.isSame(new Date,"day")?" vis-today":t.isSame(s().add(1,"day"),"day")?" vis-tomorrow":t.isSame(s().add(-1,"day"),"day")?" vis-yesterday":""}function i(t){return t.isSame(new Date,"week")?" vis-current-week":""}function o(t){return t.isSame(new Date,"month")?" vis-current-month":""}function n(t){return t.isSame(new Date,"year")?" vis-current-year":""}var s=this.moment,r=this.moment(this.current),a=r.locale?r.locale("en"):r.lang("en"),h=this.step;switch(this.scale){case"millisecond":return t(a.milliseconds()).trim();case"second":return t(a.seconds()).trim();case"minute":return t(a.minutes()).trim();case"hour":var d=a.hours();return 4==this.step&&(d=d+"-h"+(d+4)),"vis-h"+d+e(a)+t(a.hours());case"weekday":return"vis-"+a.format("dddd").toLowerCase()+e(a)+i(a)+t(a.date());case"day":var l=a.date(),c=a.format("MMMM").toLowerCase();return"vis-day"+l+" vis-"+c+o(a)+t(l-1);case"month":return"vis-"+a.format("MMMM").toLowerCase()+o(a)+t(a.month());case"year":var u=a.year();return"vis-year"+u+n(a)+t(u);default:return""}},t.exports=o},function(t,e,i){function o(t,e,i){this.groupId=t,this.subgroups={},this.subgroupIndex=0,this.subgroupOrderer=e&&e.subgroupOrder,this.itemSet=i,this.dom={},this.props={label:{width:0,height:0}},this.className=null,this.items={},this.visibleItems=[],this.orderedItems={byStart:[],byEnd:[]},this.checkRangedItems=!1;var o=this;this.itemSet.body.emitter.on("checkRangedItems",function(){o.checkRangedItems=!0}),this._create(),this.setData(e)}var n=i(1),s=i(37);i(38);o.prototype._create=function(){var t=document.createElement("div");this.itemSet.options.groupEditable.order?t.className="vis-label draggable":t.className="vis-label",this.dom.label=t;var e=document.createElement("div");e.className="vis-inner",t.appendChild(e),this.dom.inner=e;var i=document.createElement("div");i.className="vis-group",i["timeline-group"]=this,this.dom.foreground=i,this.dom.background=document.createElement("div"),this.dom.background.className="vis-group",this.dom.axis=document.createElement("div"),this.dom.axis.className="vis-group",this.dom.marker=document.createElement("div"),this.dom.marker.style.visibility="hidden",this.dom.marker.innerHTML="?",this.dom.background.appendChild(this.dom.marker)},o.prototype.setData=function(t){var e;if(e=this.itemSet.options&&this.itemSet.options.groupTemplate?this.itemSet.options.groupTemplate(t):t&&t.content,e instanceof Element){for(this.dom.inner.appendChild(e);this.dom.inner.firstChild;)this.dom.inner.removeChild(this.dom.inner.firstChild);this.dom.inner.appendChild(e)}else void 0!==e&&null!==e?this.dom.inner.innerHTML=e:this.dom.inner.innerHTML=this.groupId||"";this.dom.label.title=t&&t.title||"",this.dom.inner.firstChild?n.removeClassName(this.dom.inner,"vis-hidden"):n.addClassName(this.dom.inner,"vis-hidden");var i=t&&t.className||null;i!=this.className&&(this.className&&(n.removeClassName(this.dom.label,this.className),n.removeClassName(this.dom.foreground,this.className),n.removeClassName(this.dom.background,this.className),n.removeClassName(this.dom.axis,this.className)),n.addClassName(this.dom.label,i),n.addClassName(this.dom.foreground,i),n.addClassName(this.dom.background,i),n.addClassName(this.dom.axis,i),this.className=i),this.style&&(n.removeCssText(this.dom.label,this.style),this.style=null),t&&t.style&&(n.addCssText(this.dom.label,t.style),this.style=t.style)},o.prototype.getLabelWidth=function(){return this.props.label.width},o.prototype.redraw=function(t,e,i){var o=!1,r=this.dom.marker.clientHeight;if(r!=this.lastMarkerHeight&&(this.lastMarkerHeight=r,n.forEach(this.items,function(t){t.dirty=!0,t.displayed&&t.redraw()}),i=!0),this._calculateSubGroupHeights(),"function"==typeof this.itemSet.options.order){if(i){var a=this,h=!1;n.forEach(this.items,function(t){t.displayed||(t.redraw(),a.visibleItems.push(t)),t.repositionX(h)});var d=this.orderedItems.byStart.slice().sort(function(t,e){return a.itemSet.options.order(t.data,e.data)});s.stack(d,e,!0)}this.visibleItems=this._updateVisibleItems(this.orderedItems,this.visibleItems,t)}else this.visibleItems=this._updateVisibleItems(this.orderedItems,this.visibleItems,t),this.itemSet.options.stack?s.stack(this.visibleItems,e,i):s.nostack(this.visibleItems,e,this.subgroups);var l=this._calculateHeight(e),c=this.dom.foreground;this.top=c.offsetTop,this.right=c.offsetLeft,this.width=c.offsetWidth,o=n.updateProperty(this,"height",l)||o,o=n.updateProperty(this.props.label,"width",this.dom.inner.clientWidth)||o,o=n.updateProperty(this.props.label,"height",this.dom.inner.clientHeight)||o,this.dom.background.style.height=l+"px",this.dom.foreground.style.height=l+"px",this.dom.label.style.height=l+"px";for(var u=0,p=this.visibleItems.length;p>u;u++){var f=this.visibleItems[u];f.repositionY(e)}return o},o.prototype._calculateSubGroupHeights=function(){if(Object.keys(this.subgroups).length>0){var t=this;this.resetSubgroups(),n.forEach(this.visibleItems,function(e){void 0!==e.data.subgroup&&(t.subgroups[e.data.subgroup].height=Math.max(t.subgroups[e.data.subgroup].height,e.height),t.subgroups[e.data.subgroup].visible=!0)})}},o.prototype._calculateHeight=function(t){var e,i=this.visibleItems;if(i.length>0){var o=i[0].top,s=i[0].top+i[0].height;if(n.forEach(i,function(t){o=Math.min(o,t.top),s=Math.max(s,t.top+t.height)}),o>t.axis){var r=o-t.axis;s-=r,n.forEach(i,function(t){t.top-=r})}e=s+t.item.vertical/2}else e=0;return e=Math.max(e,this.props.label.height)},o.prototype.show=function(){this.dom.label.parentNode||this.itemSet.dom.labelSet.appendChild(this.dom.label),this.dom.foreground.parentNode||this.itemSet.dom.foreground.appendChild(this.dom.foreground),this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background),this.dom.axis.parentNode||this.itemSet.dom.axis.appendChild(this.dom.axis)},o.prototype.hide=function(){var t=this.dom.label;t.parentNode&&t.parentNode.removeChild(t);var e=this.dom.foreground;e.parentNode&&e.parentNode.removeChild(e);var i=this.dom.background;i.parentNode&&i.parentNode.removeChild(i);var o=this.dom.axis;o.parentNode&&o.parentNode.removeChild(o)},o.prototype.add=function(t){if(this.items[t.id]=t,t.setParent(this),void 0!==t.data.subgroup&&(void 0===this.subgroups[t.data.subgroup]&&(this.subgroups[t.data.subgroup]={height:0,visible:!1,index:this.subgroupIndex,items:[]},this.subgroupIndex++),this.subgroups[t.data.subgroup].items.push(t)),this.orderSubgroups(),-1==this.visibleItems.indexOf(t)){var e=this.itemSet.body.range;this._checkIfVisible(t,this.visibleItems,e)}},o.prototype.orderSubgroups=function(){if(void 0!==this.subgroupOrderer){var t=[];if("string"==typeof this.subgroupOrderer){for(var e in this.subgroups)t.push({subgroup:e,sortField:this.subgroups[e].items[0].data[this.subgroupOrderer]});t.sort(function(t,e){return t.sortField-e.sortField})}else if("function"==typeof this.subgroupOrderer){for(var e in this.subgroups)t.push(this.subgroups[e].items[0].data);t.sort(this.subgroupOrderer)}if(t.length>0)for(var i=0;i<t.length;i++)this.subgroups[t[i].subgroup].index=i}},o.prototype.resetSubgroups=function(){for(var t in this.subgroups)this.subgroups.hasOwnProperty(t)&&(this.subgroups[t].visible=!1)},o.prototype.remove=function(t){delete this.items[t.id],t.setParent(null);var e=this.visibleItems.indexOf(t);if(-1!=e&&this.visibleItems.splice(e,1),void 0!==t.data.subgroup){var i=this.subgroups[t.data.subgroup];if(i){var o=i.items.indexOf(t);i.items.splice(o,1),i.items.length||(delete this.subgroups[t.data.subgroup],this.subgroupIndex--),this.orderSubgroups()}}},o.prototype.removeFromDataSet=function(t){this.itemSet.removeItem(t.id)},o.prototype.order=function(){for(var t=n.toArray(this.items),e=[],i=[],o=0;o<t.length;o++)void 0!==t[o].data.end&&i.push(t[o]),e.push(t[o]);this.orderedItems={byStart:e,byEnd:i},s.orderByStart(this.orderedItems.byStart),s.orderByEnd(this.orderedItems.byEnd)},o.prototype._updateVisibleItems=function(t,e,i){var o,s,r=[],a={},h=(i.end-i.start)/4,d=i.start-h,l=i.end+h,c=function(t){
-return d>t?-1:l>=t?0:1};if(e.length>0)for(s=0;s<e.length;s++)this._checkIfVisibleWithReference(e[s],r,a,i);var u=n.binarySearchCustom(t.byStart,c,"data","start");if(this._traceVisible(u,t.byStart,r,a,function(t){return t.data.start<d||t.data.start>l}),1==this.checkRangedItems)for(this.checkRangedItems=!1,s=0;s<t.byEnd.length;s++)this._checkIfVisibleWithReference(t.byEnd[s],r,a,i);else{var p=n.binarySearchCustom(t.byEnd,c,"data","end");this._traceVisible(p,t.byEnd,r,a,function(t){return t.data.end<d||t.data.end>l})}for(s=0;s<r.length;s++)o=r[s],o.displayed||o.show(),o.repositionX();return r},o.prototype._traceVisible=function(t,e,i,o,n){var s,r;if(-1!=t){for(r=t;r>=0&&(s=e[r],!n(s));r--)void 0===o[s.id]&&(o[s.id]=!0,i.push(s));for(r=t+1;r<e.length&&(s=e[r],!n(s));r++)void 0===o[s.id]&&(o[s.id]=!0,i.push(s))}},o.prototype._checkIfVisible=function(t,e,i){t.isVisible(i)?(t.displayed||t.show(),t.repositionX(),e.push(t)):t.displayed&&t.hide()},o.prototype._checkIfVisibleWithReference=function(t,e,i,o){t.isVisible(o)?void 0===i[t.id]&&(i[t.id]=!0,e.push(t)):t.displayed&&t.hide()},t.exports=o},function(t,e){var i=.001;e.orderByStart=function(t){t.sort(function(t,e){return t.data.start-e.data.start})},e.orderByEnd=function(t){t.sort(function(t,e){var i="end"in t.data?t.data.end:t.data.start,o="end"in e.data?e.data.end:e.data.start;return i-o})},e.stack=function(t,i,o){var n,s;if(o)for(n=0,s=t.length;s>n;n++)t[n].top=null;for(n=0,s=t.length;s>n;n++){var r=t[n];if(r.stack&&null===r.top){r.top=i.axis;do{for(var a=null,h=0,d=t.length;d>h;h++){var l=t[h];if(null!==l.top&&l!==r&&l.stack&&e.collision(r,l,i.item,l.options.rtl)){a=l;break}}null!=a&&(r.top=a.top+a.height+i.item.vertical)}while(a)}}},e.nostack=function(t,e,i){var o,n,s;for(o=0,n=t.length;n>o;o++)if(void 0!==t[o].data.subgroup){s=e.axis;for(var r in i)i.hasOwnProperty(r)&&1==i[r].visible&&i[r].index<i[t[o].data.subgroup].index&&(s+=i[r].height+e.item.vertical);t[o].top=s}else t[o].top=e.axis},e.collision=function(t,e,o,n){return n?t.right-o.horizontal+i<e.right+e.width&&t.right+t.width+o.horizontal-i>e.right&&t.top-o.vertical+i<e.top+e.height&&t.top+t.height+o.vertical-i>e.top:t.left-o.horizontal+i<e.left+e.width&&t.left+t.width+o.horizontal-i>e.left&&t.top-o.vertical+i<e.top+e.height&&t.top+t.height+o.vertical-i>e.top}},function(t,e,i){function o(t,e,i){if(this.props={content:{width:0}},this.overflow=!1,this.options=i,t){if(void 0==t.start)throw new Error('Property "start" missing in item '+t.id);if(void 0==t.end)throw new Error('Property "end" missing in item '+t.id)}n.call(this,t,e,i)}var n=(i(20),i(39));o.prototype=new n(null,null,null),o.prototype.baseClassName="vis-item vis-range",o.prototype.isVisible=function(t){return this.data.start<t.end&&this.data.end>t.start},o.prototype.redraw=function(){var t=this.dom;if(t||(this.dom={},t=this.dom,t.box=document.createElement("div"),t.frame=document.createElement("div"),t.frame.className="vis-item-overflow",t.box.appendChild(t.frame),t.content=document.createElement("div"),t.content.className="vis-item-content",t.frame.appendChild(t.content),t.box["timeline-item"]=this,this.dirty=!0),!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!t.box.parentNode){var e=this.parent.dom.foreground;if(!e)throw new Error("Cannot redraw item: parent has no foreground container element");e.appendChild(t.box)}if(this.displayed=!0,this.dirty){this._updateContents(this.dom.content),this._updateTitle(this.dom.box),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var i=(this.options.editable.updateTime||this.options.editable.updateGroup||this.editable===!0)&&this.editable!==!1,o=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(i?" vis-editable":" vis-readonly");t.box.className=this.baseClassName+o,this.overflow="hidden"!==window.getComputedStyle(t.frame).overflow,this.dom.content.style.maxWidth="none",this.props.content.width=this.dom.content.offsetWidth,this.height=this.dom.box.offsetHeight,this.dom.content.style.maxWidth="",this.dirty=!1}this._repaintDeleteButton(t.box),this._repaintDragLeft(),this._repaintDragRight()},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){if(this.displayed){var t=this.dom.box;t.parentNode&&t.parentNode.removeChild(t),this.displayed=!1}},o.prototype.repositionX=function(t){var e,i,o=this.parent.width,n=this.conversion.toScreen(this.data.start),s=this.conversion.toScreen(this.data.end);void 0!==t&&t!==!0||(-o>n&&(n=-o),s>2*o&&(s=2*o));var r=Math.max(s-n,1);switch(this.overflow?(this.options.rtl?this.right=n:this.left=n,this.width=r+this.props.content.width,i=this.props.content.width):(this.options.rtl?this.right=n:this.left=n,this.width=r,i=Math.min(s-n,this.props.content.width)),this.options.rtl?this.dom.box.style.right=this.right+"px":this.dom.box.style.left=this.left+"px",this.dom.box.style.width=r+"px",this.options.align){case"left":this.options.rtl?this.dom.content.style.right="0":this.dom.content.style.left="0";break;case"right":this.options.rtl?this.dom.content.style.right=Math.max(r-i,0)+"px":this.dom.content.style.left=Math.max(r-i,0)+"px";break;case"center":this.options.rtl?this.dom.content.style.right=Math.max((r-i)/2,0)+"px":this.dom.content.style.left=Math.max((r-i)/2,0)+"px";break;default:e=this.overflow?s>0?Math.max(-n,0):-i:0>n?-n:0,this.options.rtl?this.dom.content.style.right=e+"px":this.dom.content.style.left=e+"px"}},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.box;"top"==t?e.style.top=this.top+"px":e.style.top=this.parent.height-this.top-this.height+"px"},o.prototype._repaintDragLeft=function(){if(this.selected&&this.options.editable.updateTime&&!this.dom.dragLeft){var t=document.createElement("div");t.className="vis-drag-left",t.dragLeftItem=this,this.dom.box.appendChild(t),this.dom.dragLeft=t}else!this.selected&&this.dom.dragLeft&&(this.dom.dragLeft.parentNode&&this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft),this.dom.dragLeft=null)},o.prototype._repaintDragRight=function(){if(this.selected&&this.options.editable.updateTime&&!this.dom.dragRight){var t=document.createElement("div");t.className="vis-drag-right",t.dragRightItem=this,this.dom.box.appendChild(t),this.dom.dragRight=t}else!this.selected&&this.dom.dragRight&&(this.dom.dragRight.parentNode&&this.dom.dragRight.parentNode.removeChild(this.dom.dragRight),this.dom.dragRight=null)},t.exports=o},function(t,e,i){function o(t,e,i){this.id=null,this.parent=null,this.data=t,this.dom=null,this.conversion=e||{},this.options=i||{},this.selected=!1,this.displayed=!1,this.dirty=!0,this.top=null,this.right=null,this.left=null,this.width=null,this.height=null,this.editable=null,this.data&&this.data.hasOwnProperty("editable")&&"boolean"==typeof this.data.editable&&(this.editable=t.editable)}var n=i(20),s=i(1);o.prototype.stack=!0,o.prototype.select=function(){this.selected=!0,this.dirty=!0,this.displayed&&this.redraw()},o.prototype.unselect=function(){this.selected=!1,this.dirty=!0,this.displayed&&this.redraw()},o.prototype.setData=function(t){var e=void 0!=t.group&&this.data.group!=t.group;e&&this.parent.itemSet._moveToGroup(this,t.group),t.hasOwnProperty("editable")&&"boolean"==typeof t.editable&&(this.editable=t.editable),this.data=t,this.dirty=!0,this.displayed&&this.redraw()},o.prototype.setParent=function(t){this.displayed?(this.hide(),this.parent=t,this.parent&&this.show()):this.parent=t},o.prototype.isVisible=function(t){return!1},o.prototype.show=function(){return!1},o.prototype.hide=function(){return!1},o.prototype.redraw=function(){},o.prototype.repositionX=function(){},o.prototype.repositionY=function(){},o.prototype._repaintDeleteButton=function(t){var e=(this.options.editable.remove||this.data.editable===!0)&&this.data.editable!==!1;if(this.selected&&e&&!this.dom.deleteButton){var i=this,o=document.createElement("div");this.options.rtl?o.className="vis-delete-rtl":o.className="vis-delete",o.title="Delete this item",new n(o).on("tap",function(t){t.stopPropagation(),i.parent.removeFromDataSet(i)}),t.appendChild(o),this.dom.deleteButton=o}else!this.selected&&this.dom.deleteButton&&(this.dom.deleteButton.parentNode&&this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton),this.dom.deleteButton=null)},o.prototype._updateContents=function(t){var e;if(this.options.template){var i=this.parent.itemSet.itemsData.get(this.id);e=this.options.template(i)}else e=this.data.content;var o=this._contentToString(this.content)!==this._contentToString(e);if(o){if(e instanceof Element)t.innerHTML="",t.appendChild(e);else if(void 0!=e)t.innerHTML=e;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '+this.id);this.content=e}},o.prototype._updateTitle=function(t){null!=this.data.title?t.title=this.data.title||"":t.removeAttribute("vis-title")},o.prototype._updateDataAttributes=function(t){if(this.options.dataAttributes&&this.options.dataAttributes.length>0){var e=[];if(Array.isArray(this.options.dataAttributes))e=this.options.dataAttributes;else{if("all"!=this.options.dataAttributes)return;e=Object.keys(this.data)}for(var i=0;i<e.length;i++){var o=e[i],n=this.data[o];null!=n?t.setAttribute("data-"+o,n):t.removeAttribute("data-"+o)}}},o.prototype._updateStyle=function(t){this.style&&(s.removeCssText(t,this.style),this.style=null),this.data.style&&(s.addCssText(t,this.data.style),this.style=this.data.style)},o.prototype._contentToString=function(t){return"string"==typeof t?t:t&&"outerHTML"in t?t.outerHTML:t},o.prototype.getWidthLeft=function(){return 0},o.prototype.getWidthRight=function(){return 0},t.exports=o},function(t,e,i){function o(t,e,i){n.call(this,t,e,i),this.width=0,this.height=0,this.top=0,this.left=0}var n=(i(1),i(36));o.prototype=Object.create(n.prototype),o.prototype.redraw=function(t,e,i){var o=!1;this.visibleItems=this._updateVisibleItems(this.orderedItems,this.visibleItems,t),this.width=this.dom.background.offsetWidth,this.dom.background.style.height="0";for(var n=0,s=this.visibleItems.length;s>n;n++){var r=this.visibleItems[n];r.repositionY(e)}return o},o.prototype.show=function(){this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background)},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={dot:{width:0,height:0},line:{width:0,height:0}},this.options=i,t&&void 0==t.start)throw new Error('Property "start" missing in item '+t);n.call(this,t,e,i)}var n=i(39);i(1);o.prototype=new n(null,null,null),o.prototype.isVisible=function(t){var e=(t.end-t.start)/4;return this.data.start>t.start-e&&this.data.start<t.end+e},o.prototype.redraw=function(){var t=this.dom;if(t||(this.dom={},t=this.dom,t.box=document.createElement("DIV"),t.content=document.createElement("DIV"),t.content.className="vis-item-content",t.box.appendChild(t.content),t.line=document.createElement("DIV"),t.line.className="vis-line",t.dot=document.createElement("DIV"),t.dot.className="vis-dot",t.box["timeline-item"]=this,this.dirty=!0),!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!t.box.parentNode){var e=this.parent.dom.foreground;if(!e)throw new Error("Cannot redraw item: parent has no foreground container element");e.appendChild(t.box)}if(!t.line.parentNode){var i=this.parent.dom.background;if(!i)throw new Error("Cannot redraw item: parent has no background container element");i.appendChild(t.line)}if(!t.dot.parentNode){var o=this.parent.dom.axis;if(!i)throw new Error("Cannot redraw item: parent has no axis container element");o.appendChild(t.dot)}if(this.displayed=!0,this.dirty){this._updateContents(this.dom.content),this._updateTitle(this.dom.box),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var n=(this.options.editable.updateTime||this.options.editable.updateGroup||this.editable===!0)&&this.editable!==!1,s=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(n?" vis-editable":" vis-readonly");t.box.className="vis-item vis-box"+s,t.line.className="vis-item vis-line"+s,t.dot.className="vis-item vis-dot"+s,this.props.dot.height=t.dot.offsetHeight,this.props.dot.width=t.dot.offsetWidth,this.props.line.width=t.line.offsetWidth,this.width=t.box.offsetWidth,this.height=t.box.offsetHeight,this.dirty=!1}this._repaintDeleteButton(t.box)},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){if(this.displayed){var t=this.dom;t.box.parentNode&&t.box.parentNode.removeChild(t.box),t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot),this.displayed=!1}},o.prototype.repositionX=function(){var t=this.conversion.toScreen(this.data.start),e=this.options.align;"right"==e?this.options.rtl?(this.right=t-this.width,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t-this.props.line.width+"px",this.dom.dot.style.right=t-this.props.line.width/2-this.props.dot.width/2+"px"):(this.left=t-this.width,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t-this.props.line.width+"px",this.dom.dot.style.left=t-this.props.line.width/2-this.props.dot.width/2+"px"):"left"==e?this.options.rtl?(this.right=t,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t+"px",this.dom.dot.style.right=t+this.props.line.width/2-this.props.dot.width/2+"px"):(this.left=t,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t+"px",this.dom.dot.style.left=t+this.props.line.width/2-this.props.dot.width/2+"px"):this.options.rtl?(this.right=t-this.width/2,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t-this.props.line.width+"px",this.dom.dot.style.right=t-this.props.dot.width/2+"px"):(this.left=t-this.width/2,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t-this.props.line.width/2+"px",this.dom.dot.style.left=t-this.props.dot.width/2+"px")},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.box,i=this.dom.line,o=this.dom.dot;if("top"==t)e.style.top=(this.top||0)+"px",i.style.top="0",i.style.height=this.parent.top+this.top+1+"px",i.style.bottom="";else{var n=this.parent.itemSet.props.height,s=n-this.parent.top-this.parent.height+this.top;e.style.top=(this.parent.height-this.top-this.height||0)+"px",i.style.top=n-s+"px",i.style.bottom="0"}o.style.top=-this.props.dot.height/2+"px"},o.prototype.getWidthLeft=function(){return this.width/2},o.prototype.getWidthRight=function(){return this.width/2},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={dot:{top:0,width:0,height:0},content:{height:0,marginLeft:0,marginRight:0}},this.options=i,t&&void 0==t.start)throw new Error('Property "start" missing in item '+t);n.call(this,t,e,i)}var n=i(39);o.prototype=new n(null,null,null),o.prototype.isVisible=function(t){var e=(t.end-t.start)/4;return this.data.start>t.start-e&&this.data.start<t.end+e},o.prototype.redraw=function(){var t=this.dom;if(t||(this.dom={},t=this.dom,t.point=document.createElement("div"),t.content=document.createElement("div"),t.content.className="vis-item-content",t.point.appendChild(t.content),t.dot=document.createElement("div"),t.point.appendChild(t.dot),t.point["timeline-item"]=this,this.dirty=!0),!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!t.point.parentNode){var e=this.parent.dom.foreground;if(!e)throw new Error("Cannot redraw item: parent has no foreground container element");e.appendChild(t.point)}if(this.displayed=!0,this.dirty){this._updateContents(this.dom.content),this._updateTitle(this.dom.point),this._updateDataAttributes(this.dom.point),this._updateStyle(this.dom.point);var i=(this.options.editable.updateTime||this.options.editable.updateGroup||this.editable===!0)&&this.editable!==!1,o=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(i?" vis-editable":" vis-readonly");t.point.className="vis-item vis-point"+o,t.dot.className="vis-item vis-dot"+o,this.props.dot.width=t.dot.offsetWidth,this.props.dot.height=t.dot.offsetHeight,this.props.content.height=t.content.offsetHeight,this.options.rtl?t.content.style.marginRight=2*this.props.dot.width+"px":t.content.style.marginLeft=2*this.props.dot.width+"px",this.width=t.point.offsetWidth,this.height=t.point.offsetHeight,t.dot.style.top=(this.height-this.props.dot.height)/2+"px",this.options.rtl?t.dot.style.right=this.props.dot.width/2+"px":t.dot.style.left=this.props.dot.width/2+"px",this.dirty=!1}this._repaintDeleteButton(t.point)},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){this.displayed&&(this.dom.point.parentNode&&this.dom.point.parentNode.removeChild(this.dom.point),this.displayed=!1)},o.prototype.repositionX=function(){var t=this.conversion.toScreen(this.data.start);this.options.rtl?(this.right=t-this.props.dot.width,this.dom.point.style.right=this.right+"px"):(this.left=t-this.props.dot.width,this.dom.point.style.left=this.left+"px")},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.point;"top"==t?e.style.top=this.top+"px":e.style.top=this.parent.height-this.top-this.height+"px"},o.prototype.getWidthLeft=function(){return this.props.dot.width},o.prototype.getWidthRight=function(){return this.props.dot.width},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={content:{width:0}},this.overflow=!1,t){if(void 0==t.start)throw new Error('Property "start" missing in item '+t.id);if(void 0==t.end)throw new Error('Property "end" missing in item '+t.id)}n.call(this,t,e,i)}var n=(i(20),i(39)),s=i(40),r=i(38);o.prototype=new n(null,null,null),o.prototype.baseClassName="vis-item vis-background",o.prototype.stack=!1,o.prototype.isVisible=function(t){return this.data.start<t.end&&this.data.end>t.start},o.prototype.redraw=function(){var t=this.dom;if(t||(this.dom={},t=this.dom,t.box=document.createElement("div"),t.frame=document.createElement("div"),t.frame.className="vis-item-overflow",t.box.appendChild(t.frame),t.content=document.createElement("div"),t.content.className="vis-item-content",t.frame.appendChild(t.content),this.dirty=!0),!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!t.box.parentNode){var e=this.parent.dom.background;if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(t.box)}if(this.displayed=!0,this.dirty){this._updateContents(this.dom.content),this._updateTitle(this.dom.content),this._updateDataAttributes(this.dom.content),this._updateStyle(this.dom.box);var i=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"");t.box.className=this.baseClassName+i,this.overflow="hidden"!==window.getComputedStyle(t.content).overflow,this.props.content.width=this.dom.content.offsetWidth,this.height=0,this.dirty=!1}},o.prototype.show=r.prototype.show,o.prototype.hide=r.prototype.hide,o.prototype.repositionX=r.prototype.repositionX,o.prototype.repositionY=function(t){var e="top"===this.options.orientation.item;this.dom.content.style.top=e?"":"0",this.dom.content.style.bottom=e?"0":"";var i;if(void 0!==this.data.subgroup){var o=this.data.subgroup,n=this.parent.subgroups,r=n[o].index;if(1==e){i=this.parent.subgroups[o].height+t.item.vertical,i+=0==r?t.axis-.5*t.item.vertical:0;var a=this.parent.top;for(var h in n)n.hasOwnProperty(h)&&1==n[h].visible&&n[h].index<r&&(a+=n[h].height+t.item.vertical);a+=0!=r?t.axis-.5*t.item.vertical:0,this.dom.box.style.top=a+"px",this.dom.box.style.bottom=""}else{var a=this.parent.top,d=0;for(var h in n)if(n.hasOwnProperty(h)&&1==n[h].visible){var l=n[h].height+t.item.vertical;d+=l,n[h].index>r&&(a+=l)}i=this.parent.subgroups[o].height+t.item.vertical,this.dom.box.style.top=this.parent.height-d+a+"px",this.dom.box.style.bottom=""}}else this.parent instanceof s?(i=Math.max(this.parent.height,this.parent.itemSet.body.domProps.center.height,this.parent.itemSet.body.domProps.centerContainer.height),this.dom.box.style.top=e?"0":"",this.dom.box.style.bottom=e?"":"0"):(i=this.parent.height,this.dom.box.style.top=this.parent.top+"px",this.dom.box.style.bottom="");this.dom.box.style.height=i+"px"},t.exports=o},function(t,e,i){function o(t,e){this.dom={foreground:null,lines:[],majorTexts:[],minorTexts:[],redundant:{lines:[],majorTexts:[],minorTexts:[]}},this.props={range:{start:0,end:0,minimumStep:0},lineTop:0},this.defaultOptions={orientation:{axis:"bottom"},showMinorLabels:!0,showMajorLabels:!0,maxMinorChars:7,format:a.FORMAT,moment:d,timeAxis:null},this.options=s.extend({},this.defaultOptions),this.body=t,this._create(),this.setOptions(e)}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(1),r=i(31),a=i(35),h=i(32),d=i(2);o.prototype=new r,o.prototype.setOptions=function(t){t&&(s.selectiveExtend(["showMinorLabels","showMajorLabels","maxMinorChars","hiddenDates","timeAxis","moment","rtl"],this.options,t),s.selectiveDeepExtend(["format"],this.options,t),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.axis=t.orientation:"object"===n(t.orientation)&&"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis)),"locale"in t&&("function"==typeof d.locale?d.locale(t.locale):d.lang(t.locale)))},o.prototype._create=function(){this.dom.foreground=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.foreground.className="vis-time-axis vis-foreground",this.dom.background.className="vis-time-axis vis-background"},o.prototype.destroy=function(){this.dom.foreground.parentNode&&this.dom.foreground.parentNode.removeChild(this.dom.foreground),this.dom.background.parentNode&&this.dom.background.parentNode.removeChild(this.dom.background),this.body=null},o.prototype.redraw=function(){var t=this.props,e=this.dom.foreground,i=this.dom.background,o="top"==this.options.orientation.axis?this.body.dom.top:this.body.dom.bottom,n=e.parentNode!==o;this._calculateCharSize();var s=this.options.showMinorLabels&&"none"!==this.options.orientation.axis,r=this.options.showMajorLabels&&"none"!==this.options.orientation.axis;t.minorLabelHeight=s?t.minorCharHeight:0,t.majorLabelHeight=r?t.majorCharHeight:0,t.height=t.minorLabelHeight+t.majorLabelHeight,t.width=e.offsetWidth,t.minorLineHeight=this.body.domProps.root.height-t.majorLabelHeight-("top"==this.options.orientation.axis?this.body.domProps.bottom.height:this.body.domProps.top.height),t.minorLineWidth=1,t.majorLineHeight=t.minorLineHeight+t.majorLabelHeight,t.majorLineWidth=1;var a=e.nextSibling,h=i.nextSibling;return e.parentNode&&e.parentNode.removeChild(e),i.parentNode&&i.parentNode.removeChild(i),e.style.height=this.props.height+"px",this._repaintLabels(),a?o.insertBefore(e,a):o.appendChild(e),h?this.body.dom.backgroundVertical.insertBefore(i,h):this.body.dom.backgroundVertical.appendChild(i),this._isResized()||n},o.prototype._repaintLabels=function(){var t=this.options.orientation.axis,e=s.convert(this.body.range.start,"Number"),i=s.convert(this.body.range.end,"Number"),o=this.body.util.toTime((this.props.minorCharWidth||10)*this.options.maxMinorChars).valueOf(),n=o-h.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this.body.range,o);n-=this.body.util.toTime(0).valueOf();var r=new a(new Date(e),new Date(i),n,this.body.hiddenDates);r.setMoment(this.options.moment),this.options.format&&r.setFormat(this.options.format),this.options.timeAxis&&r.setScale(this.options.timeAxis),this.step=r;var d=this.dom;d.redundant.lines=d.lines,d.redundant.majorTexts=d.majorTexts,d.redundant.minorTexts=d.minorTexts,d.lines=[],d.majorTexts=[],d.minorTexts=[];var c,u,p,f,m,v,g,y,b,w,_=0,x=void 0,k=0,O=1e3;for(r.start(),u=r.getCurrent(),f=this.body.util.toScreen(u);r.hasNext()&&O>k;){k++,m=r.isMajor(),w=r.getClassName(),b=r.getLabelMinor(),c=u,p=f,r.next(),u=r.getCurrent(),v=r.isMajor(),f=this.body.util.toScreen(u),g=_,_=f-p;var M=_>=.4*g;if(this.options.showMinorLabels&&M){var D=this._repaintMinorText(p,b,t,w);D.style.width=_+"px"}m&&this.options.showMajorLabels?(p>0&&(void 0==x&&(x=p),D=this._repaintMajorText(p,r.getLabelMajor(),t,w)),y=this._repaintMajorLine(p,_,t,w)):M?y=this._repaintMinorLine(p,_,t,w):y&&(y.style.width=parseInt(y.style.width)+_+"px")}if(k!==O||l||(console.warn("Something is wrong with the Timeline scale. Limited drawing of grid lines to "+O+" lines."),l=!0),this.options.showMajorLabels){var S=this.body.util.toTime(0),C=r.getLabelMajor(S),T=C.length*(this.props.majorCharWidth||10)+10;(void 0==x||x>T)&&this._repaintMajorText(0,C,t,w)}s.forEach(this.dom.redundant,function(t){for(;t.length;){var e=t.pop();e&&e.parentNode&&e.parentNode.removeChild(e)}})},o.prototype._repaintMinorText=function(t,e,i,o){var n=this.dom.redundant.minorTexts.shift();if(!n){var s=document.createTextNode("");n=document.createElement("div"),n.appendChild(s),this.dom.foreground.appendChild(n)}return this.dom.minorTexts.push(n),n.childNodes[0].nodeValue=e,n.style.top="top"==i?this.props.majorLabelHeight+"px":"0",this.options.rtl?(n.style.left="",n.style.right=t+"px"):n.style.left=t+"px",n.className="vis-text vis-minor "+o,n},o.prototype._repaintMajorText=function(t,e,i,o){var n=this.dom.redundant.majorTexts.shift();if(!n){var s=document.createTextNode(e);n=document.createElement("div"),n.appendChild(s),this.dom.foreground.appendChild(n)}return this.dom.majorTexts.push(n),n.childNodes[0].nodeValue=e,n.className="vis-text vis-major "+o,n.style.top="top"==i?"0":this.props.minorLabelHeight+"px",this.options.rtl?(n.style.left="",n.style.right=t+"px"):n.style.left=t+"px",n},o.prototype._repaintMinorLine=function(t,e,i,o){var n=this.dom.redundant.lines.shift();n||(n=document.createElement("div"),this.dom.background.appendChild(n)),this.dom.lines.push(n);var s=this.props;return"top"==i?n.style.top=s.majorLabelHeight+"px":n.style.top=this.body.domProps.top.height+"px",n.style.height=s.minorLineHeight+"px",this.options.rtl?(n.style.left="",n.style.right=t-s.minorLineWidth/2+"px",n.className="vis-grid vis-vertical-rtl vis-minor "+o):(n.style.left=t-s.minorLineWidth/2+"px",n.className="vis-grid vis-vertical vis-minor "+o),n.style.width=e+"px",n},o.prototype._repaintMajorLine=function(t,e,i,o){var n=this.dom.redundant.lines.shift();n||(n=document.createElement("div"),this.dom.background.appendChild(n)),this.dom.lines.push(n);var s=this.props;return"top"==i?n.style.top="0":n.style.top=this.body.domProps.top.height+"px",this.options.rtl?(n.style.left="",n.style.right=t-s.majorLineWidth/2+"px",n.className="vis-grid vis-vertical-rtl vis-major "+o):(n.style.left=t-s.majorLineWidth/2+"px",n.className="vis-grid vis-vertical vis-major "+o),n.style.height=s.majorLineHeight+"px",n.style.width=e+"px",n},o.prototype._calculateCharSize=function(){this.dom.measureCharMinor||(this.dom.measureCharMinor=document.createElement("DIV"),this.dom.measureCharMinor.className="vis-text vis-minor vis-measure",this.dom.measureCharMinor.style.position="absolute",this.dom.measureCharMinor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMinor)),this.props.minorCharHeight=this.dom.measureCharMinor.clientHeight,this.props.minorCharWidth=this.dom.measureCharMinor.clientWidth,this.dom.measureCharMajor||(this.dom.measureCharMajor=document.createElement("DIV"),this.dom.measureCharMajor.className="vis-text vis-major vis-measure",this.dom.measureCharMajor.style.position="absolute",this.dom.measureCharMajor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMajor)),this.props.majorCharHeight=this.dom.measureCharMajor.clientHeight,this.props.majorCharWidth=this.dom.measureCharMajor.clientWidth};var l=!1;t.exports=o},function(t,e,i){function o(t){this.active=!1,this.dom={container:t},this.dom.overlay=document.createElement("div"),this.dom.overlay.className="vis-overlay",this.dom.container.appendChild(this.dom.overlay),this.hammer=a(this.dom.overlay),this.hammer.on("tap",this._onTapOverlay.bind(this));var e=this,i=["tap","doubletap","press","pinch","pan","panstart","panmove","panend"];i.forEach(function(t){e.hammer.on(t,function(t){t.stopPropagation()})}),document&&document.body&&(this.onClick=function(i){n(i.target,t)||e.deactivate()},document.body.addEventListener("click",this.onClick)),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=s(),this.escListener=this.deactivate.bind(this)}function n(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1}var s=i(23),r=i(13),a=i(20),h=i(1);r(o.prototype),o.current=null,o.prototype.destroy=function(){this.deactivate(),this.dom.overlay.parentNode.removeChild(this.dom.overlay),this.onClick&&document.body.removeEventListener("click",this.onClick),this.hammer.destroy(),this.hammer=null},o.prototype.activate=function(){o.current&&o.current.deactivate(),o.current=this,this.active=!0,this.dom.overlay.style.display="none",h.addClassName(this.dom.container,"vis-active"),this.emit("change"),this.emit("activate"),this.keycharm.bind("esc",this.escListener)},o.prototype.deactivate=function(){this.active=!1,this.dom.overlay.style.display="",h.removeClassName(this.dom.container,"vis-active"),this.keycharm.unbind("esc",this.escListener),this.emit("change"),this.emit("deactivate")},o.prototype._onTapOverlay=function(t){this.activate(),t.stopPropagation()},t.exports=o},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={moment:a,locales:h,locale:"en",id:void 0,title:void 0},this.options=s.extend({},this.defaultOptions),e&&e.time?this.customTime=e.time:this.customTime=new Date,this.eventParams={},this.setOptions(e),this._create()}var n=i(20),s=i(1),r=i(31),a=i(2),h=i(47);o.prototype=new r,o.prototype.setOptions=function(t){t&&s.selectiveExtend(["moment","locale","locales","id"],this.options,t)},o.prototype._create=function(){var t=document.createElement("div");t["custom-time"]=this,t.className="vis-custom-time "+(this.options.id||""),t.style.position="absolute",t.style.top="0px",t.style.height="100%",this.bar=t;var e=document.createElement("div");e.style.position="relative",e.style.top="0px",e.style.left="-10px",e.style.height="100%",e.style.width="20px",t.appendChild(e),this.hammer=new n(e),this.hammer.on("panstart",this._onDragStart.bind(this)),this.hammer.on("panmove",this._onDrag.bind(this)),this.hammer.on("panend",this._onDragEnd.bind(this)),this.hammer.get("pan").set({threshold:5,direction:n.DIRECTION_HORIZONTAL})},o.prototype.destroy=function(){this.hide(),this.hammer.destroy(),this.hammer=null,this.body=null},o.prototype.redraw=function(){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar));var e=this.body.util.toScreen(this.customTime),i=this.options.locales[this.options.locale];i||(this.warned||(console.log("WARNING: options.locales['"+this.options.locale+"'] not found. See http://visjs.org/docs/timeline.html#Localization"),this.warned=!0),i=this.options.locales.en);var o=this.options.title;return void 0===o&&(o=i.time+": "+this.options.moment(this.customTime).format("dddd, MMMM Do YYYY, H:mm:ss"),o=o.charAt(0).toUpperCase()+o.substring(1)),this.bar.style.left=e+"px",this.bar.title=o,!1},o.prototype.hide=function(){this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar)},o.prototype.setCustomTime=function(t){this.customTime=s.convert(t,"Date"),this.redraw()},o.prototype.getCustomTime=function(){return new Date(this.customTime.valueOf())},o.prototype.setCustomTitle=function(t){this.options.title=t},o.prototype._onDragStart=function(t){this.eventParams.dragging=!0,this.eventParams.customTime=this.customTime,t.stopPropagation()},o.prototype._onDrag=function(t){if(this.eventParams.dragging){var e=this.body.util.toScreen(this.eventParams.customTime)+t.deltaX,i=this.body.util.toTime(e);this.setCustomTime(i),this.body.emitter.emit("timechange",{id:this.options.id,time:new Date(this.customTime.valueOf())
-}),t.stopPropagation()}},o.prototype._onDragEnd=function(t){this.eventParams.dragging&&(this.body.emitter.emit("timechanged",{id:this.options.id,time:new Date(this.customTime.valueOf())}),t.stopPropagation())},o.customTimeFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("custom-time"))return e["custom-time"];e=e.parentNode}return null},t.exports=o},function(t,e){e.en={current:"current",time:"time"},e.en_EN=e.en,e.en_US=e.en,e.nl={current:"huidige",time:"tijd"},e.nl_NL=e.nl,e.nl_BE=e.nl},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={rtl:!1,showCurrentTime:!0,moment:r,locales:a,locale:"en"},this.options=n.extend({},this.defaultOptions),this.offset=0,this._create(),this.setOptions(e)}var n=i(1),s=i(31),r=i(2),a=i(47);o.prototype=new s,o.prototype._create=function(){var t=document.createElement("div");t.className="vis-current-time",t.style.position="absolute",t.style.top="0px",t.style.height="100%",this.bar=t},o.prototype.destroy=function(){this.options.showCurrentTime=!1,this.redraw(),this.body=null},o.prototype.setOptions=function(t){t&&n.selectiveExtend(["rtl","showCurrentTime","moment","locale","locales"],this.options,t)},o.prototype.redraw=function(){if(this.options.showCurrentTime){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar),this.start());var e=this.options.moment((new Date).valueOf()+this.offset),i=this.body.util.toScreen(e),o=this.options.locales[this.options.locale];o||(this.warned||(console.log("WARNING: options.locales['"+this.options.locale+"'] not found. See http://visjs.org/docs/timeline/#Localization"),this.warned=!0),o=this.options.locales.en);var n=o.current+" "+o.time+": "+e.format("dddd, MMMM Do YYYY, H:mm:ss");n=n.charAt(0).toUpperCase()+n.substring(1),this.options.rtl?this.bar.style.right=i+"px":this.bar.style.left=i+"px",this.bar.title=n}else this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),this.stop();return!1},o.prototype.start=function(){function t(){e.stop();var i=e.body.range.conversion(e.body.domProps.center.width).scale,o=1/i/10;30>o&&(o=30),o>1e3&&(o=1e3),e.redraw(),e.body.emitter.emit("currentTimeTick"),e.currentTimeTimer=setTimeout(t,o)}var e=this;t()},o.prototype.stop=function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),delete this.currentTimeTimer)},o.prototype.setCurrentTime=function(t){var e=n.convert(t,"Date").valueOf(),i=(new Date).valueOf();this.offset=e-i,this.redraw()},o.prototype.getCurrentTime=function(){return new Date((new Date).valueOf()+this.offset)},t.exports=o},function(t,e){Object.defineProperty(e,"__esModule",{value:!0});var i="string",o="boolean",n="number",s="array",r="date",a="object",h="dom",d="moment",l="any",c={configure:{enabled:{"boolean":o},filter:{"boolean":o,"function":"function"},container:{dom:h},__type__:{object:a,"boolean":o,"function":"function"}},align:{string:i},rtl:{"boolean":o,undefined:"undefined"},autoResize:{"boolean":o},throttleRedraw:{number:n},clickToUse:{"boolean":o},dataAttributes:{string:i,array:s},editable:{add:{"boolean":o,undefined:"undefined"},remove:{"boolean":o,undefined:"undefined"},updateGroup:{"boolean":o,undefined:"undefined"},updateTime:{"boolean":o,undefined:"undefined"},__type__:{"boolean":o,object:a}},end:{number:n,date:r,string:i,moment:d},format:{minorLabels:{millisecond:{string:i,undefined:"undefined"},second:{string:i,undefined:"undefined"},minute:{string:i,undefined:"undefined"},hour:{string:i,undefined:"undefined"},weekday:{string:i,undefined:"undefined"},day:{string:i,undefined:"undefined"},month:{string:i,undefined:"undefined"},year:{string:i,undefined:"undefined"},__type__:{object:a}},majorLabels:{millisecond:{string:i,undefined:"undefined"},second:{string:i,undefined:"undefined"},minute:{string:i,undefined:"undefined"},hour:{string:i,undefined:"undefined"},weekday:{string:i,undefined:"undefined"},day:{string:i,undefined:"undefined"},month:{string:i,undefined:"undefined"},year:{string:i,undefined:"undefined"},__type__:{object:a}},__type__:{object:a}},moment:{"function":"function"},groupOrder:{string:i,"function":"function"},groupEditable:{add:{"boolean":o,undefined:"undefined"},remove:{"boolean":o,undefined:"undefined"},order:{"boolean":o,undefined:"undefined"},__type__:{"boolean":o,object:a}},groupOrderSwap:{"function":"function"},height:{string:i,number:n},hiddenDates:{start:{date:r,number:n,string:i,moment:d},end:{date:r,number:n,string:i,moment:d},repeat:{string:i},__type__:{object:a,array:s}},itemsAlwaysDraggable:{"boolean":o},locale:{string:i},locales:{__any__:{any:l},__type__:{object:a}},margin:{axis:{number:n},item:{horizontal:{number:n,undefined:"undefined"},vertical:{number:n,undefined:"undefined"},__type__:{object:a,number:n}},__type__:{object:a,number:n}},max:{date:r,number:n,string:i,moment:d},maxHeight:{number:n,string:i},maxMinorChars:{number:n},min:{date:r,number:n,string:i,moment:d},minHeight:{number:n,string:i},moveable:{"boolean":o},multiselect:{"boolean":o},multiselectPerGroup:{"boolean":o},onAdd:{"function":"function"},onUpdate:{"function":"function"},onMove:{"function":"function"},onMoving:{"function":"function"},onRemove:{"function":"function"},onAddGroup:{"function":"function"},onMoveGroup:{"function":"function"},onRemoveGroup:{"function":"function"},order:{"function":"function"},orientation:{axis:{string:i,undefined:"undefined"},item:{string:i,undefined:"undefined"},__type__:{string:i,object:a}},selectable:{"boolean":o},showCurrentTime:{"boolean":o},showMajorLabels:{"boolean":o},showMinorLabels:{"boolean":o},stack:{"boolean":o},snap:{"function":"function","null":"null"},start:{date:r,number:n,string:i,moment:d},template:{"function":"function"},groupTemplate:{"function":"function"},timeAxis:{scale:{string:i,undefined:"undefined"},step:{number:n,undefined:"undefined"},__type__:{object:a}},type:{string:i},width:{string:i,number:n},zoomable:{"boolean":o},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:n},zoomMin:{number:n},__type__:{object:a}},u={global:{align:["center","left","right"],direction:!1,autoResize:!0,throttleRedraw:[10,0,1e3,10],clickToUse:!1,editable:{add:!1,remove:!1,updateGroup:!1,updateTime:!1},end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",month:"YYYY",year:""}},groupsDraggable:!1,height:"",locale:"",margin:{axis:[20,0,100,1],item:{horizontal:[10,0,100,1],vertical:[10,0,100,1]}},max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!1,multiselect:!1,multiselectPerGroup:!1,orientation:{axis:["both","bottom","top"],item:["bottom","top"]},selectable:!0,showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,stack:!0,start:"",type:["box","point","range","background"],width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1]}};e.allOptions=c,e.configureOptions=u},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e,i,o){if(!(Array.isArray(i)||i instanceof c||i instanceof u)&&i instanceof Object){var n=o;o=i,i=n}var s=this;this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:d,width:null,height:null,maxHeight:null,minHeight:null},this.options=l.deepExtend({},this.defaultOptions),this._create(t),this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this)},hiddenDates:[],util:{toScreen:s._toScreen.bind(s),toGlobalScreen:s._toGlobalScreen.bind(s),toTime:s._toTime.bind(s),toGlobalTime:s._toGlobalTime.bind(s)}},this.range=new p(this.body),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new m(this.body),this.components.push(this.timeAxis),this.currentTime=new v(this.body),this.components.push(this.currentTime),this.linegraph=new y(this.body),this.components.push(this.linegraph),this.itemsData=null,this.groupsData=null,this.on("tap",function(t){s.emit("click",s.getEventProperties(t))}),this.on("doubletap",function(t){s.emit("doubleClick",s.getEventProperties(t))}),this.dom.root.oncontextmenu=function(t){s.emit("contextmenu",s.getEventProperties(t))},o&&this.setOptions(o),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}var s=i(26),r=o(s),a=i(29),h=o(a),d=(i(13),i(20),i(2)),l=i(1),c=i(9),u=i(11),p=i(30),f=i(33),m=i(44),v=i(48),g=i(46),y=i(51),b=i(29).printStyle,w=i(59).allOptions,_=i(59).configureOptions;n.prototype=new f,n.prototype.setOptions=function(t){var e=h["default"].validate(t,w);e===!0&&console.log("%cErrors have been found in the supplied options object.",b),f.prototype.setOptions.call(this,t)},n.prototype.setItems=function(t){var e,i=null==this.itemsData;if(e=t?t instanceof c||t instanceof u?t:new c(t,{type:{start:"Date",end:"Date"}}):null,this.itemsData=e,this.linegraph&&this.linegraph.setItems(e),i)if(void 0!=this.options.start||void 0!=this.options.end){var o=void 0!=this.options.start?this.options.start:null,n=void 0!=this.options.end?this.options.end:null;this.setWindow(o,n,{animation:!1})}else this.fit({animation:!1})},n.prototype.setGroups=function(t){var e;e=t?t instanceof c||t instanceof u?t:new c(t):null,this.groupsData=e,this.linegraph.setGroups(e)},n.prototype.getLegend=function(t,e,i){return void 0===e&&(e=15),void 0===i&&(i=15),void 0!==this.linegraph.groups[t]?this.linegraph.groups[t].getLegend(e,i):"cannot find group:'"+t+"'"},n.prototype.isGroupVisible=function(t){return void 0!==this.linegraph.groups[t]?this.linegraph.groups[t].visible&&(void 0===this.linegraph.options.groups.visibility[t]||1==this.linegraph.options.groups.visibility[t]):!1},n.prototype.getDataRange=function(){var t=null,e=null;for(var i in this.linegraph.groups)if(this.linegraph.groups.hasOwnProperty(i)&&1==this.linegraph.groups[i].visible)for(var o=0;o<this.linegraph.groups[i].itemsData.length;o++){var n=this.linegraph.groups[i].itemsData[o],s=l.convert(n.x,"Date").valueOf();t=null==t?s:t>s?s:t,e=null==e?s:s>e?s:e}return{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},n.prototype.getEventProperties=function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,o=e-l.getAbsoluteLeft(this.dom.centerContainer),n=i-l.getAbsoluteTop(this.dom.centerContainer),s=this._toTime(o),r=g.customTimeFromTarget(t),a=l.getTarget(t),h=null;l.hasParent(a,this.timeAxis.dom.foreground)?h="axis":this.timeAxis2&&l.hasParent(a,this.timeAxis2.dom.foreground)?h="axis":l.hasParent(a,this.linegraph.yAxisLeft.dom.frame)?h="data-axis":l.hasParent(a,this.linegraph.yAxisRight.dom.frame)?h="data-axis":l.hasParent(a,this.linegraph.legendLeft.dom.frame)?h="legend":l.hasParent(a,this.linegraph.legendRight.dom.frame)?h="legend":null!=r?h="custom-time":l.hasParent(a,this.currentTime.bar)?h="current-time":l.hasParent(a,this.dom.center)&&(h="background");var d=[],c=this.linegraph.yAxisLeft,u=this.linegraph.yAxisRight;return c.hidden||d.push(c.screenToValue(n)),u.hidden||d.push(u.screenToValue(n)),{event:t,what:h,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:o,y:n,time:s,value:d}},n.prototype._createConfigurator=function(){return new r["default"](this,this.dom.container,_)},t.exports=n},function(t,e,i){function o(t,e){this.id=s.randomUUID(),this.body=t,this.defaultOptions={yAxisOrientation:"left",defaultGroup:"default",sort:!0,sampling:!0,stack:!1,graphHeight:"400px",shaded:{enabled:!1,orientation:"bottom"},style:"line",barChart:{width:50,sideBySide:!1,align:"center"},interpolation:{enabled:!0,parametrization:"centripetal",alpha:.5},drawPoints:{enabled:!0,size:6,style:"square"},dataAxis:{},legend:{},groups:{visibility:{}}},this.options=s.extend({},this.defaultOptions),this.dom={},this.props={},this.hammer=null,this.groups={},this.abortedGraphUpdate=!1,this.updateSVGheight=!1,this.updateSVGheightOnResize=!1,this.forceGraphUpdate=!0;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,o){i._onAdd(e.items)},update:function(t,e,o){i._onUpdate(e.items)},remove:function(t,e,o){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,o){i._onAddGroups(e.items)},update:function(t,e,o){i._onUpdateGroups(e.items)},remove:function(t,e,o){i._onRemoveGroups(e.items)}},this.items={},this.selection=[],this.lastStart=this.body.range.start,this.touchParams={},this.svgElements={},this.setOptions(e),this.groupsUsingDefaultStyles=[0],this.body.emitter.on("rangechanged",function(){i.lastStart=i.body.range.start,i.svg.style.left=s.option.asSize(-i.props.width),i.forceGraphUpdate=!0,i.redraw.call(i)}),this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups}}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(1),r=i(8),a=i(9),h=i(11),d=i(31),l=i(52),c=i(54),u=i(58),p=i(55),f=i(57),m=i(56),v="__ungrouped__";o.prototype=new d,o.prototype._create=function(){var t=document.createElement("div");t.className="vis-line-graph",this.dom.frame=t,this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="relative",this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",this.svg.style.display="block",t.appendChild(this.svg),this.options.dataAxis.orientation="left",this.yAxisLeft=new l(this.body,this.options.dataAxis,this.svg,this.options.groups),this.options.dataAxis.orientation="right",this.yAxisRight=new l(this.body,this.options.dataAxis,this.svg,this.options.groups),delete this.options.dataAxis.orientation,this.legendLeft=new u(this.body,this.options.legend,"left",this.options.groups),this.legendRight=new u(this.body,this.options.legend,"right",this.options.groups),this.show()},o.prototype.setOptions=function(t){if(t){var e=["sampling","defaultGroup","stack","height","graphHeight","yAxisOrientation","style","barChart","dataAxis","sort","groups"];void 0===t.graphHeight&&void 0!==t.height?(this.updateSVGheight=!0,this.updateSVGheightOnResize=!0):void 0!==this.body.domProps.centerContainer.height&&void 0!==t.graphHeight&&parseInt((t.graphHeight+"").replace("px",""))<this.body.domProps.centerContainer.height&&(this.updateSVGheight=!0),s.selectiveDeepExtend(e,this.options,t),s.mergeOptions(this.options,t,"interpolation"),s.mergeOptions(this.options,t,"drawPoints"),s.mergeOptions(this.options,t,"shaded"),s.mergeOptions(this.options,t,"legend"),t.interpolation&&"object"==n(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5)),this.yAxisLeft&&void 0!==t.dataAxis&&(this.yAxisLeft.setOptions(this.options.dataAxis),this.yAxisRight.setOptions(this.options.dataAxis)),this.legendLeft&&void 0!==t.legend&&(this.legendLeft.setOptions(this.options.legend),this.legendRight.setOptions(this.options.legend)),this.groups.hasOwnProperty(v)&&this.groups[v].setOptions(t)}this.dom.frame&&(this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0}))},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},o.prototype.setItems=function(t){var e,i=this,o=this.itemsData;if(t){if(!(t instanceof a||t instanceof h))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=t}else this.itemsData=null;if(o&&(s.forEach(this.itemListeners,function(t,e){o.off(e,t)}),e=o.getIds(),this._onRemove(e)),this.itemsData){var n=this.id;s.forEach(this.itemListeners,function(t,e){i.itemsData.on(e,t,n)}),e=this.itemsData.getIds(),this._onAdd(e)}},o.prototype.setGroups=function(t){var e,i=this;if(this.groupsData){s.forEach(this.groupListeners,function(t,e){i.groupsData.off(e,t)}),e=this.groupsData.getIds(),this.groupsData=null;for(var o=0;o<e.length;o++)this._removeGroup(e[o])}if(t){if(!(t instanceof a||t instanceof h))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var n=this.id;s.forEach(this.groupListeners,function(t,e){i.groupsData.on(e,t,n)}),e=this.groupsData.getIds(),this._onAddGroups(e)}},o.prototype._onUpdate=function(t){this._updateAllGroupData()},o.prototype._onAdd=function(t){this._onUpdate(t)},o.prototype._onRemove=function(t){this._onUpdate(t)},o.prototype._onUpdateGroups=function(t){this._updateAllGroupData()},o.prototype._onAddGroups=function(t){this._onUpdateGroups(t)},o.prototype._onRemoveGroups=function(t){for(var e=0;e<t.length;e++)this._removeGroup(t[e]);this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})},o.prototype._removeGroup=function(t){this.groups.hasOwnProperty(t)&&("right"==this.groups[t].options.yAxisOrientation?(this.yAxisRight.removeGroup(t),this.legendRight.removeGroup(t),this.legendRight.redraw()):(this.yAxisLeft.removeGroup(t),this.legendLeft.removeGroup(t),this.legendLeft.redraw()),delete this.groups[t])},o.prototype._updateGroup=function(t,e){this.groups.hasOwnProperty(e)?(this.groups[e].update(t),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.updateGroup(e,this.groups[e]),this.legendRight.updateGroup(e,this.groups[e]),this.yAxisLeft.removeGroup(e),this.legendLeft.removeGroup(e)):(this.yAxisLeft.updateGroup(e,this.groups[e]),this.legendLeft.updateGroup(e,this.groups[e]),this.yAxisRight.removeGroup(e),this.legendRight.removeGroup(e))):(this.groups[e]=new c(t,e,this.options,this.groupsUsingDefaultStyles),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.addGroup(e,this.groups[e]),this.legendRight.addGroup(e,this.groups[e])):(this.yAxisLeft.addGroup(e,this.groups[e]),this.legendLeft.addGroup(e,this.groups[e]))),this.legendLeft.redraw(),this.legendRight.redraw()},o.prototype._updateAllGroupData=function(){if(null!=this.itemsData){for(var t={},e=this.itemsData.get(),i={},o=0;o<e.length;o++){var n=e[o],r=n.group;null!==r&&void 0!==r||(r=v),i.hasOwnProperty(r)?i[r]++:i[r]=1}for(var o=0;o<e.length;o++){var n=e[o],r=n.group;null!==r&&void 0!==r||(r=v),t.hasOwnProperty(r)||(t[r]=new Array(i[r]));var a=s.bridgeObject(n);a.x=s.convert(n.x,"Date"),a.orginalY=n.y,a.y=Number(n.y);var h=t[r].length-i[r]--;t[r][h]=a}for(var r in this.groups)this.groups.hasOwnProperty(r)&&(t.hasOwnProperty(r)||(t[r]=new Array(0)));for(var r in t)if(t.hasOwnProperty(r))if(0==t[r].length)this.groups.hasOwnProperty(r)&&this._removeGroup(r);else{var d=void 0;void 0!=this.groupsData&&(d=this.groupsData.get(r)),void 0==d&&(d={id:r,content:this.options.defaultGroup+r}),this._updateGroup(d,r),this.groups[r].setItems(t[r])}this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})}},o.prototype.redraw=function(){var t=!1;this.props.width=this.dom.frame.offsetWidth,this.props.height=this.body.domProps.centerContainer.height-this.body.domProps.border.top-this.body.domProps.border.bottom,t=this._isResized()||t;var e=this.body.range.end-this.body.range.start,i=e!=this.lastVisibleInterval;if(this.lastVisibleInterval=e,1==t&&(this.svg.style.width=s.option.asSize(3*this.props.width),this.svg.style.left=s.option.asSize(-this.props.width),-1==(this.options.height+"").indexOf("%")&&1!=this.updateSVGheightOnResize||(this.updateSVGheight=!0)),1==this.updateSVGheight?(this.options.graphHeight!=this.props.height+"px"&&(this.options.graphHeight=this.props.height+"px",this.svg.style.height=this.props.height+"px"),this.updateSVGheight=!1):this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",1==t||1==i||1==this.abortedGraphUpdate||1==this.forceGraphUpdate)t=this._updateGraph()||t,this.forceGraphUpdate=!1;else if(0!=this.lastStart){var o=this.body.range.start-this.lastStart,n=this.body.range.end-this.body.range.start;if(0!=this.props.width){var r=this.props.width/n,a=o*r;this.svg.style.left=-this.props.width-a+"px"}}return this.legendLeft.redraw(),this.legendRight.redraw(),t},o.prototype._getSortedGroupIds=function(){var t=[];for(var e in this.groups)if(this.groups.hasOwnProperty(e)){var i=this.groups[e];1!=i.visible||void 0!==this.options.groups.visibility[e]&&1!=this.options.groups.visibility[e]||t.push({id:e,zIndex:i.options.zIndex})}s.insertSort(t,function(t,e){var i=t.zIndex,o=e.zIndex;return void 0===i&&(i=0),void 0===o&&(o=0),i==o?0:o>i?-1:1});for(var o=new Array(t.length),n=0;n<t.length;n++)o[n]=t[n].id;return o},o.prototype._updateGraph=function(){if(r.prepareElements(this.svgElements),0!=this.props.width&&null!=this.itemsData){var t,e,i={},o=!1,n=this.body.util.toGlobalTime(-this.body.domProps.root.width),s=this.body.util.toGlobalTime(2*this.body.domProps.root.width),a=this._getSortedGroupIds();if(a.length>0){var h={};for(this._getRelevantData(a,h,n,s),this._applySampling(a,h),e=0;e<a.length;e++)this._convertXcoordinates(h[a[e]]);if(this._getYRanges(a,h,i),o=this._updateYAxis(a,i),1==o)return r.cleanupElements(this.svgElements),this.abortedGraphUpdate=!0,!0;this.abortedGraphUpdate=!1;var d=void 0;for(e=0;e<a.length;e++)t=this.groups[a[e]],this.options.stack===!0&&"line"===this.options.style&&(void 0!=t.options.excludeFromStacking&&t.options.excludeFromStacking||(void 0!=d&&(this._stack(h[t.id],h[d.id]),1==t.options.shaded.enabled&&"group"!==t.options.shaded.orientation&&("top"==t.options.shaded.orientation&&"group"!==d.options.shaded.orientation?(d.options.shaded.orientation="group",d.options.shaded.groupId=t.id):(t.options.shaded.orientation="group",t.options.shaded.groupId=d.id))),d=t)),this._convertYcoordinates(h[a[e]],t);var l={};for(e=0;e<a.length;e++)if(t=this.groups[a[e]],"line"===t.options.style&&1==t.options.shaded.enabled){var c=h[a[e]];if(null==c||0==c.length)continue;if(l.hasOwnProperty(a[e])||(l[a[e]]=f.calcPath(c,t)),"group"===t.options.shaded.orientation){var u=t.options.shaded.groupId;if(-1===a.indexOf(u)){console.log(t.id+": Unknown shading group target given:"+u);continue}l.hasOwnProperty(u)||(l[u]=f.calcPath(h[u],this.groups[u])),f.drawShading(l[a[e]],t,l[u],this.framework)}else f.drawShading(l[a[e]],t,void 0,this.framework)}for(p.draw(a,h,this.framework),e=0;e<a.length;e++)if(t=this.groups[a[e]],h[a[e]].length>0)switch(t.options.style){case"line":l.hasOwnProperty(a[e])||(l[a[e]]=f.calcPath(h[a[e]],t)),f.draw(l[a[e]],t,this.framework);case"point":case"points":"point"!=t.options.style&&"points"!=t.options.style&&1!=t.options.drawPoints.enabled||m.draw(h[a[e]],t,this.framework);break;case"bar":}}}return r.cleanupElements(this.svgElements),!1},o.prototype._stack=function(t,e){var i,o,n,s,r;i=0;for(var a=0;a<t.length;a++){s=void 0,r=void 0;for(var h=i;h<e.length;h++){if(e[h].x===t[a].x){s=e[h],r=e[h],i=h;break}if(e[h].x>t[a].x){r=e[h],s=0==h?r:e[h-1],i=h;break}}void 0===r&&(s=e[e.length-1],r=e[e.length-1]),o=r.x-s.x,n=r.y-s.y,0==o?t[a].y=t[a].orginalY+r.y:t[a].y=t[a].orginalY+n/o*(t[a].x-s.x)+s.y}},o.prototype._getRelevantData=function(t,e,i,o){var n,r,a,h;if(t.length>0)for(r=0;r<t.length;r++){n=this.groups[t[r]];var d=n.getItems();if(1==n.options.sort){var l=function(t,e){return t.getTime()==e.getTime()?0:e>t?-1:1},c=Math.max(0,s.binarySearchValue(d,i,"x","before",l)),u=Math.min(d.length,s.binarySearchValue(d,o,"x","after",l)+1);0>=u&&(u=d.length);var p=new Array(u-c);for(a=c;u>a;a++)h=n.itemsData[a],p[a-c]=h;e[t[r]]=p}else e[t[r]]=n.itemsData}},o.prototype._applySampling=function(t,e){var i;if(t.length>0)for(var o=0;o<t.length;o++)if(i=this.groups[t[o]],1==i.options.sampling){var n=e[t[o]];if(n.length>0){var s=1,r=n.length,a=this.body.util.toGlobalScreen(n[n.length-1].x)-this.body.util.toGlobalScreen(n[0].x),h=r/a;s=Math.min(Math.ceil(.2*r),Math.max(1,Math.round(h)));for(var d=new Array(r),l=0;r>l;l+=s){var c=Math.round(l/s);d[c]=n[l]}e[t[o]]=d.splice(0,Math.round(r/s))}}},o.prototype._getYRanges=function(t,e,i){var o,n,s,r,a=[],h=[];if(t.length>0){for(s=0;s<t.length;s++)o=e[t[s]],r=this.groups[t[s]].options,o.length>0&&(n=this.groups[t[s]],r.stack===!0&&"bar"===r.style?"left"===r.yAxisOrientation?a=a.concat(n.getItems()):h=h.concat(n.getItems()):i[t[s]]=n.getYRange(o,t[s]));p.getStackedYRange(a,i,t,"__barStackLeft","left"),p.getStackedYRange(h,i,t,"__barStackRight","right")}},o.prototype._updateYAxis=function(t,e){var i,o,n=!1,s=!1,r=!1,a=1e9,h=1e9,d=-1e9,l=-1e9;if(t.length>0){for(var c=0;c<t.length;c++){var u=this.groups[t[c]];u&&"right"!=u.options.yAxisOrientation?(s=!0,a=1e9,d=-1e9):u&&u.options.yAxisOrientation&&(r=!0,h=1e9,l=-1e9)}for(var c=0;c<t.length;c++)e.hasOwnProperty(t[c])&&e[t[c]].ignore!==!0&&(i=e[t[c]].min,o=e[t[c]].max,"right"!=e[t[c]].yAxisOrientation?(s=!0,a=a>i?i:a,d=o>d?o:d):(r=!0,h=h>i?i:h,l=o>l?o:l));1==s&&this.yAxisLeft.setRange(a,d),1==r&&this.yAxisRight.setRange(h,l)}n=this._toggleAxisVisiblity(s,this.yAxisLeft)||n,n=this._toggleAxisVisiblity(r,this.yAxisRight)||n,1==r&&1==s?(this.yAxisLeft.drawIcons=!0,this.yAxisRight.drawIcons=!0):(this.yAxisLeft.drawIcons=!1,this.yAxisRight.drawIcons=!1),this.yAxisRight.master=!s,this.yAxisRight.masterAxis=this.yAxisLeft,0==this.yAxisRight.master?(1==r?this.yAxisLeft.lineOffset=this.yAxisRight.width:this.yAxisLeft.lineOffset=0,n=this.yAxisLeft.redraw()||n,n=this.yAxisRight.redraw()||n):n=this.yAxisRight.redraw()||n;for(var p=["__barStackLeft","__barStackRight","__lineStackLeft","__lineStackRight"],c=0;c<p.length;c++)-1!=t.indexOf(p[c])&&t.splice(t.indexOf(p[c]),1);return n},o.prototype._toggleAxisVisiblity=function(t,e){var i=!1;return 0==t?e.dom.frame.parentNode&&0==e.hidden&&(e.hide(),i=!0):e.dom.frame.parentNode||1!=e.hidden||(e.show(),i=!0),i},o.prototype._convertXcoordinates=function(t){for(var e=this.body.util.toScreen,i=0;i<t.length;i++)t[i].screen_x=e(t[i].x)+this.props.width,t[i].screen_y=t[i].y},o.prototype._convertYcoordinates=function(t,e){var i=this.yAxisLeft,o=Number(this.svg.style.height.replace("px",""));"right"==e.options.yAxisOrientation&&(i=this.yAxisRight);for(var n=0;n<t.length;n++)t[n].screen_y=Math.round(i.convertValue(t[n].y));e.setZeroPosition(Math.min(o,i.convertValue(0)))},t.exports=o},function(t,e,i){function o(t,e,i,o){this.id=n.randomUUID(),this.body=t,this.defaultOptions={orientation:"left",showMinorLabels:!0,showMajorLabels:!0,icons:!1,majorLinesOffset:7,minorLinesOffset:4,labelOffsetX:10,labelOffsetY:2,iconWidth:20,width:"40px",visible:!0,alignZeros:!0,left:{range:{min:void 0,max:void 0},format:function(t){return""+parseFloat(t.toPrecision(3))},title:{text:void 0,style:void 0}},right:{range:{min:void 0,max:void 0},format:function(t){return""+parseFloat(t.toPrecision(3))},title:{text:void 0,style:void 0}}},this.linegraphOptions=o,this.linegraphSVG=i,this.props={},this.DOMelements={lines:{},labels:{},title:{}},this.dom={},this.scale=void 0,this.range={start:0,end:0},this.options=n.extend({},this.defaultOptions),this.conversionFactor=1,this.setOptions(e),this.width=Number((""+this.options.width).replace("px","")),this.minWidth=this.width,this.height=this.linegraphSVG.getBoundingClientRect().height,this.hidden=!1,this.stepPixels=25,this.zeroCrossing=-1,this.amountOfSteps=-1,this.lineOffset=0,this.master=!0,this.masterAxis=null,this.svgElements={},this.iconsRemoved=!1,this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups};var s=this;this.body.emitter.on("verticalDrag",function(){s.dom.lineContainer.style.top=s.body.domProps.scrollTop+"px"})}var n=i(1),s=i(8),r=i(31),a=i(53);o.prototype=new r,o.prototype.addGroup=function(t,e){this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1},o.prototype.updateGroup=function(t,e){this.groups.hasOwnProperty(t)||(this.amountOfGroups+=1),this.groups[t]=e},o.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},o.prototype.setOptions=function(t){if(t){var e=!1;this.options.orientation!=t.orientation&&void 0!==t.orientation&&(e=!0);var i=["orientation","showMinorLabels","showMajorLabels","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","width","visible","left","right","alignZeros"];n.selectiveDeepExtend(i,this.options,t),this.minWidth=Number((""+this.options.width).replace("px","")),e===!0&&this.dom.frame&&(this.hide(),this.show())}},o.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.style.width=this.options.width,this.dom.frame.style.height=this.height,this.dom.lineContainer=document.createElement("div"),this.dom.lineContainer.style.width="100%",this.dom.lineContainer.style.height=this.height,this.dom.lineContainer.style.position="relative",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.height="100%",this.svg.style.width="100%",this.svg.style.display="block",this.dom.frame.appendChild(this.svg)},o.prototype._redrawGroupIcons=function(){s.prepareElements(this.svgElements);var t,e=this.options.iconWidth,i=15,o=4,n=o+.5*i;t="left"===this.options.orientation?o:this.width-e-o;var r=Object.keys(this.groups);r.sort(function(t,e){return e>t?-1:1});for(var a=0;a<r.length;a++){var h=r[a];this.groups[h].visible!==!0||void 0!==this.linegraphOptions.visibility[h]&&this.linegraphOptions.visibility[h]!==!0||(this.groups[h].getLegend(e,i,this.framework,t,n),n+=i+o)}s.cleanupElements(this.svgElements),this.iconsRemoved=!1},o.prototype._cleanupIcons=function(){this.iconsRemoved===!1&&(s.prepareElements(this.svgElements),s.cleanupElements(this.svgElements),this.iconsRemoved=!0)},o.prototype.show=function(){this.hidden=!1,this.dom.frame.parentNode||(this.options.rtl?this.body.dom.left.appendChild(this.dom.frame):this.body.dom.left.appendChild(this.dom.frame)),this.dom.lineContainer.parentNode||this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer)},o.prototype.hide=function(){this.hidden=!0,this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.lineContainer.parentNode&&this.dom.lineContainer.parentNode.removeChild(this.dom.lineContainer)},o.prototype.setRange=function(t,e){this.range.start=t,this.range.end=e},o.prototype.redraw=function(){var t=!1,e=0;this.dom.lineContainer.style.top=this.body.domProps.scrollTop+"px";for(var i in this.groups)this.groups.hasOwnProperty(i)&&(this.groups[i].visible!==!0||void 0!==this.linegraphOptions.visibility[i]&&this.linegraphOptions.visibility[i]!==!0||e++);if(0===this.amountOfGroups||0===e)this.hide();else{this.show(),this.height=Number(this.linegraphSVG.style.height.replace("px","")),this.dom.lineContainer.style.height=this.height+"px",this.width=this.options.visible===!0?Number((""+this.options.width).replace("px","")):0;var o=this.props,n=this.dom.frame;n.className="vis-data-axis",this._calculateCharSize();var s=this.options.orientation,r=this.options.showMinorLabels,a=this.options.showMajorLabels;o.minorLabelHeight=r?o.minorCharHeight:0,o.majorLabelHeight=a?o.majorCharHeight:0,o.minorLineWidth=this.body.dom.backgroundHorizontal.offsetWidth-this.lineOffset-this.width+2*this.options.minorLinesOffset,o.minorLineHeight=1,o.majorLineWidth=this.body.dom.backgroundHorizontal.offsetWidth-this.lineOffset-this.width+2*this.options.majorLinesOffset,o.majorLineHeight=1,"left"===s?(n.style.top="0",n.style.left="0",n.style.bottom="",n.style.width=this.width+"px",n.style.height=this.height+"px",this.props.width=this.body.domProps.left.width,this.props.height=this.body.domProps.left.height):(n.style.top="",n.style.bottom="0",n.style.left="0",n.style.width=this.width+"px",n.style.height=this.height+"px",this.props.width=this.body.domProps.right.width,
-this.props.height=this.body.domProps.right.height),t=this._redrawLabels(),t=this._isResized()||t,this.options.icons===!0?this._redrawGroupIcons():this._cleanupIcons(),this._redrawTitle(s)}return t},o.prototype._redrawLabels=function(){var t=this,e=!1;s.prepareElements(this.DOMelements.lines),s.prepareElements(this.DOMelements.labels);var i=this.options.orientation,o=void 0!=this.options[i].range?this.options[i].range:{},n=!0;void 0!=o.max&&(this.range.end=o.max,n=!1);var r=!0;void 0!=o.min&&(this.range.start=o.min,r=!1),this.scale=new a(this.range.start,this.range.end,r,n,this.dom.frame.offsetHeight,this.props.majorCharHeight,this.options.alignZeros,this.options[i].format),this.master===!1&&void 0!=this.masterAxis&&this.scale.followScale(this.masterAxis.scale),this.maxLabelSize=0;var h=this.scale.getLines();h.forEach(function(e){var o=e.y,n=e.major;t.options.showMinorLabels&&n===!1&&t._redrawLabel(o-2,e.val,i,"vis-y-axis vis-minor",t.props.minorCharHeight),n&&o>=0&&t._redrawLabel(o-2,e.val,i,"vis-y-axis vis-major",t.props.majorCharHeight),t.master===!0&&(n?t._redrawLine(o,i,"vis-grid vis-horizontal vis-major",t.options.majorLinesOffset,t.props.majorLineWidth):t._redrawLine(o,i,"vis-grid vis-horizontal vis-minor",t.options.minorLinesOffset,t.props.minorLineWidth))});var d=0;void 0!==this.options[i].title&&void 0!==this.options[i].title.text&&(d=this.props.titleCharHeight);var l=this.options.icons===!0?Math.max(this.options.iconWidth,d)+this.options.labelOffsetX+15:d+this.options.labelOffsetX+15;return this.maxLabelSize>this.width-l&&this.options.visible===!0?(this.width=this.maxLabelSize+l,this.options.width=this.width+"px",s.cleanupElements(this.DOMelements.lines),s.cleanupElements(this.DOMelements.labels),this.redraw(),e=!0):this.maxLabelSize<this.width-l&&this.options.visible===!0&&this.width>this.minWidth?(this.width=Math.max(this.minWidth,this.maxLabelSize+l),this.options.width=this.width+"px",s.cleanupElements(this.DOMelements.lines),s.cleanupElements(this.DOMelements.labels),this.redraw(),e=!0):(s.cleanupElements(this.DOMelements.lines),s.cleanupElements(this.DOMelements.labels),e=!1),e},o.prototype.convertValue=function(t){return this.scale.convertValue(t)},o.prototype.screenToValue=function(t){return this.scale.screenToValue(t)},o.prototype._redrawLabel=function(t,e,i,o,n){var r=s.getDOMElement("div",this.DOMelements.labels,this.dom.frame);r.className=o,r.innerHTML=e,"left"===i?(r.style.left="-"+this.options.labelOffsetX+"px",r.style.textAlign="right"):(r.style.right="-"+this.options.labelOffsetX+"px",r.style.textAlign="left"),r.style.top=t-.5*n+this.options.labelOffsetY+"px",e+="";var a=Math.max(this.props.majorCharWidth,this.props.minorCharWidth);this.maxLabelSize<e.length*a&&(this.maxLabelSize=e.length*a)},o.prototype._redrawLine=function(t,e,i,o,n){if(this.master===!0){var r=s.getDOMElement("div",this.DOMelements.lines,this.dom.lineContainer);r.className=i,r.innerHTML="","left"===e?r.style.left=this.width-o+"px":r.style.right=this.width-o+"px",r.style.width=n+"px",r.style.top=t+"px"}},o.prototype._redrawTitle=function(t){if(s.prepareElements(this.DOMelements.title),void 0!==this.options[t].title&&void 0!==this.options[t].title.text){var e=s.getDOMElement("div",this.DOMelements.title,this.dom.frame);e.className="vis-y-axis vis-title vis-"+t,e.innerHTML=this.options[t].title.text,void 0!==this.options[t].title.style&&n.addCssText(e,this.options[t].title.style),"left"===t?e.style.left=this.props.titleCharHeight+"px":e.style.right=this.props.titleCharHeight+"px",e.style.width=this.height+"px"}s.cleanupElements(this.DOMelements.title)},o.prototype._calculateCharSize=function(){if(!("minorCharHeight"in this.props)){var t=document.createTextNode("0"),e=document.createElement("div");e.className="vis-y-axis vis-minor vis-measure",e.appendChild(t),this.dom.frame.appendChild(e),this.props.minorCharHeight=e.clientHeight,this.props.minorCharWidth=e.clientWidth,this.dom.frame.removeChild(e)}if(!("majorCharHeight"in this.props)){var i=document.createTextNode("0"),o=document.createElement("div");o.className="vis-y-axis vis-major vis-measure",o.appendChild(i),this.dom.frame.appendChild(o),this.props.majorCharHeight=o.clientHeight,this.props.majorCharWidth=o.clientWidth,this.dom.frame.removeChild(o)}if(!("titleCharHeight"in this.props)){var n=document.createTextNode("0"),s=document.createElement("div");s.className="vis-y-axis vis-title vis-measure",s.appendChild(n),this.dom.frame.appendChild(s),this.props.titleCharHeight=s.clientHeight,this.props.titleCharWidth=s.clientWidth,this.dom.frame.removeChild(s)}},t.exports=o},function(t,e){function i(t,e,i,o,n,s){var r=arguments.length<=6||void 0===arguments[6]?!1:arguments[6],a=arguments.length<=7||void 0===arguments[7]?!1:arguments[7];if(this.majorSteps=[1,2,5,10],this.minorSteps=[.25,.5,1,2],this.customLines=null,this.containerHeight=n,this.majorCharHeight=s,this._start=t,this._end=e,this.scale=1,this.minorStepIdx=-1,this.magnitudefactor=1,this.determineScale(),this.zeroAlign=r,this.autoScaleStart=i,this.autoScaleEnd=o,this.formattingFunction=a,i||o){var h=this,d=function(t){var e=t-t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx]);return t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx])>.5*(h.magnitudefactor*h.minorSteps[h.minorStepIdx])?e+h.magnitudefactor*h.minorSteps[h.minorStepIdx]:e};i&&(this._start-=2*this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._start=d(this._start)),o&&(this._end+=this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._end=d(this._end)),this.determineScale()}}i.prototype.setCharHeight=function(t){this.majorCharHeight=t},i.prototype.setHeight=function(t){this.containerHeight=t},i.prototype.determineScale=function(){var t=this._end-this._start;this.scale=this.containerHeight/t;var e=this.majorCharHeight/this.scale,i=t>0?Math.round(Math.log(t)/Math.LN10):0;this.minorStepIdx=-1,this.magnitudefactor=Math.pow(10,i);var o=0;0>i&&(o=i);for(var n=!1,s=o;Math.abs(s)<=Math.abs(i);s++){this.magnitudefactor=Math.pow(10,s);for(var r=0;r<this.minorSteps.length;r++){var a=this.magnitudefactor*this.minorSteps[r];if(a>=e){n=!0,this.minorStepIdx=r;break}}if(n===!0)break}},i.prototype.is_major=function(t){return t%(this.magnitudefactor*this.majorSteps[this.minorStepIdx])===0},i.prototype.getStep=function(){return this.magnitudefactor*this.minorSteps[this.minorStepIdx]},i.prototype.getFirstMajor=function(){var t=this.magnitudefactor*this.majorSteps[this.minorStepIdx];return this.convertValue(this._start+(t-this._start%t)%t)},i.prototype.formatValue=function(t){var e=t.toPrecision(5);return"function"==typeof this.formattingFunction&&(e=this.formattingFunction(t)),"number"==typeof e?""+e:"string"==typeof e?e:t.toPrecision(5)},i.prototype.getLines=function(){for(var t=[],e=this.getStep(),i=(e-this._start%e)%e,o=this._start+i;this._end-o>1e-5;o+=e)o!=this._start&&t.push({major:this.is_major(o),y:this.convertValue(o),val:this.formatValue(o)});return t},i.prototype.followScale=function(t){var e=this.minorStepIdx,i=this._start,o=this._end,n=this,s=function(){n.magnitudefactor*=2},r=function(){n.magnitudefactor/=2};t.minorStepIdx<=1&&this.minorStepIdx<=1||t.minorStepIdx>1&&this.minorStepIdx>1||(t.minorStepIdx<this.minorStepIdx?(this.minorStepIdx=1,2==e?s():(s(),s())):(this.minorStepIdx=2,1==e?r():(r(),r())));for(var a=(t.getLines(),t.convertValue(0)),h=t.getStep()*t.scale,d=!1,l=0;!d&&l++<5;){this.scale=h/(this.minorSteps[this.minorStepIdx]*this.magnitudefactor);var c=this.containerHeight/this.scale;this._start=i,this._end=this._start+c;var u=this._end*this.scale,p=this.magnitudefactor*this.majorSteps[this.minorStepIdx],f=this.getFirstMajor()-t.getFirstMajor();if(this.zeroAlign){var m=a-u;this._end+=m/this.scale,this._start=this._end-c}else this.autoScaleStart?(this._start-=f/this.scale,this._end=this._start+c):(this._start+=p-f/this.scale,this._end=this._start+c);if(!this.autoScaleEnd&&this._end>o+1e-5)r(),d=!1;else{if(!this.autoScaleStart&&this._start<i-1e-5){if(!(this.zeroAlign&&i>=0)){r(),d=!1;continue}console.warn("Can't adhere to given 'min' range, due to zeroalign")}this.autoScaleStart&&this.autoScaleEnd&&o-i>c?(s(),d=!1):d=!0}}},i.prototype.convertValue=function(t){return this.containerHeight-(t-this._start)*this.scale},i.prototype.screenToValue=function(t){return(this.containerHeight-t)/this.scale+this._start},t.exports=i},function(t,e,i){function o(t,e,i,o){this.id=e;var n=["sampling","style","sort","yAxisOrientation","barChart","drawPoints","shaded","interpolation","zIndex","excludeFromStacking","excludeFromLegend"];this.options=s.selectiveBridgeObject(n,i),this.usingDefaultStyle=void 0===t.className,this.groupsUsingDefaultStyles=o,this.zeroPosition=0,this.update(t),1==this.usingDefaultStyle&&(this.groupsUsingDefaultStyles[0]+=1),this.itemsData=[],this.visible=void 0===t.visible?!0:t.visible}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(1),r=(i(8),i(55)),a=i(57),h=i(56);o.prototype.setItems=function(t){null!=t?(this.itemsData=t,1==this.options.sort&&s.insertSort(this.itemsData,function(t,e){return t.x>e.x?1:-1})):this.itemsData=[]},o.prototype.getItems=function(){return this.itemsData},o.prototype.setZeroPosition=function(t){this.zeroPosition=t},o.prototype.setOptions=function(t){if(void 0!==t){var e=["sampling","style","sort","yAxisOrientation","barChart","zIndex","excludeFromStacking","excludeFromLegend"];s.selectiveDeepExtend(e,this.options,t),"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),s.mergeOptions(this.options,t,"interpolation"),s.mergeOptions(this.options,t,"drawPoints"),s.mergeOptions(this.options,t,"shaded"),t.interpolation&&"object"==n(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5))}},o.prototype.update=function(t){this.group=t,this.content=t.content||"graph",this.className=t.className||this.className||"vis-graph-group"+this.groupsUsingDefaultStyles[0]%10,this.visible=void 0===t.visible?!0:t.visible,this.style=t.style,this.setOptions(t.options)},o.prototype.getLegend=function(t,e,i,o,n){if(void 0==i||null==i){var s=document.createElementNS("http://www.w3.org/2000/svg","svg");i={svg:s,svgElements:{},options:this.options,groups:[this]}}switch(void 0!=o&&null!=o||(o=0),void 0!=n&&null!=n||(n=.5*e),this.options.style){case"line":a.drawIcon(this,o,n,t,e,i);break;case"points":case"point":h.drawIcon(this,o,n,t,e,i);break;case"bar":r.drawIcon(this,o,n,t,e,i)}return{icon:i.svg,label:this.content,orientation:this.options.yAxisOrientation}},o.prototype.getYRange=function(t){for(var e=t[0].y,i=t[0].y,o=0;o<t.length;o++)e=e>t[o].y?t[o].y:e,i=i<t[o].y?t[o].y:i;return{min:e,max:i,yAxisOrientation:this.options.yAxisOrientation}},t.exports=o},function(t,e,i){function o(t,e){}var n=i(8),s=i(56);o.drawIcon=function(t,e,i,o,s,r){var a=.5*s,h=n.getSVGElement("rect",r.svgElements,r.svg);h.setAttributeNS(null,"x",e),h.setAttributeNS(null,"y",i-a),h.setAttributeNS(null,"width",o),h.setAttributeNS(null,"height",2*a),h.setAttributeNS(null,"class","vis-outline");var d=Math.round(.3*o),l=t.options.barChart.width,c=l/d,u=Math.round(.4*s),p=Math.round(.75*s),f=Math.round((o-2*d)/3);if(n.drawBar(e+.5*d+f,i+a-u-1,d,u,t.className+" vis-bar",r.svgElements,r.svg,t.style),n.drawBar(e+1.5*d+f+2,i+a-p-1,d,p,t.className+" vis-bar",r.svgElements,r.svg,t.style),1==t.options.drawPoints.enabled){var m={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size/c,className:t.className};n.drawPoint(e+.5*d+f,i+a-u-1,m,r.svgElements,r.svg),n.drawPoint(e+1.5*d+f+2,i+a-p-1,m,r.svgElements,r.svg)}},o.draw=function(t,e,i){var r,a,h,d,l,c,u=[],p={},f=0;for(l=0;l<t.length;l++)if(d=i.groups[t[l]],"bar"===d.options.style&&d.visible===!0&&(void 0===i.options.groups.visibility[t[l]]||i.options.groups.visibility[t[l]]===!0))for(c=0;c<e[t[l]].length;c++)u.push({screen_x:e[t[l]][c].screen_x,screen_y:e[t[l]][c].screen_y,x:e[t[l]][c].x,y:e[t[l]][c].y,groupId:t[l],label:e[t[l]][c].label}),f+=1;if(0!==f)for(u.sort(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x}),o._getDataIntersections(p,u),l=0;l<u.length;l++){d=i.groups[u[l].groupId];var m=void 0!=d.options.barChart.minWidth?d.options.barChart.minWidth:.1*d.options.barChart.width;a=u[l].screen_x;var v=0;if(void 0===p[a])l+1<u.length&&(r=Math.abs(u[l+1].screen_x-a)),h=o._getSafeDrawData(r,d,m);else{var g=l+(p[a].amount-p[a].resolved);l-(p[a].resolved+1);g<u.length&&(r=Math.abs(u[g].screen_x-a)),h=o._getSafeDrawData(r,d,m),p[a].resolved+=1,d.options.stack===!0&&d.options.excludeFromStacking!==!0?u[l].screen_y<d.zeroPosition?(v=p[a].accumulatedNegative,p[a].accumulatedNegative+=d.zeroPosition-u[l].screen_y):(v=p[a].accumulatedPositive,p[a].accumulatedPositive+=d.zeroPosition-u[l].screen_y):d.options.barChart.sideBySide===!0&&(h.width=h.width/p[a].amount,h.offset+=p[a].resolved*h.width-.5*h.width*(p[a].amount+1))}if(n.drawBar(u[l].screen_x+h.offset,u[l].screen_y-v,h.width,d.zeroPosition-u[l].screen_y,d.className+" vis-bar",i.svgElements,i.svg,d.style),d.options.drawPoints.enabled===!0){var y={screen_x:u[l].screen_x,screen_y:u[l].screen_y-v,x:u[l].x,y:u[l].y,groupId:u[l].groupId,label:u[l].label};s.draw([y],d,i,h.offset)}}},o._getDataIntersections=function(t,e){for(var i,o=0;o<e.length;o++)o+1<e.length&&(i=Math.abs(e[o+1].screen_x-e[o].screen_x)),o>0&&(i=Math.min(i,Math.abs(e[o-1].screen_x-e[o].screen_x))),0===i&&(void 0===t[e[o].screen_x]&&(t[e[o].screen_x]={amount:0,resolved:0,accumulatedPositive:0,accumulatedNegative:0}),t[e[o].screen_x].amount+=1)},o._getSafeDrawData=function(t,e,i){var o,n;return t<e.options.barChart.width&&t>0?(o=i>t?i:t,n=0,"left"===e.options.barChart.align?n-=.5*t:"right"===e.options.barChart.align&&(n+=.5*t)):(o=e.options.barChart.width,n=0,"left"===e.options.barChart.align?n-=.5*e.options.barChart.width:"right"===e.options.barChart.align&&(n+=.5*e.options.barChart.width)),{width:o,offset:n}},o.getStackedYRange=function(t,e,i,n,s){if(t.length>0){t.sort(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x});var r={};o._getDataIntersections(r,t),e[n]=o._getStackedYRange(r,t),e[n].yAxisOrientation=s,i.push(n)}},o._getStackedYRange=function(t,e){for(var i,o=e[0].screen_y,n=e[0].screen_y,s=0;s<e.length;s++)i=e[s].screen_x,void 0===t[i]?(o=o>e[s].screen_y?e[s].screen_y:o,n=n<e[s].screen_y?e[s].screen_y:n):e[s].screen_y<0?t[i].accumulatedNegative+=e[s].screen_y:t[i].accumulatedPositive+=e[s].screen_y;for(var r in t)t.hasOwnProperty(r)&&(o=o>t[r].accumulatedNegative?t[r].accumulatedNegative:o,o=o>t[r].accumulatedPositive?t[r].accumulatedPositive:o,n=n<t[r].accumulatedNegative?t[r].accumulatedNegative:n,n=n<t[r].accumulatedPositive?t[r].accumulatedPositive:n);return{min:o,max:n}},t.exports=o},function(t,e,i){function o(t,e){}function n(t,e){return e="undefined"==typeof e?{}:e,{style:e.style||t.options.drawPoints.style,styles:e.styles||t.options.drawPoints.styles,size:e.size||t.options.drawPoints.size,className:e.className||t.className}}function s(t,e){var i=void 0;return t.options&&t.options.drawPoints&&t.options.drawPoints.onRender&&"function"==typeof t.options.drawPoints.onRender&&(i=t.options.drawPoints.onRender),e.group.options&&e.group.options.drawPoints&&e.group.options.drawPoints.onRender&&"function"==typeof e.group.options.drawPoints.onRender&&(i=e.group.options.drawPoints.onRender),i}var r="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},a=i(8);o.draw=function(t,e,i,o){o=o||0;for(var h=s(i,e),d=0;d<t.length;d++)if(h){var l=h(t[d],e);l!==!0&&"object"!==("undefined"==typeof l?"undefined":r(l))||a.drawPoint(t[d].screen_x+o,t[d].screen_y,n(e,l),i.svgElements,i.svg,t[d].label)}else a.drawPoint(t[d].screen_x+o,t[d].screen_y,n(e),i.svgElements,i.svg,t[d].label)},o.drawIcon=function(t,e,i,o,s,r){var h=.5*s,d=a.getSVGElement("rect",r.svgElements,r.svg);d.setAttributeNS(null,"x",e),d.setAttributeNS(null,"y",i-h),d.setAttributeNS(null,"width",o),d.setAttributeNS(null,"height",2*h),d.setAttributeNS(null,"class","vis-outline"),a.drawPoint(e+.5*o,i,n(t),r.svgElements,r.svg)},t.exports=o},function(t,e,i){function o(t,e){}var n=i(8);o.calcPath=function(t,e){if(null!=t&&t.length>0){var i=[];return i=1==e.options.interpolation.enabled?o._catmullRom(t,e):o._linear(t)}},o.drawIcon=function(t,e,i,o,s,r){var a,h,d=.5*s,l=n.getSVGElement("rect",r.svgElements,r.svg);if(l.setAttributeNS(null,"x",e),l.setAttributeNS(null,"y",i-d),l.setAttributeNS(null,"width",o),l.setAttributeNS(null,"height",2*d),l.setAttributeNS(null,"class","vis-outline"),a=n.getSVGElement("path",r.svgElements,r.svg),a.setAttributeNS(null,"class",t.className),void 0!==t.style&&a.setAttributeNS(null,"style",t.style),a.setAttributeNS(null,"d","M"+e+","+i+" L"+(e+o)+","+i),1==t.options.shaded.enabled&&(h=n.getSVGElement("path",r.svgElements,r.svg),"top"==t.options.shaded.orientation?h.setAttributeNS(null,"d","M"+e+", "+(i-d)+"L"+e+","+i+" L"+(e+o)+","+i+" L"+(e+o)+","+(i-d)):h.setAttributeNS(null,"d","M"+e+","+i+" L"+e+","+(i+d)+" L"+(e+o)+","+(i+d)+"L"+(e+o)+","+i),h.setAttributeNS(null,"class",t.className+" vis-icon-fill"),void 0!==t.options.shaded.style&&""!==t.options.shaded.style&&h.setAttributeNS(null,"style",t.options.shaded.style)),1==t.options.drawPoints.enabled){var c={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size,className:t.className};n.drawPoint(e+.5*o,i,c,r.svgElements,r.svg)}},o.drawShading=function(t,e,i,o){if(1==e.options.shaded.enabled){var s=Number(o.svg.style.height.replace("px","")),r=n.getSVGElement("path",o.svgElements,o.svg),a="L";1==e.options.interpolation.enabled&&(a="C");var h,d=0;d="top"==e.options.shaded.orientation?0:"bottom"==e.options.shaded.orientation?s:Math.min(Math.max(0,e.zeroPosition),s),h="group"==e.options.shaded.orientation&&null!=i&&void 0!=i?"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" L"+i[i.length-1][0]+","+i[i.length-1][1]+" "+this.serializePath(i,a,!0)+i[0][0]+","+i[0][1]+" Z":"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" V"+d+" H"+t[0][0]+" Z",r.setAttributeNS(null,"class",e.className+" vis-fill"),void 0!==e.options.shaded.style&&r.setAttributeNS(null,"style",e.options.shaded.style),r.setAttributeNS(null,"d",h)}},o.draw=function(t,e,i){if(null!=t&&void 0!=t){var o=n.getSVGElement("path",i.svgElements,i.svg);o.setAttributeNS(null,"class",e.className),void 0!==e.style&&o.setAttributeNS(null,"style",e.style);var s="L";1==e.options.interpolation.enabled&&(s="C"),o.setAttributeNS(null,"d","M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,s,!1))}},o.serializePath=function(t,e,i){if(t.length<2)return"";var o=e;if(i)for(var n=t.length-2;n>0;n--)o+=t[n][0]+","+t[n][1]+" ";else for(var n=1;n<t.length;n++)o+=t[n][0]+","+t[n][1]+" ";return o},o._catmullRomUniform=function(t){var e,i,o,n,s,r,a=[];a.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var h=1/6,d=t.length,l=0;d-1>l;l++)e=0==l?t[0]:t[l-1],i=t[l],o=t[l+1],n=d>l+2?t[l+2]:o,s={screen_x:(-e.screen_x+6*i.screen_x+o.screen_x)*h,screen_y:(-e.screen_y+6*i.screen_y+o.screen_y)*h},r={screen_x:(i.screen_x+6*o.screen_x-n.screen_x)*h,screen_y:(i.screen_y+6*o.screen_y-n.screen_y)*h},a.push([s.screen_x,s.screen_y]),a.push([r.screen_x,r.screen_y]),a.push([o.screen_x,o.screen_y]);return a},o._catmullRom=function(t,e){var i=e.options.interpolation.alpha;if(0==i||void 0===i)return this._catmullRomUniform(t);var o,n,s,r,a,h,d,l,c,u,p,f,m,v,g,y,b,w,_,x=[];x.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var k=t.length,O=0;k-1>O;O++)o=0==O?t[0]:t[O-1],n=t[O],s=t[O+1],r=k>O+2?t[O+2]:s,d=Math.sqrt(Math.pow(o.screen_x-n.screen_x,2)+Math.pow(o.screen_y-n.screen_y,2)),l=Math.sqrt(Math.pow(n.screen_x-s.screen_x,2)+Math.pow(n.screen_y-s.screen_y,2)),c=Math.sqrt(Math.pow(s.screen_x-r.screen_x,2)+Math.pow(s.screen_y-r.screen_y,2)),v=Math.pow(c,i),y=Math.pow(c,2*i),g=Math.pow(l,i),b=Math.pow(l,2*i),_=Math.pow(d,i),w=Math.pow(d,2*i),u=2*w+3*_*g+b,p=2*y+3*v*g+b,f=3*_*(_+g),f>0&&(f=1/f),m=3*v*(v+g),m>0&&(m=1/m),a={screen_x:(-b*o.screen_x+u*n.screen_x+w*s.screen_x)*f,screen_y:(-b*o.screen_y+u*n.screen_y+w*s.screen_y)*f},h={screen_x:(y*n.screen_x+p*s.screen_x-b*r.screen_x)*m,screen_y:(y*n.screen_y+p*s.screen_y-b*r.screen_y)*m},0==a.screen_x&&0==a.screen_y&&(a=n),0==h.screen_x&&0==h.screen_y&&(h=s),x.push([a.screen_x,a.screen_y]),x.push([h.screen_x,h.screen_y]),x.push([s.screen_x,s.screen_y]);return x},o._linear=function(t){for(var e=[],i=0;i<t.length;i++)e.push([t[i].screen_x,t[i].screen_y]);return e},t.exports=o},function(t,e,i){function o(t,e,i,o){this.body=t,this.defaultOptions={enabled:!1,icons:!0,iconSize:20,iconSpacing:6,left:{visible:!0,position:"top-left"},right:{visible:!0,position:"top-right"}},this.side=i,this.options=n.extend({},this.defaultOptions),this.linegraphOptions=o,this.svgElements={},this.dom={},this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups},this.setOptions(e)}var n=i(1),s=i(8),r=i(31);o.prototype=new r,o.prototype.clear=function(){this.groups={},this.amountOfGroups=0},o.prototype.addGroup=function(t,e){1!=e.options.excludeFromLegend&&(this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1)},o.prototype.updateGroup=function(t,e){this.groups[t]=e},o.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},o.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-legend",this.dom.frame.style.position="absolute",this.dom.frame.style.top="10px",this.dom.frame.style.display="block",this.dom.textArea=document.createElement("div"),this.dom.textArea.className="vis-legend-text",this.dom.textArea.style.position="relative",this.dom.textArea.style.top="0px",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.width=this.options.iconSize+5+"px",this.svg.style.height="100%",this.dom.frame.appendChild(this.svg),this.dom.frame.appendChild(this.dom.textArea)},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},o.prototype.setOptions=function(t){var e=["enabled","orientation","icons","left","right"];n.selectiveDeepExtend(e,this.options,t)},o.prototype.redraw=function(){var t=0,e=Object.keys(this.groups);e.sort(function(t,e){return e>t?-1:1});for(var i=0;i<e.length;i++){var o=e[i];1!=this.groups[o].visible||void 0!==this.linegraphOptions.visibility[o]&&1!=this.linegraphOptions.visibility[o]||t++}if(0==this.options[this.side].visible||0==this.amountOfGroups||0==this.options.enabled||0==t)this.hide();else{if(this.show(),"top-left"==this.options[this.side].position||"bottom-left"==this.options[this.side].position?(this.dom.frame.style.left="4px",this.dom.frame.style.textAlign="left",this.dom.textArea.style.textAlign="left",this.dom.textArea.style.left=this.options.iconSize+15+"px",this.dom.textArea.style.right="",this.svg.style.left="0px",this.svg.style.right=""):(this.dom.frame.style.right="4px",this.dom.frame.style.textAlign="right",this.dom.textArea.style.textAlign="right",this.dom.textArea.style.right=this.options.iconSize+15+"px",this.dom.textArea.style.left="",this.svg.style.right="0px",this.svg.style.left=""),"top-left"==this.options[this.side].position||"top-right"==this.options[this.side].position)this.dom.frame.style.top=4-Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.bottom="";else{var n=this.body.domProps.center.height-this.body.domProps.centerContainer.height;this.dom.frame.style.bottom=4+n+Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.top=""}0==this.options.icons?(this.dom.frame.style.width=this.dom.textArea.offsetWidth+10+"px",this.dom.textArea.style.right="",this.dom.textArea.style.left="",this.svg.style.width="0px"):(this.dom.frame.style.width=this.options.iconSize+15+this.dom.textArea.offsetWidth+10+"px",this.drawLegendIcons());for(var s="",i=0;i<e.length;i++){var o=e[i];1!=this.groups[o].visible||void 0!==this.linegraphOptions.visibility[o]&&1!=this.linegraphOptions.visibility[o]||(s+=this.groups[o].content+"<br />")}this.dom.textArea.innerHTML=s,this.dom.textArea.style.lineHeight=.75*this.options.iconSize+this.options.iconSpacing+"px"}},o.prototype.drawLegendIcons=function(){if(this.dom.frame.parentNode){var t=Object.keys(this.groups);t.sort(function(t,e){return e>t?-1:1}),s.resetElements(this.svgElements);var e=window.getComputedStyle(this.dom.frame).paddingTop,i=Number(e.replace("px","")),o=i,n=this.options.iconSize,r=.75*this.options.iconSize,a=i+.5*r+3;this.svg.style.width=n+5+i+"px";for(var h=0;h<t.length;h++){var d=t[h];1!=this.groups[d].visible||void 0!==this.linegraphOptions.visibility[d]&&1!=this.linegraphOptions.visibility[d]||(this.groups[d].getLegend(n,r,this.framework,o,a),a+=r+this.options.iconSpacing)}}},t.exports=o},function(t,e){Object.defineProperty(e,"__esModule",{value:!0});var i="string",o="boolean",n="number",s="array",r="date",a="object",h="dom",d="moment",l="any",c={configure:{enabled:{"boolean":o},filter:{"boolean":o,"function":"function"},container:{dom:h},__type__:{object:a,"boolean":o,"function":"function"}},yAxisOrientation:{string:["left","right"]},defaultGroup:{string:i},sort:{"boolean":o},sampling:{"boolean":o},stack:{"boolean":o},graphHeight:{string:i,number:n},shaded:{enabled:{"boolean":o},orientation:{string:["bottom","top","zero","group"]},groupId:{object:a},__type__:{"boolean":o,object:a}},style:{string:["line","bar","points"]},barChart:{width:{number:n},minWidth:{number:n},sideBySide:{"boolean":o},align:{string:["left","center","right"]},__type__:{object:a}},interpolation:{enabled:{"boolean":o},parametrization:{string:["centripetal","chordal","uniform"]},alpha:{number:n},__type__:{object:a,"boolean":o}},drawPoints:{enabled:{"boolean":o},onRender:{"function":"function"},size:{number:n},style:{string:["square","circle"]},__type__:{object:a,"boolean":o,"function":"function"}},dataAxis:{showMinorLabels:{"boolean":o},showMajorLabels:{"boolean":o},icons:{"boolean":o},width:{string:i,number:n},visible:{"boolean":o},alignZeros:{"boolean":o},left:{range:{min:{number:n},max:{number:n},__type__:{object:a}},format:{"function":"function"},title:{text:{string:i,number:n},style:{string:i},__type__:{object:a}},__type__:{object:a}},right:{range:{min:{number:n},max:{number:n},__type__:{object:a}},format:{"function":"function"},title:{text:{string:i,number:n},style:{string:i},__type__:{object:a}},__type__:{object:a}},__type__:{object:a}},legend:{enabled:{"boolean":o},icons:{"boolean":o},left:{visible:{"boolean":o},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:a}},right:{visible:{"boolean":o},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:a}},__type__:{object:a,"boolean":o}},groups:{visibility:{any:l},__type__:{object:a}},autoResize:{"boolean":o},throttleRedraw:{number:n},clickToUse:{"boolean":o},end:{number:n,date:r,string:i,moment:d},format:{minorLabels:{millisecond:{string:i,undefined:"undefined"},second:{string:i,undefined:"undefined"},minute:{string:i,undefined:"undefined"},hour:{string:i,undefined:"undefined"},weekday:{string:i,undefined:"undefined"},day:{string:i,undefined:"undefined"},month:{string:i,undefined:"undefined"},year:{string:i,undefined:"undefined"},__type__:{object:a}},majorLabels:{millisecond:{string:i,undefined:"undefined"},second:{string:i,undefined:"undefined"},minute:{string:i,undefined:"undefined"},hour:{string:i,undefined:"undefined"},weekday:{string:i,undefined:"undefined"},day:{string:i,undefined:"undefined"},month:{string:i,undefined:"undefined"},year:{string:i,undefined:"undefined"},__type__:{object:a}},__type__:{object:a}},moment:{"function":"function"},height:{string:i,number:n},hiddenDates:{start:{date:r,number:n,string:i,moment:d},end:{date:r,number:n,string:i,moment:d},repeat:{string:i},__type__:{object:a,array:s}},locale:{string:i},locales:{__any__:{any:l},__type__:{object:a}},max:{date:r,number:n,string:i,moment:d},maxHeight:{number:n,string:i},maxMinorChars:{number:n},min:{date:r,number:n,string:i,moment:d},minHeight:{number:n,string:i},moveable:{"boolean":o},multiselect:{"boolean":o},orientation:{string:i},showCurrentTime:{"boolean":o},showMajorLabels:{"boolean":o},showMinorLabels:{"boolean":o},start:{date:r,number:n,string:i,moment:d},timeAxis:{scale:{string:i,undefined:"undefined"},step:{number:n,undefined:"undefined"},__type__:{object:a}},width:{string:i,number:n},zoomable:{"boolean":o},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:n},zoomMin:{number:n},zIndex:{number:n},__type__:{object:a}},u={global:{sort:!0,sampling:!0,stack:!1,shaded:{enabled:!1,orientation:["zero","top","bottom","group"]},style:["line","bar","points"],barChart:{width:[50,5,100,5],minWidth:[50,5,100,5],sideBySide:!1,align:["left","center","right"]},interpolation:{enabled:!0,parametrization:["centripetal","chordal","uniform"]},drawPoints:{enabled:!0,size:[6,2,30,1],style:["square","circle"]},dataAxis:{showMinorLabels:!0,showMajorLabels:!0,icons:!1,width:[40,0,200,1],visible:!0,alignZeros:!0,left:{title:{text:"",style:""}},right:{title:{text:"",style:""}}},legend:{enabled:!1,icons:!0,left:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]},right:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]}},autoResize:!0,throttleRedraw:[10,0,1e3,10],clickToUse:!1,end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",month:"YYYY",year:""}},height:"",locale:"",max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!0,orientation:["both","bottom","top"],showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,start:"",width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1],zIndex:0}};e.allOptions=c,e.configureOptions=u},function(t,e,i){e.util=i(1),e.DOMutil=i(8),e.DataSet=i(9),e.DataView=i(11),e.Queue=i(10),e.Network=i(61),e.network={Images:i(62),dotparser:i(118),gephiParser:i(119),allOptions:i(114)},e.network.convertDot=function(t){return e.network.dotparser.DOTToGraph(t)},e.network.convertGephi=function(t,i){return e.network.gephiParser.parseGephi(t,i)},e.moment=i(2),e.Hammer=i(20),e.keycharm=i(23)},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e,i){var o=this;if(!(this instanceof n))throw new SyntaxError("Constructor must be called with the new operator");this.options={},this.defaultOptions={locale:"en",locales:Y,clickToUse:!1},F.extend(this.options,this.defaultOptions),this.body={container:t,nodes:{},nodeIndices:[],edges:{},edgeIndices:[],emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this),once:this.once.bind(this)},eventListeners:{onTap:function(){},onTouch:function(){},onDoubleTap:function(){},onHold:function(){},onDragStart:function(){},onDrag:function(){},onDragEnd:function(){},onMouseWheel:function(){},onPinch:function(){},onMouseMove:function(){},onRelease:function(){},onContext:function(){}},data:{nodes:null,edges:null
-},functions:{createNode:function(){},createEdge:function(){},getPointer:function(){}},modules:{},view:{scale:1,translation:{x:0,y:0}}},this.bindEventListeners(),this.images=new r["default"](function(){return o.body.emitter.emit("_requestRedraw")}),this.groups=new h["default"],this.canvas=new w["default"](this.body),this.selectionHandler=new D["default"](this.body,this.canvas),this.interactionHandler=new O["default"](this.body,this.canvas,this.selectionHandler),this.view=new x["default"](this.body,this.canvas),this.renderer=new y["default"](this.body,this.canvas),this.physics=new f["default"](this.body),this.layoutEngine=new C["default"](this.body),this.clustering=new v["default"](this.body),this.manipulation=new E["default"](this.body,this.canvas,this.selectionHandler),this.nodesHandler=new l["default"](this.body,this.images,this.groups,this.layoutEngine),this.edgesHandler=new u["default"](this.body,this.images,this.groups),this.body.modules.kamadaKawai=new A["default"](this.body,150,.05),this.body.modules.clustering=this.clustering,this.canvas._create(),this.setOptions(i),this.setData(e)}var s=i(62),r=o(s),a=i(63),h=o(a),d=i(64),l=o(d),c=i(84),u=o(c),p=i(93),f=o(p),m=i(102),v=o(m),g=i(105),y=o(g),b=i(106),w=o(b),_=i(107),x=o(_),k=i(108),O=o(k),M=i(111),D=o(M),S=i(112),C=o(S),T=i(113),E=o(T),P=i(26),I=o(P),N=i(29),R=o(N),z=i(114),L=i(115),A=o(L);i(117);var B=i(13),F=i(1),j=(i(9),i(11),i(118)),H=i(119),W=i(45),Y=i(120);B(n.prototype),n.prototype.setOptions=function(t){var e=this;if(void 0!==t){var i=R["default"].validate(t,z.allOptions);i===!0&&console.log("%cErrors have been found in the supplied options object.",N.printStyle);var o=["locale","locales","clickToUse"];if(F.selectiveDeepExtend(o,this.options,t),t=this.layoutEngine.setOptions(t.layout,t),this.canvas.setOptions(t),this.groups.setOptions(t.groups),this.nodesHandler.setOptions(t.nodes),this.edgesHandler.setOptions(t.edges),this.physics.setOptions(t.physics),this.manipulation.setOptions(t.manipulation,t,this.options),this.interactionHandler.setOptions(t.interaction),this.renderer.setOptions(t.interaction),this.selectionHandler.setOptions(t.interaction),void 0!==t.groups&&this.body.emitter.emit("refreshNodes"),"configure"in t&&(this.configurator||(this.configurator=new I["default"](this,this.body.container,z.configureOptions,this.canvas.pixelRatio)),this.configurator.setOptions(t.configure)),this.configurator&&this.configurator.options.enabled===!0){var n={nodes:{},edges:{},layout:{},interaction:{},manipulation:{},physics:{},global:{}};F.deepExtend(n.nodes,this.nodesHandler.options),F.deepExtend(n.edges,this.edgesHandler.options),F.deepExtend(n.layout,this.layoutEngine.options),F.deepExtend(n.interaction,this.selectionHandler.options),F.deepExtend(n.interaction,this.renderer.options),F.deepExtend(n.interaction,this.interactionHandler.options),F.deepExtend(n.manipulation,this.manipulation.options),F.deepExtend(n.physics,this.physics.options),F.deepExtend(n.global,this.canvas.options),F.deepExtend(n.global,this.options),this.configurator.setModuleOptions(n)}void 0!==t.clickToUse?t.clickToUse===!0?void 0===this.activator&&(this.activator=new W(this.canvas.frame),this.activator.on("change",function(){e.body.emitter.emit("activate")})):(void 0!==this.activator&&(this.activator.destroy(),delete this.activator),this.body.emitter.emit("activate")):this.body.emitter.emit("activate"),this.canvas.setSize(),this.body.emitter.emit("startSimulation")}},n.prototype._updateVisibleIndices=function(){var t=this.body.nodes,e=this.body.edges;this.body.nodeIndices=[],this.body.edgeIndices=[];for(var i in t)t.hasOwnProperty(i)&&t[i].options.hidden===!1&&this.body.nodeIndices.push(t[i].id);for(var o in e)e.hasOwnProperty(o)&&e[o].options.hidden===!1&&this.body.edgeIndices.push(e[o].id)},n.prototype.bindEventListeners=function(){var t=this;this.body.emitter.on("_dataChanged",function(){t._updateVisibleIndices(),t.body.emitter.emit("_requestRedraw"),t.body.emitter.emit("_dataUpdated")}),this.body.emitter.on("_dataUpdated",function(){t._updateValueRange(t.body.nodes),t._updateValueRange(t.body.edges),t.body.emitter.emit("startSimulation"),t.body.emitter.emit("_requestRedraw")})},n.prototype.setData=function(t){if(this.body.emitter.emit("resetPhysics"),this.body.emitter.emit("_resetData"),this.selectionHandler.unselectAll(),t&&t.dot&&(t.nodes||t.edges))throw new SyntaxError('Data must contain either parameter "dot" or  parameter pair "nodes" and "edges", but not both.');if(this.setOptions(t&&t.options),t&&t.dot){console.log("The dot property has been depricated. Please use the static convertDot method to convert DOT into vis.network format and use the normal data format with nodes and edges. This converter is used like this: var data = vis.network.convertDot(dotString);");var e=j.DOTToGraph(t.dot);return void this.setData(e)}if(t&&t.gephi){console.log("The gephi property has been depricated. Please use the static convertGephi method to convert gephi into vis.network format and use the normal data format with nodes and edges. This converter is used like this: var data = vis.network.convertGephi(gephiJson);");var i=H.parseGephi(t.gephi);return void this.setData(i)}this.nodesHandler.setData(t&&t.nodes,!0),this.edgesHandler.setData(t&&t.edges,!0),this.body.emitter.emit("_dataChanged"),this.body.emitter.emit("_dataLoaded"),this.body.emitter.emit("initPhysics")},n.prototype.destroy=function(){this.body.emitter.emit("destroy"),this.body.emitter.off(),this.off(),delete this.groups,delete this.canvas,delete this.selectionHandler,delete this.interactionHandler,delete this.view,delete this.renderer,delete this.physics,delete this.layoutEngine,delete this.clustering,delete this.manipulation,delete this.nodesHandler,delete this.edgesHandler,delete this.configurator,delete this.images;for(var t in this.body.nodes)delete this.body.nodes[t];for(var e in this.body.edges)delete this.body.edges[e];F.recursiveDOMDelete(this.body.container)},n.prototype._updateValueRange=function(t){var e,i=void 0,o=void 0,n=0;for(e in t)if(t.hasOwnProperty(e)){var s=t[e].getValue();void 0!==s&&(i=void 0===i?s:Math.min(s,i),o=void 0===o?s:Math.max(s,o),n+=s)}if(void 0!==i&&void 0!==o)for(e in t)t.hasOwnProperty(e)&&t[e].setValueRange(i,o,n)},n.prototype.isActive=function(){return!this.activator||this.activator.active},n.prototype.setSize=function(){return this.canvas.setSize.apply(this.canvas,arguments)},n.prototype.canvasToDOM=function(){return this.canvas.canvasToDOM.apply(this.canvas,arguments)},n.prototype.DOMtoCanvas=function(){return this.canvas.DOMtoCanvas.apply(this.canvas,arguments)},n.prototype.findNode=function(){return this.clustering.findNode.apply(this.clustering,arguments)},n.prototype.isCluster=function(){return this.clustering.isCluster.apply(this.clustering,arguments)},n.prototype.openCluster=function(){return this.clustering.openCluster.apply(this.clustering,arguments)},n.prototype.cluster=function(){return this.clustering.cluster.apply(this.clustering,arguments)},n.prototype.getNodesInCluster=function(){return this.clustering.getNodesInCluster.apply(this.clustering,arguments)},n.prototype.clusterByConnection=function(){return this.clustering.clusterByConnection.apply(this.clustering,arguments)},n.prototype.clusterByHubsize=function(){return this.clustering.clusterByHubsize.apply(this.clustering,arguments)},n.prototype.clusterOutliers=function(){return this.clustering.clusterOutliers.apply(this.clustering,arguments)},n.prototype.getSeed=function(){return this.layoutEngine.getSeed.apply(this.layoutEngine,arguments)},n.prototype.enableEditMode=function(){return this.manipulation.enableEditMode.apply(this.manipulation,arguments)},n.prototype.disableEditMode=function(){return this.manipulation.disableEditMode.apply(this.manipulation,arguments)},n.prototype.addNodeMode=function(){return this.manipulation.addNodeMode.apply(this.manipulation,arguments)},n.prototype.editNode=function(){return this.manipulation.editNode.apply(this.manipulation,arguments)},n.prototype.editNodeMode=function(){return console.log("Deprecated: Please use editNode instead of editNodeMode."),this.manipulation.editNode.apply(this.manipulation,arguments)},n.prototype.addEdgeMode=function(){return this.manipulation.addEdgeMode.apply(this.manipulation,arguments)},n.prototype.editEdgeMode=function(){return this.manipulation.editEdgeMode.apply(this.manipulation,arguments)},n.prototype.deleteSelected=function(){return this.manipulation.deleteSelected.apply(this.manipulation,arguments)},n.prototype.getPositions=function(){return this.nodesHandler.getPositions.apply(this.nodesHandler,arguments)},n.prototype.storePositions=function(){return this.nodesHandler.storePositions.apply(this.nodesHandler,arguments)},n.prototype.moveNode=function(){return this.nodesHandler.moveNode.apply(this.nodesHandler,arguments)},n.prototype.getBoundingBox=function(){return this.nodesHandler.getBoundingBox.apply(this.nodesHandler,arguments)},n.prototype.getConnectedNodes=function(t){return void 0!==this.body.nodes[t]?this.nodesHandler.getConnectedNodes.apply(this.nodesHandler,arguments):this.edgesHandler.getConnectedNodes.apply(this.edgesHandler,arguments)},n.prototype.getConnectedEdges=function(){return this.nodesHandler.getConnectedEdges.apply(this.nodesHandler,arguments)},n.prototype.startSimulation=function(){return this.physics.startSimulation.apply(this.physics,arguments)},n.prototype.stopSimulation=function(){return this.physics.stopSimulation.apply(this.physics,arguments)},n.prototype.stabilize=function(){return this.physics.stabilize.apply(this.physics,arguments)},n.prototype.getSelection=function(){return this.selectionHandler.getSelection.apply(this.selectionHandler,arguments)},n.prototype.setSelection=function(){return this.selectionHandler.setSelection.apply(this.selectionHandler,arguments)},n.prototype.getSelectedNodes=function(){return this.selectionHandler.getSelectedNodes.apply(this.selectionHandler,arguments)},n.prototype.getSelectedEdges=function(){return this.selectionHandler.getSelectedEdges.apply(this.selectionHandler,arguments)},n.prototype.getNodeAt=function(){var t=this.selectionHandler.getNodeAt.apply(this.selectionHandler,arguments);return void 0!==t&&void 0!==t.id?t.id:t},n.prototype.getEdgeAt=function(){var t=this.selectionHandler.getEdgeAt.apply(this.selectionHandler,arguments);return void 0!==t&&void 0!==t.id?t.id:t},n.prototype.selectNodes=function(){return this.selectionHandler.selectNodes.apply(this.selectionHandler,arguments)},n.prototype.selectEdges=function(){return this.selectionHandler.selectEdges.apply(this.selectionHandler,arguments)},n.prototype.unselectAll=function(){this.selectionHandler.unselectAll.apply(this.selectionHandler,arguments),this.redraw()},n.prototype.redraw=function(){return this.renderer.redraw.apply(this.renderer,arguments)},n.prototype.getScale=function(){return this.view.getScale.apply(this.view,arguments)},n.prototype.getViewPosition=function(){return this.view.getViewPosition.apply(this.view,arguments)},n.prototype.fit=function(){return this.view.fit.apply(this.view,arguments)},n.prototype.moveTo=function(){return this.view.moveTo.apply(this.view,arguments)},n.prototype.focus=function(){return this.view.focus.apply(this.view,arguments)},n.prototype.releaseNode=function(){return this.view.releaseNode.apply(this.view,arguments)},n.prototype.getOptionsFromConfigurator=function(){var t={};return this.configurator&&(t=this.configurator.getOptions.apply(this.configurator)),t},t.exports=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e){i(this,t),this.images={},this.imageBroken={},this.callback=e}return o(t,[{key:"_addImageToCache",value:function(t,e){0===e.width&&(document.body.appendChild(e),e.width=e.offsetWidth,e.height=e.offsetHeight,document.body.removeChild(e)),this.images[t]=e}},{key:"_tryloadBrokenUrl",value:function(t,e,i){var o=this;void 0!==t&&void 0!==e&&void 0!==i&&(i.onerror=function(){console.error("Could not load brokenImage:",e),o._addImageToCache(t,new Image)},i.src=e)}},{key:"_redrawWithImage",value:function(t){this.callback&&this.callback(t)}},{key:"load",value:function(t,e,i){var o=this,n=this.images[t];if(n)return n;var s=new Image;return s.onload=function(){o._addImageToCache(t,s),o._redrawWithImage(s)},s.onerror=function(){console.error("Could not load image:",t),o._tryloadBrokenUrl(t,e,s)},s.src=t,s}}]),t}();e["default"]=n},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=i(1),r=function(){function t(){o(this,t),this.clear(),this.defaultIndex=0,this.groupsArray=[],this.groupIndex=0,this.defaultGroups=[{border:"#2B7CE9",background:"#97C2FC",highlight:{border:"#2B7CE9",background:"#D2E5FF"},hover:{border:"#2B7CE9",background:"#D2E5FF"}},{border:"#FFA500",background:"#FFFF00",highlight:{border:"#FFA500",background:"#FFFFA3"},hover:{border:"#FFA500",background:"#FFFFA3"}},{border:"#FA0A10",background:"#FB7E81",highlight:{border:"#FA0A10",background:"#FFAFB1"},hover:{border:"#FA0A10",background:"#FFAFB1"}},{border:"#41A906",background:"#7BE141",highlight:{border:"#41A906",background:"#A1EC76"},hover:{border:"#41A906",background:"#A1EC76"}},{border:"#E129F0",background:"#EB7DF4",highlight:{border:"#E129F0",background:"#F0B3F5"},hover:{border:"#E129F0",background:"#F0B3F5"}},{border:"#7C29F0",background:"#AD85E4",highlight:{border:"#7C29F0",background:"#D3BDF0"},hover:{border:"#7C29F0",background:"#D3BDF0"}},{border:"#C37F00",background:"#FFA807",highlight:{border:"#C37F00",background:"#FFCA66"},hover:{border:"#C37F00",background:"#FFCA66"}},{border:"#4220FB",background:"#6E6EFD",highlight:{border:"#4220FB",background:"#9B9BFD"},hover:{border:"#4220FB",background:"#9B9BFD"}},{border:"#FD5A77",background:"#FFC0CB",highlight:{border:"#FD5A77",background:"#FFD1D9"},hover:{border:"#FD5A77",background:"#FFD1D9"}},{border:"#4AD63A",background:"#C2FABC",highlight:{border:"#4AD63A",background:"#E6FFE3"},hover:{border:"#4AD63A",background:"#E6FFE3"}},{border:"#990000",background:"#EE0000",highlight:{border:"#BB0000",background:"#FF3333"},hover:{border:"#BB0000",background:"#FF3333"}},{border:"#FF6000",background:"#FF6000",highlight:{border:"#FF6000",background:"#FF6000"},hover:{border:"#FF6000",background:"#FF6000"}},{border:"#97C2FC",background:"#2B7CE9",highlight:{border:"#D2E5FF",background:"#2B7CE9"},hover:{border:"#D2E5FF",background:"#2B7CE9"}},{border:"#399605",background:"#255C03",highlight:{border:"#399605",background:"#255C03"},hover:{border:"#399605",background:"#255C03"}},{border:"#B70054",background:"#FF007E",highlight:{border:"#B70054",background:"#FF007E"},hover:{border:"#B70054",background:"#FF007E"}},{border:"#AD85E4",background:"#7C29F0",highlight:{border:"#D3BDF0",background:"#7C29F0"},hover:{border:"#D3BDF0",background:"#7C29F0"}},{border:"#4557FA",background:"#000EA1",highlight:{border:"#6E6EFD",background:"#000EA1"},hover:{border:"#6E6EFD",background:"#000EA1"}},{border:"#FFC0CB",background:"#FD5A77",highlight:{border:"#FFD1D9",background:"#FD5A77"},hover:{border:"#FFD1D9",background:"#FD5A77"}},{border:"#C2FABC",background:"#74D66A",highlight:{border:"#E6FFE3",background:"#74D66A"},hover:{border:"#E6FFE3",background:"#74D66A"}},{border:"#EE0000",background:"#990000",highlight:{border:"#FF3333",background:"#BB0000"},hover:{border:"#FF3333",background:"#BB0000"}}],this.options={},this.defaultOptions={useDefaultGroups:!0},s.extend(this.options,this.defaultOptions)}return n(t,[{key:"setOptions",value:function(t){var e=["useDefaultGroups"];if(void 0!==t)for(var i in t)if(t.hasOwnProperty(i)&&-1===e.indexOf(i)){var o=t[i];this.add(i,o)}}},{key:"clear",value:function(){this.groups={},this.groupsArray=[]}},{key:"get",value:function(t){var e=this.groups[t];if(void 0===e)if(this.options.useDefaultGroups===!1&&this.groupsArray.length>0){var i=this.groupIndex%this.groupsArray.length;this.groupIndex++,e={},e.color=this.groups[this.groupsArray[i]],this.groups[t]=e}else{var o=this.defaultIndex%this.defaultGroups.length;this.defaultIndex++,e={},e.color=this.defaultGroups[o],this.groups[t]=e}return e}},{key:"add",value:function(t,e){return this.groups[t]=e,this.groupsArray.push(t),e}}]),t}();e["default"]=r},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(65),a=o(r),h=i(66),d=o(h),l=i(1),c=i(9),u=i(11),p=function(){function t(e,i,o,s){var r=this;n(this,t),this.body=e,this.images=i,this.groups=o,this.layoutEngine=s,this.body.functions.createNode=this.create.bind(this),this.nodesListeners={add:function(t,e){r.add(e.items)},update:function(t,e){r.update(e.items,e.data)},remove:function(t,e){r.remove(e.items)}},this.options={},this.defaultOptions={borderWidth:1,borderWidthSelected:2,brokenImage:void 0,color:{border:"#2B7CE9",background:"#97C2FC",highlight:{border:"#2B7CE9",background:"#D2E5FF"},hover:{border:"#2B7CE9",background:"#D2E5FF"}},fixed:{x:!1,y:!1},font:{color:"#343434",size:14,face:"arial",background:"none",strokeWidth:0,strokeColor:"#ffffff",align:"center"},group:void 0,hidden:!1,icon:{face:"FontAwesome",code:void 0,size:50,color:"#2B7CE9"},image:void 0,label:void 0,labelHighlightBold:!0,level:void 0,mass:1,physics:!0,scaling:{min:10,max:30,label:{enabled:!1,min:14,max:30,maxVisible:30,drawThreshold:5},customScalingFunction:function(t,e,i,o){if(e===t)return.5;var n=1/(e-t);return Math.max(0,(o-t)*n)}},shadow:{enabled:!1,color:"rgba(0,0,0,0.5)",size:10,x:5,y:5},shape:"ellipse",shapeProperties:{borderDashes:!1,borderRadius:6,interpolation:!0,useImageSize:!1,useBorderWithImage:!1},size:25,title:void 0,value:void 0,x:void 0,y:void 0},l.extend(this.options,this.defaultOptions),this.bindEventListeners()}return s(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("refreshNodes",this.refresh.bind(this)),this.body.emitter.on("refresh",this.refresh.bind(this)),this.body.emitter.on("destroy",function(){l.forEach(t.nodesListeners,function(e,i){t.body.data.nodes&&t.body.data.nodes.off(i,e)}),delete t.body.functions.createNode,delete t.nodesListeners.add,delete t.nodesListeners.update,delete t.nodesListeners.remove,delete t.nodesListeners})}},{key:"setOptions",value:function(t){if(void 0!==t){if(a["default"].parseOptions(this.options,t),void 0!==t.shape)for(var e in this.body.nodes)this.body.nodes.hasOwnProperty(e)&&this.body.nodes[e].updateShape();if(void 0!==t.font){d["default"].parseOptions(this.options.font,t);for(var i in this.body.nodes)this.body.nodes.hasOwnProperty(i)&&(this.body.nodes[i].updateLabelModule(),this.body.nodes[i]._reset())}if(void 0!==t.size)for(var o in this.body.nodes)this.body.nodes.hasOwnProperty(o)&&this.body.nodes[o]._reset();void 0===t.hidden&&void 0===t.physics||this.body.emitter.emit("_dataChanged")}}},{key:"setData",value:function(t){var e=this,i=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],o=this.body.data.nodes;if(t instanceof c||t instanceof u)this.body.data.nodes=t;else if(Array.isArray(t))this.body.data.nodes=new c,this.body.data.nodes.add(t);else{if(t)throw new TypeError("Array or DataSet expected");this.body.data.nodes=new c}o&&l.forEach(this.nodesListeners,function(t,e){o.off(e,t)}),this.body.nodes={},this.body.data.nodes&&!function(){var t=e;l.forEach(e.nodesListeners,function(e,i){t.body.data.nodes.on(i,e)});var i=e.body.data.nodes.getIds();e.add(i,!0)}(),i===!1&&this.body.emitter.emit("_dataChanged")}},{key:"add",value:function(t){for(var e=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],i=void 0,o=[],n=0;n<t.length;n++){i=t[n];var s=this.body.data.nodes.get(i),r=this.create(s);o.push(r),this.body.nodes[i]=r}this.layoutEngine.positionInitially(o),e===!1&&this.body.emitter.emit("_dataChanged")}},{key:"update",value:function(t,e){for(var i=this.body.nodes,o=!1,n=0;n<t.length;n++){var s=t[n],r=i[s],a=e[n];void 0!==r?o=r.setOptions(a):(o=!0,r=this.create(a),i[s]=r)}o===!0?this.body.emitter.emit("_dataChanged"):this.body.emitter.emit("_dataUpdated")}},{key:"remove",value:function(t){for(var e=this.body.nodes,i=0;i<t.length;i++){var o=t[i];delete e[o]}this.body.emitter.emit("_dataChanged")}},{key:"create",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?a["default"]:arguments[1];return new e(t,this.body,this.images,this.groups,this.options)}},{key:"refresh",value:function(){var t=arguments.length<=0||void 0===arguments[0]?!1:arguments[0],e=this.body.nodes;for(var i in e){var o=void 0;e.hasOwnProperty(i)&&(o=e[i]);var n=this.body.data.nodes._data[i];void 0!==o&&void 0!==n&&(t===!0&&o.setOptions({x:null,y:null}),o.setOptions({fixed:!1}),o.setOptions(n))}}},{key:"getPositions",value:function(t){var e={};if(void 0!==t){if(Array.isArray(t)===!0){for(var i=0;i<t.length;i++)if(void 0!==this.body.nodes[t[i]]){var o=this.body.nodes[t[i]];e[t[i]]={x:Math.round(o.x),y:Math.round(o.y)}}}else if(void 0!==this.body.nodes[t]){var n=this.body.nodes[t];e[t]={x:Math.round(n.x),y:Math.round(n.y)}}}else for(var s=0;s<this.body.nodeIndices.length;s++){var r=this.body.nodes[this.body.nodeIndices[s]];e[this.body.nodeIndices[s]]={x:Math.round(r.x),y:Math.round(r.y)}}return e}},{key:"storePositions",value:function(){var t=[],e=this.body.data.nodes.getDataSet();for(var i in e._data)if(e._data.hasOwnProperty(i)){var o=this.body.nodes[i];e._data[i].x==Math.round(o.x)&&e._data[i].y==Math.round(o.y)||t.push({id:o.id,x:Math.round(o.x),y:Math.round(o.y)})}e.update(t)}},{key:"getBoundingBox",value:function(t){return void 0!==this.body.nodes[t]?this.body.nodes[t].shape.boundingBox:void 0}},{key:"getConnectedNodes",value:function(t){var e=[];if(void 0!==this.body.nodes[t])for(var i=this.body.nodes[t],o={},n=0;n<i.edges.length;n++){var s=i.edges[n];s.toId==i.id?void 0===o[s.fromId]&&(e.push(s.fromId),o[s.fromId]=!0):s.fromId==i.id&&void 0===o[s.toId]&&(e.push(s.toId),o[s.toId]=!0)}return e}},{key:"getConnectedEdges",value:function(t){var e=[];if(void 0!==this.body.nodes[t])for(var i=this.body.nodes[t],o=0;o<i.edges.length;o++)e.push(i.edges[o].id);else console.log("NodeId provided for getConnectedEdges does not exist. Provided: ",t);return e}},{key:"moveNode",value:function(t,e,i){var o=this;void 0!==this.body.nodes[t]?(this.body.nodes[t].x=Number(e),this.body.nodes[t].y=Number(i),setTimeout(function(){o.body.emitter.emit("startSimulation")},0)):console.log("Node id supplied to moveNode does not exist. Provided: ",t)}}]),t}();e["default"]=p},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(66),a=o(r),h=i(67),d=o(h),l=i(69),c=o(l),u=i(71),p=o(u),f=i(72),m=o(f),v=i(73),g=o(v),y=i(75),b=o(y),w=i(76),_=o(w),x=i(77),k=o(x),O=i(78),M=o(O),D=i(79),S=o(D),C=i(80),T=o(C),E=i(81),P=o(E),I=i(82),N=o(I),R=i(83),z=o(R),L=i(29),A=(o(L),i(1)),B=function(){function t(e,i,o,s,r){n(this,t),this.options=A.bridgeObject(r),this.globalOptions=r,this.body=i,this.edges=[],this.id=void 0,this.imagelist=o,this.grouplist=s,this.x=void 0,this.y=void 0,this.baseSize=this.options.size,this.baseFontSize=this.options.font.size,this.predefinedPosition=!1,this.selected=!1,this.hover=!1,this.labelModule=new a["default"](this.body,this.options,!1),this.setOptions(e)}return s(t,[{key:"attachEdge",value:function(t){-1===this.edges.indexOf(t)&&this.edges.push(t)}},{key:"detachEdge",value:function(t){var e=this.edges.indexOf(t);-1!=e&&this.edges.splice(e,1)}},{key:"setOptions",value:function(e){var i=this.options.shape;if(e){if(void 0!==e.id&&(this.id=e.id),void 0===this.id)throw"Node must have an id";if(void 0!==e.x&&(null===e.x?(this.x=void 0,this.predefinedPosition=!1):(this.x=parseInt(e.x),this.predefinedPosition=!0)),void 0!==e.y&&(null===e.y?(this.y=void 0,this.predefinedPosition=!1):(this.y=parseInt(e.y),this.predefinedPosition=!0)),void 0!==e.size&&(this.baseSize=e.size),void 0!==e.value&&(e.value=parseFloat(e.value)),"number"==typeof e.group||"string"==typeof e.group&&""!=e.group){var o=this.grouplist.get(e.group);A.deepExtend(this.options,o),this.options.color=A.parseColor(this.options.color)}if(t.parseOptions(this.options,e,!0,this.globalOptions),void 0!==this.options.image){if(!this.imagelist)throw"No imagelist provided";this.imageObj=this.imagelist.load(this.options.image,this.options.brokenImage,this.id)}return this.updateLabelModule(),this.updateShape(i),void 0!==e.hidden||void 0!==e.physics}}},{key:"updateLabelModule",value:function(){void 0!==this.options.label&&null!==this.options.label||(this.options.label=""),this.labelModule.setOptions(this.options,!0),void 0!==this.labelModule.baseSize&&(this.baseFontSize=this.labelModule.baseSize)}},{key:"updateShape",value:function(t){if(t===this.options.shape&&this.shape)this.shape.setOptions(this.options,this.imageObj);else switch(this.options.shape){case"box":this.shape=new d["default"](this.options,this.body,this.labelModule);break;case"circle":this.shape=new c["default"](this.options,this.body,this.labelModule);break;case"circularImage":this.shape=new p["default"](this.options,this.body,this.labelModule,this.imageObj);break;case"database":this.shape=new m["default"](this.options,this.body,this.labelModule);break;case"diamond":this.shape=new g["default"](this.options,this.body,this.labelModule);break;case"dot":this.shape=new b["default"](this.options,this.body,this.labelModule);break;case"ellipse":this.shape=new _["default"](this.options,this.body,this.labelModule);break;case"icon":this.shape=new k["default"](this.options,this.body,this.labelModule);break;case"image":this.shape=new M["default"](this.options,this.body,this.labelModule,this.imageObj);break;case"square":this.shape=new S["default"](this.options,this.body,this.labelModule);break;case"star":this.shape=new T["default"](this.options,this.body,this.labelModule);break;case"text":this.shape=new P["default"](this.options,this.body,this.labelModule);break;case"triangle":this.shape=new N["default"](this.options,this.body,this.labelModule);break;case"triangleDown":this.shape=new z["default"](this.options,this.body,this.labelModule);break;default:this.shape=new _["default"](this.options,this.body,this.labelModule)}this._reset()}},{key:"select",value:function(){this.selected=!0,this._reset()}},{key:"unselect",value:function(){this.selected=!1,this._reset()}},{key:"_reset",value:function(){this.shape.width=void 0,this.shape.height=void 0}},{key:"getTitle",value:function(){return this.options.title}},{key:"distanceToBorder",value:function(t,e){return this.shape.distanceToBorder(t,e)}},{key:"isFixed",value:function(){return this.options.fixed.x&&this.options.fixed.y}},{key:"isSelected",value:function(){return this.selected}},{key:"getValue",value:function(){return this.options.value}},{key:"setValueRange",value:function(t,e,i){if(void 0!==this.options.value){var o=this.options.scaling.customScalingFunction(t,e,i,this.options.value),n=this.options.scaling.max-this.options.scaling.min;if(this.options.scaling.label.enabled===!0){var s=this.options.scaling.label.max-this.options.scaling.label.min;this.options.font.size=this.options.scaling.label.min+o*s}this.options.size=this.options.scaling.min+o*n}else this.options.size=this.baseSize,this.options.font.size=this.baseFontSize;this.updateLabelModule()}},{key:"draw",value:function(t){this.shape.draw(t,this.x,this.y,this.selected,this.hover)}},{key:"updateBoundingBox",value:function(t){this.shape.updateBoundingBox(this.x,this.y,t)}},{key:"resize",value:function(t){this.shape.resize(t,this.selected)}},{key:"isOverlappingWith",value:function(t){return this.shape.left<t.right&&this.shape.left+this.shape.width>t.left&&this.shape.top<t.bottom&&this.shape.top+this.shape.height>t.top}},{key:"isBoundingBoxOverlappingWith",value:function(t){return this.shape.boundingBox.left<t.right&&this.shape.boundingBox.right>t.left&&this.shape.boundingBox.top<t.bottom&&this.shape.boundingBox.bottom>t.top}}],[{key:"parseOptions",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=arguments.length<=3||void 0===arguments[3]?{}:arguments[3],n=["color","font","fixed","shadow"];if(A.selectiveNotDeepExtend(n,t,e,i),A.mergeOptions(t,e,"shadow",i,o),void 0!==e.color&&null!==e.color){var s=A.parseColor(e.color);A.fillIfDefined(t.color,s)}else i===!0&&null===e.color&&(t.color=A.bridgeObject(o.color));void 0!==e.fixed&&null!==e.fixed&&("boolean"==typeof e.fixed?(t.fixed.x=e.fixed,t.fixed.y=e.fixed):(void 0!==e.fixed.x&&"boolean"==typeof e.fixed.x&&(t.fixed.x=e.fixed.x),void 0!==e.fixed.y&&"boolean"==typeof e.fixed.y&&(t.fixed.y=e.fixed.y))),void 0!==e.font&&null!==e.font?a["default"].parseOptions(t.font,e):i===!0&&null===e.font&&(t.font=A.bridgeObject(o.font)),void 0!==e.scaling&&A.mergeOptions(t.scaling,e.scaling,"label",i,o.scaling)}}]),t}();e["default"]=B},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){var i=[],o=!0,n=!1,s=void 0;try{for(var r,a=t[Symbol.iterator]();!(o=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);o=!0);}catch(h){n=!0,s=h}finally{try{!o&&a["return"]&&a["return"]()}finally{if(n)throw s}}return i}return function(e,i){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return t(e,i);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},r=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),a=i(1),h=function(){function t(e,i){var n=arguments.length<=2||void 0===arguments[2]?!1:arguments[2];o(this,t),this.body=e,this.pointToSelf=!1,this.baseSize=void 0,this.fontOptions={},this.setOptions(i),this.size={top:0,left:0,width:0,height:0,yLine:0},this.isEdgeLabel=n}return r(t,[{key:"setOptions",value:function(e){var i=arguments.length<=1||void 0===arguments[1]?!1:arguments[1];this.nodeOptions=e,this.fontOptions=a.deepExtend({},e.font,!0),void 0!==e.label&&(this.labelDirty=!0),void 0!==e.font&&(t.parseOptions(this.fontOptions,e,i),"string"==typeof e.font?this.baseSize=this.fontOptions.size:"object"===s(e.font)&&void 0!==e.font.size&&(this.baseSize=e.font.size))}},{key:"draw",value:function(t,e,i,o){var n=arguments.length<=4||void 0===arguments[4]?"middle":arguments[4];if(void 0!==this.nodeOptions.label){var s=this.fontOptions.size*this.body.view.scale;this.nodeOptions.label&&s<this.nodeOptions.scaling.label.drawThreshold-1||(this.calculateLabelSize(t,o,e,i,n),
-this._drawBackground(t),this._drawText(t,o,e,i,n))}}},{key:"_drawBackground",value:function(t){if(void 0!==this.fontOptions.background&&"none"!==this.fontOptions.background){t.fillStyle=this.fontOptions.background;var e=2;if(this.isEdgeLabel)switch(this.fontOptions.align){case"middle":t.fillRect(.5*-this.size.width,.5*-this.size.height,this.size.width,this.size.height);break;case"top":t.fillRect(.5*-this.size.width,-(this.size.height+e),this.size.width,this.size.height);break;case"bottom":t.fillRect(.5*-this.size.width,e,this.size.width,this.size.height);break;default:t.fillRect(this.size.left,this.size.top-.5*e,this.size.width,this.size.height)}else t.fillRect(this.size.left,this.size.top-.5*e,this.size.width,this.size.height)}}},{key:"_drawText",value:function(t,e,i,o){var s=arguments.length<=4||void 0===arguments[4]?"middle":arguments[4],r=this.fontOptions.size,a=r*this.body.view.scale;a>=this.nodeOptions.scaling.label.maxVisible&&(r=Number(this.nodeOptions.scaling.label.maxVisible)/this.body.view.scale);var h=this.size.yLine,d=this._getColor(a),l=n(d,2),c=l[0],u=l[1],p=this._setAlignment(t,i,h,s),f=n(p,2);i=f[0],h=f[1],t.font=(e&&this.nodeOptions.labelHighlightBold?"bold ":"")+r+"px "+this.fontOptions.face,t.fillStyle=c,this.isEdgeLabel||"left"!==this.fontOptions.align?t.textAlign="center":(t.textAlign=this.fontOptions.align,i-=.5*this.size.width),this.fontOptions.strokeWidth>0&&(t.lineWidth=this.fontOptions.strokeWidth,t.strokeStyle=u,t.lineJoin="round");for(var m=0;m<this.lineCount;m++)this.fontOptions.strokeWidth>0&&t.strokeText(this.lines[m],i,h),t.fillText(this.lines[m],i,h),h+=r}},{key:"_setAlignment",value:function(t,e,i,o){if(this.isEdgeLabel&&"horizontal"!==this.fontOptions.align&&this.pointToSelf===!1){e=0,i=0;var n=2;"top"===this.fontOptions.align?(t.textBaseline="alphabetic",i-=2*n):"bottom"===this.fontOptions.align?(t.textBaseline="hanging",i+=2*n):t.textBaseline="middle"}else t.textBaseline=o;return[e,i]}},{key:"_getColor",value:function(t){var e=this.fontOptions.color||"#000000",i=this.fontOptions.strokeColor||"#ffffff";if(t<=this.nodeOptions.scaling.label.drawThreshold){var o=Math.max(0,Math.min(1,1-(this.nodeOptions.scaling.label.drawThreshold-t)));e=a.overrideOpacity(e,o),i=a.overrideOpacity(i,o)}return[e,i]}},{key:"getTextSize",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],i={width:this._processLabel(t,e),height:this.fontOptions.size*this.lineCount,lineCount:this.lineCount};return i}},{key:"calculateLabelSize",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?0:arguments[2],o=arguments.length<=3||void 0===arguments[3]?0:arguments[3],n=arguments.length<=4||void 0===arguments[4]?"middle":arguments[4];this.labelDirty===!0&&(this.size.width=this._processLabel(t,e)),this.size.height=this.fontOptions.size*this.lineCount,this.size.left=i-.5*this.size.width,this.size.top=o-.5*this.size.height,this.size.yLine=o+.5*(1-this.lineCount)*this.fontOptions.size,"hanging"===n&&(this.size.top+=.5*this.fontOptions.size,this.size.top+=4,this.size.yLine+=4),this.labelDirty=!1}},{key:"_processLabel",value:function(t,e){var i=0,o=[""],n=0;if(void 0!==this.nodeOptions.label){o=String(this.nodeOptions.label).split("\n"),n=o.length,t.font=(e&&this.nodeOptions.labelHighlightBold?"bold ":"")+this.fontOptions.size+"px "+this.fontOptions.face,i=t.measureText(o[0]).width;for(var s=1;n>s;s++){var r=t.measureText(o[s]).width;i=r>i?r:i}}return this.lines=o,this.lineCount=n,i}}],[{key:"parseOptions",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2];if("string"==typeof e.font){var o=e.font.split(" ");t.size=o[0].replace("px",""),t.face=o[1],t.color=o[2]}else"object"===s(e.font)&&a.fillIfDefined(t,e.font,i);t.size=Number(t.size)}}]),t}();e["default"]=h},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t,e){if(void 0===this.width){var i=5,o=this.labelModule.getTextSize(t,e);this.width=o.width+2*i,this.height=o.height+2*i,this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){this.resize(t,o),this.left=e-this.width/2,this.top=i-this.height/2;var s=this.options.borderWidth,r=this.options.borderWidthSelected||2*this.options.borderWidth;t.strokeStyle=o?this.options.color.highlight.border:n?this.options.color.hover.border:this.options.color.border,t.lineWidth=o?r:s,t.lineWidth/=this.body.view.scale,t.lineWidth=Math.min(this.width,t.lineWidth),t.fillStyle=o?this.options.color.highlight.background:n?this.options.color.hover.background:this.options.color.background;var a=this.options.shapeProperties.borderRadius;t.roundRect(this.left,this.top,this.width,this.height,a),this.enableShadow(t),t.fill(),this.disableShadow(t),t.save(),s>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore(),this.updateBoundingBox(e,i,t,o),this.labelModule.draw(t,e,i,o)}},{key:"updateBoundingBox",value:function(t,e,i,o){this.resize(i,o),this.left=t-.5*this.width,this.top=e-.5*this.height;var n=this.options.shapeProperties.borderRadius;this.boundingBox.left=this.left-n,this.boundingBox.top=this.top-n,this.boundingBox.bottom=this.top+this.height+n,this.boundingBox.right=this.left+this.width+n}},{key:"distanceToBorder",value:function(t,e){this.resize(t);var i=this.options.borderWidth;return Math.min(Math.abs(this.width/2/Math.cos(e)),Math.abs(this.height/2/Math.sin(e)))+i}}]),e}(d["default"]);e["default"]=l},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=o,this.labelModule=n,this.setOptions(e),this.top=void 0,this.left=void 0,this.height=void 0,this.width=void 0,this.radius=void 0,this.boundingBox={top:0,left:0,right:0,bottom:0}}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"_distanceToBorder",value:function(t,e){var i=this.options.borderWidth;return this.resize(t),Math.min(Math.abs(this.width/2/Math.cos(e)),Math.abs(this.height/2/Math.sin(e)))+i}},{key:"enableShadow",value:function(t){this.options.shadow.enabled===!0&&(t.shadowColor=this.options.shadow.color,t.shadowBlur=this.options.shadow.size,t.shadowOffsetX=this.options.shadow.x,t.shadowOffsetY=this.options.shadow.y)}},{key:"disableShadow",value:function(t){this.options.shadow.enabled===!0&&(t.shadowColor="rgba(0,0,0,0)",t.shadowBlur=0,t.shadowOffsetX=0,t.shadowOffsetY=0)}},{key:"enableBorderDashes",value:function(t){if(this.options.shapeProperties.borderDashes!==!1)if(void 0!==t.setLineDash){var e=this.options.shapeProperties.borderDashes;e===!0&&(e=[5,15]),t.setLineDash(e)}else console.warn("setLineDash is not supported in this browser. The dashed borders cannot be used."),this.options.shapeProperties.borderDashes=!1}},{key:"disableBorderDashes",value:function(t){this.options.shapeProperties.borderDashes!==!1&&(void 0!==t.setLineDash?t.setLineDash([0]):(console.warn("setLineDash is not supported in this browser. The dashed borders cannot be used."),this.options.shapeProperties.borderDashes=!1))}}]),t}();e["default"]=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(70),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t,e){if(void 0===this.width){var i=5,o=this.labelModule.getTextSize(t,e),n=Math.max(o.width,o.height)+2*i;this.options.size=n/2,this.width=n,this.height=n,this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){this.resize(t,o),this.left=e-this.width/2,this.top=i-this.height/2,this._drawRawCircle(t,e,i,o,n,this.options.size),this.boundingBox.top=i-this.options.size,this.boundingBox.left=e-this.options.size,this.boundingBox.right=e+this.options.size,this.boundingBox.bottom=i+this.options.size,this.updateBoundingBox(e,i),this.labelModule.draw(t,e,i,o)}},{key:"updateBoundingBox",value:function(t,e){this.boundingBox.top=e-this.options.size,this.boundingBox.left=t-this.options.size,this.boundingBox.right=t+this.options.size,this.boundingBox.bottom=e+this.options.size}},{key:"distanceToBorder",value:function(t,e){return this.resize(t),.5*this.width}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){n(this,e);var r=s(this,Object.getPrototypeOf(e).call(this,t,i,o));return r.labelOffset=0,r.imageLoaded=!1,r}return r(e,t),a(e,[{key:"setOptions",value:function(t,e){this.options=t,e&&(this.imageObj=e)}},{key:"_resizeImage",value:function(){var t=!1;if(this.imageObj.width&&this.imageObj.height?this.imageLoaded===!1&&(this.imageLoaded=!0,t=!0):this.imageLoaded=!1,!this.width||!this.height||t===!0){var e,i,o;this.imageObj.width&&this.imageObj.height&&(e=0,i=0),this.options.shapeProperties.useImageSize===!1?this.imageObj.width>this.imageObj.height?(o=this.imageObj.width/this.imageObj.height,e=2*this.options.size*o||this.imageObj.width,i=2*this.options.size||this.imageObj.height):(o=this.imageObj.width&&this.imageObj.height?this.imageObj.height/this.imageObj.width:1,e=2*this.options.size,i=2*this.options.size*o):(e=this.imageObj.width,i=this.imageObj.height),this.width=e,this.height=i,this.radius=.5*this.width}}},{key:"_drawRawCircle",value:function(t,e,i,o,n,s){var r=this.options.borderWidth,a=this.options.borderWidthSelected||2*this.options.borderWidth,h=(o?a:r)/this.body.view.scale;t.lineWidth=Math.min(this.width,h),t.strokeStyle=o?this.options.color.highlight.border:n?this.options.color.hover.border:this.options.color.border,t.fillStyle=o?this.options.color.highlight.background:n?this.options.color.hover.background:this.options.color.background,t.circle(e,i,s),this.enableShadow(t),t.fill(),this.disableShadow(t),t.save(),h>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore()}},{key:"_drawImageAtPosition",value:function(t){if(0!=this.imageObj.width){t.globalAlpha=1,this.enableShadow(t);var e=this.imageObj.width/this.width/this.body.view.scale;if(e>2&&this.options.shapeProperties.interpolation===!0){var i=this.imageObj.width,o=this.imageObj.height,n=document.createElement("canvas");n.width=i,n.height=i;var s=n.getContext("2d");e*=.5,i*=.5,o*=.5,s.drawImage(this.imageObj,0,0,i,o);for(var r=0,a=1;e>2&&4>a;)s.drawImage(n,r,0,i,o,r+i,0,i/2,o/2),r+=i,e*=.5,i*=.5,o*=.5,a+=1;t.drawImage(n,r,0,i,o,this.left,this.top,this.width,this.height)}else t.drawImage(this.imageObj,this.left,this.top,this.width,this.height);this.disableShadow(t)}}},{key:"_drawImageLabel",value:function(t,e,i,o){var n,s=0;if(void 0!==this.height){s=.5*this.height;var r=this.labelModule.getTextSize(t);r.lineCount>=1&&(s+=r.height/2)}n=i+s,this.options.label&&(this.labelOffset=s),this.labelModule.draw(t,e,n,o,"hanging")}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(70),d=o(h),l=function(t){function e(t,i,o,r){n(this,e);var a=s(this,Object.getPrototypeOf(e).call(this,t,i,o));return a.imageObj=r,a._swapToImageResizeWhenImageLoaded=!0,a}return r(e,t),a(e,[{key:"resize",value:function(){if(void 0===this.imageObj.src||void 0===this.imageObj.width||void 0===this.imageObj.height){if(!this.width){var t=2*this.options.size;this.width=t,this.height=t,this._swapToImageResizeWhenImageLoaded=!0,this.radius=.5*this.width}}else this._swapToImageResizeWhenImageLoaded&&(this.width=void 0,this.height=void 0,this._swapToImageResizeWhenImageLoaded=!1),this._resizeImage()}},{key:"draw",value:function(t,e,i,o,n){this.resize(),this.left=e-this.width/2,this.top=i-this.height/2;var s=Math.min(.5*this.height,.5*this.width);this._drawRawCircle(t,e,i,o,n,s),t.save(),t.clip(),this._drawImageAtPosition(t),t.restore(),this._drawImageLabel(t,e,i,o),this.updateBoundingBox(e,i)}},{key:"updateBoundingBox",value:function(t,e){this.boundingBox.top=e-this.options.size,this.boundingBox.left=t-this.options.size,this.boundingBox.right=t+this.options.size,this.boundingBox.bottom=e+this.options.size,this.boundingBox.left=Math.min(this.boundingBox.left,this.labelModule.size.left),this.boundingBox.right=Math.max(this.boundingBox.right,this.labelModule.size.left+this.labelModule.size.width),this.boundingBox.bottom=Math.max(this.boundingBox.bottom,this.boundingBox.bottom+this.labelOffset)}},{key:"distanceToBorder",value:function(t,e){return this.resize(t),.5*this.width}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t,e){if(void 0===this.width){var i=5,o=this.labelModule.getTextSize(t,e),n=o.width+2*i;this.width=n,this.height=n,this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){this.resize(t,o),this.left=e-this.width/2,this.top=i-this.height/2;var s=this.options.borderWidth,r=this.options.borderWidthSelected||2*this.options.borderWidth,a=(o?r:s)/this.body.view.scale;t.lineWidth=Math.min(this.width,a),t.strokeStyle=o?this.options.color.highlight.border:n?this.options.color.hover.border:this.options.color.border,t.fillStyle=o?this.options.color.highlight.background:n?this.options.color.hover.background:this.options.color.background,t.database(e-this.width/2,i-.5*this.height,this.width,this.height),this.enableShadow(t),t.fill(),this.disableShadow(t),t.save(),a>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore(),this.updateBoundingBox(e,i,t,o),this.labelModule.draw(t,e,i,o)}},{key:"updateBoundingBox",value:function(t,e,i,o){this.resize(i,o),this.left=t-.5*this.width,this.top=e-.5*this.height,this.boundingBox.left=this.left,this.boundingBox.top=this.top,this.boundingBox.bottom=this.top+this.height,this.boundingBox.right=this.left+this.width}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"diamond",4,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_resizeShape",value:function(){if(void 0===this.width){var t=2*this.options.size;this.width=t,this.height=t,this.radius=.5*this.width}}},{key:"_drawShape",value:function(t,e,i,o,n,s,r){this._resizeShape(),this.left=o-this.width/2,this.top=n-this.height/2;var a=this.options.borderWidth,h=this.options.borderWidthSelected||2*this.options.borderWidth,d=(s?h:a)/this.body.view.scale;if(t.lineWidth=Math.min(this.width,d),t.strokeStyle=s?this.options.color.highlight.border:r?this.options.color.hover.border:this.options.color.border,t.fillStyle=s?this.options.color.highlight.background:r?this.options.color.hover.background:this.options.color.background,t[e](o,n,this.options.size),this.enableShadow(t),t.fill(),this.disableShadow(t),t.save(),d>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore(),void 0!==this.options.label){var l=n+.5*this.height+3;this.labelModule.draw(t,o,l,s,"hanging")}this.updateBoundingBox(o,n)}},{key:"updateBoundingBox",value:function(t,e){this.boundingBox.top=e-this.options.size,this.boundingBox.left=t-this.options.size,this.boundingBox.right=t+this.options.size,this.boundingBox.bottom=e+this.options.size,void 0!==this.options.label&&this.labelModule.size.width>0&&(this.boundingBox.left=Math.min(this.boundingBox.left,this.labelModule.size.left),this.boundingBox.right=Math.max(this.boundingBox.right,this.labelModule.size.left+this.labelModule.size.width),this.boundingBox.bottom=Math.max(this.boundingBox.bottom,this.boundingBox.bottom+this.labelModule.size.height+3))}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"circle",2,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this.resize(t),this.options.size}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t,e){if(void 0===this.width){var i=this.labelModule.getTextSize(t,e);this.width=1.5*i.width,this.height=2*i.height,this.width<this.height&&(this.width=this.height),this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){this.resize(t,o),this.left=e-.5*this.width,this.top=i-.5*this.height;var s=this.options.borderWidth,r=this.options.borderWidthSelected||2*this.options.borderWidth,a=(o?r:s)/this.body.view.scale;t.lineWidth=Math.min(this.width,a),t.strokeStyle=o?this.options.color.highlight.border:n?this.options.color.hover.border:this.options.color.border,t.fillStyle=o?this.options.color.highlight.background:n?this.options.color.hover.background:this.options.color.background,t.ellipse(this.left,this.top,this.width,this.height),this.enableShadow(t),t.fill(),this.disableShadow(t),t.save(),a>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore(),this.updateBoundingBox(e,i,t,o),this.labelModule.draw(t,e,i,o)}},{key:"updateBoundingBox",value:function(t,e,i,o){this.resize(i,o),this.left=t-.5*this.width,this.top=e-.5*this.height,this.boundingBox.left=this.left,this.boundingBox.top=this.top,this.boundingBox.bottom=this.top+this.height,this.boundingBox.right=this.left+this.width}},{key:"distanceToBorder",value:function(t,e){this.resize(t);var i=.5*this.width,o=.5*this.height,n=Math.sin(e)*i,s=Math.cos(e)*o;return i*o/Math.sqrt(n*n+s*s)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){if(void 0===this.width){var e=5,i={width:Number(this.options.icon.size),height:Number(this.options.icon.size)};this.width=i.width+2*e,this.height=i.height+2*e,this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){if(this.resize(t),this.options.icon.size=this.options.icon.size||50,this.left=e-.5*this.width,this.top=i-.5*this.height,this._icon(t,e,i,o),void 0!==this.options.label){var s=5;this.labelModule.draw(t,e,i+.5*this.height+s,o)}this.updateBoundingBox(e,i)}},{key:"updateBoundingBox",value:function(t,e){if(this.boundingBox.top=e-.5*this.options.icon.size,this.boundingBox.left=t-.5*this.options.icon.size,this.boundingBox.right=t+.5*this.options.icon.size,this.boundingBox.bottom=e+.5*this.options.icon.size,void 0!==this.options.label&&this.labelModule.size.width>0){var i=5;this.boundingBox.left=Math.min(this.boundingBox.left,this.labelModule.size.left),this.boundingBox.right=Math.max(this.boundingBox.right,this.labelModule.size.left+this.labelModule.size.width),this.boundingBox.bottom=Math.max(this.boundingBox.bottom,this.boundingBox.bottom+this.labelModule.size.height+i)}}},{key:"_icon",value:function(t,e,i,o){var n=Number(this.options.icon.size);void 0!==this.options.icon.code?(t.font=(o?"bold ":"")+n+"px "+this.options.icon.face,t.fillStyle=this.options.icon.color||"black",t.textAlign="center",t.textBaseline="middle",this.enableShadow(t),t.fillText(this.options.icon.code,e,i),this.disableShadow(t)):console.error("When using the icon shape, you need to define the code in the icon options object. This can be done per node or globally.")}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(70),d=o(h),l=function(t){function e(t,i,o,r){n(this,e);var a=s(this,Object.getPrototypeOf(e).call(this,t,i,o));return a.imageObj=r,a}return r(e,t),a(e,[{key:"resize",value:function(){this._resizeImage()}},{key:"draw",value:function(t,e,i,o,n){if(this.resize(),this.left=e-this.width/2,this.top=i-this.height/2,this.options.shapeProperties.useBorderWithImage===!0){var s=this.options.borderWidth,r=this.options.borderWidthSelected||2*this.options.borderWidth,a=(o?r:s)/this.body.view.scale;t.lineWidth=Math.min(this.width,a),t.beginPath(),t.strokeStyle=o?this.options.color.highlight.border:n?this.options.color.hover.border:this.options.color.border,t.fillStyle=o?this.options.color.highlight.background:n?this.options.color.hover.background:this.options.color.background,t.rect(this.left-.5*t.lineWidth,this.top-.5*t.lineWidth,this.width+t.lineWidth,this.height+t.lineWidth),t.fill(),t.save(),a>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore(),t.closePath()}this._drawImageAtPosition(t),this._drawImageLabel(t,e,i,o||n),this.updateBoundingBox(e,i)}},{key:"updateBoundingBox",value:function(t,e){this.resize(),this.left=t-this.width/2,this.top=e-this.height/2,this.boundingBox.top=this.top,this.boundingBox.left=this.left,this.boundingBox.right=this.left+this.width,this.boundingBox.bottom=this.top+this.height,void 0!==this.options.label&&this.labelModule.size.width>0&&(this.boundingBox.left=Math.min(this.boundingBox.left,this.labelModule.size.left),this.boundingBox.right=Math.max(this.boundingBox.right,this.labelModule.size.left+this.labelModule.size.width),this.boundingBox.bottom=Math.max(this.boundingBox.bottom,this.boundingBox.bottom+this.labelOffset))}},{key:"distanceToBorder",
-value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"square",2,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"star",4,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t,e){if(void 0===this.width){var i=5,o=this.labelModule.getTextSize(t,e);this.width=o.width+2*i,this.height=o.height+2*i,this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){this.resize(t,o||n),this.left=e-this.width/2,this.top=i-this.height/2,this.enableShadow(t),this.labelModule.draw(t,e,i,o||n),this.disableShadow(t),this.updateBoundingBox(e,i,t,o)}},{key:"updateBoundingBox",value:function(t,e,i,o){this.resize(i,o),this.left=t-this.width/2,this.top=e-this.height/2,this.boundingBox.top=this.top,this.boundingBox.left=this.left,this.boundingBox.right=this.left+this.width,this.boundingBox.bottom=this.top+this.height}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"triangle",3,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"triangleDown",3,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(85),a=o(r),h=i(66),d=o(h),l=i(1),c=i(9),u=i(11),p=function(){function t(e,i,o){var s=this;n(this,t),this.body=e,this.images=i,this.groups=o,this.body.functions.createEdge=this.create.bind(this),this.edgesListeners={add:function(t,e){s.add(e.items)},update:function(t,e){s.update(e.items)},remove:function(t,e){s.remove(e.items)}},this.options={},this.defaultOptions={arrows:{to:{enabled:!1,scaleFactor:1},middle:{enabled:!1,scaleFactor:1},from:{enabled:!1,scaleFactor:1}},arrowStrikethrough:!0,color:{color:"#848484",highlight:"#848484",hover:"#848484",inherit:"from",opacity:1},dashes:!1,font:{color:"#343434",size:14,face:"arial",background:"none",strokeWidth:2,strokeColor:"#ffffff",align:"horizontal"},hidden:!1,hoverWidth:1.5,label:void 0,labelHighlightBold:!0,length:void 0,physics:!0,scaling:{min:1,max:15,label:{enabled:!0,min:14,max:30,maxVisible:30,drawThreshold:5},customScalingFunction:function(t,e,i,o){if(e===t)return.5;var n=1/(e-t);return Math.max(0,(o-t)*n)}},selectionWidth:1.5,selfReferenceSize:20,shadow:{enabled:!1,color:"rgba(0,0,0,0.5)",size:10,x:5,y:5},smooth:{enabled:!0,type:"dynamic",forceDirection:"none",roundness:.5},title:void 0,width:1,value:void 0},l.extend(this.options,this.defaultOptions),this.bindEventListeners()}return s(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("_forceDisableDynamicCurves",function(e){"dynamic"===e&&(e="continuous");var i=!1;for(var o in t.body.edges)if(t.body.edges.hasOwnProperty(o)){var n=t.body.edges[o],s=t.body.data.edges._data[o];if(void 0!==s){var r=s.smooth;void 0!==r&&r.enabled===!0&&"dynamic"===r.type&&(void 0===e?n.setOptions({smooth:!1}):n.setOptions({smooth:{type:e}}),i=!0)}}i===!0&&t.body.emitter.emit("_dataChanged")}),this.body.emitter.on("_dataUpdated",function(){t.reconnectEdges(),t.markAllEdgesAsDirty()}),this.body.emitter.on("refreshEdges",this.refresh.bind(this)),this.body.emitter.on("refresh",this.refresh.bind(this)),this.body.emitter.on("destroy",function(){l.forEach(t.edgesListeners,function(e,i){t.body.data.edges&&t.body.data.edges.off(i,e)}),delete t.body.functions.createEdge,delete t.edgesListeners.add,delete t.edgesListeners.update,delete t.edgesListeners.remove,delete t.edgesListeners})}},{key:"setOptions",value:function(t){if(void 0!==t){a["default"].parseOptions(this.options,t),void 0!==t.color&&this.markAllEdgesAsDirty();var e=!1;if(void 0!==t.smooth)for(var i in this.body.edges)this.body.edges.hasOwnProperty(i)&&(e=this.body.edges[i].updateEdgeType()||e);if(void 0!==t.font){d["default"].parseOptions(this.options.font,t);for(var o in this.body.edges)this.body.edges.hasOwnProperty(o)&&this.body.edges[o].updateLabelModule()}void 0===t.hidden&&void 0===t.physics&&e!==!0||this.body.emitter.emit("_dataChanged")}}},{key:"setData",value:function(t){var e=this,i=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],o=this.body.data.edges;if(t instanceof c||t instanceof u)this.body.data.edges=t;else if(Array.isArray(t))this.body.data.edges=new c,this.body.data.edges.add(t);else{if(t)throw new TypeError("Array or DataSet expected");this.body.data.edges=new c}if(o&&l.forEach(this.edgesListeners,function(t,e){o.off(e,t)}),this.body.edges={},this.body.data.edges){l.forEach(this.edgesListeners,function(t,i){e.body.data.edges.on(i,t)});var n=this.body.data.edges.getIds();this.add(n,!0)}i===!1&&this.body.emitter.emit("_dataChanged")}},{key:"add",value:function(t){for(var e=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],i=this.body.edges,o=this.body.data.edges,n=0;n<t.length;n++){var s=t[n],r=i[s];r&&r.disconnect();var a=o.get(s,{showInternalIds:!0});i[s]=this.create(a)}e===!1&&this.body.emitter.emit("_dataChanged")}},{key:"update",value:function(t){for(var e=this.body.edges,i=this.body.data.edges,o=!1,n=0;n<t.length;n++){var s=t[n],r=i.get(s),a=e[s];void 0!==a?(a.disconnect(),o=a.setOptions(r)||o,a.connect()):(this.body.edges[s]=this.create(r),o=!0)}o===!0?this.body.emitter.emit("_dataChanged"):this.body.emitter.emit("_dataUpdated")}},{key:"remove",value:function(t){for(var e=this.body.edges,i=0;i<t.length;i++){var o=t[i],n=e[o];void 0!==n&&(n.cleanup(),n.disconnect(),delete e[o])}this.body.emitter.emit("_dataChanged")}},{key:"refresh",value:function(){var t=this.body.edges;for(var e in t){var i=void 0;t.hasOwnProperty(e)&&(i=t[e]);var o=this.body.data.edges._data[e];void 0!==i&&void 0!==o&&i.setOptions(o)}}},{key:"create",value:function(t){return new a["default"](t,this.body,this.options)}},{key:"markAllEdgesAsDirty",value:function(){for(var t in this.body.edges)this.body.edges[t].edgeType.colorDirty=!0}},{key:"reconnectEdges",value:function(){var t,e=this.body.nodes,i=this.body.edges;for(t in e)e.hasOwnProperty(t)&&(e[t].edges=[]);for(t in i)if(i.hasOwnProperty(t)){var o=i[t];o.from=null,o.to=null,o.connect()}}},{key:"getConnectedNodes",value:function(t){var e=[];if(void 0!==this.body.edges[t]){var i=this.body.edges[t];i.fromId&&e.push(i.fromId),i.toId&&e.push(i.toId)}return e}}]),t}();e["default"]=p},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},r=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),a=i(66),h=o(a),d=i(86),l=o(d),c=i(90),u=o(c),p=i(91),f=o(p),m=i(92),v=o(m),g=i(1),y=function(){function t(e,i,o){if(n(this,t),void 0===i)throw"No body provided";this.options=g.bridgeObject(o),this.globalOptions=o,this.body=i,this.id=void 0,this.fromId=void 0,this.toId=void 0,this.selected=!1,this.hover=!1,this.labelDirty=!0,this.colorDirty=!0,this.baseWidth=this.options.width,this.baseFontSize=this.options.font.size,this.from=void 0,this.to=void 0,this.edgeType=void 0,this.connected=!1,this.labelModule=new h["default"](this.body,this.options,!0),this.setOptions(e)}return r(t,[{key:"setOptions",value:function(e){if(e){this.colorDirty=!0,t.parseOptions(this.options,e,!0,this.globalOptions),void 0!==e.id&&(this.id=e.id),void 0!==e.from&&(this.fromId=e.from),void 0!==e.to&&(this.toId=e.to),void 0!==e.title&&(this.title=e.title),void 0!==e.value&&(e.value=parseFloat(e.value)),this.updateLabelModule();var i=this.updateEdgeType();return this._setInteractionWidths(),this.connect(),void 0===e.hidden&&void 0===e.physics||(i=!0),i}}},{key:"updateLabelModule",value:function(){this.labelModule.setOptions(this.options,!0),void 0!==this.labelModule.baseSize&&(this.baseFontSize=this.labelModule.baseSize)}},{key:"updateEdgeType",value:function(){var t=!1,e=!0,i=this.options.smooth;return void 0!==this.edgeType&&(this.edgeType instanceof u["default"]&&i.enabled===!0&&"dynamic"===i.type&&(e=!1),this.edgeType instanceof l["default"]&&i.enabled===!0&&"cubicBezier"===i.type&&(e=!1),this.edgeType instanceof f["default"]&&i.enabled===!0&&"dynamic"!==i.type&&"cubicBezier"!==i.type&&(e=!1),this.edgeType instanceof v["default"]&&i.enabled===!1&&(e=!1),e===!0&&(t=this.cleanup())),e===!0?this.options.smooth.enabled===!0?"dynamic"===this.options.smooth.type?(t=!0,this.edgeType=new u["default"](this.options,this.body,this.labelModule)):"cubicBezier"===this.options.smooth.type?this.edgeType=new l["default"](this.options,this.body,this.labelModule):this.edgeType=new f["default"](this.options,this.body,this.labelModule):this.edgeType=new v["default"](this.options,this.body,this.labelModule):this.edgeType.setOptions(this.options),t}},{key:"connect",value:function(){this.disconnect(),this.from=this.body.nodes[this.fromId]||void 0,this.to=this.body.nodes[this.toId]||void 0,this.connected=void 0!==this.from&&void 0!==this.to,this.connected===!0?(this.from.attachEdge(this),this.to.attachEdge(this)):(this.from&&this.from.detachEdge(this),this.to&&this.to.detachEdge(this)),this.edgeType.connect()}},{key:"disconnect",value:function(){this.from&&(this.from.detachEdge(this),this.from=void 0),this.to&&(this.to.detachEdge(this),this.to=void 0),this.connected=!1}},{key:"getTitle",value:function(){return this.title}},{key:"isSelected",value:function(){return this.selected}},{key:"getValue",value:function(){return this.options.value}},{key:"setValueRange",value:function(t,e,i){if(void 0!==this.options.value){var o=this.options.scaling.customScalingFunction(t,e,i,this.options.value),n=this.options.scaling.max-this.options.scaling.min;if(this.options.scaling.label.enabled===!0){var s=this.options.scaling.label.max-this.options.scaling.label.min;this.options.font.size=this.options.scaling.label.min+o*s}this.options.width=this.options.scaling.min+o*n}else this.options.width=this.baseWidth,this.options.font.size=this.baseFontSize;this._setInteractionWidths(),this.updateLabelModule()}},{key:"_setInteractionWidths",value:function(){"function"==typeof this.options.hoverWidth?this.edgeType.hoverWidth=this.options.hoverWidth(this.options.width):this.edgeType.hoverWidth=this.options.hoverWidth+this.options.width,"function"==typeof this.options.selectionWidth?this.edgeType.selectionWidth=this.options.selectionWidth(this.options.width):this.edgeType.selectionWidth=this.options.selectionWidth+this.options.width}},{key:"draw",value:function(t){var e=this.edgeType.getViaNode(),i={};this.edgeType.fromPoint=this.edgeType.from,this.edgeType.toPoint=this.edgeType.to,this.options.arrows.from.enabled===!0&&(i.from=this.edgeType.getArrowData(t,"from",e,this.selected,this.hover),this.options.arrowStrikethrough===!1&&(this.edgeType.fromPoint=i.from.core)),this.options.arrows.to.enabled===!0&&(i.to=this.edgeType.getArrowData(t,"to",e,this.selected,this.hover),this.options.arrowStrikethrough===!1&&(this.edgeType.toPoint=i.to.core)),this.options.arrows.middle.enabled===!0&&(i.middle=this.edgeType.getArrowData(t,"middle",e,this.selected,this.hover)),this.edgeType.drawLine(t,this.selected,this.hover,e),this.drawArrows(t,i),this.drawLabel(t,e)}},{key:"drawArrows",value:function(t,e){this.options.arrows.from.enabled===!0&&this.edgeType.drawArrowHead(t,this.selected,this.hover,e.from),this.options.arrows.middle.enabled===!0&&this.edgeType.drawArrowHead(t,this.selected,this.hover,e.middle),this.options.arrows.to.enabled===!0&&this.edgeType.drawArrowHead(t,this.selected,this.hover,e.to)}},{key:"drawLabel",value:function(t,e){if(void 0!==this.options.label){var i=this.from,o=this.to,n=this.from.selected||this.to.selected||this.selected;if(i.id!=o.id){this.labelModule.pointToSelf=!1;var s=this.edgeType.getPoint(.5,e);t.save(),"horizontal"!==this.options.font.align&&(this.labelModule.calculateLabelSize(t,n,s.x,s.y),t.translate(s.x,this.labelModule.size.yLine),this._rotateForLabelAlignment(t)),this.labelModule.draw(t,s.x,s.y,n),t.restore()}else{this.labelModule.pointToSelf=!0;var r,a,h=this.options.selfReferenceSize;i.shape.width>i.shape.height?(r=i.x+.5*i.shape.width,a=i.y-h):(r=i.x+h,a=i.y-.5*i.shape.height),s=this._pointOnCircle(r,a,h,.125),this.labelModule.draw(t,s.x,s.y,n)}}}},{key:"isOverlappingWith",value:function(t){if(this.connected){var e=10,i=this.from.x,o=this.from.y,n=this.to.x,s=this.to.y,r=t.left,a=t.top,h=this.edgeType.getDistanceToEdge(i,o,n,s,r,a);return e>h}return!1}},{key:"_rotateForLabelAlignment",value:function(t){var e=this.from.y-this.to.y,i=this.from.x-this.to.x,o=Math.atan2(e,i);(-1>o&&0>i||o>0&&0>i)&&(o+=Math.PI),t.rotate(o)}},{key:"_pointOnCircle",value:function(t,e,i,o){var n=2*o*Math.PI;return{x:t+i*Math.cos(n),y:e-i*Math.sin(n)}}},{key:"select",value:function(){this.selected=!0}},{key:"unselect",value:function(){this.selected=!1}},{key:"cleanup",value:function(){return this.edgeType.cleanup()}}],[{key:"parseOptions",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=arguments.length<=3||void 0===arguments[3]?{}:arguments[3],n=["arrowStrikethrough","id","from","hidden","hoverWidth","label","labelHighlightBold","length","line","opacity","physics","scaling","selectionWidth","selfReferenceSize","to","title","value","width"];if(g.selectiveDeepExtend(n,t,e,i),g.mergeOptions(t,e,"smooth",i,o),g.mergeOptions(t,e,"shadow",i,o),void 0!==e.dashes&&null!==e.dashes?t.dashes=e.dashes:i===!0&&null===e.dashes&&(t.dashes=Object.create(o.dashes)),void 0!==e.scaling&&null!==e.scaling?(void 0!==e.scaling.min&&(t.scaling.min=e.scaling.min),void 0!==e.scaling.max&&(t.scaling.max=e.scaling.max),g.mergeOptions(t.scaling,e.scaling,"label",i,o.scaling)):i===!0&&null===e.scaling&&(t.scaling=Object.create(o.scaling)),void 0!==e.arrows&&null!==e.arrows)if("string"==typeof e.arrows){var r=e.arrows.toLowerCase();t.arrows.to.enabled=-1!=r.indexOf("to"),t.arrows.middle.enabled=-1!=r.indexOf("middle"),t.arrows.from.enabled=-1!=r.indexOf("from")}else{if("object"!==s(e.arrows))throw new Error("The arrow newOptions can only be an object or a string. Refer to the documentation. You used:"+JSON.stringify(e.arrows));g.mergeOptions(t.arrows,e.arrows,"to",i,o.arrows),g.mergeOptions(t.arrows,e.arrows,"middle",i,o.arrows),g.mergeOptions(t.arrows,e.arrows,"from",i,o.arrows)}else i===!0&&null===e.arrows&&(t.arrows=Object.create(o.arrows));if(void 0!==e.color&&null!==e.color)if(t.color=g.deepExtend({},t.color,!0),g.isString(e.color))t.color.color=e.color,t.color.highlight=e.color,t.color.hover=e.color,t.color.inherit=!1;else{var a=!1;void 0!==e.color.color&&(t.color.color=e.color.color,a=!0),void 0!==e.color.highlight&&(t.color.highlight=e.color.highlight,a=!0),void 0!==e.color.hover&&(t.color.hover=e.color.hover,a=!0),void 0!==e.color.inherit&&(t.color.inherit=e.color.inherit),void 0!==e.color.opacity&&(t.color.opacity=Math.min(1,Math.max(0,e.color.opacity))),void 0===e.color.inherit&&a===!0&&(t.color.inherit=!1)}else i===!0&&null===e.color&&(t.color=g.bridgeObject(o.color));void 0!==e.font&&null!==e.font?h["default"].parseOptions(t.font,e):i===!0&&null===e.font&&(t.font=g.bridgeObject(o.font))}}]),t}();e["default"]=y},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){var i=[],o=!0,n=!1,s=void 0;try{for(var r,a=t[Symbol.iterator]();!(o=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);o=!0);}catch(h){n=!0,s=h}finally{try{!o&&a["return"]&&a["return"]()}finally{if(n)throw s}}return i}return function(e,i){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return t(e,i);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),h=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),d=i(87),l=o(d),c=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),h(e,[{key:"_line",value:function(t,e){var i=e[0],o=e[1];t.beginPath(),t.moveTo(this.fromPoint.x,this.fromPoint.y),void 0===e||void 0===i.x?t.lineTo(this.toPoint.x,this.toPoint.y):t.bezierCurveTo(i.x,i.y,o.x,o.y,this.toPoint.x,this.toPoint.y),this.enableShadow(t),t.stroke(),this.disableShadow(t)}},{key:"_getViaCoordinates",value:function(){var t=this.from.x-this.to.x,e=this.from.y-this.to.y,i=void 0,o=void 0,n=void 0,s=void 0,r=this.options.smooth.roundness;return(Math.abs(t)>Math.abs(e)||this.options.smooth.forceDirection===!0||"horizontal"===this.options.smooth.forceDirection)&&"vertical"!==this.options.smooth.forceDirection?(o=this.from.y,s=this.to.y,i=this.from.x-r*t,n=this.to.x+r*t):(o=this.from.y-r*e,s=this.to.y+r*e,i=this.from.x,n=this.to.x),[{x:i,y:o},{x:n,y:s}]}},{key:"getViaNode",value:function(){return this._getViaCoordinates()}},{key:"_findBorderPosition",value:function(t,e){return this._findBorderPositionBezier(t,e)}},{key:"_getDistanceToEdge",value:function(t,e,i,o,n,s){var r=arguments.length<=6||void 0===arguments[6]?this._getViaCoordinates():arguments[6],h=a(r,2),d=h[0],l=h[1];return this._getDistanceToBezierEdge(t,e,i,o,n,s,d,l)}},{key:"getPoint",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?this._getViaCoordinates():arguments[1],i=a(e,2),o=i[0],n=i[1],s=t,r=[];r[0]=Math.pow(1-s,3),r[1]=3*s*Math.pow(1-s,2),r[2]=3*Math.pow(s,2)*(1-s),r[3]=Math.pow(s,3);var h=r[0]*this.fromPoint.x+r[1]*o.x+r[2]*n.x+r[3]*this.toPoint.x,d=r[0]*this.fromPoint.y+r[1]*o.y+r[2]*n.y+r[3]*this.toPoint.y;return{x:h,y:d}}}]),e}(l["default"]);e["default"]=c},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(88),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_getDistanceToBezierEdge",value:function(t,e,i,o,n,s,r,a){var h=1e9,d=void 0,l=void 0,c=void 0,u=void 0,p=void 0,f=t,m=e,v=[0,0,0,0];for(l=1;10>l;l++)c=.1*l,v[0]=Math.pow(1-c,3),v[1]=3*c*Math.pow(1-c,2),v[2]=3*Math.pow(c,2)*(1-c),v[3]=Math.pow(c,3),u=v[0]*t+v[1]*r.x+v[2]*a.x+v[3]*i,p=v[0]*e+v[1]*r.y+v[2]*a.y+v[3]*o,l>0&&(d=this._getDistanceToLine(f,m,u,p,n,s),h=h>d?d:h),f=u,m=p;return h}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(89),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_findBorderPositionBezier",value:function(t,e){var i,o,n,s,r,a=arguments.length<=2||void 0===arguments[2]?this._getViaCoordinates():arguments[2],h=10,d=0,l=0,c=1,u=.2,p=this.to,f=!1;for(t.id===this.from.id&&(p=this.from,f=!0);c>=l&&h>d;){var m=.5*(l+c);if(i=this.getPoint(m,a),o=Math.atan2(p.y-i.y,p.x-i.x),n=p.distanceToBorder(e,o),s=Math.sqrt(Math.pow(i.x-p.x,2)+Math.pow(i.y-p.y,2)),r=n-s,Math.abs(r)<u)break;0>r?f===!1?l=m:c=m:f===!1?c=m:l=m,d++}return i.t=m,i}},{key:"_getDistanceToBezierEdge",value:function(t,e,i,o,n,s,r){var a=1e9,h=void 0,d=void 0,l=void 0,c=void 0,u=void 0,p=t,f=e;for(d=1;10>d;d++)l=.1*d,c=Math.pow(1-l,2)*t+2*l*(1-l)*r.x+Math.pow(l,2)*i,u=Math.pow(1-l,2)*e+2*l*(1-l)*r.y+Math.pow(l,2)*o,d>0&&(h=this._getDistanceToLine(p,f,c,u,n,s),a=a>h?h:a),p=c,f=u;return a}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){var i=[],o=!0,n=!1,s=void 0;try{for(var r,a=t[Symbol.iterator]();!(o=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);o=!0);}catch(h){n=!0,s=h}finally{try{!o&&a["return"]&&a["return"]()}finally{if(n)throw s}}return i}return function(e,i){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return t(e,i);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(1),a=function(){function t(e,i,n){o(this,t),this.body=i,this.labelModule=n,this.options={},this.setOptions(e),this.colorDirty=!0,this.color={},this.selectionWidth=2,this.hoverWidth=1.5,this.fromPoint=this.from,this.toPoint=this.to}return s(t,[{key:"connect",value:function(){this.from=this.body.nodes[this.options.from],this.to=this.body.nodes[this.options.to]}},{key:"cleanup",value:function(){return!1}},{key:"setOptions",value:function(t){this.options=t,this.from=this.body.nodes[this.options.from],this.to=this.body.nodes[this.options.to],this.id=this.options.id}},{key:"drawLine",value:function(t,e,i,o){t.strokeStyle=this.getColor(t,e,i),t.lineWidth=this.getLineWidth(e,i),this.options.dashes!==!1?this._drawDashedLine(t,o):this._drawLine(t,o)}},{key:"_drawLine",value:function(t,e,i,o){if(this.from!=this.to)this._line(t,e,i,o);else{var s=this._getCircleData(t),r=n(s,3),a=r[0],h=r[1],d=r[2];this._circle(t,a,h,d)}}},{key:"_drawDashedLine",value:function(t,e,i,o){t.lineCap="round";var s=[5,5];if(Array.isArray(this.options.dashes)===!0&&(s=this.options.dashes),void 0!==t.setLineDash){if(t.save(),t.setLineDash(s),t.lineDashOffset=0,this.from!=this.to)this._line(t,e);else{var r=this._getCircleData(t),a=n(r,3),h=a[0],d=a[1],l=a[2];this._circle(t,h,d,l)}t.setLineDash([0]),t.lineDashOffset=0,t.restore()}else{if(this.from!=this.to)t.dashedLine(this.from.x,this.from.y,this.to.x,this.to.y,s);else{var c=this._getCircleData(t),u=n(c,3),p=u[0],f=u[1],m=u[2];this._circle(t,p,f,m)}this.enableShadow(t),t.stroke(),this.disableShadow(t)}}},{key:"findBorderPosition",value:function(t,e,i){return this.from!=this.to?this._findBorderPosition(t,e,i):this._findBorderPositionCircle(t,e,i)}},{key:"findBorderPositions",value:function(t){var e={},i={};if(this.from!=this.to)e=this._findBorderPosition(this.from,t),i=this._findBorderPosition(this.to,t);else{var o=this._getCircleData(t),s=n(o,3),r=s[0],a=s[1];s[2];e=this._findBorderPositionCircle(this.from,t,{x:r,y:a,low:.25,high:.6,direction:-1}),i=this._findBorderPositionCircle(this.from,t,{x:r,y:a,low:.6,high:.8,direction:1})}return{from:e,to:i}}},{key:"_getCircleData",value:function(t){var e=void 0,i=void 0,o=this.from,n=this.options.selfReferenceSize;return void 0!==t&&void 0===o.shape.width&&o.shape.resize(t),o.shape.width>o.shape.height?(e=o.x+.5*o.shape.width,i=o.y-n):(e=o.x+n,i=o.y-.5*o.shape.height),[e,i,n]}},{key:"_pointOnCircle",value:function(t,e,i,o){var n=2*o*Math.PI;return{x:t+i*Math.cos(n),y:e-i*Math.sin(n)}}},{key:"_findBorderPositionCircle",value:function(t,e,i){for(var o=i.x,n=i.y,s=i.low,r=i.high,a=i.direction,h=10,d=0,l=this.options.selfReferenceSize,c=void 0,u=void 0,p=void 0,f=void 0,m=void 0,v=.05,g=.5*(s+r);r>=s&&h>d&&(g=.5*(s+r),c=this._pointOnCircle(o,n,l,g),u=Math.atan2(t.y-c.y,t.x-c.x),p=t.distanceToBorder(e,u),f=Math.sqrt(Math.pow(c.x-t.x,2)+Math.pow(c.y-t.y,2)),
-m=p-f,!(Math.abs(m)<v));)m>0?a>0?s=g:r=g:a>0?r=g:s=g,d++;return c.t=g,c}},{key:"getLineWidth",value:function(t,e){return t===!0?Math.max(this.selectionWidth,.3/this.body.view.scale):e===!0?Math.max(this.hoverWidth,.3/this.body.view.scale):Math.max(this.options.width,.3/this.body.view.scale)}},{key:"getColor",value:function(t,e,i){var o=this.options.color;if(o.inherit!==!1){if("both"===o.inherit&&this.from.id!==this.to.id){var n=t.createLinearGradient(this.from.x,this.from.y,this.to.x,this.to.y),s=void 0,a=void 0;return s=this.from.options.color.highlight.border,a=this.to.options.color.highlight.border,this.from.selected===!1&&this.to.selected===!1?(s=r.overrideOpacity(this.from.options.color.border,this.options.color.opacity),a=r.overrideOpacity(this.to.options.color.border,this.options.color.opacity)):this.from.selected===!0&&this.to.selected===!1?a=this.to.options.color.border:this.from.selected===!1&&this.to.selected===!0&&(s=this.from.options.color.border),n.addColorStop(0,s),n.addColorStop(1,a),n}this.colorDirty===!0&&("to"===o.inherit?(this.color.highlight=this.to.options.color.highlight.border,this.color.hover=this.to.options.color.hover.border,this.color.color=r.overrideOpacity(this.to.options.color.border,o.opacity)):(this.color.highlight=this.from.options.color.highlight.border,this.color.hover=this.from.options.color.hover.border,this.color.color=r.overrideOpacity(this.from.options.color.border,o.opacity)))}else this.colorDirty===!0&&(this.color.highlight=o.highlight,this.color.hover=o.hover,this.color.color=r.overrideOpacity(o.color,o.opacity));return this.colorDirty=!1,e===!0?this.color.highlight:i===!0?this.color.hover:this.color.color}},{key:"_circle",value:function(t,e,i,o){this.enableShadow(t),t.beginPath(),t.arc(e,i,o,0,2*Math.PI,!1),t.stroke(),this.disableShadow(t)}},{key:"getDistanceToEdge",value:function(t,e,i,o,s,r,a){var h=0;if(this.from!=this.to)h=this._getDistanceToEdge(t,e,i,o,s,r,a);else{var d=this._getCircleData(),l=n(d,3),c=l[0],u=l[1],p=l[2],f=c-s,m=u-r;h=Math.abs(Math.sqrt(f*f+m*m)-p)}return this.labelModule.size.left<s&&this.labelModule.size.left+this.labelModule.size.width>s&&this.labelModule.size.top<r&&this.labelModule.size.top+this.labelModule.size.height>r?0:h}},{key:"_getDistanceToLine",value:function(t,e,i,o,n,s){var r=i-t,a=o-e,h=r*r+a*a,d=((n-t)*r+(s-e)*a)/h;d>1?d=1:0>d&&(d=0);var l=t+d*r,c=e+d*a,u=l-n,p=c-s;return Math.sqrt(u*u+p*p)}},{key:"getArrowData",value:function(t,e,i,o,s){var r=void 0,a=void 0,h=void 0,d=void 0,l=void 0,c=void 0,u=this.getLineWidth(o,s);if("from"===e?(h=this.from,d=this.to,l=.1,c=this.options.arrows.from.scaleFactor):"to"===e?(h=this.to,d=this.from,l=-.1,c=this.options.arrows.to.scaleFactor):(h=this.to,d=this.from,c=this.options.arrows.middle.scaleFactor),h!=d)if("middle"!==e)if(this.options.smooth.enabled===!0){a=this.findBorderPosition(h,t,{via:i});var p=this.getPoint(Math.max(0,Math.min(1,a.t+l)),i);r=Math.atan2(a.y-p.y,a.x-p.x)}else r=Math.atan2(h.y-d.y,h.x-d.x),a=this.findBorderPosition(h,t);else r=Math.atan2(h.y-d.y,h.x-d.x),a=this.getPoint(.5,i);else{var f=this._getCircleData(t),m=n(f,3),v=m[0],g=m[1],y=m[2];"from"===e?(a=this.findBorderPosition(this.from,t,{x:v,y:g,low:.25,high:.6,direction:-1}),r=-2*a.t*Math.PI+1.5*Math.PI+.1*Math.PI):"to"===e?(a=this.findBorderPosition(this.from,t,{x:v,y:g,low:.6,high:1,direction:1}),r=-2*a.t*Math.PI+1.5*Math.PI-1.1*Math.PI):(a=this._pointOnCircle(v,g,y,.175),r=3.9269908169872414)}var b=15*c+3*u,w=a.x-.9*b*Math.cos(r),_=a.y-.9*b*Math.sin(r),x={x:w,y:_};return{point:a,core:x,angle:r,length:b}}},{key:"drawArrowHead",value:function(t,e,i,o){t.strokeStyle=this.getColor(t,e,i),t.fillStyle=t.strokeStyle,t.lineWidth=this.getLineWidth(e,i),t.arrow(o.point.x,o.point.y,o.angle,o.length),this.enableShadow(t),t.fill(),this.disableShadow(t)}},{key:"enableShadow",value:function(t){this.options.shadow.enabled===!0&&(t.shadowColor=this.options.shadow.color,t.shadowBlur=this.options.shadow.size,t.shadowOffsetX=this.options.shadow.x,t.shadowOffsetY=this.options.shadow.y)}},{key:"disableShadow",value:function(t){this.options.shadow.enabled===!0&&(t.shadowColor="rgba(0,0,0,0)",t.shadowBlur=0,t.shadowOffsetX=0,t.shadowOffsetY=0)}}]),t}();e["default"]=a},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(88),d=o(h),l=function(t){function e(t,i,o){n(this,e);var r=s(this,Object.getPrototypeOf(e).call(this,t,i,o));return r._boundFunction=function(){r.positionBezierNode()},r.body.emitter.on("_repositionBezierNodes",r._boundFunction),r}return r(e,t),a(e,[{key:"setOptions",value:function(t){var e=!1;this.options.physics!==t.physics&&(e=!0),this.options=t,this.id=this.options.id,this.from=this.body.nodes[this.options.from],this.to=this.body.nodes[this.options.to],this.setupSupportNode(),this.connect(),e===!0&&(this.via.setOptions({physics:this.options.physics}),this.positionBezierNode())}},{key:"connect",value:function(){this.from=this.body.nodes[this.options.from],this.to=this.body.nodes[this.options.to],void 0===this.from||void 0===this.to||this.options.physics===!1?this.via.setOptions({physics:!1}):this.from.id===this.to.id?this.via.setOptions({physics:!1}):this.via.setOptions({physics:!0})}},{key:"cleanup",value:function(){return this.body.emitter.off("_repositionBezierNodes",this._boundFunction),void 0!==this.via?(delete this.body.nodes[this.via.id],this.via=void 0,!0):!1}},{key:"setupSupportNode",value:function(){if(void 0===this.via){var t="edgeId:"+this.id,e=this.body.functions.createNode({id:t,shape:"circle",physics:!0,hidden:!0});this.body.nodes[t]=e,this.via=e,this.via.parentEdgeId=this.id,this.positionBezierNode()}}},{key:"positionBezierNode",value:function(){void 0!==this.via&&void 0!==this.from&&void 0!==this.to?(this.via.x=.5*(this.from.x+this.to.x),this.via.y=.5*(this.from.y+this.to.y)):void 0!==this.via&&(this.via.x=0,this.via.y=0)}},{key:"_line",value:function(t,e){t.beginPath(),t.moveTo(this.fromPoint.x,this.fromPoint.y),void 0===e.x?t.lineTo(this.toPoint.x,this.toPoint.y):t.quadraticCurveTo(e.x,e.y,this.toPoint.x,this.toPoint.y),this.enableShadow(t),t.stroke(),this.disableShadow(t)}},{key:"getViaNode",value:function(){return this.via}},{key:"getPoint",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?this.via:arguments[1],i=t,o=Math.pow(1-i,2)*this.fromPoint.x+2*i*(1-i)*e.x+Math.pow(i,2)*this.toPoint.x,n=Math.pow(1-i,2)*this.fromPoint.y+2*i*(1-i)*e.y+Math.pow(i,2)*this.toPoint.y;return{x:o,y:n}}},{key:"_findBorderPosition",value:function(t,e){return this._findBorderPositionBezier(t,e,this.via)}},{key:"_getDistanceToEdge",value:function(t,e,i,o,n,s){return this._getDistanceToBezierEdge(t,e,i,o,n,s,this.via)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(88),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_line",value:function(t,e){t.beginPath(),t.moveTo(this.fromPoint.x,this.fromPoint.y),void 0===e.x?t.lineTo(this.toPoint.x,this.toPoint.y):t.quadraticCurveTo(e.x,e.y,this.toPoint.x,this.toPoint.y),this.enableShadow(t),t.stroke(),this.disableShadow(t)}},{key:"getViaNode",value:function(){return this._getViaCoordinates()}},{key:"_getViaCoordinates",value:function(){var t=void 0,e=void 0,i=this.options.smooth.roundness,o=this.options.smooth.type,n=Math.abs(this.from.x-this.to.x),s=Math.abs(this.from.y-this.to.y);if("discrete"===o||"diagonalCross"===o)Math.abs(this.from.x-this.to.x)<=Math.abs(this.from.y-this.to.y)?(this.from.y>=this.to.y?this.from.x<=this.to.x?(t=this.from.x+i*s,e=this.from.y-i*s):this.from.x>this.to.x&&(t=this.from.x-i*s,e=this.from.y-i*s):this.from.y<this.to.y&&(this.from.x<=this.to.x?(t=this.from.x+i*s,e=this.from.y+i*s):this.from.x>this.to.x&&(t=this.from.x-i*s,e=this.from.y+i*s)),"discrete"===o&&(t=i*s>n?this.from.x:t)):Math.abs(this.from.x-this.to.x)>Math.abs(this.from.y-this.to.y)&&(this.from.y>=this.to.y?this.from.x<=this.to.x?(t=this.from.x+i*n,e=this.from.y-i*n):this.from.x>this.to.x&&(t=this.from.x-i*n,e=this.from.y-i*n):this.from.y<this.to.y&&(this.from.x<=this.to.x?(t=this.from.x+i*n,e=this.from.y+i*n):this.from.x>this.to.x&&(t=this.from.x-i*n,e=this.from.y+i*n)),"discrete"===o&&(e=i*n>s?this.from.y:e));else if("straightCross"===o)Math.abs(this.from.x-this.to.x)<=Math.abs(this.from.y-this.to.y)?(t=this.from.x,e=this.from.y<this.to.y?this.to.y-(1-i)*s:this.to.y+(1-i)*s):Math.abs(this.from.x-this.to.x)>Math.abs(this.from.y-this.to.y)&&(t=this.from.x<this.to.x?this.to.x-(1-i)*n:this.to.x+(1-i)*n,e=this.from.y);else if("horizontal"===o)t=this.from.x<this.to.x?this.to.x-(1-i)*n:this.to.x+(1-i)*n,e=this.from.y;else if("vertical"===o)t=this.from.x,e=this.from.y<this.to.y?this.to.y-(1-i)*s:this.to.y+(1-i)*s;else if("curvedCW"===o){n=this.to.x-this.from.x,s=this.from.y-this.to.y;var r=Math.sqrt(n*n+s*s),a=Math.PI,h=Math.atan2(s,n),d=(h+(.5*i+.5)*a)%(2*a);t=this.from.x+(.5*i+.5)*r*Math.sin(d),e=this.from.y+(.5*i+.5)*r*Math.cos(d)}else if("curvedCCW"===o){n=this.to.x-this.from.x,s=this.from.y-this.to.y;var l=Math.sqrt(n*n+s*s),c=Math.PI,u=Math.atan2(s,n),p=(u+(.5*-i+.5)*c)%(2*c);t=this.from.x+(.5*i+.5)*l*Math.sin(p),e=this.from.y+(.5*i+.5)*l*Math.cos(p)}else Math.abs(this.from.x-this.to.x)<=Math.abs(this.from.y-this.to.y)?this.from.y>=this.to.y?this.from.x<=this.to.x?(t=this.from.x+i*s,e=this.from.y-i*s,t=this.to.x<t?this.to.x:t):this.from.x>this.to.x&&(t=this.from.x-i*s,e=this.from.y-i*s,t=this.to.x>t?this.to.x:t):this.from.y<this.to.y&&(this.from.x<=this.to.x?(t=this.from.x+i*s,e=this.from.y+i*s,t=this.to.x<t?this.to.x:t):this.from.x>this.to.x&&(t=this.from.x-i*s,e=this.from.y+i*s,t=this.to.x>t?this.to.x:t)):Math.abs(this.from.x-this.to.x)>Math.abs(this.from.y-this.to.y)&&(this.from.y>=this.to.y?this.from.x<=this.to.x?(t=this.from.x+i*n,e=this.from.y-i*n,e=this.to.y>e?this.to.y:e):this.from.x>this.to.x&&(t=this.from.x-i*n,e=this.from.y-i*n,e=this.to.y>e?this.to.y:e):this.from.y<this.to.y&&(this.from.x<=this.to.x?(t=this.from.x+i*n,e=this.from.y+i*n,e=this.to.y<e?this.to.y:e):this.from.x>this.to.x&&(t=this.from.x-i*n,e=this.from.y+i*n,e=this.to.y<e?this.to.y:e)));return{x:t,y:e}}},{key:"_findBorderPosition",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?{}:arguments[2];return this._findBorderPositionBezier(t,e,i.via)}},{key:"_getDistanceToEdge",value:function(t,e,i,o,n,s){var r=arguments.length<=6||void 0===arguments[6]?this._getViaCoordinates():arguments[6];return this._getDistanceToBezierEdge(t,e,i,o,n,s,r)}},{key:"getPoint",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?this._getViaCoordinates():arguments[1],i=t,o=Math.pow(1-i,2)*this.fromPoint.x+2*i*(1-i)*e.x+Math.pow(i,2)*this.toPoint.x,n=Math.pow(1-i,2)*this.fromPoint.y+2*i*(1-i)*e.y+Math.pow(i,2)*this.toPoint.y;return{x:o,y:n}}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(89),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_line",value:function(t){t.beginPath(),t.moveTo(this.fromPoint.x,this.fromPoint.y),t.lineTo(this.toPoint.x,this.toPoint.y),this.enableShadow(t),t.stroke(),this.disableShadow(t)}},{key:"getViaNode",value:function(){}},{key:"getPoint",value:function(t){return{x:(1-t)*this.fromPoint.x+t*this.toPoint.x,y:(1-t)*this.fromPoint.y+t*this.toPoint.y}}},{key:"_findBorderPosition",value:function(t,e){var i=this.to,o=this.from;t.id===this.from.id&&(i=this.from,o=this.to);var n=Math.atan2(i.y-o.y,i.x-o.x),s=i.x-o.x,r=i.y-o.y,a=Math.sqrt(s*s+r*r),h=t.distanceToBorder(e,n),d=(a-h)/a,l={};return l.x=(1-d)*o.x+d*i.x,l.y=(1-d)*o.y+d*i.y,l}},{key:"_getDistanceToEdge",value:function(t,e,i,o,n,s){return this._getDistanceToLine(t,e,i,o,n,s)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(94),a=o(r),h=i(95),d=o(h),l=i(96),c=o(l),u=i(97),p=o(u),f=i(98),m=o(f),v=i(99),g=o(v),y=i(100),b=o(y),w=i(101),_=o(w),x=i(1),k=function(){function t(e){n(this,t),this.body=e,this.physicsBody={physicsNodeIndices:[],physicsEdgeIndices:[],forces:{},velocities:{}},this.physicsEnabled=!0,this.simulationInterval=1e3/60,this.requiresTimeout=!0,this.previousStates={},this.referenceState={},this.freezeCache={},this.renderTimer=void 0,this.adaptiveTimestep=!1,this.adaptiveTimestepEnabled=!1,this.adaptiveCounter=0,this.adaptiveInterval=3,this.stabilized=!1,this.startedStabilization=!1,this.stabilizationIterations=0,this.ready=!1,this.options={},this.defaultOptions={enabled:!0,barnesHut:{theta:.5,gravitationalConstant:-2e3,centralGravity:.3,springLength:95,springConstant:.04,damping:.09,avoidOverlap:0},forceAtlas2Based:{theta:.5,gravitationalConstant:-50,centralGravity:.01,springConstant:.08,springLength:100,damping:.4,avoidOverlap:0},repulsion:{centralGravity:.2,springLength:200,springConstant:.05,nodeDistance:100,damping:.09,avoidOverlap:0},hierarchicalRepulsion:{centralGravity:0,springLength:100,springConstant:.01,nodeDistance:120,damping:.09},maxVelocity:50,minVelocity:.75,solver:"barnesHut",stabilization:{enabled:!0,iterations:1e3,updateInterval:50,onlyDynamicEdges:!1,fit:!0},timestep:.5,adaptiveTimestep:!0},x.extend(this.options,this.defaultOptions),this.timestep=.5,this.layoutFailed=!1,this.bindEventListeners()}return s(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("initPhysics",function(){t.initPhysics()}),this.body.emitter.on("_layoutFailed",function(){t.layoutFailed=!0}),this.body.emitter.on("resetPhysics",function(){t.stopSimulation(),t.ready=!1}),this.body.emitter.on("disablePhysics",function(){t.physicsEnabled=!1,t.stopSimulation()}),this.body.emitter.on("restorePhysics",function(){t.setOptions(t.options),t.ready===!0&&t.startSimulation()}),this.body.emitter.on("startSimulation",function(){t.ready===!0&&t.startSimulation()}),this.body.emitter.on("stopSimulation",function(){t.stopSimulation()}),this.body.emitter.on("destroy",function(){t.stopSimulation(!1),t.body.emitter.off()}),this.body.emitter.on("_dataChanged",function(){t.updatePhysicsData()})}},{key:"setOptions",value:function(t){void 0!==t&&(t===!1?(this.options.enabled=!1,this.physicsEnabled=!1,this.stopSimulation()):(this.physicsEnabled=!0,x.selectiveNotDeepExtend(["stabilization"],this.options,t),x.mergeOptions(this.options,t,"stabilization"),void 0===t.enabled&&(this.options.enabled=!0),this.options.enabled===!1&&(this.physicsEnabled=!1,this.stopSimulation()),this.timestep=this.options.timestep)),this.init()}},{key:"init",value:function(){var t;"forceAtlas2Based"===this.options.solver?(t=this.options.forceAtlas2Based,this.nodesSolver=new b["default"](this.body,this.physicsBody,t),this.edgesSolver=new p["default"](this.body,this.physicsBody,t),this.gravitySolver=new _["default"](this.body,this.physicsBody,t)):"repulsion"===this.options.solver?(t=this.options.repulsion,this.nodesSolver=new d["default"](this.body,this.physicsBody,t),this.edgesSolver=new p["default"](this.body,this.physicsBody,t),this.gravitySolver=new g["default"](this.body,this.physicsBody,t)):"hierarchicalRepulsion"===this.options.solver?(t=this.options.hierarchicalRepulsion,this.nodesSolver=new c["default"](this.body,this.physicsBody,t),this.edgesSolver=new m["default"](this.body,this.physicsBody,t),this.gravitySolver=new g["default"](this.body,this.physicsBody,t)):(t=this.options.barnesHut,this.nodesSolver=new a["default"](this.body,this.physicsBody,t),this.edgesSolver=new p["default"](this.body,this.physicsBody,t),this.gravitySolver=new g["default"](this.body,this.physicsBody,t)),this.modelOptions=t}},{key:"initPhysics",value:function(){this.physicsEnabled===!0&&this.options.enabled===!0?this.options.stabilization.enabled===!0?this.stabilize():(this.stabilized=!1,this.ready=!0,this.body.emitter.emit("fit",{},this.layoutFailed),this.startSimulation()):(this.ready=!0,this.body.emitter.emit("fit"))}},{key:"startSimulation",value:function(){this.physicsEnabled===!0&&this.options.enabled===!0?(this.stabilized=!1,this.adaptiveTimestep=!1,this.body.emitter.emit("_resizeNodes"),void 0===this.viewFunction&&(this.viewFunction=this.simulationStep.bind(this),this.body.emitter.on("initRedraw",this.viewFunction),this.body.emitter.emit("_startRendering"))):this.body.emitter.emit("_redraw")}},{key:"stopSimulation",value:function(){var t=arguments.length<=0||void 0===arguments[0]?!0:arguments[0];this.stabilized=!0,t===!0&&this._emitStabilized(),void 0!==this.viewFunction&&(this.body.emitter.off("initRedraw",this.viewFunction),this.viewFunction=void 0,t===!0&&this.body.emitter.emit("_stopRendering"))}},{key:"simulationStep",value:function(){var t=Date.now();this.physicsTick();var e=Date.now()-t;(e<.4*this.simulationInterval||this.runDoubleSpeed===!0)&&this.stabilized===!1&&(this.physicsTick(),this.runDoubleSpeed=!0),this.stabilized===!0&&this.stopSimulation()}},{key:"_emitStabilized",value:function(){var t=this,e=arguments.length<=0||void 0===arguments[0]?this.stabilizationIterations:arguments[0];(this.stabilizationIterations>1||this.startedStabilization===!0)&&setTimeout(function(){t.body.emitter.emit("stabilized",{iterations:e}),t.startedStabilization=!1,t.stabilizationIterations=0},0)}},{key:"physicsTick",value:function(){if(this.startedStabilization===!1&&(this.body.emitter.emit("startStabilizing"),this.startedStabilization=!0),this.stabilized===!1){if(this.adaptiveTimestep===!0&&this.adaptiveTimestepEnabled===!0){var t=1.2;this.adaptiveCounter%this.adaptiveInterval===0?(this.timestep=2*this.timestep,this.calculateForces(),this.moveNodes(),this.revert(),this.timestep=.5*this.timestep,this.calculateForces(),this.moveNodes(),this.calculateForces(),this.moveNodes(),this._evaluateStepQuality()===!0?this.timestep=t*this.timestep:this.timestep/t<this.options.timestep?this.timestep=this.options.timestep:(this.adaptiveCounter=-1,this.timestep=Math.max(this.options.timestep,this.timestep/t))):(this.calculateForces(),this.moveNodes()),this.adaptiveCounter+=1}else this.timestep=this.options.timestep,this.calculateForces(),this.moveNodes();this.stabilized===!0&&this.revert(),this.stabilizationIterations++}}},{key:"updatePhysicsData",value:function(){this.physicsBody.forces={},this.physicsBody.physicsNodeIndices=[],this.physicsBody.physicsEdgeIndices=[];var t=this.body.nodes,e=this.body.edges;for(var i in t)t.hasOwnProperty(i)&&t[i].options.physics===!0&&this.physicsBody.physicsNodeIndices.push(t[i].id);for(var o in e)e.hasOwnProperty(o)&&e[o].options.physics===!0&&this.physicsBody.physicsEdgeIndices.push(e[o].id);for(var n=0;n<this.physicsBody.physicsNodeIndices.length;n++){var s=this.physicsBody.physicsNodeIndices[n];this.physicsBody.forces[s]={x:0,y:0},void 0===this.physicsBody.velocities[s]&&(this.physicsBody.velocities[s]={x:0,y:0})}for(var r in this.physicsBody.velocities)void 0===t[r]&&delete this.physicsBody.velocities[r]}},{key:"revert",value:function(){var t=Object.keys(this.previousStates),e=this.body.nodes,i=this.physicsBody.velocities;this.referenceState={};for(var o=0;o<t.length;o++){var n=t[o];void 0!==e[n]?e[n].options.physics===!0&&(this.referenceState[n]={positions:{x:e[n].x,y:e[n].y}},i[n].x=this.previousStates[n].vx,i[n].y=this.previousStates[n].vy,e[n].x=this.previousStates[n].x,e[n].y=this.previousStates[n].y):delete this.previousStates[n]}}},{key:"_evaluateStepQuality",value:function(){var t=void 0,e=void 0,i=void 0,o=this.body.nodes,n=this.referenceState,s=.3;for(var r in this.referenceState)if(this.referenceState.hasOwnProperty(r)&&void 0!==o[r]&&(t=o[r].x-n[r].positions.x,e=o[r].y-n[r].positions.y,i=Math.sqrt(Math.pow(t,2)+Math.pow(e,2)),i>s))return!1;return!0}},{key:"moveNodes",value:function(){for(var t=this.physicsBody.physicsNodeIndices,e=this.options.maxVelocity?this.options.maxVelocity:1e9,i=0,o=0,n=5,s=0;s<t.length;s++){var r=t[s],a=this._performStep(r,e);i=Math.max(i,a),o+=a}this.adaptiveTimestepEnabled=o/t.length<n,this.stabilized=i<this.options.minVelocity}},{key:"_performStep",value:function(t,e){var i=this.body.nodes[t],o=this.timestep,n=this.physicsBody.forces,s=this.physicsBody.velocities;if(this.previousStates[t]={x:i.x,y:i.y,vx:s[t].x,vy:s[t].y},i.options.fixed.x===!1){var r=this.modelOptions.damping*s[t].x,a=(n[t].x-r)/i.options.mass;s[t].x+=a*o,s[t].x=Math.abs(s[t].x)>e?s[t].x>0?e:-e:s[t].x,i.x+=s[t].x*o}else n[t].x=0,s[t].x=0;if(i.options.fixed.y===!1){var h=this.modelOptions.damping*s[t].y,d=(n[t].y-h)/i.options.mass;s[t].y+=d*o,s[t].y=Math.abs(s[t].y)>e?s[t].y>0?e:-e:s[t].y,i.y+=s[t].y*o}else n[t].y=0,s[t].y=0;var l=Math.sqrt(Math.pow(s[t].x,2)+Math.pow(s[t].y,2));return l}},{key:"calculateForces",value:function(){this.gravitySolver.solve(),this.nodesSolver.solve(),this.edgesSolver.solve()}},{key:"_freezeNodes",value:function(){var t=this.body.nodes;for(var e in t)t.hasOwnProperty(e)&&t[e].x&&t[e].y&&(this.freezeCache[e]={x:t[e].options.fixed.x,y:t[e].options.fixed.y},t[e].options.fixed.x=!0,t[e].options.fixed.y=!0)}},{key:"_restoreFrozenNodes",value:function(){var t=this.body.nodes;for(var e in t)t.hasOwnProperty(e)&&void 0!==this.freezeCache[e]&&(t[e].options.fixed.x=this.freezeCache[e].x,t[e].options.fixed.y=this.freezeCache[e].y);this.freezeCache={}}},{key:"stabilize",value:function(){var t=this,e=arguments.length<=0||void 0===arguments[0]?this.options.stabilization.iterations:arguments[0];return"number"!=typeof e&&(console.log("The stabilize method needs a numeric amount of iterations. Switching to default: ",this.options.stabilization.iterations),e=this.options.stabilization.iterations),0===this.physicsBody.physicsNodeIndices.length?void(this.ready=!0):(this.adaptiveTimestep=this.options.adaptiveTimestep,this.body.emitter.emit("_resizeNodes"),this.stopSimulation(),this.stabilized=!1,this.body.emitter.emit("_blockRedraw"),this.targetIterations=e,this.options.stabilization.onlyDynamicEdges===!0&&this._freezeNodes(),this.stabilizationIterations=0,void setTimeout(function(){return t._stabilizationBatch()},0))}},{key:"_stabilizationBatch",value:function(){this.startedStabilization===!1&&(this.body.emitter.emit("startStabilizing"),this.startedStabilization=!0);for(var t=0;this.stabilized===!1&&t<this.options.stabilization.updateInterval&&this.stabilizationIterations<this.targetIterations;)this.physicsTick(),t++;this.stabilized===!1&&this.stabilizationIterations<this.targetIterations?(this.body.emitter.emit("stabilizationProgress",{iterations:this.stabilizationIterations,total:this.targetIterations}),setTimeout(this._stabilizationBatch.bind(this),0)):this._finalizeStabilization()}},{key:"_finalizeStabilization",value:function(){this.body.emitter.emit("_allowRedraw"),this.options.stabilization.fit===!0&&this.body.emitter.emit("fit"),this.options.stabilization.onlyDynamicEdges===!0&&this._restoreFrozenNodes(),this.body.emitter.emit("stabilizationIterationsDone"),this.body.emitter.emit("_requestRedraw"),this.stabilized===!0?this._emitStabilized():this.startSimulation(),this.ready=!0}},{key:"_drawForces",value:function(t){for(var e=0;e<this.physicsBody.physicsNodeIndices.length;e++){var i=this.body.nodes[this.physicsBody.physicsNodeIndices[e]],o=this.physicsBody.forces[this.physicsBody.physicsNodeIndices[e]],n=20,s=.03,r=Math.sqrt(Math.pow(o.x,2)+Math.pow(o.x,2)),a=Math.min(Math.max(5,r),15),h=3*a,d=x.HSVToHex((180-180*Math.min(1,Math.max(0,s*r)))/360,1,1);t.lineWidth=a,t.strokeStyle=d,t.beginPath(),t.moveTo(i.x,i.y),t.lineTo(i.x+n*o.x,i.y+n*o.y),t.stroke();var l=Math.atan2(o.y,o.x);t.fillStyle=d,t.arrow(i.x+n*o.x+Math.cos(l)*h,i.y+n*o.y+Math.sin(l)*h,l,h),t.fill()}}}]),t}();e["default"]=k},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.barnesHutTree,this.setOptions(n),this.randomSeed=5}return o(t,[{key:"setOptions",value:function(t){this.options=t,this.thetaInversed=1/this.options.theta,this.overlapAvoidanceFactor=1-Math.max(0,Math.min(1,this.options.avoidOverlap))}},{key:"seededRandom",value:function(){var t=1e4*Math.sin(this.randomSeed++);return t-Math.floor(t)}},{key:"solve",value:function(){if(0!==this.options.gravitationalConstant&&this.physicsBody.physicsNodeIndices.length>0){var t=void 0,e=this.body.nodes,i=this.physicsBody.physicsNodeIndices,o=i.length,n=this._formBarnesHutTree(e,i);this.barnesHutTree=n;for(var s=0;o>s;s++)t=e[i[s]],t.options.mass>0&&(this._getForceContribution(n.root.children.NW,t),this._getForceContribution(n.root.children.NE,t),this._getForceContribution(n.root.children.SW,t),this._getForceContribution(n.root.children.SE,t))}}},{key:"_getForceContribution",value:function(t,e){if(t.childrenCount>0){var i=void 0,o=void 0,n=void 0;i=t.centerOfMass.x-e.x,o=t.centerOfMass.y-e.y,n=Math.sqrt(i*i+o*o),n*t.calcSize>this.thetaInversed?this._calculateForces(n,i,o,e,t):4===t.childrenCount?(this._getForceContribution(t.children.NW,e),this._getForceContribution(t.children.NE,e),this._getForceContribution(t.children.SW,e),this._getForceContribution(t.children.SE,e)):t.children.data.id!=e.id&&this._calculateForces(n,i,o,e,t)}}},{key:"_calculateForces",value:function(t,e,i,o,n){0===t&&(t=.1,e=t),this.overlapAvoidanceFactor<1&&(t=Math.max(.1+this.overlapAvoidanceFactor*o.shape.radius,t-o.shape.radius));var s=this.options.gravitationalConstant*n.mass*o.options.mass/Math.pow(t,3),r=e*s,a=i*s;this.physicsBody.forces[o.id].x+=r,this.physicsBody.forces[o.id].y+=a}},{key:"_formBarnesHutTree",value:function(t,e){for(var i=void 0,o=e.length,n=t[e[0]].x,s=t[e[0]].y,r=t[e[0]].x,a=t[e[0]].y,h=1;o>h;h++){var d=t[e[h]].x,l=t[e[h]].y;t[e[h]].options.mass>0&&(n>d&&(n=d),d>r&&(r=d),s>l&&(s=l),l>a&&(a=l))}var c=Math.abs(r-n)-Math.abs(a-s);c>0?(s-=.5*c,a+=.5*c):(n+=.5*c,r-=.5*c);var u=1e-5,p=Math.max(u,Math.abs(r-n)),f=.5*p,m=.5*(n+r),v=.5*(s+a),g={root:{centerOfMass:{x:0,y:0},mass:0,range:{minX:m-f,maxX:m+f,minY:v-f,maxY:v+f},size:p,calcSize:1/p,children:{data:null},maxWidth:0,level:0,childrenCount:4}};this._splitBranch(g.root);for(var y=0;o>y;y++)i=t[e[y]],i.options.mass>0&&this._placeInTree(g.root,i);return g}},{key:"_updateBranchMass",value:function(t,e){var i=t.mass+e.options.mass,o=1/i;t.centerOfMass.x=t.centerOfMass.x*t.mass+e.x*e.options.mass,t.centerOfMass.x*=o,t.centerOfMass.y=t.centerOfMass.y*t.mass+e.y*e.options.mass,t.centerOfMass.y*=o,t.mass=i;var n=Math.max(Math.max(e.height,e.radius),e.width);t.maxWidth=t.maxWidth<n?n:t.maxWidth}},{key:"_placeInTree",value:function(t,e,i){1==i&&void 0!==i||this._updateBranchMass(t,e),t.children.NW.range.maxX>e.x?t.children.NW.range.maxY>e.y?this._placeInRegion(t,e,"NW"):this._placeInRegion(t,e,"SW"):t.children.NW.range.maxY>e.y?this._placeInRegion(t,e,"NE"):this._placeInRegion(t,e,"SE")}},{key:"_placeInRegion",value:function(t,e,i){switch(t.children[i].childrenCount){case 0:t.children[i].children.data=e,t.children[i].childrenCount=1,this._updateBranchMass(t.children[i],e);break;case 1:t.children[i].children.data.x===e.x&&t.children[i].children.data.y===e.y?(e.x+=this.seededRandom(),e.y+=this.seededRandom()):(this._splitBranch(t.children[i]),this._placeInTree(t.children[i],e));break;case 4:this._placeInTree(t.children[i],e)}}},{key:"_splitBranch",value:function(t){var e=null;1===t.childrenCount&&(e=t.children.data,t.mass=0,t.centerOfMass.x=0,t.centerOfMass.y=0),t.childrenCount=4,t.children.data=null,this._insertRegion(t,"NW"),this._insertRegion(t,"NE"),this._insertRegion(t,"SW"),this._insertRegion(t,"SE"),null!=e&&this._placeInTree(t,e)}},{key:"_insertRegion",value:function(t,e){var i=void 0,o=void 0,n=void 0,s=void 0,r=.5*t.size;switch(e){case"NW":i=t.range.minX,o=t.range.minX+r,n=t.range.minY,s=t.range.minY+r;break;case"NE":i=t.range.minX+r,o=t.range.maxX,n=t.range.minY,s=t.range.minY+r;break;case"SW":i=t.range.minX,o=t.range.minX+r,n=t.range.minY+r,s=t.range.maxY;break;case"SE":i=t.range.minX+r,o=t.range.maxX,n=t.range.minY+r,s=t.range.maxY}t.children[e]={centerOfMass:{x:0,y:0},mass:0,range:{minX:i,maxX:o,minY:n,maxY:s},size:.5*t.size,calcSize:2*t.calcSize,children:{data:null},maxWidth:0,level:t.level+1,childrenCount:0}}},{key:"_debug",value:function(t,e){void 0!==this.barnesHutTree&&(t.lineWidth=1,this._drawBranch(this.barnesHutTree.root,t,e))}},{key:"_drawBranch",value:function(t,e,i){void 0===i&&(i="#FF0000"),4===t.childrenCount&&(this._drawBranch(t.children.NW,e),this._drawBranch(t.children.NE,e),this._drawBranch(t.children.SE,e),this._drawBranch(t.children.SW,e)),e.strokeStyle=i,e.beginPath(),e.moveTo(t.range.minX,t.range.minY),e.lineTo(t.range.maxX,t.range.minY),e.stroke(),e.beginPath(),e.moveTo(t.range.maxX,t.range.minY),e.lineTo(t.range.maxX,t.range.maxY),e.stroke(),e.beginPath(),e.moveTo(t.range.maxX,t.range.maxY),
-e.lineTo(t.range.minX,t.range.maxY),e.stroke(),e.beginPath(),e.moveTo(t.range.minX,t.range.maxY),e.lineTo(t.range.minX,t.range.minY),e.stroke()}}]),t}();e["default"]=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.setOptions(n)}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"solve",value:function(){for(var t,e,i,o,n,s,r,a,h=this.body.nodes,d=this.physicsBody.physicsNodeIndices,l=this.physicsBody.forces,c=this.options.nodeDistance,u=-2/3/c,p=4/3,f=0;f<d.length-1;f++){r=h[d[f]];for(var m=f+1;m<d.length;m++)a=h[d[m]],t=a.x-r.x,e=a.y-r.y,i=Math.sqrt(t*t+e*e),0===i&&(i=.1*Math.random(),t=i),2*c>i&&(s=.5*c>i?1:u*i+p,s/=i,o=t*s,n=e*s,l[r.id].x-=o,l[r.id].y-=n,l[a.id].x+=o,l[a.id].y+=n)}}}]),t}();e["default"]=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.setOptions(n)}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"solve",value:function(){var t,e,i,o,n,s,r,a,h,d,l=this.body.nodes,c=this.physicsBody.physicsNodeIndices,u=this.physicsBody.forces,p=this.options.nodeDistance;for(h=0;h<c.length-1;h++)for(r=l[c[h]],d=h+1;d<c.length;d++)if(a=l[c[d]],r.level===a.level){t=a.x-r.x,e=a.y-r.y,i=Math.sqrt(t*t+e*e);var f=.05;s=p>i?-Math.pow(f*i,2)+Math.pow(f*p,2):0,0===i?i=.01:s/=i,o=t*s,n=e*s,u[r.id].x-=o,u[r.id].y-=n,u[a.id].x+=o,u[a.id].y+=n}}}]),t}();e["default"]=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.setOptions(n)}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"solve",value:function(){for(var t=void 0,e=void 0,i=this.physicsBody.physicsEdgeIndices,o=this.body.edges,n=void 0,s=void 0,r=void 0,a=0;a<i.length;a++)e=o[i[a]],e.connected===!0&&e.toId!==e.fromId&&void 0!==this.body.nodes[e.toId]&&void 0!==this.body.nodes[e.fromId]&&(void 0!==e.edgeType.via?(t=void 0===e.options.length?this.options.springLength:e.options.length,n=e.to,s=e.edgeType.via,r=e.from,this._calculateSpringForce(n,s,.5*t),this._calculateSpringForce(s,r,.5*t)):(t=void 0===e.options.length?1.5*this.options.springLength:e.options.length,this._calculateSpringForce(e.from,e.to,t)))}},{key:"_calculateSpringForce",value:function(t,e,i){var o=t.x-e.x,n=t.y-e.y,s=Math.max(Math.sqrt(o*o+n*n),.01),r=this.options.springConstant*(i-s)/s,a=o*r,h=n*r;void 0!==this.physicsBody.forces[t.id]&&(this.physicsBody.forces[t.id].x+=a,this.physicsBody.forces[t.id].y+=h),void 0!==this.physicsBody.forces[e.id]&&(this.physicsBody.forces[e.id].x-=a,this.physicsBody.forces[e.id].y-=h)}}]),t}();e["default"]=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.setOptions(n)}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"solve",value:function(){for(var t,e,i,o,n,s,r,a,h=this.body.edges,d=.5,l=this.physicsBody.physicsEdgeIndices,c=this.physicsBody.physicsNodeIndices,u=this.physicsBody.forces,p=0;p<c.length;p++){var f=c[p];u[f].springFx=0,u[f].springFy=0}for(var m=0;m<l.length;m++)e=h[l[m]],e.connected===!0&&(t=void 0===e.options.length?this.options.springLength:e.options.length,i=e.from.x-e.to.x,o=e.from.y-e.to.y,a=Math.sqrt(i*i+o*o),a=0===a?.01:a,r=this.options.springConstant*(t-a)/a,n=i*r,s=o*r,e.to.level!=e.from.level?(void 0!==u[e.toId]&&(u[e.toId].springFx-=n,u[e.toId].springFy-=s),void 0!==u[e.fromId]&&(u[e.fromId].springFx+=n,u[e.fromId].springFy+=s)):(void 0!==u[e.toId]&&(u[e.toId].x-=d*n,u[e.toId].y-=d*s),void 0!==u[e.fromId]&&(u[e.fromId].x+=d*n,u[e.fromId].y+=d*s)));for(var v,g,r=1,y=0;y<c.length;y++){var b=c[y];v=Math.min(r,Math.max(-r,u[b].springFx)),g=Math.min(r,Math.max(-r,u[b].springFy)),u[b].x+=v,u[b].y+=g}for(var w=0,_=0,x=0;x<c.length;x++){var k=c[x];w+=u[k].x,_+=u[k].y}for(var O=w/c.length,M=_/c.length,D=0;D<c.length;D++){var S=c[D];u[S].x-=O,u[S].y-=M}}}]),t}();e["default"]=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.setOptions(n)}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"solve",value:function(){for(var t=void 0,e=void 0,i=void 0,o=void 0,n=this.body.nodes,s=this.physicsBody.physicsNodeIndices,r=this.physicsBody.forces,a=0;a<s.length;a++){var h=s[a];o=n[h],t=-o.x,e=-o.y,i=Math.sqrt(t*t+e*e),this._calculateForces(i,t,e,r,o)}}},{key:"_calculateForces",value:function(t,e,i,o,n){var s=0===t?0:this.options.centralGravity/t;o[n.id].x=e*s,o[n.id].y=i*s}}]),t}();e["default"]=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(94),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_calculateForces",value:function(t,e,i,o,n){0===t&&(t=.1*Math.random(),e=t),this.overlapAvoidanceFactor<1&&(t=Math.max(.1+this.overlapAvoidanceFactor*o.shape.radius,t-o.shape.radius));var s=o.edges.length+1,r=this.options.gravitationalConstant*n.mass*o.options.mass*s/Math.pow(t,2),a=e*r,h=i*r;this.physicsBody.forces[o.id].x+=a,this.physicsBody.forces[o.id].y+=h}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(99),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_calculateForces",value:function(t,e,i,o,n){if(t>0){var s=n.edges.length+1,r=this.options.centralGravity*s*n.options.mass;o[n.id].x=e*r,o[n.id].y=i*r}}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},r=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),a=i(103),h=o(a),d=i(104),l=o(d),c=i(1),u=function(){function t(e){var i=this;n(this,t),this.body=e,this.clusteredNodes={},this.clusteredEdges={},this.options={},this.defaultOptions={},c.extend(this.options,this.defaultOptions),this.body.emitter.on("_resetData",function(){i.clusteredNodes={},i.clusteredEdges={}})}return r(t,[{key:"setOptions",value:function(t){}},{key:"clusterByHubsize",value:function(t,e){void 0===t?t=this._getHubSize():"object"===("undefined"==typeof t?"undefined":s(t))&&(e=this._checkOptions(t),t=this._getHubSize());for(var i=[],o=0;o<this.body.nodeIndices.length;o++){var n=this.body.nodes[this.body.nodeIndices[o]];n.edges.length>=t&&i.push(n.id)}for(var r=0;r<i.length;r++)this.clusterByConnection(i[r],e,!0);this.body.emitter.emit("_dataChanged")}},{key:"cluster",value:function(){var t=arguments.length<=0||void 0===arguments[0]?{}:arguments[0],e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];if(void 0===t.joinCondition)throw new Error("Cannot call clusterByNodeData without a joinCondition function in the options.");t=this._checkOptions(t);for(var i={},o={},n=0;n<this.body.nodeIndices.length;n++){var s=this.body.nodeIndices[n],r=this.body.nodes[s],a=h["default"].cloneOptions(r);if(t.joinCondition(a)===!0){i[s]=this.body.nodes[s];for(var d=0;d<r.edges.length;d++){var l=r.edges[d];void 0===this.clusteredEdges[l.id]&&(o[l.id]=l)}}}this._cluster(i,o,t,e)}},{key:"clusterByEdgeCount",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!0:arguments[2];e=this._checkOptions(e);for(var o=[],n={},s=void 0,r=void 0,a=void 0,d=void 0,l=void 0,c=0;c<this.body.nodeIndices.length;c++){var u={},p={};if(d=this.body.nodeIndices[c],void 0===n[d]){l=0,a=this.body.nodes[d],r=[];for(var f=0;f<a.edges.length;f++)s=a.edges[f],void 0===this.clusteredEdges[s.id]&&(s.toId!==s.fromId&&l++,r.push(s));if(l===t){for(var m=!0,v=0;v<r.length;v++){s=r[v];var g=this._getConnectedId(s,d);if(void 0===e.joinCondition)p[s.id]=s,u[d]=this.body.nodes[d],u[g]=this.body.nodes[g],n[d]=!0;else{var y=h["default"].cloneOptions(this.body.nodes[d]);if(e.joinCondition(y)!==!0){m=!1;break}p[s.id]=s,u[d]=this.body.nodes[d],n[d]=!0}}Object.keys(u).length>0&&Object.keys(p).length>0&&m===!0&&o.push({nodes:u,edges:p})}}}for(var b=0;b<o.length;b++)this._cluster(o[b].nodes,o[b].edges,e,!1);i===!0&&this.body.emitter.emit("_dataChanged")}},{key:"clusterOutliers",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];this.clusterByEdgeCount(1,t,e)}},{key:"clusterBridges",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];this.clusterByEdgeCount(2,t,e)}},{key:"clusterByConnection",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!0:arguments[2];if(void 0===t)throw new Error("No nodeId supplied to clusterByConnection!");if(void 0===this.body.nodes[t])throw new Error("The nodeId given to clusterByConnection does not exist!");var o=this.body.nodes[t];e=this._checkOptions(e,o),void 0===e.clusterNodeProperties.x&&(e.clusterNodeProperties.x=o.x),void 0===e.clusterNodeProperties.y&&(e.clusterNodeProperties.y=o.y),void 0===e.clusterNodeProperties.fixed&&(e.clusterNodeProperties.fixed={},e.clusterNodeProperties.fixed.x=o.options.fixed.x,e.clusterNodeProperties.fixed.y=o.options.fixed.y);var n={},s={},r=o.id,a=h["default"].cloneOptions(o);n[r]=o;for(var d=0;d<o.edges.length;d++){var l=o.edges[d];if(void 0===this.clusteredEdges[l.id]){var c=this._getConnectedId(l,r);if(void 0===this.clusteredNodes[c])if(c!==r)if(void 0===e.joinCondition)s[l.id]=l,n[c]=this.body.nodes[c];else{var u=h["default"].cloneOptions(this.body.nodes[c]);e.joinCondition(a,u)===!0&&(s[l.id]=l,n[c]=this.body.nodes[c])}else s[l.id]=l}}this._cluster(n,s,e,i)}},{key:"_createClusterEdges",value:function(t,e,i,o){for(var n=void 0,s=void 0,r=void 0,a=void 0,d=void 0,l=void 0,u=Object.keys(t),p=[],f=0;f<u.length;f++){s=u[f],r=t[s];for(var m=0;m<r.edges.length;m++)n=r.edges[m],void 0===this.clusteredEdges[n.id]&&(n.toId==n.fromId?e[n.id]=n:n.toId==s?(a=i.id,d=n.fromId,l=d):(a=n.toId,d=i.id,l=a),void 0===t[l]&&p.push({edge:n,fromId:d,toId:a}))}for(var v=0;v<p.length;v++){var g=p[v].edge,y=h["default"].cloneOptions(g,"edge");c.deepExtend(y,o),y.from=p[v].fromId,y.to=p[v].toId,y.id="clusterEdge:"+c.randomUUID();var b=this.body.functions.createEdge(y);b.clusteringEdgeReplacingId=g.id,this.body.edges[b.id]=b,b.connect(),this._backupEdgeOptions(g),g.setOptions({physics:!1,hidden:!0})}}},{key:"_checkOptions",value:function(){var t=arguments.length<=0||void 0===arguments[0]?{}:arguments[0];return void 0===t.clusterEdgeProperties&&(t.clusterEdgeProperties={}),void 0===t.clusterNodeProperties&&(t.clusterNodeProperties={}),t}},{key:"_cluster",value:function(t,e,i){var o=arguments.length<=3||void 0===arguments[3]?!0:arguments[3];if(!(Object.keys(t).length<2)){for(var n in t)if(t.hasOwnProperty(n)&&void 0!==this.clusteredNodes[n])return;var s=c.deepExtend({},i.clusterNodeProperties);if(void 0!==i.processProperties){var r=[];for(var a in t)if(t.hasOwnProperty(a)){var d=h["default"].cloneOptions(t[a]);r.push(d)}var u=[];for(var p in e)if(e.hasOwnProperty(p)&&"clusterEdge:"!==p.substr(0,12)){var f=h["default"].cloneOptions(e[p],"edge");u.push(f)}if(s=i.processProperties(s,r,u),!s)throw new Error("The processProperties function does not return properties!")}void 0===s.id&&(s.id="cluster:"+c.randomUUID());var m=s.id;void 0===s.label&&(s.label="cluster");var v=void 0;void 0===s.x&&(v=this._getClusterPosition(t),s.x=v.x),void 0===s.y&&(void 0===v&&(v=this._getClusterPosition(t)),s.y=v.y),s.id=m;var g=this.body.functions.createNode(s,l["default"]);g.isCluster=!0,g.containedNodes=t,g.containedEdges=e,g.clusterEdgeProperties=i.clusterEdgeProperties,this.body.nodes[s.id]=g,this._createClusterEdges(t,e,s,i.clusterEdgeProperties);for(var y in e)if(e.hasOwnProperty(y)&&void 0!==this.body.edges[y]){var b=this.body.edges[y];this._backupEdgeOptions(b),b.setOptions({physics:!1,hidden:!0})}for(var w in t)t.hasOwnProperty(w)&&(this.clusteredNodes[w]={clusterId:s.id,node:this.body.nodes[w]},this.body.nodes[w].setOptions({hidden:!0,physics:!1}));s.id=void 0,o===!0&&this.body.emitter.emit("_dataChanged")}}},{key:"_backupEdgeOptions",value:function(t){void 0===this.clusteredEdges[t.id]&&(this.clusteredEdges[t.id]={physics:t.options.physics,hidden:t.options.hidden})}},{key:"_restoreEdge",value:function(t){var e=this.clusteredEdges[t.id];void 0!==e&&(t.setOptions({physics:e.physics,hidden:e.hidden}),delete this.clusteredEdges[t.id])}},{key:"isCluster",value:function(t){return void 0!==this.body.nodes[t]?this.body.nodes[t].isCluster===!0:(console.log("Node does not exist."),!1)}},{key:"_getClusterPosition",value:function(t){for(var e=Object.keys(t),i=t[e[0]].x,o=t[e[0]].x,n=t[e[0]].y,s=t[e[0]].y,r=void 0,a=1;a<e.length;a++)r=t[e[a]],i=r.x<i?r.x:i,o=r.x>o?r.x:o,n=r.y<n?r.y:n,s=r.y>s?r.y:s;return{x:.5*(i+o),y:.5*(n+s)}}},{key:"openCluster",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!0:arguments[2];if(void 0===t)throw new Error("No clusterNodeId supplied to openCluster.");if(void 0===this.body.nodes[t])throw new Error("The clusterNodeId supplied to openCluster does not exist.");if(void 0===this.body.nodes[t].containedNodes)return void console.log("The node:"+t+" is not a cluster.");var o=this.body.nodes[t],n=o.containedNodes,s=o.containedEdges;if(void 0!==e&&void 0!==e.releaseFunction&&"function"==typeof e.releaseFunction){var r={},a={x:o.x,y:o.y};for(var d in n)if(n.hasOwnProperty(d)){var l=this.body.nodes[d];r[d]={x:l.x,y:l.y}}var u=e.releaseFunction(a,r);for(var p in n)if(n.hasOwnProperty(p)){var f=this.body.nodes[p];void 0!==u[p]&&(f.x=void 0===u[p].x?o.x:u[p].x,f.y=void 0===u[p].y?o.y:u[p].y)}}else for(var m in n)if(n.hasOwnProperty(m)){var v=this.body.nodes[m];v=n[m],v.options.fixed.x===!1&&(v.x=o.x),v.options.fixed.y===!1&&(v.y=o.y)}for(var g in n)if(n.hasOwnProperty(g)){var y=this.body.nodes[g];y.vx=o.vx,y.vy=o.vy,y.setOptions({hidden:!1,physics:!0}),delete this.clusteredNodes[g]}for(var b=[],w=0;w<o.edges.length;w++)b.push(o.edges[w]);for(var _=0;_<b.length;_++){var x=b[_],k=this._getConnectedId(x,t);if(void 0!==this.clusteredNodes[k]){var O=this.body.nodes[this.clusteredNodes[k].clusterId],M=this.body.edges[x.clusteringEdgeReplacingId];if(void 0!==M){O.containedEdges[M.id]=M,delete s[M.id];var D=M.fromId,S=M.toId;M.toId==k?S=this.clusteredNodes[k].clusterId:D=this.clusteredNodes[k].clusterId;var C=h["default"].cloneOptions(M,"edge");c.deepExtend(C,O.clusterEdgeProperties);var T="clusterEdge:"+c.randomUUID();c.deepExtend(C,{from:D,to:S,hidden:!1,physics:!0,id:T});var E=this.body.functions.createEdge(C);E.clusteringEdgeReplacingId=M.id,this.body.edges[T]=E,this.body.edges[T].connect()}}else{var P=this.body.edges[x.clusteringEdgeReplacingId];void 0!==P&&this._restoreEdge(P)}x.cleanup(),x.disconnect(),delete this.body.edges[x.id]}for(var I in s)s.hasOwnProperty(I)&&this._restoreEdge(s[I]);delete this.body.nodes[t],i===!0&&this.body.emitter.emit("_dataChanged")}},{key:"getNodesInCluster",value:function(t){var e=[];if(this.isCluster(t)===!0){var i=this.body.nodes[t].containedNodes;for(var o in i)i.hasOwnProperty(o)&&e.push(this.body.nodes[o].id)}return e}},{key:"findNode",value:function(t){for(var e=[],i=100,o=0;void 0!==this.clusteredNodes[t]&&i>o;)e.push(this.body.nodes[t].id),t=this.clusteredNodes[t].clusterId,o++;return e.push(this.body.nodes[t].id),e.reverse(),e}},{key:"_getConnectedId",value:function(t,e){return t.toId!=e?t.toId:t.fromId!=e?t.fromId:t.fromId}},{key:"_getHubSize",value:function(){for(var t=0,e=0,i=0,o=0,n=0;n<this.body.nodeIndices.length;n++){var s=this.body.nodes[this.body.nodeIndices[n]];s.edges.length>o&&(o=s.edges.length),t+=s.edges.length,e+=Math.pow(s.edges.length,2),i+=1}t/=i,e/=i;var r=e-Math.pow(t,2),a=Math.sqrt(r),h=Math.floor(t+2*a);return h>o&&(h=o),h}}]),t}();e["default"]=u},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=i(1),r=function(){function t(){o(this,t)}return n(t,null,[{key:"getRange",value:function(t){var e,i=arguments.length<=1||void 0===arguments[1]?[]:arguments[1],o=1e9,n=-1e9,s=1e9,r=-1e9;if(i.length>0)for(var a=0;a<i.length;a++)e=t[i[a]],s>e.shape.boundingBox.left&&(s=e.shape.boundingBox.left),r<e.shape.boundingBox.right&&(r=e.shape.boundingBox.right),o>e.shape.boundingBox.top&&(o=e.shape.boundingBox.top),n<e.shape.boundingBox.bottom&&(n=e.shape.boundingBox.bottom);return 1e9===s&&-1e9===r&&1e9===o&&-1e9===n&&(o=0,n=0,s=0,r=0),{minX:s,maxX:r,minY:o,maxY:n}}},{key:"getRangeCore",value:function(t){var e,i=arguments.length<=1||void 0===arguments[1]?[]:arguments[1],o=1e9,n=-1e9,s=1e9,r=-1e9;if(i.length>0)for(var a=0;a<i.length;a++)e=t[i[a]],s>e.x&&(s=e.x),r<e.x&&(r=e.x),o>e.y&&(o=e.y),n<e.y&&(n=e.y);return 1e9===s&&-1e9===r&&1e9===o&&-1e9===n&&(o=0,n=0,s=0,r=0),{minX:s,maxX:r,minY:o,maxY:n}}},{key:"findCenter",value:function(t){return{x:.5*(t.maxX+t.minX),y:.5*(t.maxY+t.minY)}}},{key:"cloneOptions",value:function(t,e){var i={};return void 0===e||"node"===e?(s.deepExtend(i,t.options,!0),i.x=t.x,i.y=t.y,i.amountOfConnections=t.edges.length):s.deepExtend(i,t.options,!0),i}}]),t}();e["default"]=r},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=i(65),h=o(a),d=function(t){function e(t,i,o,r,a){n(this,e);var h=s(this,Object.getPrototypeOf(e).call(this,t,i,o,r,a));return h.isCluster=!0,h.containedNodes={},h.containedEdges={},h}return r(e,t),e}(h["default"]);e["default"]=d},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}();"undefined"!=typeof window&&(window.requestAnimationFrame=window.requestAnimationFrame||window.mozRequestAnimationFrame||window.webkitRequestAnimationFrame||window.msRequestAnimationFrame);var s=i(1),r=function(){function t(e,i){o(this,t),this.body=e,this.canvas=i,this.redrawRequested=!1,this.renderTimer=void 0,this.requiresTimeout=!0,this.renderingActive=!1,this.renderRequests=0,this.pixelRatio=void 0,this.allowRedraw=!0,this.dragging=!1,this.options={},this.defaultOptions={hideEdgesOnDrag:!1,hideNodesOnDrag:!1},s.extend(this.options,this.defaultOptions),this._determineBrowserMethod(),this.bindEventListeners()}return n(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("dragStart",function(){t.dragging=!0}),this.body.emitter.on("dragEnd",function(){return t.dragging=!1}),this.body.emitter.on("_resizeNodes",function(){return t._resizeNodes()}),this.body.emitter.on("_redraw",function(){t.renderingActive===!1&&t._redraw()}),this.body.emitter.on("_blockRedraw",function(){t.allowRedraw=!1}),this.body.emitter.on("_allowRedraw",function(){t.allowRedraw=!0,t.redrawRequested=!1}),this.body.emitter.on("_requestRedraw",this._requestRedraw.bind(this)),this.body.emitter.on("_startRendering",function(){t.renderRequests+=1,t.renderingActive=!0,t._startRendering()}),this.body.emitter.on("_stopRendering",function(){t.renderRequests-=1,t.renderingActive=t.renderRequests>0,t.renderTimer=void 0}),this.body.emitter.on("destroy",function(){t.renderRequests=0,t.allowRedraw=!1,t.renderingActive=!1,t.requiresTimeout===!0?clearTimeout(t.renderTimer):cancelAnimationFrame(t.renderTimer),t.body.emitter.off()})}},{key:"setOptions",value:function(t){if(void 0!==t){var e=["hideEdgesOnDrag","hideNodesOnDrag"];s.selectiveDeepExtend(e,this.options,t)}}},{key:"_startRendering",value:function(){this.renderingActive===!0&&void 0===this.renderTimer&&(this.requiresTimeout===!0?this.renderTimer=window.setTimeout(this._renderStep.bind(this),this.simulationInterval):this.renderTimer=window.requestAnimationFrame(this._renderStep.bind(this)))}},{key:"_renderStep",value:function(){this.renderingActive===!0&&(this.renderTimer=void 0,this.requiresTimeout===!0&&this._startRendering(),this._redraw(),this.requiresTimeout===!1&&this._startRendering())}},{key:"redraw",value:function(){this.body.emitter.emit("setSize"),this._redraw()}},{key:"_requestRedraw",value:function(){var t=this;this.redrawRequested!==!0&&this.renderingActive===!1&&this.allowRedraw===!0&&(this.redrawRequested=!0,this.requiresTimeout===!0?window.setTimeout(function(){t._redraw(!1)},0):window.requestAnimationFrame(function(){t._redraw(!1)}))}},{key:"_redraw",value:function(){var t=arguments.length<=0||void 0===arguments[0]?!1:arguments[0];if(this.allowRedraw===!0){this.body.emitter.emit("initRedraw"),this.redrawRequested=!1;var e=this.canvas.frame.canvas.getContext("2d");0!==this.canvas.frame.canvas.width&&0!==this.canvas.frame.canvas.height||this.canvas.setSize(),this.pixelRatio=(window.devicePixelRatio||1)/(e.webkitBackingStorePixelRatio||e.mozBackingStorePixelRatio||e.msBackingStorePixelRatio||e.oBackingStorePixelRatio||e.backingStorePixelRatio||1),e.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var i=this.canvas.frame.canvas.clientWidth,o=this.canvas.frame.canvas.clientHeight;if(e.clearRect(0,0,i,o),0===this.canvas.frame.clientWidth)return;e.save(),e.translate(this.body.view.translation.x,this.body.view.translation.y),e.scale(this.body.view.scale,this.body.view.scale),e.beginPath(),this.body.emitter.emit("beforeDrawing",e),e.closePath(),t===!1&&(this.dragging===!1||this.dragging===!0&&this.options.hideEdgesOnDrag===!1)&&this._drawEdges(e),(this.dragging===!1||this.dragging===!0&&this.options.hideNodesOnDrag===!1)&&this._drawNodes(e,t),e.beginPath(),this.body.emitter.emit("afterDrawing",e),e.closePath(),e.restore(),t===!0&&e.clearRect(0,0,i,o)}}},{key:"_resizeNodes",value:function(){var t=this.canvas.frame.canvas.getContext("2d");void 0===this.pixelRatio&&(this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1)),t.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0),t.save(),t.translate(this.body.view.translation.x,this.body.view.translation.y),t.scale(this.body.view.scale,this.body.view.scale);var e=this.body.nodes,i=void 0;for(var o in e)e.hasOwnProperty(o)&&(i=e[o],i.resize(t),i.updateBoundingBox(t,i.selected));t.restore()}},{key:"_drawNodes",value:function(t){for(var e=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],i=this.body.nodes,o=this.body.nodeIndices,n=void 0,s=[],r=20,a=this.canvas.DOMtoCanvas({x:-r,y:-r}),h=this.canvas.DOMtoCanvas({x:this.canvas.frame.canvas.clientWidth+r,y:this.canvas.frame.canvas.clientHeight+r}),d={top:a.y,left:a.x,bottom:h.y,right:h.x},l=0;l<o.length;l++)n=i[o[l]],n.isSelected()?s.push(o[l]):e===!0?n.draw(t):n.isBoundingBoxOverlappingWith(d)===!0?n.draw(t):n.updateBoundingBox(t,n.selected);for(var c=0;c<s.length;c++)n=i[s[c]],n.draw(t)}},{key:"_drawEdges",value:function(t){for(var e=this.body.edges,i=this.body.edgeIndices,o=void 0,n=0;n<i.length;n++)o=e[i[n]],o.connected===!0&&o.draw(t)}},{key:"_determineBrowserMethod",value:function(){if("undefined"!=typeof window){var t=navigator.userAgent.toLowerCase();this.requiresTimeout=!1,-1!=t.indexOf("msie 9.0")?this.requiresTimeout=!0:-1!=t.indexOf("safari")&&t.indexOf("chrome")<=-1&&(this.requiresTimeout=!0)}else this.requiresTimeout=!0}}]),t}();e["default"]=r},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=i(20),r=i(28),a=i(1),h=function(){function t(e){o(this,t),this.body=e,this.pixelRatio=1,this.resizeTimer=void 0,this.resizeFunction=this._onResize.bind(this),this.cameraState={},this.initialized=!1,this.options={},this.defaultOptions={autoResize:!0,height:"100%",width:"100%"},a.extend(this.options,this.defaultOptions),this.bindEventListeners()}return n(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.once("resize",function(e){0!==e.width&&(t.body.view.translation.x=.5*e.width),0!==e.height&&(t.body.view.translation.y=.5*e.height)}),this.body.emitter.on("setSize",this.setSize.bind(this)),this.body.emitter.on("destroy",function(){t.hammerFrame.destroy(),t.hammer.destroy(),t._cleanUp()})}},{key:"setOptions",value:function(t){var e=this;if(void 0!==t){var i=["width","height","autoResize"];a.selectiveDeepExtend(i,this.options,t)}this.options.autoResize===!0&&(this._cleanUp(),this.resizeTimer=setInterval(function(){var t=e.setSize();t===!0&&e.body.emitter.emit("_requestRedraw")},1e3),this.resizeFunction=this._onResize.bind(this),a.addEventListener(window,"resize",this.resizeFunction))}},{key:"_cleanUp",value:function(){void 0!==this.resizeTimer&&clearInterval(this.resizeTimer),a.removeEventListener(window,"resize",this.resizeFunction),this.resizeFunction=void 0}},{key:"_onResize",value:function(){this.setSize(),this.body.emitter.emit("_redraw")}},{key:"_getCameraState",value:function(){var t=arguments.length<=0||void 0===arguments[0]?this.pixelRatio:arguments[0];this.initialized===!0&&(this.cameraState.previousWidth=this.frame.canvas.width/t,this.cameraState.previousHeight=this.frame.canvas.height/t,this.cameraState.scale=this.body.view.scale,this.cameraState.position=this.DOMtoCanvas({x:.5*this.frame.canvas.width/t,y:.5*this.frame.canvas.height/t}))}},{key:"_setCameraState",value:function(){if(void 0!==this.cameraState.scale&&0!==this.frame.canvas.clientWidth&&0!==this.frame.canvas.clientHeight&&0!==this.pixelRatio&&this.cameraState.previousWidth>0){var t=this.frame.canvas.width/this.pixelRatio/this.cameraState.previousWidth,e=this.frame.canvas.height/this.pixelRatio/this.cameraState.previousHeight,i=this.cameraState.scale;1!=t&&1!=e?i=.5*this.cameraState.scale*(t+e):1!=t?i=this.cameraState.scale*t:1!=e&&(i=this.cameraState.scale*e),this.body.view.scale=i;var o=this.DOMtoCanvas({x:.5*this.frame.canvas.clientWidth,y:.5*this.frame.canvas.clientHeight}),n={x:o.x-this.cameraState.position.x,y:o.y-this.cameraState.position.y};this.body.view.translation.x+=n.x*this.body.view.scale,this.body.view.translation.y+=n.y*this.body.view.scale}}},{key:"_prepareValue",value:function(t){if("number"==typeof t)return t+"px";if("string"==typeof t){if(-1!==t.indexOf("%")||-1!==t.indexOf("px"))return t;if(-1===t.indexOf("%"))return t+"px"}throw new Error("Could not use the value supplied for width or height:"+t)}},{key:"_create",value:function(){for(;this.body.container.hasChildNodes();)this.body.container.removeChild(this.body.container.firstChild);if(this.frame=document.createElement("div"),this.frame.className="vis-network",this.frame.style.position="relative",this.frame.style.overflow="hidden",this.frame.tabIndex=900,this.frame.canvas=document.createElement("canvas"),this.frame.canvas.style.position="relative",this.frame.appendChild(this.frame.canvas),this.frame.canvas.getContext){var t=this.frame.canvas.getContext("2d");this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1),
-this.frame.canvas.getContext("2d").setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0)}else{var e=document.createElement("DIV");e.style.color="red",e.style.fontWeight="bold",e.style.padding="10px",e.innerHTML="Error: your browser does not support HTML canvas",this.frame.canvas.appendChild(e)}this.body.container.appendChild(this.frame),this.body.view.scale=1,this.body.view.translation={x:.5*this.frame.canvas.clientWidth,y:.5*this.frame.canvas.clientHeight},this._bindHammer()}},{key:"_bindHammer",value:function(){var t=this;void 0!==this.hammer&&this.hammer.destroy(),this.drag={},this.pinch={},this.hammer=new s(this.frame.canvas),this.hammer.get("pinch").set({enable:!0}),this.hammer.get("pan").set({threshold:5,direction:s.DIRECTION_ALL}),r.onTouch(this.hammer,function(e){t.body.eventListeners.onTouch(e)}),this.hammer.on("tap",function(e){t.body.eventListeners.onTap(e)}),this.hammer.on("doubletap",function(e){t.body.eventListeners.onDoubleTap(e)}),this.hammer.on("press",function(e){t.body.eventListeners.onHold(e)}),this.hammer.on("panstart",function(e){t.body.eventListeners.onDragStart(e)}),this.hammer.on("panmove",function(e){t.body.eventListeners.onDrag(e)}),this.hammer.on("panend",function(e){t.body.eventListeners.onDragEnd(e)}),this.hammer.on("pinch",function(e){t.body.eventListeners.onPinch(e)}),this.frame.canvas.addEventListener("mousewheel",function(e){t.body.eventListeners.onMouseWheel(e)}),this.frame.canvas.addEventListener("DOMMouseScroll",function(e){t.body.eventListeners.onMouseWheel(e)}),this.frame.canvas.addEventListener("mousemove",function(e){t.body.eventListeners.onMouseMove(e)}),this.frame.canvas.addEventListener("contextmenu",function(e){t.body.eventListeners.onContext(e)}),this.hammerFrame=new s(this.frame),r.onRelease(this.hammerFrame,function(e){t.body.eventListeners.onRelease(e)})}},{key:"setSize",value:function(){var t=arguments.length<=0||void 0===arguments[0]?this.options.width:arguments[0],e=arguments.length<=1||void 0===arguments[1]?this.options.height:arguments[1];t=this._prepareValue(t),e=this._prepareValue(e);var i=!1,o=this.frame.canvas.width,n=this.frame.canvas.height,s=this.frame.canvas.getContext("2d"),r=this.pixelRatio;return this.pixelRatio=(window.devicePixelRatio||1)/(s.webkitBackingStorePixelRatio||s.mozBackingStorePixelRatio||s.msBackingStorePixelRatio||s.oBackingStorePixelRatio||s.backingStorePixelRatio||1),t!=this.options.width||e!=this.options.height||this.frame.style.width!=t||this.frame.style.height!=e?(this._getCameraState(r),this.frame.style.width=t,this.frame.style.height=e,this.frame.canvas.style.width="100%",this.frame.canvas.style.height="100%",this.frame.canvas.width=Math.round(this.frame.canvas.clientWidth*this.pixelRatio),this.frame.canvas.height=Math.round(this.frame.canvas.clientHeight*this.pixelRatio),this.options.width=t,this.options.height=e,i=!0):(this.frame.canvas.width==Math.round(this.frame.canvas.clientWidth*this.pixelRatio)&&this.frame.canvas.height==Math.round(this.frame.canvas.clientHeight*this.pixelRatio)||this._getCameraState(r),this.frame.canvas.width!=Math.round(this.frame.canvas.clientWidth*this.pixelRatio)&&(this.frame.canvas.width=Math.round(this.frame.canvas.clientWidth*this.pixelRatio),i=!0),this.frame.canvas.height!=Math.round(this.frame.canvas.clientHeight*this.pixelRatio)&&(this.frame.canvas.height=Math.round(this.frame.canvas.clientHeight*this.pixelRatio),i=!0)),i===!0&&(this.body.emitter.emit("resize",{width:Math.round(this.frame.canvas.width/this.pixelRatio),height:Math.round(this.frame.canvas.height/this.pixelRatio),oldWidth:Math.round(o/this.pixelRatio),oldHeight:Math.round(n/this.pixelRatio)}),this._setCameraState()),this.initialized=!0,i}},{key:"_XconvertDOMtoCanvas",value:function(t){return(t-this.body.view.translation.x)/this.body.view.scale}},{key:"_XconvertCanvasToDOM",value:function(t){return t*this.body.view.scale+this.body.view.translation.x}},{key:"_YconvertDOMtoCanvas",value:function(t){return(t-this.body.view.translation.y)/this.body.view.scale}},{key:"_YconvertCanvasToDOM",value:function(t){return t*this.body.view.scale+this.body.view.translation.y}},{key:"canvasToDOM",value:function(t){return{x:this._XconvertCanvasToDOM(t.x),y:this._YconvertCanvasToDOM(t.y)}}},{key:"DOMtoCanvas",value:function(t){return{x:this._XconvertDOMtoCanvas(t.x),y:this._YconvertDOMtoCanvas(t.y)}}}]),t}();e["default"]=h},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(103),a=o(r),h=i(1),d=function(){function t(e,i){var o=this;n(this,t),this.body=e,this.canvas=i,this.animationSpeed=1/this.renderRefreshRate,this.animationEasingFunction="easeInOutQuint",this.easingTime=0,this.sourceScale=0,this.targetScale=0,this.sourceTranslation=0,this.targetTranslation=0,this.lockedOnNodeId=void 0,this.lockedOnNodeOffset=void 0,this.touchTime=0,this.viewFunction=void 0,this.body.emitter.on("fit",this.fit.bind(this)),this.body.emitter.on("animationFinished",function(){o.body.emitter.emit("_stopRendering")}),this.body.emitter.on("unlockNode",this.releaseNode.bind(this))}return s(t,[{key:"setOptions",value:function(){var t=arguments.length<=0||void 0===arguments[0]?{}:arguments[0];this.options=t}},{key:"fit",value:function(){var t=arguments.length<=0||void 0===arguments[0]?{nodes:[]}:arguments[0],e=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],i=void 0,o=void 0;if(void 0!==t.nodes&&0!==t.nodes.length||(t.nodes=this.body.nodeIndices),e===!0){var n=0;for(var s in this.body.nodes)if(this.body.nodes.hasOwnProperty(s)){var r=this.body.nodes[s];r.predefinedPosition===!0&&(n+=1)}if(n>.5*this.body.nodeIndices.length)return void this.fit(t,!1);i=a["default"].getRange(this.body.nodes,t.nodes);var h=this.body.nodeIndices.length;o=12.662/(h+7.4147)+.0964822;var d=Math.min(this.canvas.frame.canvas.clientWidth/600,this.canvas.frame.canvas.clientHeight/600);o*=d}else{this.body.emitter.emit("_resizeNodes"),i=a["default"].getRange(this.body.nodes,t.nodes);var l=1.1*Math.abs(i.maxX-i.minX),c=1.1*Math.abs(i.maxY-i.minY),u=this.canvas.frame.canvas.clientWidth/l,p=this.canvas.frame.canvas.clientHeight/c;o=p>=u?u:p}o>1?o=1:0===o&&(o=1);var f=a["default"].findCenter(i),m={position:f,scale:o,animation:t.animation};this.moveTo(m)}},{key:"focus",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?{}:arguments[1];if(void 0!==this.body.nodes[t]){var i={x:this.body.nodes[t].x,y:this.body.nodes[t].y};e.position=i,e.lockedOnNode=t,this.moveTo(e)}else console.log("Node: "+t+" cannot be found.")}},{key:"moveTo",value:function(t){return void 0===t?void(t={}):(void 0===t.offset&&(t.offset={x:0,y:0}),void 0===t.offset.x&&(t.offset.x=0),void 0===t.offset.y&&(t.offset.y=0),void 0===t.scale&&(t.scale=this.body.view.scale),void 0===t.position&&(t.position=this.getViewPosition()),void 0===t.animation&&(t.animation={duration:0}),t.animation===!1&&(t.animation={duration:0}),t.animation===!0&&(t.animation={}),void 0===t.animation.duration&&(t.animation.duration=1e3),void 0===t.animation.easingFunction&&(t.animation.easingFunction="easeInOutQuad"),void this.animateView(t))}},{key:"animateView",value:function(t){if(void 0!==t){this.animationEasingFunction=t.animation.easingFunction,this.releaseNode(),t.locked===!0&&(this.lockedOnNodeId=t.lockedOnNode,this.lockedOnNodeOffset=t.offset),0!=this.easingTime&&this._transitionRedraw(!0),this.sourceScale=this.body.view.scale,this.sourceTranslation=this.body.view.translation,this.targetScale=t.scale,this.body.view.scale=this.targetScale;var e=this.canvas.DOMtoCanvas({x:.5*this.canvas.frame.canvas.clientWidth,y:.5*this.canvas.frame.canvas.clientHeight}),i={x:e.x-t.position.x,y:e.y-t.position.y};this.targetTranslation={x:this.sourceTranslation.x+i.x*this.targetScale+t.offset.x,y:this.sourceTranslation.y+i.y*this.targetScale+t.offset.y},0===t.animation.duration?void 0!=this.lockedOnNodeId?(this.viewFunction=this._lockedRedraw.bind(this),this.body.emitter.on("initRedraw",this.viewFunction)):(this.body.view.scale=this.targetScale,this.body.view.translation=this.targetTranslation,this.body.emitter.emit("_requestRedraw")):(this.animationSpeed=1/(60*t.animation.duration*.001)||1/60,this.animationEasingFunction=t.animation.easingFunction,this.viewFunction=this._transitionRedraw.bind(this),this.body.emitter.on("initRedraw",this.viewFunction),this.body.emitter.emit("_startRendering"))}}},{key:"_lockedRedraw",value:function(){var t={x:this.body.nodes[this.lockedOnNodeId].x,y:this.body.nodes[this.lockedOnNodeId].y},e=this.canvas.DOMtoCanvas({x:.5*this.canvas.frame.canvas.clientWidth,y:.5*this.canvas.frame.canvas.clientHeight}),i={x:e.x-t.x,y:e.y-t.y},o=this.body.view.translation,n={x:o.x+i.x*this.body.view.scale+this.lockedOnNodeOffset.x,y:o.y+i.y*this.body.view.scale+this.lockedOnNodeOffset.y};this.body.view.translation=n}},{key:"releaseNode",value:function(){void 0!==this.lockedOnNodeId&&void 0!==this.viewFunction&&(this.body.emitter.off("initRedraw",this.viewFunction),this.lockedOnNodeId=void 0,this.lockedOnNodeOffset=void 0)}},{key:"_transitionRedraw",value:function(){var t=arguments.length<=0||void 0===arguments[0]?!1:arguments[0];this.easingTime+=this.animationSpeed,this.easingTime=t===!0?1:this.easingTime;var e=h.easingFunctions[this.animationEasingFunction](this.easingTime);this.body.view.scale=this.sourceScale+(this.targetScale-this.sourceScale)*e,this.body.view.translation={x:this.sourceTranslation.x+(this.targetTranslation.x-this.sourceTranslation.x)*e,y:this.sourceTranslation.y+(this.targetTranslation.y-this.sourceTranslation.y)*e},this.easingTime>=1&&(this.body.emitter.off("initRedraw",this.viewFunction),this.easingTime=0,void 0!=this.lockedOnNodeId&&(this.viewFunction=this._lockedRedraw.bind(this),this.body.emitter.on("initRedraw",this.viewFunction)),this.body.emitter.emit("animationFinished"))}},{key:"getScale",value:function(){return this.body.view.scale}},{key:"getViewPosition",value:function(){return this.canvas.DOMtoCanvas({x:.5*this.canvas.frame.canvas.clientWidth,y:.5*this.canvas.frame.canvas.clientHeight})}}]),t}();e["default"]=d},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(109),a=o(r),h=i(110),d=o(h),l=i(1),c=function(){function t(e,i,o){n(this,t),this.body=e,this.canvas=i,this.selectionHandler=o,this.navigationHandler=new a["default"](e,i),this.body.eventListeners.onTap=this.onTap.bind(this),this.body.eventListeners.onTouch=this.onTouch.bind(this),this.body.eventListeners.onDoubleTap=this.onDoubleTap.bind(this),this.body.eventListeners.onHold=this.onHold.bind(this),this.body.eventListeners.onDragStart=this.onDragStart.bind(this),this.body.eventListeners.onDrag=this.onDrag.bind(this),this.body.eventListeners.onDragEnd=this.onDragEnd.bind(this),this.body.eventListeners.onMouseWheel=this.onMouseWheel.bind(this),this.body.eventListeners.onPinch=this.onPinch.bind(this),this.body.eventListeners.onMouseMove=this.onMouseMove.bind(this),this.body.eventListeners.onRelease=this.onRelease.bind(this),this.body.eventListeners.onContext=this.onContext.bind(this),this.touchTime=0,this.drag={},this.pinch={},this.popup=void 0,this.popupObj=void 0,this.popupTimer=void 0,this.body.functions.getPointer=this.getPointer.bind(this),this.options={},this.defaultOptions={dragNodes:!0,dragView:!0,hover:!1,keyboard:{enabled:!1,speed:{x:10,y:10,zoom:.02},bindToWindow:!0},navigationButtons:!1,tooltipDelay:300,zoomView:!0},l.extend(this.options,this.defaultOptions),this.bindEventListeners()}return s(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("destroy",function(){clearTimeout(t.popupTimer),delete t.body.functions.getPointer})}},{key:"setOptions",value:function(t){if(void 0!==t){var e=["hideEdgesOnDrag","hideNodesOnDrag","keyboard","multiselect","selectable","selectConnectedEdges"];l.selectiveNotDeepExtend(e,this.options,t),l.mergeOptions(this.options,t,"keyboard"),t.tooltip&&(l.extend(this.options.tooltip,t.tooltip),t.tooltip.color&&(this.options.tooltip.color=l.parseColor(t.tooltip.color)))}this.navigationHandler.setOptions(this.options)}},{key:"getPointer",value:function(t){return{x:t.x-l.getAbsoluteLeft(this.canvas.frame.canvas),y:t.y-l.getAbsoluteTop(this.canvas.frame.canvas)}}},{key:"onTouch",value:function(t){(new Date).valueOf()-this.touchTime>50&&(this.drag.pointer=this.getPointer(t.center),this.drag.pinched=!1,this.pinch.scale=this.body.view.scale,this.touchTime=(new Date).valueOf())}},{key:"onTap",value:function(t){var e=this.getPointer(t.center),i=this.selectionHandler.options.multiselect&&(t.changedPointers[0].ctrlKey||t.changedPointers[0].metaKey);this.checkSelectionChanges(e,t,i),this.selectionHandler._generateClickEvent("click",t,e)}},{key:"onDoubleTap",value:function(t){var e=this.getPointer(t.center);this.selectionHandler._generateClickEvent("doubleClick",t,e)}},{key:"onHold",value:function(t){var e=this.getPointer(t.center),i=this.selectionHandler.options.multiselect;this.checkSelectionChanges(e,t,i),this.selectionHandler._generateClickEvent("click",t,e),this.selectionHandler._generateClickEvent("hold",t,e)}},{key:"onRelease",value:function(t){if((new Date).valueOf()-this.touchTime>10){var e=this.getPointer(t.center);this.selectionHandler._generateClickEvent("release",t,e),this.touchTime=(new Date).valueOf()}}},{key:"onContext",value:function(t){var e=this.getPointer({x:t.clientX,y:t.clientY});this.selectionHandler._generateClickEvent("oncontext",t,e)}},{key:"checkSelectionChanges",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=this.selectionHandler._getSelectedEdgeCount(),n=this.selectionHandler._getSelectedNodeCount(),s=this.selectionHandler.getSelection(),r=void 0;r=i===!0?this.selectionHandler.selectAdditionalOnPoint(t):this.selectionHandler.selectOnPoint(t);var a=this.selectionHandler._getSelectedEdgeCount(),h=this.selectionHandler._getSelectedNodeCount(),d=this.selectionHandler.getSelection(),l=this._determineIfDifferent(s,d),c=l.nodesChanged,u=l.edgesChanged,p=!1;h-n>0?(this.selectionHandler._generateClickEvent("selectNode",e,t),r=!0,p=!0):c===!0&&h>0?(this.selectionHandler._generateClickEvent("deselectNode",e,t,s),this.selectionHandler._generateClickEvent("selectNode",e,t),p=!0,r=!0):0>h-n&&(this.selectionHandler._generateClickEvent("deselectNode",e,t,s),r=!0),a-o>0&&p===!1?(this.selectionHandler._generateClickEvent("selectEdge",e,t),r=!0):a>0&&u===!0?(this.selectionHandler._generateClickEvent("deselectEdge",e,t,s),this.selectionHandler._generateClickEvent("selectEdge",e,t),r=!0):0>a-o&&(this.selectionHandler._generateClickEvent("deselectEdge",e,t,s),r=!0),r===!0&&this.selectionHandler._generateClickEvent("select",e,t)}},{key:"_determineIfDifferent",value:function(t,e){for(var i=!1,o=!1,n=0;n<t.nodes.length;n++)-1===e.nodes.indexOf(t.nodes[n])&&(i=!0);for(var s=0;s<e.nodes.length;s++)-1===t.nodes.indexOf(t.nodes[s])&&(i=!0);for(var r=0;r<t.edges.length;r++)-1===e.edges.indexOf(t.edges[r])&&(o=!0);for(var a=0;a<e.edges.length;a++)-1===t.edges.indexOf(t.edges[a])&&(o=!0);return{nodesChanged:i,edgesChanged:o}}},{key:"onDragStart",value:function(t){void 0===this.drag.pointer&&this.onTouch(t);var e=this.selectionHandler.getNodeAt(this.drag.pointer);if(this.drag.dragging=!0,this.drag.selection=[],this.drag.translation=l.extend({},this.body.view.translation),this.drag.nodeId=void 0,void 0!==e&&this.options.dragNodes===!0){this.drag.nodeId=e.id,e.isSelected()===!1&&(this.selectionHandler.unselectAll(),this.selectionHandler.selectObject(e)),this.selectionHandler._generateClickEvent("dragStart",t,this.drag.pointer);var i=this.selectionHandler.selectionObj.nodes;for(var o in i)if(i.hasOwnProperty(o)){var n=i[o],s={id:n.id,node:n,x:n.x,y:n.y,xFixed:n.options.fixed.x,yFixed:n.options.fixed.y};n.options.fixed.x=!0,n.options.fixed.y=!0,this.drag.selection.push(s)}}else this.selectionHandler._generateClickEvent("dragStart",t,this.drag.pointer,void 0,!0)}},{key:"onDrag",value:function(t){var e=this;if(this.drag.pinched!==!0){this.body.emitter.emit("unlockNode");var i=this.getPointer(t.center),o=this.drag.selection;if(o&&o.length&&this.options.dragNodes===!0)!function(){e.selectionHandler._generateClickEvent("dragging",t,i);var n=i.x-e.drag.pointer.x,s=i.y-e.drag.pointer.y;o.forEach(function(t){var i=t.node;t.xFixed===!1&&(i.x=e.canvas._XconvertDOMtoCanvas(e.canvas._XconvertCanvasToDOM(t.x)+n)),t.yFixed===!1&&(i.y=e.canvas._YconvertDOMtoCanvas(e.canvas._YconvertCanvasToDOM(t.y)+s))}),e.body.emitter.emit("startSimulation")}();else if(this.options.dragView===!0){if(this.selectionHandler._generateClickEvent("dragging",t,i,void 0,!0),void 0===this.drag.pointer)return void this.onDragStart(t);var n=i.x-this.drag.pointer.x,s=i.y-this.drag.pointer.y;this.body.view.translation={x:this.drag.translation.x+n,y:this.drag.translation.y+s},this.body.emitter.emit("_redraw")}}}},{key:"onDragEnd",value:function(t){this.drag.dragging=!1;var e=this.drag.selection;e&&e.length?(e.forEach(function(t){t.node.options.fixed.x=t.xFixed,t.node.options.fixed.y=t.yFixed}),this.selectionHandler._generateClickEvent("dragEnd",t,this.getPointer(t.center)),this.body.emitter.emit("startSimulation")):(this.selectionHandler._generateClickEvent("dragEnd",t,this.getPointer(t.center),void 0,!0),this.body.emitter.emit("_requestRedraw"))}},{key:"onPinch",value:function(t){var e=this.getPointer(t.center);this.drag.pinched=!0,void 0===this.pinch.scale&&(this.pinch.scale=1);var i=this.pinch.scale*t.scale;this.zoom(i,e)}},{key:"zoom",value:function(t,e){if(this.options.zoomView===!0){var i=this.body.view.scale;1e-5>t&&(t=1e-5),t>10&&(t=10);var o=void 0;void 0!==this.drag&&this.drag.dragging===!0&&(o=this.canvas.DOMtoCanvas(this.drag.pointer));var n=this.body.view.translation,s=t/i,r=(1-s)*e.x+n.x*s,a=(1-s)*e.y+n.y*s;if(this.body.view.scale=t,this.body.view.translation={x:r,y:a},void 0!=o){var h=this.canvas.canvasToDOM(o);this.drag.pointer.x=h.x,this.drag.pointer.y=h.y}this.body.emitter.emit("_requestRedraw"),t>i?this.body.emitter.emit("zoom",{direction:"+",scale:this.body.view.scale}):this.body.emitter.emit("zoom",{direction:"-",scale:this.body.view.scale})}}},{key:"onMouseWheel",value:function(t){if(this.options.zoomView===!0){var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail&&(e=-t.detail/3),0!==e){var i=this.body.view.scale,o=e/10;0>e&&(o/=1-o),i*=1+o;var n=this.getPointer({x:t.clientX,y:t.clientY});this.zoom(i,n)}t.preventDefault()}}},{key:"onMouseMove",value:function(t){var e=this,i=this.getPointer({x:t.clientX,y:t.clientY}),o=!1;if(void 0!==this.popup&&(this.popup.hidden===!1&&this._checkHidePopup(i),this.popup.hidden===!1&&(o=!0,this.popup.setPosition(i.x+3,i.y-5),this.popup.show())),this.options.keyboard.bindToWindow===!1&&this.options.keyboard.enabled===!0&&this.canvas.frame.focus(),o===!1&&(void 0!==this.popupTimer&&(clearInterval(this.popupTimer),this.popupTimer=void 0),this.drag.dragging||(this.popupTimer=setTimeout(function(){return e._checkShowPopup(i)},this.options.tooltipDelay))),this.options.hover===!0){var n=this.selectionHandler.getNodeAt(i);void 0===n&&(n=this.selectionHandler.getEdgeAt(i)),this.selectionHandler.hoverObject(n)}}},{key:"_checkShowPopup",value:function(t){var e=this.canvas._XconvertDOMtoCanvas(t.x),i=this.canvas._YconvertDOMtoCanvas(t.y),o={left:e,top:i,right:e,bottom:i},n=void 0===this.popupObj?void 0:this.popupObj.id,s=!1,r="node";if(void 0===this.popupObj){for(var a=this.body.nodeIndices,h=this.body.nodes,l=void 0,c=[],u=0;u<a.length;u++)l=h[a[u]],l.isOverlappingWith(o)===!0&&void 0!==l.getTitle()&&c.push(a[u]);c.length>0&&(this.popupObj=h[c[c.length-1]],s=!0)}if(void 0===this.popupObj&&s===!1){for(var p=this.body.edgeIndices,f=this.body.edges,m=void 0,v=[],g=0;g<p.length;g++)m=f[p[g]],m.isOverlappingWith(o)===!0&&m.connected===!0&&void 0!==m.getTitle()&&v.push(p[g]);v.length>0&&(this.popupObj=f[v[v.length-1]],r="edge")}void 0!==this.popupObj?this.popupObj.id!==n&&(void 0===this.popup&&(this.popup=new d["default"](this.canvas.frame)),this.popup.popupTargetType=r,this.popup.popupTargetId=this.popupObj.id,this.popup.setPosition(t.x+3,t.y-5),this.popup.setText(this.popupObj.getTitle()),this.popup.show(),this.body.emitter.emit("showPopup",this.popupObj.id)):void 0!==this.popup&&(this.popup.hide(),this.body.emitter.emit("hidePopup"))}},{key:"_checkHidePopup",value:function(t){var e=this.selectionHandler._pointerToPositionObject(t),i=!1;if("node"===this.popup.popupTargetType){if(void 0!==this.body.nodes[this.popup.popupTargetId]&&(i=this.body.nodes[this.popup.popupTargetId].isOverlappingWith(e),i===!0)){var o=this.selectionHandler.getNodeAt(t);i=o.id===this.popup.popupTargetId}}else void 0===this.selectionHandler.getNodeAt(t)&&void 0!==this.body.edges[this.popup.popupTargetId]&&(i=this.body.edges[this.popup.popupTargetId].isOverlappingWith(e));i===!1&&(this.popupObj=void 0,this.popup.hide(),this.body.emitter.emit("hidePopup"))}}]),t}();e["default"]=c},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=(i(1),i(20)),r=i(28),a=i(23),h=function(){function t(e,i){var n=this;o(this,t),this.body=e,this.canvas=i,this.iconsCreated=!1,this.navigationHammers=[],this.boundFunctions={},this.touchTime=0,this.activated=!1,this.body.emitter.on("activate",function(){n.activated=!0,n.configureKeyboardBindings()}),this.body.emitter.on("deactivate",function(){n.activated=!1,n.configureKeyboardBindings()}),this.body.emitter.on("destroy",function(){void 0!==n.keycharm&&n.keycharm.destroy()}),this.options={}}return n(t,[{key:"setOptions",value:function(t){void 0!==t&&(this.options=t,this.create())}},{key:"create",value:function(){this.options.navigationButtons===!0?this.iconsCreated===!1&&this.loadNavigationElements():this.iconsCreated===!0&&this.cleanNavigation(),this.configureKeyboardBindings()}},{key:"cleanNavigation",value:function(){if(0!=this.navigationHammers.length){for(var t=0;t<this.navigationHammers.length;t++)this.navigationHammers[t].destroy();this.navigationHammers=[]}this.navigationDOM&&this.navigationDOM.wrapper&&this.navigationDOM.wrapper.parentNode&&this.navigationDOM.wrapper.parentNode.removeChild(this.navigationDOM.wrapper),this.iconsCreated=!1}},{key:"loadNavigationElements",value:function(){var t=this;this.cleanNavigation(),this.navigationDOM={};var e=["up","down","left","right","zoomIn","zoomOut","zoomExtends"],i=["_moveUp","_moveDown","_moveLeft","_moveRight","_zoomIn","_zoomOut","_fit"];this.navigationDOM.wrapper=document.createElement("div"),this.navigationDOM.wrapper.className="vis-navigation",this.canvas.frame.appendChild(this.navigationDOM.wrapper);for(var o=0;o<e.length;o++){this.navigationDOM[e[o]]=document.createElement("div"),this.navigationDOM[e[o]].className="vis-button vis-"+e[o],this.navigationDOM.wrapper.appendChild(this.navigationDOM[e[o]]);var n=new s(this.navigationDOM[e[o]]);"_fit"===i[o]?r.onTouch(n,this._fit.bind(this)):r.onTouch(n,this.bindToRedraw.bind(this,i[o])),this.navigationHammers.push(n)}var a=new s(this.canvas.frame);r.onRelease(a,function(){t._stopMovement()}),this.navigationHammers.push(a),this.iconsCreated=!0}},{key:"bindToRedraw",value:function(t){void 0===this.boundFunctions[t]&&(this.boundFunctions[t]=this[t].bind(this),this.body.emitter.on("initRedraw",this.boundFunctions[t]),this.body.emitter.emit("_startRendering"))}},{key:"unbindFromRedraw",value:function(t){void 0!==this.boundFunctions[t]&&(this.body.emitter.off("initRedraw",this.boundFunctions[t]),this.body.emitter.emit("_stopRendering"),delete this.boundFunctions[t])}},{key:"_fit",value:function(){(new Date).valueOf()-this.touchTime>700&&(this.body.emitter.emit("fit",{duration:700}),this.touchTime=(new Date).valueOf())}},{key:"_stopMovement",value:function(){for(var t in this.boundFunctions)this.boundFunctions.hasOwnProperty(t)&&(this.body.emitter.off("initRedraw",this.boundFunctions[t]),this.body.emitter.emit("_stopRendering"));this.boundFunctions={}}},{key:"_moveUp",value:function(){this.body.view.translation.y+=this.options.keyboard.speed.y}},{key:"_moveDown",value:function(){this.body.view.translation.y-=this.options.keyboard.speed.y}},{key:"_moveLeft",value:function(){this.body.view.translation.x+=this.options.keyboard.speed.x}},{key:"_moveRight",value:function(){this.body.view.translation.x-=this.options.keyboard.speed.x}},{key:"_zoomIn",value:function(){this.body.view.scale*=1+this.options.keyboard.speed.zoom,this.body.emitter.emit("zoom",{direction:"+",scale:this.body.view.scale})}},{key:"_zoomOut",value:function(){this.body.view.scale/=1+this.options.keyboard.speed.zoom,this.body.emitter.emit("zoom",{direction:"-",scale:this.body.view.scale})}},{key:"configureKeyboardBindings",value:function(){var t=this;void 0!==this.keycharm&&this.keycharm.destroy(),this.options.keyboard.enabled===!0&&(this.options.keyboard.bindToWindow===!0?this.keycharm=a({container:window,preventDefault:!0}):this.keycharm=a({container:this.canvas.frame,preventDefault:!0}),this.keycharm.reset(),this.activated===!0&&(this.keycharm.bind("up",function(){t.bindToRedraw("_moveUp")},"keydown"),this.keycharm.bind("down",function(){t.bindToRedraw("_moveDown")},"keydown"),this.keycharm.bind("left",function(){t.bindToRedraw("_moveLeft")},"keydown"),this.keycharm.bind("right",function(){t.bindToRedraw("_moveRight")},"keydown"),this.keycharm.bind("=",function(){t.bindToRedraw("_zoomIn")},"keydown"),this.keycharm.bind("num+",function(){t.bindToRedraw("_zoomIn")},"keydown"),this.keycharm.bind("num-",function(){t.bindToRedraw("_zoomOut")},"keydown"),this.keycharm.bind("-",function(){t.bindToRedraw("_zoomOut")},"keydown"),this.keycharm.bind("[",function(){t.bindToRedraw("_zoomOut")},"keydown"),this.keycharm.bind("]",function(){t.bindToRedraw("_zoomIn")},"keydown"),this.keycharm.bind("pageup",function(){t.bindToRedraw("_zoomIn")},"keydown"),this.keycharm.bind("pagedown",function(){t.bindToRedraw("_zoomOut")},"keydown"),this.keycharm.bind("up",function(){t.unbindFromRedraw("_moveUp")},"keyup"),this.keycharm.bind("down",function(){t.unbindFromRedraw("_moveDown")},"keyup"),this.keycharm.bind("left",function(){t.unbindFromRedraw("_moveLeft")},"keyup"),this.keycharm.bind("right",function(){t.unbindFromRedraw("_moveRight")},"keyup"),this.keycharm.bind("=",function(){t.unbindFromRedraw("_zoomIn")},"keyup"),this.keycharm.bind("num+",function(){t.unbindFromRedraw("_zoomIn")},"keyup"),this.keycharm.bind("num-",function(){t.unbindFromRedraw("_zoomOut")},"keyup"),this.keycharm.bind("-",function(){t.unbindFromRedraw("_zoomOut")},"keyup"),this.keycharm.bind("[",function(){t.unbindFromRedraw("_zoomOut")},"keyup"),this.keycharm.bind("]",function(){t.unbindFromRedraw("_zoomIn")},"keyup"),this.keycharm.bind("pageup",function(){t.unbindFromRedraw("_zoomIn")},"keyup"),this.keycharm.bind("pagedown",function(){t.unbindFromRedraw("_zoomOut")},"keyup")))}}]),t}();e["default"]=h},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e){i(this,t),this.container=e,this.x=0,this.y=0,this.padding=5,this.hidden=!1,this.frame=document.createElement("div"),this.frame.className="vis-network-tooltip",this.container.appendChild(this.frame)}return o(t,[{key:"setPosition",value:function(t,e){this.x=parseInt(t),this.y=parseInt(e)}},{key:"setText",value:function(t){t instanceof Element?(this.frame.innerHTML="",this.frame.appendChild(t)):this.frame.innerHTML=t}},{key:"show",value:function(t){if(void 0===t&&(t=!0),t===!0){var e=this.frame.clientHeight,i=this.frame.clientWidth,o=this.frame.parentNode.clientHeight,n=this.frame.parentNode.clientWidth,s=this.y-e;s+e+this.padding>o&&(s=o-e-this.padding),s<this.padding&&(s=this.padding);var r=this.x;r+i+this.padding>n&&(r=n-i-this.padding),r<this.padding&&(r=this.padding),this.frame.style.left=r+"px",this.frame.style.top=s+"px",this.frame.style.visibility="visible",this.hidden=!1}else this.hide()}},{key:"hide",value:function(){this.hidden=!0,this.frame.style.visibility="hidden"}}]),t}();e["default"]=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(65),a=o(r),h=i(85),d=o(h),l=i(1),c=function(){function t(e,i){var o=this;n(this,t),this.body=e,this.canvas=i,this.selectionObj={nodes:[],edges:[]},this.hoverObj={nodes:{},edges:{}},this.options={},this.defaultOptions={multiselect:!1,selectable:!0,selectConnectedEdges:!0,hoverConnectedEdges:!0},l.extend(this.options,this.defaultOptions),this.body.emitter.on("_dataChanged",function(){o.updateSelection()})}return s(t,[{key:"setOptions",value:function(t){if(void 0!==t){var e=["multiselect","hoverConnectedEdges","selectable","selectConnectedEdges"];l.selectiveDeepExtend(e,this.options,t)}}},{key:"selectOnPoint",value:function(t){var e=!1;if(this.options.selectable===!0){var i=this.getNodeAt(t)||this.getEdgeAt(t);this.unselectAll(),void 0!==i&&(e=this.selectObject(i)),this.body.emitter.emit("_requestRedraw")}return e}},{key:"selectAdditionalOnPoint",value:function(t){var e=!1;if(this.options.selectable===!0){var i=this.getNodeAt(t)||this.getEdgeAt(t);void 0!==i&&(e=!0,i.isSelected()===!0?this.deselectObject(i):this.selectObject(i),this.body.emitter.emit("_requestRedraw"))}return e}},{key:"_generateClickEvent",value:function(t,e,i,o){var n=arguments.length<=4||void 0===arguments[4]?!1:arguments[4],s=void 0;s=n===!0?{nodes:[],edges:[]}:this.getSelection(),s.pointer={DOM:{x:i.x,y:i.y},canvas:this.canvas.DOMtoCanvas(i)},s.event=e,void 0!==o&&(s.previousSelection=o),this.body.emitter.emit(t,s)}},{key:"selectObject",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?this.options.selectConnectedEdges:arguments[1];return void 0!==t?(t instanceof a["default"]&&e===!0&&this._selectConnectedEdges(t),t.select(),this._addToSelection(t),!0):!1}},{key:"deselectObject",value:function(t){t.isSelected()===!0&&(t.selected=!1,this._removeFromSelection(t))}},{key:"_getAllNodesOverlappingWith",value:function(t){for(var e=[],i=this.body.nodes,o=0;o<this.body.nodeIndices.length;o++){var n=this.body.nodeIndices[o];i[n].isOverlappingWith(t)&&e.push(n)}return e}},{key:"_pointerToPositionObject",value:function(t){var e=this.canvas.DOMtoCanvas(t);return{left:e.x-1,top:e.y+1,right:e.x+1,bottom:e.y-1}}},{key:"getNodeAt",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1],i=this._pointerToPositionObject(t),o=this._getAllNodesOverlappingWith(i);
-return o.length>0?e===!0?this.body.nodes[o[o.length-1]]:o[o.length-1]:void 0}},{key:"_getEdgesOverlappingWith",value:function(t,e){for(var i=this.body.edges,o=0;o<this.body.edgeIndices.length;o++){var n=this.body.edgeIndices[o];i[n].isOverlappingWith(t)&&e.push(n)}}},{key:"_getAllEdgesOverlappingWith",value:function(t){var e=[];return this._getEdgesOverlappingWith(t,e),e}},{key:"getEdgeAt",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1],i=this._pointerToPositionObject(t),o=this._getAllEdgesOverlappingWith(i);return o.length>0?e===!0?this.body.edges[o[o.length-1]]:o[o.length-1]:void 0}},{key:"_addToSelection",value:function(t){t instanceof a["default"]?this.selectionObj.nodes[t.id]=t:this.selectionObj.edges[t.id]=t}},{key:"_addToHover",value:function(t){t instanceof a["default"]?this.hoverObj.nodes[t.id]=t:this.hoverObj.edges[t.id]=t}},{key:"_removeFromSelection",value:function(t){t instanceof a["default"]?(delete this.selectionObj.nodes[t.id],this._unselectConnectedEdges(t)):delete this.selectionObj.edges[t.id]}},{key:"unselectAll",value:function(){for(var t in this.selectionObj.nodes)this.selectionObj.nodes.hasOwnProperty(t)&&this.selectionObj.nodes[t].unselect();for(var e in this.selectionObj.edges)this.selectionObj.edges.hasOwnProperty(e)&&this.selectionObj.edges[e].unselect();this.selectionObj={nodes:{},edges:{}}}},{key:"_getSelectedNodeCount",value:function(){var t=0;for(var e in this.selectionObj.nodes)this.selectionObj.nodes.hasOwnProperty(e)&&(t+=1);return t}},{key:"_getSelectedNode",value:function(){for(var t in this.selectionObj.nodes)if(this.selectionObj.nodes.hasOwnProperty(t))return this.selectionObj.nodes[t]}},{key:"_getSelectedEdge",value:function(){for(var t in this.selectionObj.edges)if(this.selectionObj.edges.hasOwnProperty(t))return this.selectionObj.edges[t]}},{key:"_getSelectedEdgeCount",value:function(){var t=0;for(var e in this.selectionObj.edges)this.selectionObj.edges.hasOwnProperty(e)&&(t+=1);return t}},{key:"_getSelectedObjectCount",value:function(){var t=0;for(var e in this.selectionObj.nodes)this.selectionObj.nodes.hasOwnProperty(e)&&(t+=1);for(var i in this.selectionObj.edges)this.selectionObj.edges.hasOwnProperty(i)&&(t+=1);return t}},{key:"_selectionIsEmpty",value:function(){for(var t in this.selectionObj.nodes)if(this.selectionObj.nodes.hasOwnProperty(t))return!1;for(var e in this.selectionObj.edges)if(this.selectionObj.edges.hasOwnProperty(e))return!1;return!0}},{key:"_clusterInSelection",value:function(){for(var t in this.selectionObj.nodes)if(this.selectionObj.nodes.hasOwnProperty(t)&&this.selectionObj.nodes[t].clusterSize>1)return!0;return!1}},{key:"_selectConnectedEdges",value:function(t){for(var e=0;e<t.edges.length;e++){var i=t.edges[e];i.select(),this._addToSelection(i)}}},{key:"_hoverConnectedEdges",value:function(t){for(var e=0;e<t.edges.length;e++){var i=t.edges[e];i.hover=!0,this._addToHover(i)}}},{key:"_unselectConnectedEdges",value:function(t){for(var e=0;e<t.edges.length;e++){var i=t.edges[e];i.unselect(),this._removeFromSelection(i)}}},{key:"blurObject",value:function(t){t.hover===!0&&(t.hover=!1,t instanceof a["default"]?this.body.emitter.emit("blurNode",{node:t.id}):this.body.emitter.emit("blurEdge",{edge:t.id}))}},{key:"hoverObject",value:function(t){var e=!1;for(var i in this.hoverObj.nodes)this.hoverObj.nodes.hasOwnProperty(i)&&(void 0===t||t instanceof a["default"]&&t.id!=i||t instanceof d["default"])&&(this.blurObject(this.hoverObj.nodes[i]),delete this.hoverObj.nodes[i],e=!0);for(var o in this.hoverObj.edges)this.hoverObj.edges.hasOwnProperty(o)&&(e===!0?(this.hoverObj.edges[o].hover=!1,delete this.hoverObj.edges[o]):void 0===t&&(this.blurObject(this.hoverObj.edges[o]),delete this.hoverObj.edges[o],e=!0));void 0!==t&&(t.hover===!1&&(t.hover=!0,this._addToHover(t),e=!0,t instanceof a["default"]?this.body.emitter.emit("hoverNode",{node:t.id}):this.body.emitter.emit("hoverEdge",{edge:t.id})),t instanceof a["default"]&&this.options.hoverConnectedEdges===!0&&this._hoverConnectedEdges(t)),e===!0&&this.body.emitter.emit("_requestRedraw")}},{key:"getSelection",value:function(){var t=this.getSelectedNodes(),e=this.getSelectedEdges();return{nodes:t,edges:e}}},{key:"getSelectedNodes",value:function(){var t=[];if(this.options.selectable===!0)for(var e in this.selectionObj.nodes)this.selectionObj.nodes.hasOwnProperty(e)&&t.push(this.selectionObj.nodes[e].id);return t}},{key:"getSelectedEdges",value:function(){var t=[];if(this.options.selectable===!0)for(var e in this.selectionObj.edges)this.selectionObj.edges.hasOwnProperty(e)&&t.push(this.selectionObj.edges[e].id);return t}},{key:"setSelection",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?{}:arguments[1],i=void 0,o=void 0;if(!t||!t.nodes&&!t.edges)throw"Selection must be an object with nodes and/or edges properties";if((e.unselectAll||void 0===e.unselectAll)&&this.unselectAll(),t.nodes)for(i=0;i<t.nodes.length;i++){o=t.nodes[i];var n=this.body.nodes[o];if(!n)throw new RangeError('Node with id "'+o+'" not found');this.selectObject(n,e.highlightEdges)}if(t.edges)for(i=0;i<t.edges.length;i++){o=t.edges[i];var s=this.body.edges[o];if(!s)throw new RangeError('Edge with id "'+o+'" not found');this.selectObject(s)}this.body.emitter.emit("_requestRedraw")}},{key:"selectNodes",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];if(!t||void 0===t.length)throw"Selection must be an array with ids";this.setSelection({nodes:t},{highlightEdges:e})}},{key:"selectEdges",value:function(t){if(!t||void 0===t.length)throw"Selection must be an array with ids";this.setSelection({edges:t})}},{key:"updateSelection",value:function(){for(var t in this.selectionObj.nodes)this.selectionObj.nodes.hasOwnProperty(t)&&(this.body.nodes.hasOwnProperty(t)||delete this.selectionObj.nodes[t]);for(var e in this.selectionObj.edges)this.selectionObj.edges.hasOwnProperty(e)&&(this.body.edges.hasOwnProperty(e)||delete this.selectionObj.edges[e])}}]),t}();e["default"]=c},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){var i=[],o=!0,n=!1,s=void 0;try{for(var r,a=t[Symbol.iterator]();!(o=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);o=!0);}catch(h){n=!0,s=h}finally{try{!o&&a["return"]&&a["return"]()}finally{if(n)throw s}}return i}return function(e,i){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return t(e,i);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),r="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(103),d=o(h),l=i(1),c=function(){function t(e){n(this,t),this.body=e,this.initialRandomSeed=Math.round(1e6*Math.random()),this.randomSeed=this.initialRandomSeed,this.setPhysics=!1,this.options={},this.optionsBackup={physics:{}},this.defaultOptions={randomSeed:void 0,improvedLayout:!0,hierarchical:{enabled:!1,levelSeparation:150,nodeSpacing:100,treeSpacing:200,blockShifting:!0,edgeMinimization:!0,parentCentralization:!0,direction:"UD",sortMethod:"hubsize"}},l.extend(this.options,this.defaultOptions),this.bindEventListeners()}return a(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("_dataChanged",function(){t.setupHierarchicalLayout()}),this.body.emitter.on("_dataLoaded",function(){t.layoutNetwork()}),this.body.emitter.on("_resetHierarchicalLayout",function(){t.setupHierarchicalLayout()})}},{key:"setOptions",value:function(t,e){if(void 0!==t){var i=this.options.hierarchical.enabled;if(l.selectiveDeepExtend(["randomSeed","improvedLayout"],this.options,t),l.mergeOptions(this.options,t,"hierarchical"),void 0!==t.randomSeed&&(this.initialRandomSeed=t.randomSeed),this.options.hierarchical.enabled===!0)return i===!0&&this.body.emitter.emit("refresh",!0),"RL"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?this.options.hierarchical.levelSeparation>0&&(this.options.hierarchical.levelSeparation*=-1):this.options.hierarchical.levelSeparation<0&&(this.options.hierarchical.levelSeparation*=-1),this.body.emitter.emit("_resetHierarchicalLayout"),this.adaptAllOptionsForHierarchicalLayout(e);if(i===!0)return this.body.emitter.emit("refresh"),l.deepExtend(e,this.optionsBackup)}return e}},{key:"adaptAllOptionsForHierarchicalLayout",value:function(t){if(this.options.hierarchical.enabled===!0){void 0===t.physics||t.physics===!0?(t.physics={enabled:void 0===this.optionsBackup.physics.enabled?!0:this.optionsBackup.physics.enabled,solver:"hierarchicalRepulsion"},this.optionsBackup.physics.enabled=void 0===this.optionsBackup.physics.enabled?!0:this.optionsBackup.physics.enabled,this.optionsBackup.physics.solver=this.optionsBackup.physics.solver||"barnesHut"):"object"===r(t.physics)?(this.optionsBackup.physics.enabled=void 0===t.physics.enabled?!0:t.physics.enabled,this.optionsBackup.physics.solver=t.physics.solver||"barnesHut",t.physics.solver="hierarchicalRepulsion"):t.physics!==!1&&(this.optionsBackup.physics.solver="barnesHut",t.physics={solver:"hierarchicalRepulsion"});var e="horizontal";"RL"!==this.options.hierarchical.direction&&"LR"!==this.options.hierarchical.direction||(e="vertical"),void 0===t.edges?(this.optionsBackup.edges={smooth:{enabled:!0,type:"dynamic"}},t.edges={smooth:!1}):void 0===t.edges.smooth?(this.optionsBackup.edges={smooth:{enabled:!0,type:"dynamic"}},t.edges.smooth=!1):"boolean"==typeof t.edges.smooth?(this.optionsBackup.edges={smooth:t.edges.smooth},t.edges.smooth={enabled:t.edges.smooth,type:e}):(void 0!==t.edges.smooth.type&&"dynamic"!==t.edges.smooth.type&&(e=t.edges.smooth.type),this.optionsBackup.edges={smooth:void 0===t.edges.smooth.enabled?!0:t.edges.smooth.enabled,type:void 0===t.edges.smooth.type?"dynamic":t.edges.smooth.type,roundness:void 0===t.edges.smooth.roundness?.5:t.edges.smooth.roundness,forceDirection:void 0===t.edges.smooth.forceDirection?!1:t.edges.smooth.forceDirection},t.edges.smooth={enabled:void 0===t.edges.smooth.enabled?!0:t.edges.smooth.enabled,type:e,roundness:void 0===t.edges.smooth.roundness?.5:t.edges.smooth.roundness,forceDirection:void 0===t.edges.smooth.forceDirection?!1:t.edges.smooth.forceDirection}),this.body.emitter.emit("_forceDisableDynamicCurves",e)}return t}},{key:"seededRandom",value:function(){var t=1e4*Math.sin(this.randomSeed++);return t-Math.floor(t)}},{key:"positionInitially",value:function(t){if(this.options.hierarchical.enabled!==!0){this.randomSeed=this.initialRandomSeed;for(var e=0;e<t.length;e++){var i=t[e],o=1*t.length+10,n=2*Math.PI*this.seededRandom();void 0===i.x&&(i.x=o*Math.cos(n)),void 0===i.y&&(i.y=o*Math.sin(n))}}}},{key:"layoutNetwork",value:function(){if(this.options.hierarchical.enabled!==!0&&this.options.improvedLayout===!0){for(var t=0,e=0;e<this.body.nodeIndices.length;e++){var i=this.body.nodes[this.body.nodeIndices[e]];i.predefinedPosition===!0&&(t+=1)}if(t<.5*this.body.nodeIndices.length){var o=10,n=0,s=100;if(this.body.nodeIndices.length>s){for(var r=this.body.nodeIndices.length;this.body.nodeIndices.length>s;){n+=1;var a=this.body.nodeIndices.length;n%3===0?this.body.modules.clustering.clusterBridges():this.body.modules.clustering.clusterOutliers();var h=this.body.nodeIndices.length;if(a==h&&n%3!==0||n>o)return this._declusterAll(),this.body.emitter.emit("_layoutFailed"),void console.info("This network could not be positioned by this version of the improved layout algorithm. Please disable improvedLayout for better performance.")}this.body.modules.kamadaKawai.setOptions({springLength:Math.max(150,2*r)})}this.body.modules.kamadaKawai.solve(this.body.nodeIndices,this.body.edgeIndices,!0),this._shiftToCenter();for(var d=70,l=0;l<this.body.nodeIndices.length;l++)this.body.nodes[this.body.nodeIndices[l]].x+=(.5-this.seededRandom())*d,this.body.nodes[this.body.nodeIndices[l]].y+=(.5-this.seededRandom())*d;this._declusterAll(),this.body.emitter.emit("_repositionBezierNodes")}}}},{key:"_shiftToCenter",value:function(){for(var t=d["default"].getRangeCore(this.body.nodes,this.body.nodeIndices),e=d["default"].findCenter(t),i=0;i<this.body.nodeIndices.length;i++)this.body.nodes[this.body.nodeIndices[i]].x-=e.x,this.body.nodes[this.body.nodeIndices[i]].y-=e.y}},{key:"_declusterAll",value:function(){for(var t=!0;t===!0;){t=!1;for(var e=0;e<this.body.nodeIndices.length;e++)this.body.nodes[this.body.nodeIndices[e]].isCluster===!0&&(t=!0,this.body.modules.clustering.openCluster(this.body.nodeIndices[e],{},!1));t===!0&&this.body.emitter.emit("_dataChanged")}}},{key:"getSeed",value:function(){return this.initialRandomSeed}},{key:"setupHierarchicalLayout",value:function(){if(this.options.hierarchical.enabled===!0&&this.body.nodeIndices.length>0){var t=void 0,e=void 0,i=!1,o=!0,n=!1;this.hierarchicalLevels={},this.lastNodeOnLevel={},this.hierarchicalChildrenReference={},this.hierarchicalParentReference={},this.hierarchicalTrees={},this.treeIndex=-1,this.distributionOrdering={},this.distributionIndex={},this.distributionOrderingPresence={};for(e in this.body.nodes)this.body.nodes.hasOwnProperty(e)&&(t=this.body.nodes[e],void 0===t.options.x&&void 0===t.options.y&&(o=!1),void 0!==t.options.level?(i=!0,this.hierarchicalLevels[e]=t.options.level):n=!0);if(n===!0&&i===!0)throw new Error("To use the hierarchical layout, nodes require either no predefined levels or levels have to be defined for all nodes.");n===!0&&("hubsize"===this.options.hierarchical.sortMethod?this._determineLevelsByHubsize():"directed"===this.options.hierarchical.sortMethod?this._determineLevelsDirected():"custom"===this.options.hierarchical.sortMethod&&this._determineLevelsCustomCallback());for(var s in this.body.nodes)this.body.nodes.hasOwnProperty(s)&&void 0===this.hierarchicalLevels[s]&&(this.hierarchicalLevels[s]=0);var r=this._getDistribution();this._generateMap(),this._placeNodesByHierarchy(r),this._condenseHierarchy(),this._shiftToCenter()}}},{key:"_condenseHierarchy",value:function(){var t=this,e=!1,i={},o=function(){for(var e=a(),i=0;i<e.length-1;i++){var o=e[i].max-e[i+1].min;n(i+1,o+t.options.hierarchical.treeSpacing)}},n=function(e,i){for(var o in t.hierarchicalTrees)if(t.hierarchicalTrees.hasOwnProperty(o)&&t.hierarchicalTrees[o]===e){var n=t.body.nodes[o],s=t._getPositionForHierarchy(n);t._setPositionForHierarchy(n,s+i,void 0,!0)}},r=function(e){var i=1e9,o=-1e9;for(var n in t.hierarchicalTrees)if(t.hierarchicalTrees.hasOwnProperty(n)&&t.hierarchicalTrees[n]===e){var s=t._getPositionForHierarchy(t.body.nodes[n]);i=Math.min(s,i),o=Math.max(s,o)}return{min:i,max:o}},a=function(){for(var e=[],i=0;i<=t.treeIndex;i++)e.push(r(i));return e},h=function w(e,i){if(i[e.id]=!0,t.hierarchicalChildrenReference[e.id]){var o=t.hierarchicalChildrenReference[e.id];if(o.length>0)for(var n=0;n<o.length;n++)w(t.body.nodes[o[n]],i)}},d=function(e){var i=arguments.length<=1||void 0===arguments[1]?1e9:arguments[1],o=1e9,n=1e9,r=1e9,a=-1e9;for(var h in e)if(e.hasOwnProperty(h)){var d=t.body.nodes[h],l=t.hierarchicalLevels[d.id],c=t._getPositionForHierarchy(d),u=t._getSpaceAroundNode(d,e),p=s(u,2),f=p[0],m=p[1];o=Math.min(f,o),n=Math.min(m,n),i>=l&&(r=Math.min(c,r),a=Math.max(c,a))}return[r,a,o,n]},l=function _(e){var i=t.hierarchicalLevels[e];if(t.hierarchicalChildrenReference[e]){var o=t.hierarchicalChildrenReference[e];if(o.length>0)for(var n=0;n<o.length;n++)i=Math.max(i,_(o[n]))}return i},c=function(t,e){var i=l(t.id),o=l(e.id);return Math.min(i,o)},u=function(e,i){var o=t.hierarchicalParentReference[e.id],n=t.hierarchicalParentReference[i.id];if(void 0===o||void 0===n)return!1;for(var s=0;s<o.length;s++)for(var r=0;r<n.length;r++)if(o[s]==n[r])return!0;return!1},p=function(e,i,o){for(var n=0;n<i.length;n++){var s=i[n],r=t.distributionOrdering[s];if(r.length>1)for(var a=0;a<r.length-1;a++)u(r[a],r[a+1])===!0&&t.hierarchicalTrees[r[a].id]===t.hierarchicalTrees[r[a+1].id]&&e(r[a],r[a+1],o)}},f=function(i,o){var n=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],r=t._getPositionForHierarchy(i),a=t._getPositionForHierarchy(o),l=Math.abs(a-r);if(l>t.options.hierarchical.nodeSpacing){var u={};u[i.id]=!0;var p={};p[o.id]=!0,h(i,u),h(o,p);var f=c(i,o),m=d(u,f),v=s(m,4),g=(v[0],v[1]),y=(v[2],v[3],d(p,f)),b=s(y,4),w=b[0],_=(b[1],b[2]),x=(b[3],Math.abs(g-w));if(x>t.options.hierarchical.nodeSpacing){var k=g-w+t.options.hierarchical.nodeSpacing;k<-_+t.options.hierarchical.nodeSpacing&&(k=-_+t.options.hierarchical.nodeSpacing),0>k&&(t._shiftBlock(o.id,k),e=!0,n===!0&&t._centerParent(o))}}},m=function(o,n){for(var r=n.id,a=n.edges,l=t.hierarchicalLevels[n.id],c=t.options.hierarchical.levelSeparation*t.options.hierarchical.levelSeparation,u={},p=[],f=0;f<a.length;f++){var m=a[f];if(m.toId!=m.fromId){var v=m.toId==r?m.from:m.to;u[a[f].id]=v,t.hierarchicalLevels[v.id]<l&&p.push(m)}}var g=function(e,i){for(var o=0,n=0;n<i.length;n++)if(void 0!==u[i[n].id]){var s=t._getPositionForHierarchy(u[i[n].id])-e;o+=s/Math.sqrt(s*s+c)}return o},y=function(e,i){for(var o=0,n=0;n<i.length;n++)if(void 0!==u[i[n].id]){var s=t._getPositionForHierarchy(u[i[n].id])-e;o-=c*Math.pow(s*s+c,-1.5)}return o},b=function(e,i){for(var o=t._getPositionForHierarchy(n),s={},r=0;e>r;r++){var a=g(o,i),h=y(o,i),d=40,l=Math.max(-d,Math.min(d,Math.round(a/h)));if(o-=l,void 0!==s[o])break;s[o]=r}return o},w=function(o){var r=t._getPositionForHierarchy(n);if(void 0===i[n.id]){var a={};a[n.id]=!0,h(n,a),i[n.id]=a}var l=d(i[n.id]),c=s(l,4),u=(c[0],c[1],c[2]),p=c[3],f=o-r,m=0;f>0?m=Math.min(f,p-t.options.hierarchical.nodeSpacing):0>f&&(m=-Math.min(-f,u-t.options.hierarchical.nodeSpacing)),0!=m&&(t._shiftBlock(n.id,m),e=!0)},_=function(i){var o=t._getPositionForHierarchy(n),r=t._getSpaceAroundNode(n),a=s(r,2),h=a[0],d=a[1],l=i-o,c=o;l>0?c=Math.min(o+(d-t.options.hierarchical.nodeSpacing),i):0>l&&(c=Math.max(o-(h-t.options.hierarchical.nodeSpacing),i)),c!==o&&(t._setPositionForHierarchy(n,c,void 0,!0),e=!0)},x=b(o,p);w(x),x=b(o,a),_(x)},v=function(i){var o=Object.keys(t.distributionOrdering);o=o.reverse();for(var n=0;i>n;n++){e=!1;for(var s=0;s<o.length;s++)for(var r=o[s],a=t.distributionOrdering[r],h=0;h<a.length;h++)m(1e3,a[h]);if(e!==!0)break}},g=function(i){var o=Object.keys(t.distributionOrdering);o=o.reverse();for(var n=0;i>n&&(e=!1,p(f,o,!0),e===!0);n++);},y=function(){for(var e in t.body.nodes)t.body.nodes.hasOwnProperty(e)&&t._centerParent(t.body.nodes[e])},b=function(){var e=Object.keys(t.distributionOrdering);e=e.reverse();for(var i=0;i<e.length;i++)for(var o=e[i],n=t.distributionOrdering[o],s=0;s<n.length;s++)t._centerParent(n[s])};this.options.hierarchical.blockShifting===!0&&(g(5),y()),this.options.hierarchical.edgeMinimization===!0&&v(20),this.options.hierarchical.parentCentralization===!0&&b(),o()}},{key:"_getSpaceAroundNode",value:function(t,e){var i=!0;void 0===e&&(i=!1);var o=this.hierarchicalLevels[t.id];if(void 0!==o){var n=this.distributionIndex[t.id],s=this._getPositionForHierarchy(t),r=1e9,a=1e9;if(0!==n){var h=this.distributionOrdering[o][n-1];if(i===!0&&void 0===e[h.id]||i===!1){var d=this._getPositionForHierarchy(h);r=s-d}}if(n!=this.distributionOrdering[o].length-1){var l=this.distributionOrdering[o][n+1];if(i===!0&&void 0===e[l.id]||i===!1){var c=this._getPositionForHierarchy(l);a=Math.min(a,c-s)}}return[r,a]}return[0,0]}},{key:"_centerParent",value:function(t){if(this.hierarchicalParentReference[t.id])for(var e=this.hierarchicalParentReference[t.id],i=0;i<e.length;i++){var o=e[i],n=this.body.nodes[o];if(this.hierarchicalChildrenReference[o]){var r=1e9,a=-1e9,h=this.hierarchicalChildrenReference[o];if(h.length>0)for(var d=0;d<h.length;d++){var l=this.body.nodes[h[d]];r=Math.min(r,this._getPositionForHierarchy(l)),a=Math.max(a,this._getPositionForHierarchy(l))}var c=this._getPositionForHierarchy(n),u=this._getSpaceAroundNode(n),p=s(u,2),f=p[0],m=p[1],v=.5*(r+a),g=c-v;(0>g&&Math.abs(g)<m-this.options.hierarchical.nodeSpacing||g>0&&Math.abs(g)<f-this.options.hierarchical.nodeSpacing)&&this._setPositionForHierarchy(n,v,void 0,!0)}}}},{key:"_placeNodesByHierarchy",value:function(t){this.positionedNodes={};for(var e in t)if(t.hasOwnProperty(e)){var i=Object.keys(t[e]);i=this._indexArrayToNodes(i),this._sortNodeArray(i);for(var o=0,n=0;n<i.length;n++){var s=i[n];if(void 0===this.positionedNodes[s.id]){var r=this.options.hierarchical.nodeSpacing*o;o>0&&(r=this._getPositionForHierarchy(i[n-1])+this.options.hierarchical.nodeSpacing),this._setPositionForHierarchy(s,r,e),this._validataPositionAndContinue(s,e,r),o++}}}}},{key:"_placeBranchNodes",value:function(t,e){if(void 0!==this.hierarchicalChildrenReference[t]){for(var i=[],o=0;o<this.hierarchicalChildrenReference[t].length;o++)i.push(this.body.nodes[this.hierarchicalChildrenReference[t][o]]);this._sortNodeArray(i);for(var n=0;n<i.length;n++){var s=i[n],r=this.hierarchicalLevels[s.id];if(!(r>e&&void 0===this.positionedNodes[s.id]))return;var a=void 0;a=0===n?this._getPositionForHierarchy(this.body.nodes[t]):this._getPositionForHierarchy(i[n-1])+this.options.hierarchical.nodeSpacing,this._setPositionForHierarchy(s,a,r),this._validataPositionAndContinue(s,r,a)}for(var h=1e9,d=-1e9,l=0;l<i.length;l++){var c=i[l].id;h=Math.min(h,this._getPositionForHierarchy(this.body.nodes[c])),d=Math.max(d,this._getPositionForHierarchy(this.body.nodes[c]))}this._setPositionForHierarchy(this.body.nodes[t],.5*(h+d),e)}}},{key:"_validataPositionAndContinue",value:function(t,e,i){if(void 0!==this.lastNodeOnLevel[e]){var o=this._getPositionForHierarchy(this.body.nodes[this.lastNodeOnLevel[e]]);if(i-o<this.options.hierarchical.nodeSpacing){var n=o+this.options.hierarchical.nodeSpacing-i,s=this._findCommonParent(this.lastNodeOnLevel[e],t.id);this._shiftBlock(s.withChild,n)}}this.lastNodeOnLevel[e]=t.id,this.positionedNodes[t.id]=!0,this._placeBranchNodes(t.id,e)}},{key:"_indexArrayToNodes",value:function(t){for(var e=[],i=0;i<t.length;i++)e.push(this.body.nodes[t[i]]);return e}},{key:"_getDistribution",value:function(){var t={},e=void 0,i=void 0;for(e in this.body.nodes)if(this.body.nodes.hasOwnProperty(e)){i=this.body.nodes[e];var o=void 0===this.hierarchicalLevels[e]?0:this.hierarchicalLevels[e];"UD"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?(i.y=this.options.hierarchical.levelSeparation*o,i.options.fixed.y=!0):(i.x=this.options.hierarchical.levelSeparation*o,i.options.fixed.x=!0),void 0===t[o]&&(t[o]={}),t[o][e]=i}return t}},{key:"_getHubSize",value:function(){var t=0;for(var e in this.body.nodes)if(this.body.nodes.hasOwnProperty(e)){var i=this.body.nodes[e];void 0===this.hierarchicalLevels[e]&&(t=i.edges.length<t?t:i.edges.length)}return t}},{key:"_determineLevelsByHubsize",value:function(){for(var t=this,e=1,i=function(e,i){void 0===t.hierarchicalLevels[i.id]&&(void 0===t.hierarchicalLevels[e.id]&&(t.hierarchicalLevels[e.id]=0),t.hierarchicalLevels[i.id]=t.hierarchicalLevels[e.id]+1)};e>0&&(e=this._getHubSize(),0!==e);)for(var o in this.body.nodes)if(this.body.nodes.hasOwnProperty(o)){var n=this.body.nodes[o];n.edges.length===e&&this._crawlNetwork(i,o)}}},{key:"_determineLevelsCustomCallback",value:function(){var t=this,e=1e5,i=function(t,e,i){},o=function(o,n,s){var r=t.hierarchicalLevels[o.id];void 0===r&&(t.hierarchicalLevels[o.id]=e);var a=i(d["default"].cloneOptions(o,"node"),d["default"].cloneOptions(n,"node"),d["default"].cloneOptions(s,"edge"));t.hierarchicalLevels[n.id]=t.hierarchicalLevels[o.id]+a};this._crawlNetwork(o),this._setMinLevelToZero()}},{key:"_determineLevelsDirected",value:function(){var t=this,e=1e4,i=function(i,o,n){var s=t.hierarchicalLevels[i.id];void 0===s&&(t.hierarchicalLevels[i.id]=e),n.toId==o.id?t.hierarchicalLevels[o.id]=t.hierarchicalLevels[i.id]+1:t.hierarchicalLevels[o.id]=t.hierarchicalLevels[i.id]-1};this._crawlNetwork(i),this._setMinLevelToZero()}},{key:"_setMinLevelToZero",value:function(){var t=1e9;for(var e in this.body.nodes)this.body.nodes.hasOwnProperty(e)&&void 0!==this.hierarchicalLevels[e]&&(t=Math.min(this.hierarchicalLevels[e],t));for(var i in this.body.nodes)this.body.nodes.hasOwnProperty(i)&&void 0!==this.hierarchicalLevels[i]&&(this.hierarchicalLevels[i]-=t)}},{key:"_generateMap",value:function(){var t=this,e=function(e,i){if(t.hierarchicalLevels[i.id]>t.hierarchicalLevels[e.id]){var o=e.id,n=i.id;void 0===t.hierarchicalChildrenReference[o]&&(t.hierarchicalChildrenReference[o]=[]),t.hierarchicalChildrenReference[o].push(n),void 0===t.hierarchicalParentReference[n]&&(t.hierarchicalParentReference[n]=[]),t.hierarchicalParentReference[n].push(o)}};this._crawlNetwork(e)}},{key:"_crawlNetwork",value:function(){var t=this,e=arguments.length<=0||void 0===arguments[0]?function(){}:arguments[0],i=arguments[1],o={},n=0,s=function d(i,n){if(void 0===o[i.id]){void 0===t.hierarchicalTrees[i.id]&&(t.hierarchicalTrees[i.id]=n,t.treeIndex=Math.max(n,t.treeIndex)),o[i.id]=!0;for(var s=void 0,r=0;r<i.edges.length;r++)i.edges[r].connected===!0&&(s=i.edges[r].toId===i.id?i.edges[r].from:i.edges[r].to,i.id!==s.id&&(e(i,s,i.edges[r]),d(s,n)))}};if(void 0===i)for(var r=0;r<this.body.nodeIndices.length;r++){var a=this.body.nodes[this.body.nodeIndices[r]];void 0===o[a.id]&&(s(a,n),n+=1)}else{var h=this.body.nodes[i];if(void 0===h)return void console.error("Node not found:",i);s(h)}}},{key:"_shiftBlock",value:function(t,e){if("UD"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?this.body.nodes[t].x+=e:this.body.nodes[t].y+=e,void 0!==this.hierarchicalChildrenReference[t])for(var i=0;i<this.hierarchicalChildrenReference[t].length;i++)this._shiftBlock(this.hierarchicalChildrenReference[t][i],e)}},{key:"_findCommonParent",value:function(t,e){var i=this,o={},n=function r(t,e){if(void 0!==i.hierarchicalParentReference[e])for(var o=0;o<i.hierarchicalParentReference[e].length;o++){var n=i.hierarchicalParentReference[e][o];t[n]=!0,r(t,n)}},s=function a(t,e){if(void 0!==i.hierarchicalParentReference[e])for(var o=0;o<i.hierarchicalParentReference[e].length;o++){var n=i.hierarchicalParentReference[e][o];if(void 0!==t[n])return{foundParent:n,withChild:e};var s=a(t,n);if(null!==s.foundParent)return s}return{foundParent:null,withChild:e}};return n(o,t),s(o,e)}},{key:"_setPositionForHierarchy",value:function(t,e,i){var o=arguments.length<=3||void 0===arguments[3]?!1:arguments[3];o!==!0&&(void 0===this.distributionOrdering[i]&&(this.distributionOrdering[i]=[],this.distributionOrderingPresence[i]={}),void 0===this.distributionOrderingPresence[i][t.id]&&(this.distributionOrdering[i].push(t),this.distributionIndex[t.id]=this.distributionOrdering[i].length-1),this.distributionOrderingPresence[i][t.id]=!0),"UD"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?t.x=e:t.y=e}},{key:"_getPositionForHierarchy",value:function(t){return"UD"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?t.x:t.y}},{key:"_sortNodeArray",value:function(t){t.length>1&&("UD"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?t.sort(function(t,e){return t.x-e.x}):t.sort(function(t,e){return t.y-e.y}))}}]),t}();e["default"]=c},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=i(1),r=i(20),a=i(28),h=function(){function t(e,i,n){var r=this;o(this,t),this.body=e,this.canvas=i,this.selectionHandler=n,this.editMode=!1,this.manipulationDiv=void 0,this.editModeDiv=void 0,this.closeDiv=void 0,this.manipulationHammers=[],this.temporaryUIFunctions={},this.temporaryEventFunctions=[],this.touchTime=0,this.temporaryIds={nodes:[],edges:[]},this.guiEnabled=!1,this.inMode=!1,this.selectedControlNode=void 0,this.options={},this.defaultOptions={enabled:!1,initiallyActive:!1,addNode:!0,addEdge:!0,editNode:void 0,editEdge:!0,deleteNode:!0,deleteEdge:!0,controlNodeStyle:{shape:"dot",size:6,color:{background:"#ff0000",border:"#3c3c3c",highlight:{background:"#07f968",border:"#3c3c3c"}},borderWidth:2,borderWidthSelected:2}},s.extend(this.options,this.defaultOptions),this.body.emitter.on("destroy",function(){r._clean()}),this.body.emitter.on("_dataChanged",this._restore.bind(this)),this.body.emitter.on("_resetData",this._restore.bind(this))}return n(t,[{key:"_restore",value:function(){this.inMode!==!1&&(this.options.initiallyActive===!0?this.enableEditMode():this.disableEditMode())}},{key:"setOptions",value:function(t,e,i){void 0!==e&&(void 0!==e.locale?this.options.locale=e.locale:this.options.locale=i.locale,void 0!==e.locales?this.options.locales=e.locales:this.options.locales=i.locales),void 0!==t&&("boolean"==typeof t?this.options.enabled=t:(this.options.enabled=!0,s.deepExtend(this.options,t)),this.options.initiallyActive===!0&&(this.editMode=!0),this._setup())}},{key:"toggleEditMode",value:function(){this.editMode===!0?this.disableEditMode():this.enableEditMode()}},{key:"enableEditMode",value:function(){this.editMode=!0,this._clean(),this.guiEnabled===!0&&(this.manipulationDiv.style.display="block",this.closeDiv.style.display="block",this.editModeDiv.style.display="none",this.showManipulatorToolbar())}},{key:"disableEditMode",value:function(){this.editMode=!1,this._clean(),this.guiEnabled===!0&&(this.manipulationDiv.style.display="none",this.closeDiv.style.display="none",this.editModeDiv.style.display="block",this._createEditButton())}},{key:"showManipulatorToolbar",value:function(){if(this._clean(),this.manipulationDOM={},this.guiEnabled===!0){this.editMode=!0,this.manipulationDiv.style.display="block",this.closeDiv.style.display="block";var t=this.selectionHandler._getSelectedNodeCount(),e=this.selectionHandler._getSelectedEdgeCount(),i=t+e,o=this.options.locales[this.options.locale],n=!1;this.options.addNode!==!1&&(this._createAddNodeButton(o),n=!0),this.options.addEdge!==!1&&(n===!0?this._createSeperator(1):n=!0,this._createAddEdgeButton(o)),1===t&&"function"==typeof this.options.editNode?(n===!0?this._createSeperator(2):n=!0,this._createEditNodeButton(o)):1===e&&0===t&&this.options.editEdge!==!1&&(n===!0?this._createSeperator(3):n=!0,this._createEditEdgeButton(o)),0!==i&&(t>0&&this.options.deleteNode!==!1?(n===!0&&this._createSeperator(4),this._createDeleteButton(o)):0===t&&this.options.deleteEdge!==!1&&(n===!0&&this._createSeperator(4),this._createDeleteButton(o))),this._bindHammerToDiv(this.closeDiv,this.toggleEditMode.bind(this)),this._temporaryBindEvent("select",this.showManipulatorToolbar.bind(this))}this.body.emitter.emit("_redraw")}},{key:"addNodeMode",value:function(){if(this.editMode!==!0&&this.enableEditMode(),this._clean(),this.inMode="addNode",this.guiEnabled===!0){var t=this.options.locales[this.options.locale];this.manipulationDOM={},this._createBackButton(t),this._createSeperator(),this._createDescription(t.addDescription||this.options.locales.en.addDescription),this._bindHammerToDiv(this.closeDiv,this.toggleEditMode.bind(this))}this._temporaryBindEvent("click",this._performAddNode.bind(this))}},{key:"editNode",value:function(){var t=this;this.editMode!==!0&&this.enableEditMode(),this._clean();var e=this.selectionHandler._getSelectedNode();if(void 0!==e){if(this.inMode="editNode","function"!=typeof this.options.editNode)throw new Error("No function has been configured to handle the editing of nodes.");if(e.isCluster!==!0){var i=s.deepExtend({},e.options,!1);
-if(i.x=e.x,i.y=e.y,2!==this.options.editNode.length)throw new Error("The function for edit does not support two arguments (data, callback)");this.options.editNode(i,function(e){null!==e&&void 0!==e&&"editNode"===t.inMode&&t.body.data.nodes.getDataSet().update(e),t.showManipulatorToolbar()})}else alert(this.options.locales[this.options.locale].editClusterError||this.options.locales.en.editClusterError)}else this.showManipulatorToolbar()}},{key:"addEdgeMode",value:function(){if(this.editMode!==!0&&this.enableEditMode(),this._clean(),this.inMode="addEdge",this.guiEnabled===!0){var t=this.options.locales[this.options.locale];this.manipulationDOM={},this._createBackButton(t),this._createSeperator(),this._createDescription(t.edgeDescription||this.options.locales.en.edgeDescription),this._bindHammerToDiv(this.closeDiv,this.toggleEditMode.bind(this))}this._temporaryBindUI("onTouch",this._handleConnect.bind(this)),this._temporaryBindUI("onDragEnd",this._finishConnect.bind(this)),this._temporaryBindUI("onDrag",this._dragControlNode.bind(this)),this._temporaryBindUI("onRelease",this._finishConnect.bind(this)),this._temporaryBindUI("onDragStart",function(){}),this._temporaryBindUI("onHold",function(){})}},{key:"editEdgeMode",value:function(){var t=this;if(this.editMode!==!0&&this.enableEditMode(),this._clean(),this.inMode="editEdge",this.guiEnabled===!0){var e=this.options.locales[this.options.locale];this.manipulationDOM={},this._createBackButton(e),this._createSeperator(),this._createDescription(e.editEdgeDescription||this.options.locales.en.editEdgeDescription),this._bindHammerToDiv(this.closeDiv,this.toggleEditMode.bind(this))}this.edgeBeingEditedId=this.selectionHandler.getSelectedEdges()[0],void 0!==this.edgeBeingEditedId?!function(){var e=t.body.edges[t.edgeBeingEditedId],i=t._getNewTargetNode(e.from.x,e.from.y),o=t._getNewTargetNode(e.to.x,e.to.y);t.temporaryIds.nodes.push(i.id),t.temporaryIds.nodes.push(o.id),t.body.nodes[i.id]=i,t.body.nodeIndices.push(i.id),t.body.nodes[o.id]=o,t.body.nodeIndices.push(o.id),t._temporaryBindUI("onTouch",t._controlNodeTouch.bind(t)),t._temporaryBindUI("onTap",function(){}),t._temporaryBindUI("onHold",function(){}),t._temporaryBindUI("onDragStart",t._controlNodeDragStart.bind(t)),t._temporaryBindUI("onDrag",t._controlNodeDrag.bind(t)),t._temporaryBindUI("onDragEnd",t._controlNodeDragEnd.bind(t)),t._temporaryBindUI("onMouseMove",function(){}),t._temporaryBindEvent("beforeDrawing",function(t){var n=e.edgeType.findBorderPositions(t);i.selected===!1&&(i.x=n.from.x,i.y=n.from.y),o.selected===!1&&(o.x=n.to.x,o.y=n.to.y)}),t.body.emitter.emit("_redraw")}():this.showManipulatorToolbar()}},{key:"deleteSelected",value:function(){var t=this;this.editMode!==!0&&this.enableEditMode(),this._clean(),this.inMode="delete";var e=this.selectionHandler.getSelectedNodes(),i=this.selectionHandler.getSelectedEdges(),o=void 0;if(e.length>0){for(var n=0;n<e.length;n++)if(this.body.nodes[e[n]].isCluster===!0)return void alert(this.options.locales[this.options.locale].deleteClusterError||this.options.locales.en.deleteClusterError);"function"==typeof this.options.deleteNode&&(o=this.options.deleteNode)}else i.length>0&&"function"==typeof this.options.deleteEdge&&(o=this.options.deleteEdge);if("function"==typeof o){var s={nodes:e,edges:i};if(2!==o.length)throw new Error("The function for delete does not support two arguments (data, callback)");o(s,function(e){null!==e&&void 0!==e&&"delete"===t.inMode?(t.body.data.edges.getDataSet().remove(e.edges),t.body.data.nodes.getDataSet().remove(e.nodes),t.body.emitter.emit("startSimulation"),t.showManipulatorToolbar()):(t.body.emitter.emit("startSimulation"),t.showManipulatorToolbar())})}else this.body.data.edges.getDataSet().remove(i),this.body.data.nodes.getDataSet().remove(e),this.body.emitter.emit("startSimulation"),this.showManipulatorToolbar()}},{key:"_setup",value:function(){this.options.enabled===!0?(this.guiEnabled=!0,this._createWrappers(),this.editMode===!1?this._createEditButton():this.showManipulatorToolbar()):(this._removeManipulationDOM(),this.guiEnabled=!1)}},{key:"_createWrappers",value:function(){void 0===this.manipulationDiv&&(this.manipulationDiv=document.createElement("div"),this.manipulationDiv.className="vis-manipulation",this.editMode===!0?this.manipulationDiv.style.display="block":this.manipulationDiv.style.display="none",this.canvas.frame.appendChild(this.manipulationDiv)),void 0===this.editModeDiv&&(this.editModeDiv=document.createElement("div"),this.editModeDiv.className="vis-edit-mode",this.editMode===!0?this.editModeDiv.style.display="none":this.editModeDiv.style.display="block",this.canvas.frame.appendChild(this.editModeDiv)),void 0===this.closeDiv&&(this.closeDiv=document.createElement("div"),this.closeDiv.className="vis-close",this.closeDiv.style.display=this.manipulationDiv.style.display,this.canvas.frame.appendChild(this.closeDiv))}},{key:"_getNewTargetNode",value:function(t,e){var i=s.deepExtend({},this.options.controlNodeStyle);i.id="targetNode"+s.randomUUID(),i.hidden=!1,i.physics=!1,i.x=t,i.y=e;var o=this.body.functions.createNode(i);return o.shape.boundingBox={left:t,right:t,top:e,bottom:e},o}},{key:"_createEditButton",value:function(){this._clean(),this.manipulationDOM={},s.recursiveDOMDelete(this.editModeDiv);var t=this.options.locales[this.options.locale],e=this._createButton("editMode","vis-button vis-edit vis-edit-mode",t.edit||this.options.locales.en.edit);this.editModeDiv.appendChild(e),this._bindHammerToDiv(e,this.toggleEditMode.bind(this))}},{key:"_clean",value:function(){this.inMode=!1,this.guiEnabled===!0&&(s.recursiveDOMDelete(this.editModeDiv),s.recursiveDOMDelete(this.manipulationDiv),this._cleanManipulatorHammers()),this._cleanupTemporaryNodesAndEdges(),this._unbindTemporaryUIs(),this._unbindTemporaryEvents(),this.body.emitter.emit("restorePhysics")}},{key:"_cleanManipulatorHammers",value:function(){if(0!=this.manipulationHammers.length){for(var t=0;t<this.manipulationHammers.length;t++)this.manipulationHammers[t].destroy();this.manipulationHammers=[]}}},{key:"_removeManipulationDOM",value:function(){this._clean(),s.recursiveDOMDelete(this.manipulationDiv),s.recursiveDOMDelete(this.editModeDiv),s.recursiveDOMDelete(this.closeDiv),this.manipulationDiv&&this.canvas.frame.removeChild(this.manipulationDiv),this.editModeDiv&&this.canvas.frame.removeChild(this.editModeDiv),this.closeDiv&&this.canvas.frame.removeChild(this.closeDiv),this.manipulationDiv=void 0,this.editModeDiv=void 0,this.closeDiv=void 0}},{key:"_createSeperator",value:function(){var t=arguments.length<=0||void 0===arguments[0]?1:arguments[0];this.manipulationDOM["seperatorLineDiv"+t]=document.createElement("div"),this.manipulationDOM["seperatorLineDiv"+t].className="vis-separator-line",this.manipulationDiv.appendChild(this.manipulationDOM["seperatorLineDiv"+t])}},{key:"_createAddNodeButton",value:function(t){var e=this._createButton("addNode","vis-button vis-add",t.addNode||this.options.locales.en.addNode);this.manipulationDiv.appendChild(e),this._bindHammerToDiv(e,this.addNodeMode.bind(this))}},{key:"_createAddEdgeButton",value:function(t){var e=this._createButton("addEdge","vis-button vis-connect",t.addEdge||this.options.locales.en.addEdge);this.manipulationDiv.appendChild(e),this._bindHammerToDiv(e,this.addEdgeMode.bind(this))}},{key:"_createEditNodeButton",value:function(t){var e=this._createButton("editNode","vis-button vis-edit",t.editNode||this.options.locales.en.editNode);this.manipulationDiv.appendChild(e),this._bindHammerToDiv(e,this.editNode.bind(this))}},{key:"_createEditEdgeButton",value:function(t){var e=this._createButton("editEdge","vis-button vis-edit",t.editEdge||this.options.locales.en.editEdge);this.manipulationDiv.appendChild(e),this._bindHammerToDiv(e,this.editEdgeMode.bind(this))}},{key:"_createDeleteButton",value:function(t){if(this.options.rtl)var e="vis-button vis-delete-rtl";else var e="vis-button vis-delete";var i=this._createButton("delete",e,t.del||this.options.locales.en.del);this.manipulationDiv.appendChild(i),this._bindHammerToDiv(i,this.deleteSelected.bind(this))}},{key:"_createBackButton",value:function(t){var e=this._createButton("back","vis-button vis-back",t.back||this.options.locales.en.back);this.manipulationDiv.appendChild(e),this._bindHammerToDiv(e,this.showManipulatorToolbar.bind(this))}},{key:"_createButton",value:function(t,e,i){var o=arguments.length<=3||void 0===arguments[3]?"vis-label":arguments[3];return this.manipulationDOM[t+"Div"]=document.createElement("div"),this.manipulationDOM[t+"Div"].className=e,this.manipulationDOM[t+"Label"]=document.createElement("div"),this.manipulationDOM[t+"Label"].className=o,this.manipulationDOM[t+"Label"].innerHTML=i,this.manipulationDOM[t+"Div"].appendChild(this.manipulationDOM[t+"Label"]),this.manipulationDOM[t+"Div"]}},{key:"_createDescription",value:function(t){this.manipulationDiv.appendChild(this._createButton("description","vis-button vis-none",t))}},{key:"_temporaryBindEvent",value:function(t,e){this.temporaryEventFunctions.push({event:t,boundFunction:e}),this.body.emitter.on(t,e)}},{key:"_temporaryBindUI",value:function(t,e){if(void 0===this.body.eventListeners[t])throw new Error("This UI function does not exist. Typo? You tried: "+t+" possible are: "+JSON.stringify(Object.keys(this.body.eventListeners)));this.temporaryUIFunctions[t]=this.body.eventListeners[t],this.body.eventListeners[t]=e}},{key:"_unbindTemporaryUIs",value:function(){for(var t in this.temporaryUIFunctions)this.temporaryUIFunctions.hasOwnProperty(t)&&(this.body.eventListeners[t]=this.temporaryUIFunctions[t],delete this.temporaryUIFunctions[t]);this.temporaryUIFunctions={}}},{key:"_unbindTemporaryEvents",value:function(){for(var t=0;t<this.temporaryEventFunctions.length;t++){var e=this.temporaryEventFunctions[t].event,i=this.temporaryEventFunctions[t].boundFunction;this.body.emitter.off(e,i)}this.temporaryEventFunctions=[]}},{key:"_bindHammerToDiv",value:function(t,e){var i=new r(t,{});a.onTouch(i,e),this.manipulationHammers.push(i)}},{key:"_cleanupTemporaryNodesAndEdges",value:function(){for(var t=0;t<this.temporaryIds.edges.length;t++){this.body.edges[this.temporaryIds.edges[t]].disconnect(),delete this.body.edges[this.temporaryIds.edges[t]];var e=this.body.edgeIndices.indexOf(this.temporaryIds.edges[t]);-1!==e&&this.body.edgeIndices.splice(e,1)}for(var i=0;i<this.temporaryIds.nodes.length;i++){delete this.body.nodes[this.temporaryIds.nodes[i]];var o=this.body.nodeIndices.indexOf(this.temporaryIds.nodes[i]);-1!==o&&this.body.nodeIndices.splice(o,1)}this.temporaryIds={nodes:[],edges:[]}}},{key:"_controlNodeTouch",value:function(t){this.selectionHandler.unselectAll(),this.lastTouch=this.body.functions.getPointer(t.center),this.lastTouch.translation=s.extend({},this.body.view.translation)}},{key:"_controlNodeDragStart",value:function(t){var e=this.lastTouch,i=this.selectionHandler._pointerToPositionObject(e),o=this.body.nodes[this.temporaryIds.nodes[0]],n=this.body.nodes[this.temporaryIds.nodes[1]],s=this.body.edges[this.edgeBeingEditedId];this.selectedControlNode=void 0;var r=o.isOverlappingWith(i),a=n.isOverlappingWith(i);r===!0?(this.selectedControlNode=o,s.edgeType.from=o):a===!0&&(this.selectedControlNode=n,s.edgeType.to=n),void 0!==this.selectedControlNode&&this.selectionHandler.selectObject(this.selectedControlNode),this.body.emitter.emit("_redraw")}},{key:"_controlNodeDrag",value:function(t){this.body.emitter.emit("disablePhysics");var e=this.body.functions.getPointer(t.center),i=this.canvas.DOMtoCanvas(e);if(void 0!==this.selectedControlNode)this.selectedControlNode.x=i.x,this.selectedControlNode.y=i.y;else{var o=e.x-this.lastTouch.x,n=e.y-this.lastTouch.y;this.body.view.translation={x:this.lastTouch.translation.x+o,y:this.lastTouch.translation.y+n}}this.body.emitter.emit("_redraw")}},{key:"_controlNodeDragEnd",value:function(t){var e=this.body.functions.getPointer(t.center),i=this.selectionHandler._pointerToPositionObject(e),o=this.body.edges[this.edgeBeingEditedId];if(void 0!==this.selectedControlNode){this.selectionHandler.unselectAll();for(var n=this.selectionHandler._getAllNodesOverlappingWith(i),s=void 0,r=n.length-1;r>=0;r--)if(n[r]!==this.selectedControlNode.id){s=this.body.nodes[n[r]];break}if(void 0!==s&&void 0!==this.selectedControlNode)if(s.isCluster===!0)alert(this.options.locales[this.options.locale].createEdgeError||this.options.locales.en.createEdgeError);else{var a=this.body.nodes[this.temporaryIds.nodes[0]];this.selectedControlNode.id===a.id?this._performEditEdge(s.id,o.to.id):this._performEditEdge(o.from.id,s.id)}else o.updateEdgeType(),this.body.emitter.emit("restorePhysics");this.body.emitter.emit("_redraw")}}},{key:"_handleConnect",value:function(t){if((new Date).valueOf()-this.touchTime>100){this.lastTouch=this.body.functions.getPointer(t.center),this.lastTouch.translation=s.extend({},this.body.view.translation);var e=this.lastTouch,i=this.selectionHandler.getNodeAt(e);if(void 0!==i)if(i.isCluster===!0)alert(this.options.locales[this.options.locale].createEdgeError||this.options.locales.en.createEdgeError);else{var o=this._getNewTargetNode(i.x,i.y);this.body.nodes[o.id]=o,this.body.nodeIndices.push(o.id);var n=this.body.functions.createEdge({id:"connectionEdge"+s.randomUUID(),from:i.id,to:o.id,physics:!1,smooth:{enabled:!0,type:"continuous",roundness:.5}});this.body.edges[n.id]=n,this.body.edgeIndices.push(n.id),this.temporaryIds.nodes.push(o.id),this.temporaryIds.edges.push(n.id)}this.touchTime=(new Date).valueOf()}}},{key:"_dragControlNode",value:function(t){var e=this.body.functions.getPointer(t.center);if(void 0!==this.temporaryIds.nodes[0]){var i=this.body.nodes[this.temporaryIds.nodes[0]];i.x=this.canvas._XconvertDOMtoCanvas(e.x),i.y=this.canvas._YconvertDOMtoCanvas(e.y),this.body.emitter.emit("_redraw")}else{var o=e.x-this.lastTouch.x,n=e.y-this.lastTouch.y;this.body.view.translation={x:this.lastTouch.translation.x+o,y:this.lastTouch.translation.y+n}}}},{key:"_finishConnect",value:function(t){var e=this.body.functions.getPointer(t.center),i=this.selectionHandler._pointerToPositionObject(e),o=void 0;void 0!==this.temporaryIds.edges[0]&&(o=this.body.edges[this.temporaryIds.edges[0]].fromId);for(var n=this.selectionHandler._getAllNodesOverlappingWith(i),s=void 0,r=n.length-1;r>=0;r--)if(-1===this.temporaryIds.nodes.indexOf(n[r])){s=this.body.nodes[n[r]];break}this._cleanupTemporaryNodesAndEdges(),void 0!==s&&(s.isCluster===!0?alert(this.options.locales[this.options.locale].createEdgeError||this.options.locales.en.createEdgeError):void 0!==this.body.nodes[o]&&void 0!==this.body.nodes[s.id]&&this._performAddEdge(o,s.id)),this.body.emitter.emit("_redraw")}},{key:"_performAddNode",value:function(t){var e=this,i={id:s.randomUUID(),x:t.pointer.canvas.x,y:t.pointer.canvas.y,label:"new"};if("function"==typeof this.options.addNode){if(2!==this.options.addNode.length)throw new Error("The function for add does not support two arguments (data,callback)");this.options.addNode(i,function(t){null!==t&&void 0!==t&&"addNode"===e.inMode&&(e.body.data.nodes.getDataSet().add(t),e.showManipulatorToolbar())})}else this.body.data.nodes.getDataSet().add(i),this.showManipulatorToolbar()}},{key:"_performAddEdge",value:function(t,e){var i=this,o={from:t,to:e};if("function"==typeof this.options.addEdge){if(2!==this.options.addEdge.length)throw new Error("The function for connect does not support two arguments (data,callback)");this.options.addEdge(o,function(t){null!==t&&void 0!==t&&"addEdge"===i.inMode&&(i.body.data.edges.getDataSet().add(t),i.selectionHandler.unselectAll(),i.showManipulatorToolbar())})}else this.body.data.edges.getDataSet().add(o),this.selectionHandler.unselectAll(),this.showManipulatorToolbar()}},{key:"_performEditEdge",value:function(t,e){var i=this,o={id:this.edgeBeingEditedId,from:t,to:e};if("function"==typeof this.options.editEdge){if(2!==this.options.editEdge.length)throw new Error("The function for edit does not support two arguments (data, callback)");this.options.editEdge(o,function(t){null===t||void 0===t||"editEdge"!==i.inMode?(i.body.edges[o.id].updateEdgeType(),i.body.emitter.emit("_redraw")):(i.body.data.edges.getDataSet().update(t),i.selectionHandler.unselectAll(),i.showManipulatorToolbar())})}else this.body.data.edges.getDataSet().update(o),this.selectionHandler.unselectAll(),this.showManipulatorToolbar()}}]),t}();e["default"]=h},function(t,e){Object.defineProperty(e,"__esModule",{value:!0});var i="string",o="boolean",n="number",s="array",r="object",a="dom",h="any",d={configure:{enabled:{"boolean":o},filter:{"boolean":o,string:i,array:s,"function":"function"},container:{dom:a},showButton:{"boolean":o},__type__:{object:r,"boolean":o,string:i,array:s,"function":"function"}},edges:{arrows:{to:{enabled:{"boolean":o},scaleFactor:{number:n},__type__:{object:r,"boolean":o}},middle:{enabled:{"boolean":o},scaleFactor:{number:n},__type__:{object:r,"boolean":o}},from:{enabled:{"boolean":o},scaleFactor:{number:n},__type__:{object:r,"boolean":o}},__type__:{string:["from","to","middle"],object:r}},arrowStrikethrough:{"boolean":o},color:{color:{string:i},highlight:{string:i},hover:{string:i},inherit:{string:["from","to","both"],"boolean":o},opacity:{number:n},__type__:{object:r,string:i}},dashes:{"boolean":o,array:s},font:{color:{string:i},size:{number:n},face:{string:i},background:{string:i},strokeWidth:{number:n},strokeColor:{string:i},align:{string:["horizontal","top","middle","bottom"]},__type__:{object:r,string:i}},hidden:{"boolean":o},hoverWidth:{"function":"function",number:n},label:{string:i,undefined:"undefined"},labelHighlightBold:{"boolean":o},length:{number:n,undefined:"undefined"},physics:{"boolean":o},scaling:{min:{number:n},max:{number:n},label:{enabled:{"boolean":o},min:{number:n},max:{number:n},maxVisible:{number:n},drawThreshold:{number:n},__type__:{object:r,"boolean":o}},customScalingFunction:{"function":"function"},__type__:{object:r}},selectionWidth:{"function":"function",number:n},selfReferenceSize:{number:n},shadow:{enabled:{"boolean":o},color:{string:i},size:{number:n},x:{number:n},y:{number:n},__type__:{object:r,"boolean":o}},smooth:{enabled:{"boolean":o},type:{string:["dynamic","continuous","discrete","diagonalCross","straightCross","horizontal","vertical","curvedCW","curvedCCW","cubicBezier"]},roundness:{number:n},forceDirection:{string:["horizontal","vertical","none"],"boolean":o},__type__:{object:r,"boolean":o}},title:{string:i,undefined:"undefined"},width:{number:n},value:{number:n,undefined:"undefined"},__type__:{object:r}},groups:{useDefaultGroups:{"boolean":o},__any__:"get from nodes, will be overwritten below",__type__:{object:r}},interaction:{dragNodes:{"boolean":o},dragView:{"boolean":o},hideEdgesOnDrag:{"boolean":o},hideNodesOnDrag:{"boolean":o},hover:{"boolean":o},keyboard:{enabled:{"boolean":o},speed:{x:{number:n},y:{number:n},zoom:{number:n},__type__:{object:r}},bindToWindow:{"boolean":o},__type__:{object:r,"boolean":o}},multiselect:{"boolean":o},navigationButtons:{"boolean":o},selectable:{"boolean":o},selectConnectedEdges:{"boolean":o},hoverConnectedEdges:{"boolean":o},tooltipDelay:{number:n},zoomView:{"boolean":o},__type__:{object:r}},layout:{randomSeed:{undefined:"undefined",number:n},improvedLayout:{"boolean":o},hierarchical:{enabled:{"boolean":o},levelSeparation:{number:n},nodeSpacing:{number:n},treeSpacing:{number:n},blockShifting:{"boolean":o},edgeMinimization:{"boolean":o},parentCentralization:{"boolean":o},direction:{string:["UD","DU","LR","RL"]},sortMethod:{string:["hubsize","directed"]},__type__:{object:r,"boolean":o}},__type__:{object:r}},manipulation:{enabled:{"boolean":o},initiallyActive:{"boolean":o},addNode:{"boolean":o,"function":"function"},addEdge:{"boolean":o,"function":"function"},editNode:{"function":"function"},editEdge:{"boolean":o,"function":"function"},deleteNode:{"boolean":o,"function":"function"},deleteEdge:{"boolean":o,"function":"function"},controlNodeStyle:"get from nodes, will be overwritten below",__type__:{object:r,"boolean":o}},nodes:{borderWidth:{number:n},borderWidthSelected:{number:n,undefined:"undefined"},brokenImage:{string:i,undefined:"undefined"},color:{border:{string:i},background:{string:i},highlight:{border:{string:i},background:{string:i},__type__:{object:r,string:i}},hover:{border:{string:i},background:{string:i},__type__:{object:r,string:i}},__type__:{object:r,string:i}},fixed:{x:{"boolean":o},y:{"boolean":o},__type__:{object:r,"boolean":o}},font:{align:{string:i},color:{string:i},size:{number:n},face:{string:i},background:{string:i},strokeWidth:{number:n},strokeColor:{string:i},__type__:{object:r,string:i}},group:{string:i,number:n,undefined:"undefined"},hidden:{"boolean":o},icon:{face:{string:i},code:{string:i},size:{number:n},color:{string:i},__type__:{object:r}},id:{string:i,number:n},image:{string:i,undefined:"undefined"},label:{string:i,undefined:"undefined"},labelHighlightBold:{"boolean":o},level:{number:n,undefined:"undefined"},mass:{number:n},physics:{"boolean":o},scaling:{min:{number:n},max:{number:n},label:{enabled:{"boolean":o},min:{number:n},max:{number:n},maxVisible:{number:n},drawThreshold:{number:n},__type__:{object:r,"boolean":o}},customScalingFunction:{"function":"function"},__type__:{object:r}},shadow:{enabled:{"boolean":o},color:{string:i},size:{number:n},x:{number:n},y:{number:n},__type__:{object:r,"boolean":o}},shape:{string:["ellipse","circle","database","box","text","image","circularImage","diamond","dot","star","triangle","triangleDown","square","icon"]},shapeProperties:{borderDashes:{"boolean":o,array:s},borderRadius:{number:n},interpolation:{"boolean":o},useImageSize:{"boolean":o},useBorderWithImage:{"boolean":o},__type__:{object:r}},size:{number:n},title:{string:i,undefined:"undefined"},value:{number:n,undefined:"undefined"},x:{number:n},y:{number:n},__type__:{object:r}},physics:{enabled:{"boolean":o},barnesHut:{gravitationalConstant:{number:n},centralGravity:{number:n},springLength:{number:n},springConstant:{number:n},damping:{number:n},avoidOverlap:{number:n},__type__:{object:r}},forceAtlas2Based:{gravitationalConstant:{number:n},centralGravity:{number:n},springLength:{number:n},springConstant:{number:n},damping:{number:n},avoidOverlap:{number:n},__type__:{object:r}},repulsion:{centralGravity:{number:n},springLength:{number:n},springConstant:{number:n},nodeDistance:{number:n},damping:{number:n},__type__:{object:r}},hierarchicalRepulsion:{centralGravity:{number:n},springLength:{number:n},springConstant:{number:n},nodeDistance:{number:n},damping:{number:n},__type__:{object:r}},maxVelocity:{number:n},minVelocity:{number:n},solver:{string:["barnesHut","repulsion","hierarchicalRepulsion","forceAtlas2Based"]},stabilization:{enabled:{"boolean":o},iterations:{number:n},updateInterval:{number:n},onlyDynamicEdges:{"boolean":o},fit:{"boolean":o},__type__:{object:r,"boolean":o}},timestep:{number:n},adaptiveTimestep:{"boolean":o},__type__:{object:r,"boolean":o}},autoResize:{"boolean":o},clickToUse:{"boolean":o},locale:{string:i},locales:{__any__:{any:h},__type__:{object:r}},height:{string:i},width:{string:i},__type__:{object:r}};d.groups.__any__=d.nodes,d.manipulation.controlNodeStyle=d.nodes;var l={nodes:{borderWidth:[1,0,10,1],borderWidthSelected:[2,0,10,1],color:{border:["color","#2B7CE9"],background:["color","#97C2FC"],highlight:{border:["color","#2B7CE9"],background:["color","#D2E5FF"]},hover:{border:["color","#2B7CE9"],background:["color","#D2E5FF"]}},fixed:{x:!1,y:!1},font:{color:["color","#343434"],size:[14,0,100,1],face:["arial","verdana","tahoma"],background:["color","none"],strokeWidth:[0,0,50,1],strokeColor:["color","#ffffff"]},hidden:!1,labelHighlightBold:!0,physics:!0,scaling:{min:[10,0,200,1],max:[30,0,200,1],label:{enabled:!1,min:[14,0,200,1],max:[30,0,200,1],maxVisible:[30,0,200,1],drawThreshold:[5,0,20,1]}},shadow:{enabled:!1,color:"rgba(0,0,0,0.5)",size:[10,0,20,1],x:[5,-30,30,1],y:[5,-30,30,1]},shape:["ellipse","box","circle","database","diamond","dot","square","star","text","triangle","triangleDown"],shapeProperties:{borderDashes:!1,borderRadius:[6,0,20,1],interpolation:!0,useImageSize:!1},size:[25,0,200,1]},edges:{arrows:{to:{enabled:!1,scaleFactor:[1,0,3,.05]},middle:{enabled:!1,scaleFactor:[1,0,3,.05]},from:{enabled:!1,scaleFactor:[1,0,3,.05]}},arrowStrikethrough:!0,color:{color:["color","#848484"],highlight:["color","#848484"],hover:["color","#848484"],inherit:["from","to","both",!0,!1],opacity:[1,0,1,.05]},dashes:!1,font:{color:["color","#343434"],size:[14,0,100,1],face:["arial","verdana","tahoma"],background:["color","none"],strokeWidth:[2,0,50,1],strokeColor:["color","#ffffff"],align:["horizontal","top","middle","bottom"]},hidden:!1,hoverWidth:[1.5,0,5,.1],labelHighlightBold:!0,physics:!0,scaling:{min:[1,0,100,1],max:[15,0,100,1],label:{enabled:!0,min:[14,0,200,1],max:[30,0,200,1],maxVisible:[30,0,200,1],drawThreshold:[5,0,20,1]}},selectionWidth:[1.5,0,5,.1],selfReferenceSize:[20,0,200,1],shadow:{enabled:!1,color:"rgba(0,0,0,0.5)",size:[10,0,20,1],x:[5,-30,30,1],y:[5,-30,30,1]},smooth:{enabled:!0,type:["dynamic","continuous","discrete","diagonalCross","straightCross","horizontal","vertical","curvedCW","curvedCCW","cubicBezier"],forceDirection:["horizontal","vertical","none"],roundness:[.5,0,1,.05]},width:[1,0,30,1]},layout:{hierarchical:{enabled:!1,levelSeparation:[150,20,500,5],nodeSpacing:[100,20,500,5],treeSpacing:[200,20,500,5],blockShifting:!0,edgeMinimization:!0,parentCentralization:!0,direction:["UD","DU","LR","RL"],sortMethod:["hubsize","directed"]}},interaction:{dragNodes:!0,dragView:!0,hideEdgesOnDrag:!1,hideNodesOnDrag:!1,hover:!1,keyboard:{enabled:!1,speed:{x:[10,0,40,1],y:[10,0,40,1],zoom:[.02,0,.1,.005]},bindToWindow:!0},multiselect:!1,navigationButtons:!1,selectable:!0,selectConnectedEdges:!0,hoverConnectedEdges:!0,tooltipDelay:[300,0,1e3,25],zoomView:!0},manipulation:{enabled:!1,initiallyActive:!1},physics:{enabled:!0,barnesHut:{gravitationalConstant:[-2e3,-3e4,0,50],centralGravity:[.3,0,10,.05],springLength:[95,0,500,5],springConstant:[.04,0,1.2,.005],damping:[.09,0,1,.01],avoidOverlap:[0,0,1,.01]},forceAtlas2Based:{gravitationalConstant:[-50,-500,0,1],centralGravity:[.01,0,1,.005],springLength:[95,0,500,5],springConstant:[.08,0,1.2,.005],damping:[.4,0,1,.01],avoidOverlap:[0,0,1,.01]},repulsion:{centralGravity:[.2,0,10,.05],springLength:[200,0,500,5],springConstant:[.05,0,1.2,.005],nodeDistance:[100,0,500,5],damping:[.09,0,1,.01]},hierarchicalRepulsion:{centralGravity:[.2,0,10,.05],springLength:[100,0,500,5],springConstant:[.01,0,1.2,.005],nodeDistance:[120,0,500,5],damping:[.09,0,1,.01]},maxVelocity:[50,0,150,1],minVelocity:[.1,.01,.5,.01],solver:["barnesHut","forceAtlas2Based","repulsion","hierarchicalRepulsion"],timestep:[.5,.01,1,.01]},global:{locale:["en","nl"]}};e.allOptions=d,e.configureOptions=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){var i=[],o=!0,n=!1,s=void 0;try{for(var r,a=t[Symbol.iterator]();!(o=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);o=!0);}catch(h){n=!0,s=h}finally{try{!o&&a["return"]&&a["return"]()}finally{if(n)throw s}}return i}return function(e,i){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return t(e,i);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),r=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),a=i(116),h=o(a),d=function(){function t(e,i,o){n(this,t),this.body=e,this.springLength=i,this.springConstant=o,this.distanceSolver=new h["default"]}return r(t,[{key:"setOptions",value:function(t){t&&(t.springLength&&(this.springLength=t.springLength),t.springConstant&&(this.springConstant=t.springConstant))}},{key:"solve",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=this.distanceSolver.getDistances(this.body,t,e);this._createL_matrix(o),this._createK_matrix(o);for(var n=.01,r=1,a=0,h=Math.max(1e3,Math.min(10*this.body.nodeIndices.length,6e3)),d=5,l=1e9,c=0,u=0,p=0,f=0,m=0;l>n&&h>a;){a+=1;var v=this._getHighestEnergyNode(i),g=s(v,4);for(c=g[0],l=g[1],u=g[2],p=g[3],f=l,m=0;f>r&&d>m;){m+=1,this._moveNode(c,u,p);var y=this._getEnergy(c),b=s(y,3);f=b[0],u=b[1],p=b[2]}}}},{key:"_getHighestEnergyNode",value:function(t){for(var e=this.body.nodeIndices,i=this.body.nodes,o=0,n=e[0],r=0,a=0,h=0;h<e.length;h++){var d=e[h];if(i[d].predefinedPosition===!1||i[d].isCluster===!0&&t===!0||i[d].options.fixed.x===!0||i[d].options.fixed.y===!0){var l=this._getEnergy(d),c=s(l,3),u=c[0],p=c[1],f=c[2];u>o&&(o=u,n=d,r=p,a=f)}}return[n,o,r,a]}},{key:"_getEnergy",value:function(t){for(var e=this.body.nodeIndices,i=this.body.nodes,o=i[t].x,n=i[t].y,s=0,r=0,a=0;a<e.length;a++){var h=e[a];if(h!==t){var d=i[h].x,l=i[h].y,c=1/Math.sqrt(Math.pow(o-d,2)+Math.pow(n-l,2));s+=this.K_matrix[t][h]*(o-d-this.L_matrix[t][h]*(o-d)*c),r+=this.K_matrix[t][h]*(n-l-this.L_matrix[t][h]*(n-l)*c)}}var u=Math.sqrt(Math.pow(s,2)+Math.pow(r,2));return[u,s,r]}},{key:"_moveNode",value:function(t,e,i){for(var o=this.body.nodeIndices,n=this.body.nodes,s=0,r=0,a=0,h=n[t].x,d=n[t].y,l=0;l<o.length;l++){var c=o[l];if(c!==t){var u=n[c].x,p=n[c].y,f=1/Math.pow(Math.pow(h-u,2)+Math.pow(d-p,2),1.5);s+=this.K_matrix[t][c]*(1-this.L_matrix[t][c]*Math.pow(d-p,2)*f),r+=this.K_matrix[t][c]*(this.L_matrix[t][c]*(h-u)*(d-p)*f),a+=this.K_matrix[t][c]*(1-this.L_matrix[t][c]*Math.pow(h-u,2)*f)}}var m=s,v=r,g=e,y=a,b=i,w=(g/m+b/v)/(v/m-y/v),_=-(v*w+g)/m;n[t].x+=_,n[t].y+=w}},{key:"_createL_matrix",value:function(t){var e=this.body.nodeIndices,i=this.springLength;this.L_matrix=[];for(var o=0;o<e.length;o++){this.L_matrix[e[o]]={};for(var n=0;n<e.length;n++)this.L_matrix[e[o]][e[n]]=i*t[e[o]][e[n]]}}},{key:"_createK_matrix",value:function(t){var e=this.body.nodeIndices,i=this.springConstant;this.K_matrix=[];for(var o=0;o<e.length;o++){this.K_matrix[e[o]]={};for(var n=0;n<e.length;n++)this.K_matrix[e[o]][e[n]]=i*Math.pow(t[e[o]][e[n]],-2)}}}]),t}();e["default"]=d},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(){i(this,t)}return o(t,[{key:"getDistances",value:function(t,e,i){for(var o={},n=t.edges,s=0;s<e.length;s++){o[e[s]]={},o[e[s]]={};for(var r=0;r<e.length;r++)o[e[s]][e[r]]=s==r?0:1e9,o[e[s]][e[r]]=s==r?0:1e9}for(var a=0;a<i.length;a++){var h=n[i[a]];h.connected===!0&&void 0!==o[h.fromId]&&void 0!==o[h.toId]&&(o[h.fromId][h.toId]=1,o[h.toId][h.fromId]=1)}for(var d=e.length,l=0;d>l;l++)for(var c=0;d-1>c;c++)for(var u=c+1;d>u;u++)o[e[c]][e[u]]=Math.min(o[e[c]][e[u]],o[e[c]][e[l]]+o[e[l]][e[u]]),o[e[u]][e[c]]=o[e[c]][e[u]];return o}}]),t}();e["default"]=n},function(t,e){"undefined"!=typeof CanvasRenderingContext2D&&(CanvasRenderingContext2D.prototype.circle=function(t,e,i){this.beginPath(),this.arc(t,e,i,0,2*Math.PI,!1),this.closePath()},CanvasRenderingContext2D.prototype.square=function(t,e,i){this.beginPath(),this.rect(t-i,e-i,2*i,2*i),this.closePath()},CanvasRenderingContext2D.prototype.triangle=function(t,e,i){this.beginPath(),i*=1.15,e+=.275*i;var o=2*i,n=o/2,s=Math.sqrt(3)/6*o,r=Math.sqrt(o*o-n*n);this.moveTo(t,e-(r-s)),this.lineTo(t+n,e+s),this.lineTo(t-n,e+s),this.lineTo(t,e-(r-s)),this.closePath()},CanvasRenderingContext2D.prototype.triangleDown=function(t,e,i){this.beginPath(),i*=1.15,e-=.275*i;var o=2*i,n=o/2,s=Math.sqrt(3)/6*o,r=Math.sqrt(o*o-n*n);this.moveTo(t,e+(r-s)),
-this.lineTo(t+n,e-s),this.lineTo(t-n,e-s),this.lineTo(t,e+(r-s)),this.closePath()},CanvasRenderingContext2D.prototype.star=function(t,e,i){this.beginPath(),i*=.82,e+=.1*i;for(var o=0;10>o;o++){var n=o%2===0?1.3*i:.5*i;this.lineTo(t+n*Math.sin(2*o*Math.PI/10),e-n*Math.cos(2*o*Math.PI/10))}this.closePath()},CanvasRenderingContext2D.prototype.diamond=function(t,e,i){this.beginPath(),this.lineTo(t,e+i),this.lineTo(t+i,e),this.lineTo(t,e-i),this.lineTo(t-i,e),this.closePath()},CanvasRenderingContext2D.prototype.roundRect=function(t,e,i,o,n){var s=Math.PI/180;0>i-2*n&&(n=i/2),0>o-2*n&&(n=o/2),this.beginPath(),this.moveTo(t+n,e),this.lineTo(t+i-n,e),this.arc(t+i-n,e+n,n,270*s,360*s,!1),this.lineTo(t+i,e+o-n),this.arc(t+i-n,e+o-n,n,0,90*s,!1),this.lineTo(t+n,e+o),this.arc(t+n,e+o-n,n,90*s,180*s,!1),this.lineTo(t,e+n),this.arc(t+n,e+n,n,180*s,270*s,!1),this.closePath()},CanvasRenderingContext2D.prototype.ellipse=function(t,e,i,o){var n=.5522848,s=i/2*n,r=o/2*n,a=t+i,h=e+o,d=t+i/2,l=e+o/2;this.beginPath(),this.moveTo(t,l),this.bezierCurveTo(t,l-r,d-s,e,d,e),this.bezierCurveTo(d+s,e,a,l-r,a,l),this.bezierCurveTo(a,l+r,d+s,h,d,h),this.bezierCurveTo(d-s,h,t,l+r,t,l),this.closePath()},CanvasRenderingContext2D.prototype.database=function(t,e,i,o){var n=1/3,s=i,r=o*n,a=.5522848,h=s/2*a,d=r/2*a,l=t+s,c=e+r,u=t+s/2,p=e+r/2,f=e+(o-r/2),m=e+o;this.beginPath(),this.moveTo(l,p),this.bezierCurveTo(l,p+d,u+h,c,u,c),this.bezierCurveTo(u-h,c,t,p+d,t,p),this.bezierCurveTo(t,p-d,u-h,e,u,e),this.bezierCurveTo(u+h,e,l,p-d,l,p),this.lineTo(l,f),this.bezierCurveTo(l,f+d,u+h,m,u,m),this.bezierCurveTo(u-h,m,t,f+d,t,f),this.lineTo(t,p)},CanvasRenderingContext2D.prototype.arrow=function(t,e,i,o){var n=t-o*Math.cos(i),s=e-o*Math.sin(i),r=t-.9*o*Math.cos(i),a=e-.9*o*Math.sin(i),h=n+o/3*Math.cos(i+.5*Math.PI),d=s+o/3*Math.sin(i+.5*Math.PI),l=n+o/3*Math.cos(i-.5*Math.PI),c=s+o/3*Math.sin(i-.5*Math.PI);this.beginPath(),this.moveTo(t,e),this.lineTo(h,d),this.lineTo(r,a),this.lineTo(l,c),this.closePath()},CanvasRenderingContext2D.prototype.dashedLine=function(t,e,i,o,n){this.beginPath(),this.moveTo(t,e);for(var s=n.length,r=i-t,a=o-e,h=a/r,d=Math.sqrt(r*r+a*a),l=0,c=!0,u=0,p=n[0];d>=.1;)p=n[l++%s],p>d&&(p=d),u=Math.sqrt(p*p/(1+h*h)),u=0>r?-u:u,t+=u,e+=h*u,c===!0?this.lineTo(t,e):this.moveTo(t,e),d-=p,c=!c})},function(t,e){function i(t){return P=t,p()}function o(){I=0,N=P.charAt(0)}function n(){I++,N=P.charAt(I)}function s(){return P.charAt(I+1)}function r(t){return L.test(t)}function a(t,e){if(t||(t={}),e)for(var i in e)e.hasOwnProperty(i)&&(t[i]=e[i]);return t}function h(t,e,i){for(var o=e.split("."),n=t;o.length;){var s=o.shift();o.length?(n[s]||(n[s]={}),n=n[s]):n[s]=i}}function d(t,e){for(var i,o,n=null,s=[t],r=t;r.parent;)s.push(r.parent),r=r.parent;if(r.nodes)for(i=0,o=r.nodes.length;o>i;i++)if(e.id===r.nodes[i].id){n=r.nodes[i];break}for(n||(n={id:e.id},t.node&&(n.attr=a(n.attr,t.node))),i=s.length-1;i>=0;i--){var h=s[i];h.nodes||(h.nodes=[]),-1===h.nodes.indexOf(n)&&h.nodes.push(n)}e.attr&&(n.attr=a(n.attr,e.attr))}function l(t,e){if(t.edges||(t.edges=[]),t.edges.push(e),t.edge){var i=a({},t.edge);e.attr=a(i,e.attr)}}function c(t,e,i,o,n){var s={from:e,to:i,type:o};return t.edge&&(s.attr=a({},t.edge)),s.attr=a(s.attr||{},n),s}function u(){for(z=T.NULL,R="";" "===N||"	"===N||"\n"===N||"\r"===N;)n();do{var t=!1;if("#"===N){for(var e=I-1;" "===P.charAt(e)||"	"===P.charAt(e);)e--;if("\n"===P.charAt(e)||""===P.charAt(e)){for(;""!=N&&"\n"!=N;)n();t=!0}}if("/"===N&&"/"===s()){for(;""!=N&&"\n"!=N;)n();t=!0}if("/"===N&&"*"===s()){for(;""!=N;){if("*"===N&&"/"===s()){n(),n();break}n()}t=!0}for(;" "===N||"	"===N||"\n"===N||"\r"===N;)n()}while(t);if(""===N)return void(z=T.DELIMITER);var i=N+s();if(E[i])return z=T.DELIMITER,R=i,n(),void n();if(E[N])return z=T.DELIMITER,R=N,void n();if(r(N)||"-"===N){for(R+=N,n();r(N);)R+=N,n();return"false"===R?R=!1:"true"===R?R=!0:isNaN(Number(R))||(R=Number(R)),void(z=T.IDENTIFIER)}if('"'===N){for(n();""!=N&&('"'!=N||'"'===N&&'"'===s());)R+=N,'"'===N&&n(),n();if('"'!=N)throw _('End of string " expected');return n(),void(z=T.IDENTIFIER)}for(z=T.UNKNOWN;""!=N;)R+=N,n();throw new SyntaxError('Syntax error in part "'+x(R,30)+'"')}function p(){var t={};if(o(),u(),"strict"===R&&(t.strict=!0,u()),"graph"!==R&&"digraph"!==R||(t.type=R,u()),z===T.IDENTIFIER&&(t.id=R,u()),"{"!=R)throw _("Angle bracket { expected");if(u(),f(t),"}"!=R)throw _("Angle bracket } expected");if(u(),""!==R)throw _("End of file expected");return u(),delete t.node,delete t.edge,delete t.graph,t}function f(t){for(;""!==R&&"}"!=R;)m(t),";"===R&&u()}function m(t){var e=v(t);if(e)return void b(t,e);var i=g(t);if(!i){if(z!=T.IDENTIFIER)throw _("Identifier expected");var o=R;if(u(),"="===R){if(u(),z!=T.IDENTIFIER)throw _("Identifier expected");t[o]=R,u()}else y(t,o)}}function v(t){var e=null;if("subgraph"===R&&(e={},e.type="subgraph",u(),z===T.IDENTIFIER&&(e.id=R,u())),"{"===R){if(u(),e||(e={}),e.parent=t,e.node=t.node,e.edge=t.edge,e.graph=t.graph,f(e),"}"!=R)throw _("Angle bracket } expected");u(),delete e.node,delete e.edge,delete e.graph,delete e.parent,t.subgraphs||(t.subgraphs=[]),t.subgraphs.push(e)}return e}function g(t){return"node"===R?(u(),t.node=w(),"node"):"edge"===R?(u(),t.edge=w(),"edge"):"graph"===R?(u(),t.graph=w(),"graph"):null}function y(t,e){var i={id:e},o=w();o&&(i.attr=o),d(t,i),b(t,e)}function b(t,e){for(;"->"===R||"--"===R;){var i,o=R;u();var n=v(t);if(n)i=n;else{if(z!=T.IDENTIFIER)throw _("Identifier or subgraph expected");i=R,d(t,{id:i}),u()}var s=w(),r=c(t,e,i,o,s);l(t,r),e=i}}function w(){for(var t=null;"["===R;){for(u(),t={};""!==R&&"]"!=R;){if(z!=T.IDENTIFIER)throw _("Attribute name expected");var e=R;if(u(),"="!=R)throw _("Equal sign = expected");if(u(),z!=T.IDENTIFIER)throw _("Attribute value expected");var i=R;h(t,e,i),u(),","==R&&u()}if("]"!=R)throw _("Bracket ] expected");u()}return t}function _(t){return new SyntaxError(t+', got "'+x(R,30)+'" (char '+I+")")}function x(t,e){return t.length<=e?t:t.substr(0,27)+"..."}function k(t,e,i){Array.isArray(t)?t.forEach(function(t){Array.isArray(e)?e.forEach(function(e){i(t,e)}):i(t,e)}):Array.isArray(e)?e.forEach(function(e){i(t,e)}):i(t,e)}function O(t,e,i){for(var o=e.split("."),n=o.pop(),s=t,r=0;r<o.length;r++){var a=o[r];a in s||(s[a]={}),s=s[a]}return s[n]=i,t}function M(t,e){var i={};for(var o in t)if(t.hasOwnProperty(o)){var n=e[o];Array.isArray(n)?n.forEach(function(e){O(i,e,t[o])}):"string"==typeof n?O(i,n,t[o]):O(i,o,t[o])}return i}function D(t){var e=i(t),o={nodes:[],edges:[],options:{}};if(e.nodes&&e.nodes.forEach(function(t){var e={id:t.id,label:String(t.label||t.id)};a(e,M(t.attr,S)),e.image&&(e.shape="image"),o.nodes.push(e)}),e.edges){var n=function(t){var e={from:t.from,to:t.to};return a(e,M(t.attr,C)),e.arrows="->"===t.type?"to":void 0,e};e.edges.forEach(function(t){var e,i;e=t.from instanceof Object?t.from.nodes:{id:t.from},i=t.to instanceof Object?t.to.nodes:{id:t.to},t.from instanceof Object&&t.from.edges&&t.from.edges.forEach(function(t){var e=n(t);o.edges.push(e)}),k(e,i,function(e,i){var s=c(o,e.id,i.id,t.type,t.attr),r=n(s);o.edges.push(r)}),t.to instanceof Object&&t.to.edges&&t.to.edges.forEach(function(t){var e=n(t);o.edges.push(e)})})}return e.attr&&(o.options=e.attr),o}var S={fontsize:"font.size",fontcolor:"font.color",labelfontcolor:"font.color",fontname:"font.face",color:["color.border","color.background"],fillcolor:"color.background",tooltip:"title",labeltooltip:"title"},C=Object.create(S);C.color="color.color";var T={NULL:0,DELIMITER:1,IDENTIFIER:2,UNKNOWN:3},E={"{":!0,"}":!0,"[":!0,"]":!0,";":!0,"=":!0,",":!0,"->":!0,"--":!0},P="",I=0,N="",R="",z=T.NULL,L=/[a-zA-Z_0-9.:#]/;e.parseDOT=i,e.DOTToGraph=D},function(t,e){function i(t,e){var i=[],o=[],n={edges:{inheritColor:!1},nodes:{fixed:!1,parseColor:!1}};void 0!==e&&(void 0!==e.fixed&&(n.nodes.fixed=e.fixed),void 0!==e.parseColor&&(n.nodes.parseColor=e.parseColor),void 0!==e.inheritColor&&(n.edges.inheritColor=e.inheritColor));for(var s=t.edges,r=t.nodes,a=0;a<s.length;a++){var h={},d=s[a];h.id=d.id,h.from=d.source,h.to=d.target,h.attributes=d.attributes,h.label=d.label,h.title=void 0!==d.attributes?d.attributes.title:void 0,"Directed"===d.type&&(h.arrows="to"),d.color&&n.inheritColor===!1&&(h.color=d.color),i.push(h)}for(var a=0;a<r.length;a++){var l={},c=r[a];l.id=c.id,l.attributes=c.attributes,l.title=c.title,l.x=c.x,l.y=c.y,l.label=c.label,l.title=void 0!==c.attributes?c.attributes.title:void 0,n.nodes.parseColor===!0?l.color=c.color:l.color=void 0!==c.color?{background:c.color,border:c.color,highlight:{background:c.color,border:c.color},hover:{background:c.color,border:c.color}}:void 0,l.size=c.size,l.fixed=n.nodes.fixed&&void 0!==c.x&&void 0!==c.y,o.push(l)}return{nodes:o,edges:i}}e.parseGephi=i},function(t,e){e.en={edit:"Edit",del:"Delete selected",back:"Back",addNode:"Add Node",addEdge:"Add Edge",editNode:"Edit Node",editEdge:"Edit Edge",addDescription:"Click in an empty space to place a new node.",edgeDescription:"Click on a node and drag the edge to another node to connect them.",editEdgeDescription:"Click on the control points and drag them to a node to connect to it.",createEdgeError:"Cannot link edges to a cluster.",deleteClusterError:"Clusters cannot be deleted.",editClusterError:"Clusters cannot be edited."},e.en_EN=e.en,e.en_US=e.en,e.de={edit:"Editieren",del:"Lösche Auswahl",back:"Zurück",addNode:"Knoten hinzufügen",addEdge:"Kante hinzufügen",editNode:"Knoten editieren",editEdge:"Kante editieren",addDescription:"Klicke auf eine freie Stelle, um einen neuen Knoten zu plazieren.",edgeDescription:"Klicke auf einen Knoten und ziehe die Kante zu einem anderen Knoten, um diese zu verbinden.",editEdgeDescription:"Klicke auf die Verbindungspunkte und ziehe diese auf einen Knoten, um sie zu verbinden.",createEdgeError:"Es ist nicht möglich, Kanten mit Clustern zu verbinden.",deleteClusterError:"Cluster können nicht gelöscht werden.",editClusterError:"Cluster können nicht editiert werden."},e.de_DE=e.de,e.es={edit:"Editar",del:"Eliminar selección",back:"Átras",addNode:"Añadir nodo",addEdge:"Añadir arista",editNode:"Editar nodo",editEdge:"Editar arista",addDescription:"Haga clic en un lugar vacío para colocar un nuevo nodo.",edgeDescription:"Haga clic en un nodo y arrastre la arista hacia otro nodo para conectarlos.",editEdgeDescription:"Haga clic en un punto de control y arrastrelo a un nodo para conectarlo.",createEdgeError:"No se puede conectar una arista a un grupo.",deleteClusterError:"No es posible eliminar grupos.",editClusterError:"No es posible editar grupos."},e.es_ES=e.es,e.nl={edit:"Wijzigen",del:"Selectie verwijderen",back:"Terug",addNode:"Node toevoegen",addEdge:"Link toevoegen",editNode:"Node wijzigen",editEdge:"Link wijzigen",addDescription:"Klik op een leeg gebied om een nieuwe node te maken.",edgeDescription:"Klik op een node en sleep de link naar een andere node om ze te verbinden.",editEdgeDescription:"Klik op de verbindingspunten en sleep ze naar een node om daarmee te verbinden.",createEdgeError:"Kan geen link maken naar een cluster.",deleteClusterError:"Clusters kunnen niet worden verwijderd.",editClusterError:"Clusters kunnen niet worden aangepast."},e.nl_NL=e.nl,e.nl_BE=e.nl}])});
-//# sourceMappingURL=vis.map
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js
index 0ba461f02317f..4f8409ca2b7c2 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js
@@ -16,11 +16,16 @@
  */
 
 var uiRoot = "";
+var appBasePath = "";
 
 function setUIRoot(val) {
     uiRoot = val;
 }
 
+function setAppBasePath(path) {
+    appBasePath = path;
+}
+
 function collapseTablePageLoad(name, table){
   if (window.localStorage.getItem(name) == "true") {
     // Set it to false so that the click function can revert it
@@ -33,7 +38,7 @@ function collapseTable(thisName, table){
     var status = window.localStorage.getItem(thisName) == "true";
     status = !status;
 
-    var thisClass = '.' + thisName
+    var thisClass = '.' + thisName;
 
     // Expand the list of additional metrics.
     var tableDiv = $(thisClass).parent().find('.' + table);
diff --git a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
index 4e417679ca663..04faf7f87cf2b 100644
--- a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
@@ -21,7 +21,7 @@ import java.util.{Timer, TimerTask}
 import java.util.concurrent.ConcurrentHashMap
 import java.util.function.Consumer
 
-import scala.collection.mutable.ArrayBuffer
+import scala.collection.mutable.{ArrayBuffer, HashSet}
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
@@ -99,10 +99,19 @@ private[spark] class BarrierCoordinator(
     // reset when a barrier() call fails due to timeout.
     private var barrierEpoch: Int = 0
 
-    // An array of RPCCallContexts for barrier tasks that are waiting for reply of a barrier()
-    // call.
+    // An Array of RPCCallContexts for barrier tasks that have made a blocking runBarrier() call
     private val requesters: ArrayBuffer[RpcCallContext] = new ArrayBuffer[RpcCallContext](numTasks)
 
+    // Messages from each barrier task that have made a blocking runBarrier() call.
+    // The messages will be replied to all tasks once sync finished.
+    private val messages = Array.ofDim[String](numTasks)
+
+    // Request methods collected from tasks inside this barrier sync. All tasks should make sure
+    // that they're calling the same method within the same barrier sync phase. In other words,
+    // the size of requestMethods should always be 1 for a legitimate barrier sync. Otherwise,
+    // the barrier sync would fail if the size of requestMethods becomes greater than 1.
+    private val requestMethods = new HashSet[RequestMethod.Value]
+
     // A timer task that ensures we may timeout for a barrier() call.
     private var timerTask: TimerTask = null
 
@@ -133,6 +142,16 @@ private[spark] class BarrierCoordinator(
     def handleRequest(requester: RpcCallContext, request: RequestToSync): Unit = synchronized {
       val taskId = request.taskAttemptId
       val epoch = request.barrierEpoch
+      val curReqMethod = request.requestMethod
+      requestMethods.add(curReqMethod)
+      if (requestMethods.size > 1) {
+        val error = new SparkException(s"Different barrier sync types found for the " +
+          s"sync $barrierId: ${requestMethods.mkString(", ")}. Please use the " +
+          s"same barrier sync type within a single sync.")
+        (requesters :+ requester).foreach(_.sendFailure(error))
+        clear()
+        return
+      }
 
       // Require the number of tasks is correctly set from the BarrierTaskContext.
       require(request.numTasks == numTasks, s"Number of tasks of $barrierId is " +
@@ -153,33 +172,23 @@ private[spark] class BarrierCoordinator(
         }
         // Add the requester to array of RPCCallContexts pending for reply.
         requesters += requester
+        messages(request.partitionId) = request.message
         logInfo(s"Barrier sync epoch $barrierEpoch from $barrierId received update from Task " +
           s"$taskId, current progress: ${requesters.size}/$numTasks.")
-        if (maybeFinishAllRequesters(requesters, numTasks)) {
+        if (requesters.size == numTasks) {
+          requesters.foreach(_.reply(messages))
           // Finished current barrier() call successfully, clean up ContextBarrierState and
           // increase the barrier epoch.
           logInfo(s"Barrier sync epoch $barrierEpoch from $barrierId received all updates from " +
             s"tasks, finished successfully.")
           barrierEpoch += 1
           requesters.clear()
+          requestMethods.clear()
           cancelTimerTask()
         }
       }
     }
 
-    // Finish all the blocking barrier sync requests from a stage attempt successfully if we
-    // have received all the sync requests.
-    private def maybeFinishAllRequesters(
-        requesters: ArrayBuffer[RpcCallContext],
-        numTasks: Int): Boolean = {
-      if (requesters.size == numTasks) {
-        requesters.foreach(_.reply(()))
-        true
-      } else {
-        false
-      }
-    }
-
     // Cleanup the internal state of a barrier stage attempt.
     def clear(): Unit = synchronized {
       // The global sync fails so the stage is expected to retry another attempt, all sync
@@ -199,7 +208,7 @@ private[spark] class BarrierCoordinator(
   }
 
   override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
-    case request @ RequestToSync(numTasks, stageId, stageAttemptId, _, _) =>
+    case request @ RequestToSync(numTasks, stageId, stageAttemptId, _, _, _, _, _) =>
       // Get or init the ContextBarrierState correspond to the stage attempt.
       val barrierId = ContextBarrierId(stageId, stageAttemptId)
       states.computeIfAbsent(barrierId,
@@ -217,18 +226,28 @@ private[spark] class BarrierCoordinator(
 private[spark] sealed trait BarrierCoordinatorMessage extends Serializable
 
 /**
- * A global sync request message from BarrierTaskContext, by `barrier()` call. Each request is
+ * A global sync request message from BarrierTaskContext. Each request is
  * identified by stageId + stageAttemptId + barrierEpoch.
  *
  * @param numTasks The number of global sync requests the BarrierCoordinator shall receive
  * @param stageId ID of current stage
  * @param stageAttemptId ID of current stage attempt
  * @param taskAttemptId Unique ID of current task
- * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls.
+ * @param barrierEpoch ID of a runBarrier() call, a task may consist multiple runBarrier() calls
+ * @param partitionId ID of the current partition the task is assigned to
+ * @param message Message sent from the BarrierTaskContext
+ * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
  */
 private[spark] case class RequestToSync(
-    numTasks: Int,
-    stageId: Int,
-    stageAttemptId: Int,
-    taskAttemptId: Long,
-    barrierEpoch: Int) extends BarrierCoordinatorMessage
+  numTasks: Int,
+  stageId: Int,
+  stageAttemptId: Int,
+  taskAttemptId: Long,
+  barrierEpoch: Int,
+  partitionId: Int,
+  message: String,
+  requestMethod: RequestMethod.Value) extends BarrierCoordinatorMessage
+
+private[spark] object RequestMethod extends Enumeration {
+  val BARRIER, ALL_GATHER = Value
+}
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 3d369802f3023..4d765481eb836 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -20,8 +20,9 @@ package org.apache.spark
 import java.util.{Properties, Timer, TimerTask}
 
 import scala.collection.JavaConverters._
-import scala.concurrent.TimeoutException
 import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.util.{Failure, Success => ScalaSuccess, Try}
 
 import org.apache.spark.annotation.{Experimental, Since}
 import org.apache.spark.executor.TaskMetrics
@@ -59,49 +60,7 @@ class BarrierTaskContext private[spark] (
   // from different tasks within the same barrier stage attempt to succeed.
   private lazy val numTasks = getTaskInfos().size
 
-  /**
-   * :: Experimental ::
-   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
-   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
-   * stage have reached this routine.
-   *
-   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
-   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
-   * timeout. Some examples of '''misuses''' are listed below:
-   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
-   * shall lead to timeout of the function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       if (context.partitionId() == 0) {
-   *           // Do nothing.
-   *       } else {
-   *           context.barrier()
-   *       }
-   *       iter
-   *   }
-   * }}}
-   *
-   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
-   * second function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       try {
-   *           // Do something that might throw an Exception.
-   *           doSomething()
-   *           context.barrier()
-   *       } catch {
-   *           case e: Exception => logWarning("...", e)
-   *       }
-   *       context.barrier()
-   *       iter
-   *   }
-   * }}}
-   */
-  @Experimental
-  @Since("2.4.0")
-  def barrier(): Unit = {
+  private def runBarrier(message: String, requestMethod: RequestMethod.Value): Array[String] = {
     logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) has entered " +
       s"the global sync, current barrier epoch is $barrierEpoch.")
     logTrace("Current callSite: " + Utils.getCallSite())
@@ -119,9 +78,9 @@ class BarrierTaskContext private[spark] (
     timer.schedule(timerTask, 60000, 60000)
 
     try {
-      val abortableRpcFuture = barrierCoordinator.askAbortable[Unit](
+      val abortableRpcFuture = barrierCoordinator.askAbortable[Array[String]](
         message = RequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
-          barrierEpoch),
+          barrierEpoch, partitionId, message, requestMethod),
         // Set a fixed timeout for RPC here, so users shall get a SparkException thrown by
         // BarrierCoordinator on timeout, instead of RPCTimeoutException from the RPC framework.
         timeout = new RpcTimeout(365.days, "barrierTimeout"))
@@ -129,29 +88,30 @@ class BarrierTaskContext private[spark] (
       // Wait the RPC future to be completed, but every 1 second it will jump out waiting
       // and check whether current spark task is killed. If killed, then throw
       // a `TaskKilledException`, otherwise continue wait RPC until it completes.
-      try {
-        while (!abortableRpcFuture.toFuture.isCompleted) {
+
+      while (!abortableRpcFuture.future.isCompleted) {
+        try {
           // wait RPC future for at most 1 second
-          try {
-            ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
-          } catch {
-            case _: TimeoutException | _: InterruptedException =>
-              // If `TimeoutException` thrown, waiting RPC future reach 1 second.
-              // If `InterruptedException` thrown, it is possible this task is killed.
-              // So in this two cases, we should check whether task is killed and then
-              // throw `TaskKilledException`
-              taskContext.killTaskIfInterrupted()
+          Thread.sleep(1000)
+        } catch {
+          case _: InterruptedException => // task is killed by driver
+        } finally {
+          Try(taskContext.killTaskIfInterrupted()) match {
+            case ScalaSuccess(_) => // task is still running healthily
+            case Failure(e) => abortableRpcFuture.abort(e)
           }
         }
-      } finally {
-        abortableRpcFuture.abort(taskContext.getKillReason().getOrElse("Unknown reason."))
       }
+      // messages which consist of all barrier tasks' messages. The future will return the
+      // desired messages if it is completed successfully. Otherwise, exception could be thrown.
+      val messages = abortableRpcFuture.future.value.get.get
 
       barrierEpoch += 1
       logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) finished " +
         "global sync successfully, waited for " +
         s"${MILLISECONDS.toSeconds(System.currentTimeMillis() - startTime)} seconds, " +
         s"current barrier epoch is $barrierEpoch.")
+      messages
     } catch {
       case e: SparkException =>
         logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) failed " +
@@ -165,6 +125,64 @@ class BarrierTaskContext private[spark] (
     }
   }
 
+  /**
+   * :: Experimental ::
+   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
+   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
+   * stage have reached this routine.
+   *
+   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
+   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
+   * timeout. Some examples of '''misuses''' are listed below:
+   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
+   * shall lead to timeout of the function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       if (context.partitionId() == 0) {
+   *           // Do nothing.
+   *       } else {
+   *           context.barrier()
+   *       }
+   *       iter
+   *   }
+   * }}}
+   *
+   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
+   * second function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       try {
+   *           // Do something that might throw an Exception.
+   *           doSomething()
+   *           context.barrier()
+   *       } catch {
+   *           case e: Exception => logWarning("...", e)
+   *       }
+   *       context.barrier()
+   *       iter
+   *   }
+   * }}}
+   */
+  @Experimental
+  @Since("2.4.0")
+  def barrier(): Unit = runBarrier("", RequestMethod.BARRIER)
+
+  /**
+   * :: Experimental ::
+   * Blocks until all tasks in the same stage have reached this routine. Each task passes in
+   * a message and returns with a list of all the messages passed in by each of those tasks.
+   *
+   * CAUTION! The allGather method requires the same precautions as the barrier method
+   *
+   * The message is type String rather than Array[Byte] because it is more convenient for
+   * the user at the cost of worse performance.
+   */
+  @Experimental
+  @Since("3.0.0")
+  def allGather(message: String): Array[String] = runBarrier(message, RequestMethod.ALL_GATHER)
+
   /**
    * :: Experimental ::
    * Returns [[BarrierTaskInfo]] for all tasks in this barrier stage, ordered by partition ID.
diff --git a/core/src/main/scala/org/apache/spark/ContextAwareIterator.scala b/core/src/main/scala/org/apache/spark/ContextAwareIterator.scala
new file mode 100644
index 0000000000000..c4d0dd8aceab0
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/ContextAwareIterator.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import org.apache.spark.annotation.DeveloperApi
+
+/**
+ * :: DeveloperApi ::
+ * A TaskContext aware iterator.
+ *
+ * As the Python evaluation consumes the parent iterator in a separate thread,
+ * it could consume more data from the parent even after the task ends and the parent is closed.
+ * If an off-heap access exists in the parent iterator, it could cause segmentation fault
+ * which crashes the executor.
+ * Thus, we should use [[ContextAwareIterator]] to stop consuming after the task ends.
+ */
+@DeveloperApi
+class ContextAwareIterator[+T](val context: TaskContext, val delegate: Iterator[T])
+  extends Iterator[T] {
+
+  override def hasNext: Boolean =
+    !context.isCompleted() && !context.isInterrupted() && delegate.hasNext
+
+  override def next(): T = delegate.next()
+}
diff --git a/core/src/main/scala/org/apache/spark/ContextCleaner.scala b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
index 9506c36bf9c8c..7c3d6d98a33ed 100644
--- a/core/src/main/scala/org/apache/spark/ContextCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
@@ -281,7 +281,7 @@ private object ContextCleaner {
 }
 
 /**
- * Listener class used for testing when any item has been cleaned by the Cleaner class.
+ * Listener class used when any item has been cleaned by the Cleaner class.
  */
 private[spark] trait CleanerListener {
   def rddCleaned(rddId: Int): Unit
diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
index 677386cc7a572..fffa7fd498b96 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -45,8 +45,8 @@ import org.apache.spark.util.{Clock, SystemClock, ThreadUtils, Utils}
  * executors that could run all current running and pending tasks at once.
  *
  * Increasing the target number of executors happens in response to backlogged tasks waiting to be
- * scheduled. If the scheduler queue is not drained in N seconds, then new executors are added. If
- * the queue persists for another M seconds, then more executors are added and so on. The number
+ * scheduled. If the scheduler queue is not drained in M seconds, then new executors are added. If
+ * the queue persists for another N seconds, then more executors are added and so on. The number
  * added in each round increases exponentially from the previous round until an upper bound has been
  * reached. The upper bound is based both on a configured property and on the current number of
  * running and pending tasks, as described above.
@@ -196,7 +196,7 @@ private[spark] class ExecutorAllocationManager(
         s"s${DYN_ALLOCATION_SUSTAINED_SCHEDULER_BACKLOG_TIMEOUT.key} must be > 0!")
     }
     if (!conf.get(config.SHUFFLE_SERVICE_ENABLED)) {
-      if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING)) {
+      if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED)) {
         logWarning("Dynamic allocation without a shuffle service is an experimental feature.")
       } else if (!testing) {
         throw new SparkException("Dynamic allocation of executors requires the external " +
@@ -618,7 +618,11 @@ private[spark] class ExecutorAllocationManager(
 
         if (taskEnd.taskInfo.speculative) {
           stageAttemptToSpeculativeTaskIndices.get(stageAttempt).foreach {_.remove{taskIndex}}
-          stageAttemptToNumSpeculativeTasks(stageAttempt) -= 1
+          // If the previous task attempt succeeded first and it was the last task in a stage,
+          // the stage may have been removed before handing this speculative TaskEnd event.
+          if (stageAttemptToNumSpeculativeTasks.contains(stageAttempt)) {
+            stageAttemptToNumSpeculativeTasks(stageAttempt) -= 1
+          }
         }
 
         taskEnd.reason match {
diff --git a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
index 2ac72e66d6f32..be630722308fa 100644
--- a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
+++ b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
@@ -80,7 +80,9 @@ private[spark] class HeartbeatReceiver(sc: SparkContext, clock: Clock)
   // executor ID -> timestamp of when the last heartbeat from this executor was received
   private val executorLastSeen = new HashMap[String, Long]
 
-  private val executorTimeoutMs = sc.conf.get(config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT)
+  private val executorTimeoutMs = sc.conf.get(
+    config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT
+  ).getOrElse(Utils.timeStringAsMs(s"${sc.conf.get(Network.NETWORK_TIMEOUT)}s"))
 
   private val checkTimeoutIntervalMs = sc.conf.get(Network.NETWORK_TIMEOUT_INTERVAL)
 
diff --git a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
index f229061a6d0f6..ec8621bc55cf3 100644
--- a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
+++ b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
@@ -696,7 +696,7 @@ private[spark] class MapOutputTrackerMaster(
    *
    * @param dep shuffle dependency object
    * @param startMapIndex the start map index
-   * @param endMapIndex the end map index
+   * @param endMapIndex the end map index (exclusive)
    * @return a sequence of locations where task runs.
    */
   def getMapLocation(
@@ -707,7 +707,8 @@ private[spark] class MapOutputTrackerMaster(
     val shuffleStatus = shuffleStatuses.get(dep.shuffleId).orNull
     if (shuffleStatus != null) {
       shuffleStatus.withMapStatuses { statuses =>
-        if (startMapIndex < endMapIndex && (startMapIndex >= 0 && endMapIndex < statuses.length)) {
+        if (startMapIndex < endMapIndex &&
+          (startMapIndex >= 0 && endMapIndex <= statuses.length)) {
           val statusesPicked = statuses.slice(startMapIndex, endMapIndex).filter(_ != null)
           statusesPicked.map(_.location.host).toSeq
         } else {
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index 0e0291d2407d1..802100e05b728 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -577,7 +577,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
     // If spark.executor.heartbeatInterval bigger than spark.network.timeout,
     // it will almost always cause ExecutorLostFailure. See SPARK-22754.
     require(executorTimeoutThresholdMs > executorHeartbeatIntervalMs, "The value of " +
-      s"${networkTimeout}=${executorTimeoutThresholdMs}ms must be no less than the value of " +
+      s"${networkTimeout}=${executorTimeoutThresholdMs}ms must be greater than the value of " +
       s"${EXECUTOR_HEARTBEAT_INTERVAL.key}=${executorHeartbeatIntervalMs}ms.")
   }
 
@@ -684,7 +684,8 @@ private[spark] object SparkConf extends Logging {
     "spark.yarn.jars" -> Seq(
       AlternateConfig("spark.yarn.jar", "2.0")),
     MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM.key -> Seq(
-      AlternateConfig("spark.reducer.maxReqSizeShuffleToMem", "2.3")),
+      AlternateConfig("spark.reducer.maxReqSizeShuffleToMem", "2.3"),
+      AlternateConfig("spark.maxRemoteBlockSizeFetchToMem", "3.0")),
     LISTENER_BUS_EVENT_QUEUE_CAPACITY.key -> Seq(
       AlternateConfig("spark.scheduler.listenerbus.eventqueue.size", "2.3")),
     DRIVER_MEMORY_OVERHEAD.key -> Seq(
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 91188d58f4201..3ccbea5136f4a 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -41,7 +41,6 @@ import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat => NewFileInputFor
 import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.deploy.{LocalSparkCluster, SparkHadoopUtil}
-import org.apache.spark.deploy.StandaloneResourceUtils._
 import org.apache.spark.executor.{ExecutorMetrics, ExecutorMetricsSource}
 import org.apache.spark.input.{FixedLengthBinaryInputFormat, PortableDataStream, StreamInputFormat, WholeTextFileInputFormat}
 import org.apache.spark.internal.Logging
@@ -83,6 +82,11 @@ class SparkContext(config: SparkConf) extends Logging {
   // The call site where this SparkContext was constructed.
   private val creationSite: CallSite = Utils.getCallSite()
 
+  if (!config.get(EXECUTOR_ALLOW_SPARK_CONTEXT)) {
+    // In order to prevent SparkContext from being created in executors.
+    SparkContext.assertOnDriver()
+  }
+
   // In order to prevent multiple SparkContexts from being active at the same time, mark this
   // context as having started construction.
   // NOTE: this must be placed at the beginning of the SparkContext constructor.
@@ -250,15 +254,6 @@ class SparkContext(config: SparkConf) extends Logging {
 
   def isLocal: Boolean = Utils.isLocalMaster(_conf)
 
-  private def isClientStandalone: Boolean = {
-    val isSparkCluster = master match {
-      case SparkMasterRegex.SPARK_REGEX(_) => true
-      case SparkMasterRegex.LOCAL_CLUSTER_REGEX(_, _, _) => true
-      case _ => false
-    }
-    deployMode == "client" && isSparkCluster
-  }
-
   /**
    * @return true if context is stopped or in the midst of stopping.
    */
@@ -385,6 +380,7 @@ class SparkContext(config: SparkConf) extends Logging {
   try {
     _conf = config.clone()
     _conf.validateSettings()
+    _conf.set("spark.app.startTime", startTime.toString)
 
     if (!_conf.contains("spark.master")) {
       throw new SparkException("A master URL must be set in your configuration")
@@ -396,17 +392,7 @@ class SparkContext(config: SparkConf) extends Logging {
     _driverLogger = DriverLogger(_conf)
 
     val resourcesFileOpt = conf.get(DRIVER_RESOURCES_FILE)
-    val allResources = getOrDiscoverAllResources(_conf, SPARK_DRIVER_PREFIX, resourcesFileOpt)
-    _resources = {
-      // driver submitted in client mode under Standalone may have conflicting resources with
-      // other drivers/workers on this host. We should sync driver's resources info into
-      // SPARK_RESOURCES/SPARK_RESOURCES_COORDINATE_DIR/ to avoid collision.
-      if (isClientStandalone) {
-        acquireResources(_conf, SPARK_DRIVER_PREFIX, allResources, Utils.getProcessId)
-      } else {
-        allResources
-      }
-    }
+    _resources = getOrDiscoverAllResources(_conf, SPARK_DRIVER_PREFIX, resourcesFileOpt)
     logResourceInfo(SPARK_DRIVER_PREFIX, _resources)
 
     // log out spark.app.name in the Spark driver logs
@@ -502,11 +488,17 @@ class SparkContext(config: SparkConf) extends Logging {
 
     // Add each JAR given through the constructor
     if (jars != null) {
-      jars.foreach(addJar)
+      jars.foreach(jar => addJar(jar, true))
+      if (addedJars.nonEmpty) {
+        _conf.set("spark.app.initial.jar.urls", addedJars.keys.toSeq.mkString(","))
+      }
     }
 
     if (files != null) {
-      files.foreach(addFile)
+      files.foreach(file => addFile(file, false, true))
+      if (addedFiles.nonEmpty) {
+        _conf.set("spark.app.initial.file.urls", addedFiles.keys.toSeq.mkString(","))
+      }
     }
 
     _executorMemory = _conf.getOption(EXECUTOR_MEMORY.key)
@@ -1510,7 +1502,7 @@ class SparkContext(config: SparkConf) extends Logging {
    * @note A path can be added only once. Subsequent additions of the same path are ignored.
    */
   def addFile(path: String): Unit = {
-    addFile(path, false)
+    addFile(path, false, false)
   }
 
   /**
@@ -1532,6 +1524,10 @@ class SparkContext(config: SparkConf) extends Logging {
    * @note A path can be added only once. Subsequent additions of the same path are ignored.
    */
   def addFile(path: String, recursive: Boolean): Unit = {
+    addFile(path, recursive, false)
+  }
+
+  private def addFile(path: String, recursive: Boolean, addedOnSubmit: Boolean): Unit = {
     val uri = new Path(path).toUri
     val schemeCorrectedURI = uri.getScheme match {
       case null => new File(path).getCanonicalFile.toURI
@@ -1569,7 +1565,7 @@ class SparkContext(config: SparkConf) extends Logging {
           path
         }
     }
-    val timestamp = System.currentTimeMillis
+    val timestamp = if (addedOnSubmit) startTime else System.currentTimeMillis
     if (addedFiles.putIfAbsent(key, timestamp).isEmpty) {
       logInfo(s"Added file $path at $key with timestamp $timestamp")
       // Fetch the file locally so that closures which are run on the driver can still use the
@@ -1579,7 +1575,7 @@ class SparkContext(config: SparkConf) extends Logging {
       postEnvironmentUpdate()
     } else {
       logWarning(s"The path $path has been added already. Overwriting of added paths " +
-       "is not supported in the current version.")
+        "is not supported in the current version.")
     }
   }
 
@@ -1612,7 +1608,8 @@ class SparkContext(config: SparkConf) extends Logging {
   }
 
   /**
-   * Get the max number of tasks that can be concurrent launched currently.
+   * Get the max number of tasks that can be concurrent launched based on the resources
+   * could be used, even if some of them are being used at the moment.
    * Note that please don't cache the value returned by this method, because the number can change
    * due to add/remove executors.
    *
@@ -1841,6 +1838,10 @@ class SparkContext(config: SparkConf) extends Logging {
    * @note A path can be added only once. Subsequent additions of the same path are ignored.
    */
   def addJar(path: String): Unit = {
+    addJar(path, false)
+  }
+
+  private def addJar(path: String, addedOnSubmit: Boolean): Unit = {
     def addLocalJarFile(file: File): String = {
       try {
         if (!file.exists()) {
@@ -1867,7 +1868,7 @@ class SparkContext(config: SparkConf) extends Logging {
           if (!fs.exists(hadoopPath)) {
             throw new FileNotFoundException(s"Jar ${path} not found")
           }
-          if (fs.isDirectory(hadoopPath)) {
+          if (fs.getFileStatus(hadoopPath).isDirectory) {
             throw new IllegalArgumentException(
               s"Directory ${path} is not allowed for addJar")
           }
@@ -1905,7 +1906,7 @@ class SparkContext(config: SparkConf) extends Logging {
         }
       }
       if (key != null) {
-        val timestamp = System.currentTimeMillis
+        val timestamp = if (addedOnSubmit) startTime else System.currentTimeMillis
         if (addedJars.putIfAbsent(key, timestamp).isEmpty) {
           logInfo(s"Added JAR $path at $key with timestamp $timestamp")
           postEnvironmentUpdate()
@@ -2019,9 +2020,6 @@ class SparkContext(config: SparkConf) extends Logging {
     Utils.tryLogNonFatalError {
       _progressBar.foreach(_.stop())
     }
-    if (isClientStandalone) {
-      releaseResources(_conf, SPARK_DRIVER_PREFIX, _resources, Utils.getProcessId)
-    }
     _taskScheduler = null
     // TODO: Cache.stop()?
     if (_env != null) {
@@ -2562,6 +2560,19 @@ object SparkContext extends Logging {
     }
   }
 
+  /**
+   * Called to ensure that SparkContext is created or accessed only on the Driver.
+   *
+   * Throws an exception if a SparkContext is about to be created in executors.
+   */
+  private def assertOnDriver(): Unit = {
+    if (TaskContext.get != null) {
+      // we're accessing it during task execution, fail.
+      throw new IllegalStateException(
+        "SparkContext should only be created and accessed on the driver.")
+    }
+  }
+
   /**
    * This function may be used to get or instantiate a SparkContext and register it as a
    * singleton object. Because we can only have one active SparkContext per JVM,
@@ -2781,8 +2792,9 @@ object SparkContext extends Logging {
       }
       // some cluster managers don't set the EXECUTOR_CORES config by default (standalone
       // and mesos coarse grained), so we can't rely on that config for those.
-      val shouldCheckExecCores = executorCores.isDefined || sc.conf.contains(EXECUTOR_CORES) ||
+      var shouldCheckExecCores = executorCores.isDefined || sc.conf.contains(EXECUTOR_CORES) ||
         (master.equalsIgnoreCase("yarn") || master.startsWith("k8s"))
+      shouldCheckExecCores &= !sc.conf.get(SKIP_VALIDATE_CORES_TESTING)
 
       // Number of cores per executor must meet at least one task requirement.
       if (shouldCheckExecCores && execCores < taskCores) {
@@ -2838,7 +2850,7 @@ object SparkContext extends Logging {
           limitingResourceName = taskReq.resourceName
         }
       }
-      if(!shouldCheckExecCores && Utils.isDynamicAllocationEnabled(sc.conf)) {
+      if(!shouldCheckExecCores) {
         // if we can't rely on the executor cores config throw a warning for user
         logWarning("Please ensure that the number of slots available on your " +
           "executors is limited by the number of cores to task cpus and not another " +
@@ -2862,7 +2874,7 @@ object SparkContext extends Logging {
             s"result in wasted resources due to resource ${limitingResourceName} limiting the " +
             s"number of runnable tasks per executor to: ${numSlots}. Please adjust " +
             s"your configuration."
-          if (Utils.isTesting) {
+          if (sc.conf.get(RESOURCES_WARNING_TESTING)) {
             throw new SparkException(message)
           } else {
             logWarning(message)
diff --git a/core/src/main/scala/org/apache/spark/SparkException.scala b/core/src/main/scala/org/apache/spark/SparkException.scala
index 4ad9a0cc4b103..41382133bd84c 100644
--- a/core/src/main/scala/org/apache/spark/SparkException.scala
+++ b/core/src/main/scala/org/apache/spark/SparkException.scala
@@ -43,3 +43,10 @@ private[spark] case class SparkUserAppException(exitCode: Int)
  */
 private[spark] case class ExecutorDeadException(message: String)
   extends SparkException(message)
+
+/**
+ * Exception thrown when Spark returns different result after upgrading to a new version.
+ */
+private[spark] class SparkUpgradeException(version: String, message: String, cause: Throwable)
+  extends RuntimeException("You may get a different result due to the upgrading of Spark" +
+    s" $version: $message", cause)
diff --git a/core/src/main/scala/org/apache/spark/TaskEndReason.scala b/core/src/main/scala/org/apache/spark/TaskEndReason.scala
index b13028f868072..6606d317e7b86 100644
--- a/core/src/main/scala/org/apache/spark/TaskEndReason.scala
+++ b/core/src/main/scala/org/apache/spark/TaskEndReason.scala
@@ -90,7 +90,8 @@ case class FetchFailed(
   extends TaskFailedReason {
   override def toErrorString: String = {
     val bmAddressString = if (bmAddress == null) "null" else bmAddress.toString
-    s"FetchFailed($bmAddressString, shuffleId=$shuffleId, mapIndex=$mapIndex, " +
+    val mapIndexString = if (mapIndex == Int.MinValue) "Unknown" else mapIndex.toString
+    s"FetchFailed($bmAddressString, shuffleId=$shuffleId, mapIndex=$mapIndexString, " +
       s"mapId=$mapId, reduceId=$reduceId, message=\n$message\n)"
   }
 
diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala
index d459627930f4c..054e7b07d1da6 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -179,11 +179,20 @@ private[spark] object TestUtils {
       destDir: File,
       toStringValue: String = "",
       baseClass: String = null,
-      classpathUrls: Seq[URL] = Seq.empty): File = {
+      classpathUrls: Seq[URL] = Seq.empty,
+      implementsClasses: Seq[String] = Seq.empty,
+      extraCodeBody: String = ""): File = {
     val extendsText = Option(baseClass).map { c => s" extends ${c}" }.getOrElse("")
+    val implementsText =
+      "implements " + (implementsClasses :+ "java.io.Serializable").mkString(", ")
     val sourceFile = new JavaSourceFromString(className,
-      "public class " + className + extendsText + " implements java.io.Serializable {" +
-      "  @Override public String toString() { return \"" + toStringValue + "\"; }}")
+      s"""
+         |public class $className $extendsText $implementsText {
+         |  @Override public String toString() { return "$toStringValue"; }
+         |
+         |  $extraCodeBody
+         |}
+        """.stripMargin)
     createCompiledClass(className, destDir, sourceFile, classpathUrls)
   }
 
@@ -240,6 +249,19 @@ private[spark] object TestUtils {
     attempt.isSuccess && attempt.get == 0
   }
 
+  def isPythonVersionAtLeast38(): Boolean = {
+    val attempt = if (Utils.isWindows) {
+      Try(Process(Seq("cmd.exe", "/C", "python3 --version"))
+        .run(ProcessLogger(s => s.startsWith("Python 3.8") || s.startsWith("Python 3.9")))
+        .exitValue())
+    } else {
+      Try(Process(Seq("sh", "-c", "python3 --version"))
+        .run(ProcessLogger(s => s.startsWith("Python 3.8") || s.startsWith("Python 3.9")))
+        .exitValue())
+    }
+    attempt.isSuccess && attempt.get == 0
+  }
+
   /**
    * Returns the response code from an HTTP(S) URL.
    */
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
index 6dc1721f56adf..a577194a48006 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
@@ -168,6 +168,21 @@ private[spark] object PythonRDD extends Logging {
     serveIterator(rdd.collect().iterator, s"serve RDD ${rdd.id}")
   }
 
+  /**
+   * A helper function to collect an RDD as an iterator, then serve it via socket.
+   * This method is similar with `PythonRDD.collectAndServe`, but user can specify job group id,
+   * job description, and interruptOnCancel option.
+   */
+  def collectAndServeWithJobGroup[T](
+      rdd: RDD[T],
+      groupId: String,
+      description: String,
+      interruptOnCancel: Boolean): Array[Any] = {
+    val sc = rdd.sparkContext
+    sc.setJobGroup(groupId, description, interruptOnCancel)
+    serveIterator(rdd.collect().iterator, s"serve RDD ${rdd.id}")
+  }
+
   /**
    * A helper function to create a local RDD iterator and serve it via socket. Partitions are
    * are collected as separate jobs, by order of index. Partition data is first requested by a
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index 658e0d593a167..820cb798050f0 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -24,8 +24,13 @@ import java.nio.charset.StandardCharsets.UTF_8
 import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 import scala.util.control.NonFatal
 
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods.{compact, render}
+
 import org.apache.spark._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{BUFFER_SIZE, EXECUTOR_CORES}
@@ -225,7 +230,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
             /* backlog */ 1,
             InetAddress.getByName("localhost")))
           // A call to accept() for ServerSocket shall block infinitely.
-          serverSocket.map(_.setSoTimeout(0))
+          serverSocket.foreach(_.setSoTimeout(0))
           new Thread("accept-connections") {
             setDaemon(true)
 
@@ -238,13 +243,18 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
                   sock.setSoTimeout(10000)
                   authHelper.authClient(sock)
                   val input = new DataInputStream(sock.getInputStream())
-                  input.readInt() match {
+                  val requestMethod = input.readInt()
+                  // The BarrierTaskContext function may wait infinitely, socket shall not timeout
+                  // before the function finishes.
+                  sock.setSoTimeout(0)
+                  requestMethod match {
                     case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
-                      // The barrier() function may wait infinitely, socket shall not timeout
-                      // before the function finishes.
-                      sock.setSoTimeout(0)
-                      barrierAndServe(sock)
-
+                      barrierAndServe(requestMethod, sock)
+                    case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
+                      val length = input.readInt()
+                      val message = new Array[Byte](length)
+                      input.readFully(message)
+                      barrierAndServe(requestMethod, sock, new String(message, UTF_8))
                     case _ =>
                       val out = new DataOutputStream(new BufferedOutputStream(
                         sock.getOutputStream))
@@ -395,15 +405,24 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
     }
 
     /**
-     * Gateway to call BarrierTaskContext.barrier().
+     * Gateway to call BarrierTaskContext methods.
      */
-    def barrierAndServe(sock: Socket): Unit = {
-      require(serverSocket.isDefined, "No available ServerSocket to redirect the barrier() call.")
-
+    def barrierAndServe(requestMethod: Int, sock: Socket, message: String = ""): Unit = {
+      require(
+        serverSocket.isDefined,
+        "No available ServerSocket to redirect the BarrierTaskContext method call."
+      )
       val out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream))
       try {
-        context.asInstanceOf[BarrierTaskContext].barrier()
-        writeUTF(BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS, out)
+        val messages = requestMethod match {
+          case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
+            context.asInstanceOf[BarrierTaskContext].barrier()
+            Array(BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS)
+          case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
+            context.asInstanceOf[BarrierTaskContext].allGather(message)
+        }
+        out.writeInt(messages.length)
+        messages.foreach(writeUTF(_, out))
       } catch {
         case e: SparkException =>
           writeUTF(e.getMessage, out)
@@ -638,6 +657,7 @@ private[spark] object SpecialLengths {
 
 private[spark] object BarrierTaskContextMessageProtocol {
   val BARRIER_FUNCTION = 1
+  val ALL_GATHER_FUNCTION = 2
   val BARRIER_RESULT_SUCCESS = "success"
   val ERROR_UNRECOGNIZED_FUNCTION = "Not recognized function call from python side."
 }
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
index 62d60475985b3..78eb6907c3898 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
@@ -27,7 +27,7 @@ import org.apache.spark.SparkContext
 import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
 
 private[spark] object PythonUtils {
-  val PY4J_ZIP_NAME = "py4j-0.10.8.1-src.zip"
+  val PY4J_ZIP_NAME = "py4j-0.10.9-src.zip"
 
   /** Get the PYTHONPATH for PySpark, either from SPARK_HOME, if it is set, or from our JAR */
   def sparkPythonPath: String = {
@@ -85,4 +85,8 @@ private[spark] object PythonUtils {
   def getBroadcastThreshold(sc: JavaSparkContext): Long = {
     sc.conf.get(org.apache.spark.internal.config.BROADCAST_FOR_UDF_COMPRESSION_THRESHOLD)
   }
+
+  def getSparkBufferSize(sc: JavaSparkContext): Int = {
+    sc.conf.get(org.apache.spark.internal.config.BUFFER_SIZE)
+  }
 }
diff --git a/core/src/main/scala/org/apache/spark/api/r/RUtils.scala b/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
index 311fade127839..784a57e7b98a8 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
@@ -43,9 +43,9 @@ private[spark] object RUtils {
    * Check if SparkR is installed before running tests that use SparkR.
    */
   def isSparkRInstalled: Boolean = {
-    localSparkRPackagePath.filter { pkgDir =>
+    localSparkRPackagePath.exists { pkgDir =>
       new File(Seq(pkgDir, "SparkR").mkString(File.separator)).exists
-    }.isDefined
+    }
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
index 77fbbc08c2103..1024d9b5060bc 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
@@ -133,22 +133,30 @@ private[spark] class TorrentBroadcast[T: ClassTag](obj: T, id: Long)
     if (!blockManager.putSingle(broadcastId, value, MEMORY_AND_DISK, tellMaster = false)) {
       throw new SparkException(s"Failed to store $broadcastId in BlockManager")
     }
-    val blocks =
-      TorrentBroadcast.blockifyObject(value, blockSize, SparkEnv.get.serializer, compressionCodec)
-    if (checksumEnabled) {
-      checksums = new Array[Int](blocks.length)
-    }
-    blocks.zipWithIndex.foreach { case (block, i) =>
+    try {
+      val blocks =
+        TorrentBroadcast.blockifyObject(value, blockSize, SparkEnv.get.serializer, compressionCodec)
       if (checksumEnabled) {
-        checksums(i) = calcChecksum(block)
+        checksums = new Array[Int](blocks.length)
       }
-      val pieceId = BroadcastBlockId(id, "piece" + i)
-      val bytes = new ChunkedByteBuffer(block.duplicate())
-      if (!blockManager.putBytes(pieceId, bytes, MEMORY_AND_DISK_SER, tellMaster = true)) {
-        throw new SparkException(s"Failed to store $pieceId of $broadcastId in local BlockManager")
+      blocks.zipWithIndex.foreach { case (block, i) =>
+        if (checksumEnabled) {
+          checksums(i) = calcChecksum(block)
+        }
+        val pieceId = BroadcastBlockId(id, "piece" + i)
+        val bytes = new ChunkedByteBuffer(block.duplicate())
+        if (!blockManager.putBytes(pieceId, bytes, MEMORY_AND_DISK_SER, tellMaster = true)) {
+          throw new SparkException(s"Failed to store $pieceId of $broadcastId " +
+            s"in local BlockManager")
+        }
       }
+      blocks.length
+    } catch {
+      case t: Throwable =>
+        logError(s"Store broadcast $broadcastId fail, remove all pieces of the broadcast")
+        blockManager.removeBroadcast(id, tellMaster = true)
+        throw t
     }
-    blocks.length
   }
 
   /** Fetch torrent blocks from the driver and/or other executors. */
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index 1180501e8c738..6f799a542bc1e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -462,6 +462,9 @@ private[spark] object SparkHadoopUtil {
     for ((key, value) <- conf.getAll if key.startsWith("spark.hadoop.")) {
       hadoopConf.set(key.substring("spark.hadoop.".length), value)
     }
+    if (conf.getOption("spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version").isEmpty) {
+      hadoopConf.set("mapreduce.fileoutputcommitter.algorithm.version", "1")
+    }
   }
 
   private def appendSparkHiveConfigs(conf: SparkConf, hadoopConf: Configuration): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index 3f7cfea778ac6..3090a3b10a97c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -513,7 +513,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
         |                              directory of each executor. File paths of these files
         |                              in executors can be accessed via SparkFiles.get(fileName).
         |
-        |  --conf PROP=VALUE           Arbitrary Spark configuration property.
+        |  --conf, -c PROP=VALUE       Arbitrary Spark configuration property.
         |  --properties-file FILE      Path to a file from which to load extra properties. If not
         |                              specified, this will look for conf/spark-defaults.conf.
         |
diff --git a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
index 65bf4351ebfd9..c7c31a85b0636 100644
--- a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
@@ -17,30 +17,21 @@
 
 package org.apache.spark.deploy
 
-import java.io.{File, RandomAccessFile}
-import java.nio.channels.{FileLock, OverlappingFileLockException}
+import java.io.File
 import java.nio.file.Files
 
 import scala.collection.mutable
-import scala.util.Random
 import scala.util.control.NonFatal
 
 import org.json4s.{DefaultFormats, Extraction}
-import org.json4s.jackson.JsonMethods.{compact, parse, render}
+import org.json4s.jackson.JsonMethods.{compact, render}
 
-import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.SparkException
 import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config.{SPARK_RESOURCES_COORDINATE, SPARK_RESOURCES_DIR}
 import org.apache.spark.resource.{ResourceAllocation, ResourceID, ResourceInformation, ResourceRequirement}
-import org.apache.spark.resource.ResourceUtils.{parseResourceRequirements, withResourcesJson}
 import org.apache.spark.util.Utils
 
 private[spark] object StandaloneResourceUtils extends Logging {
-  // These directory/files are used to coordinate the resources between
-  // the drivers/workers on the host in Spark Standalone.
-  val SPARK_RESOURCES_COORDINATE_DIR = "spark-resources"
-  val ALLOCATED_RESOURCES_FILE = "__allocated_resources__.json"
-  val RESOURCES_LOCK_FILE = "__allocated_resources__.lock"
 
   /**
    * A mutable resource information which provides more efficient modification on addresses.
@@ -86,249 +77,6 @@ private[spark] object StandaloneResourceUtils extends Logging {
     }
   }
 
-  /**
-   * Assigns (if coordinate needed) resources to workers/drivers from the same host to avoid
-   * address conflict.
-   *
-   * This function works in three steps. First, acquiring the lock on RESOURCES_LOCK_FILE
-   * to achieve synchronization among workers and drivers. Second, getting all allocated
-   * resources from ALLOCATED_RESOURCES_FILE and assigning isolated resources to the worker
-   * or driver after differentiating available resources in discovered resources from
-   * allocated resources. If available resources don't meet worker's or driver's requirement,
-   * try to update allocated resources by excluding the resource allocation if its related
-   * process has already terminated and do the assignment again. If still don't meet requirement,
-   * exception should be thrown. Third, updating ALLOCATED_RESOURCES_FILE with new allocated
-   * resources along with pid for the worker or driver. Then, return allocated resources
-   * information after releasing the lock.
-   *
-   * @param conf SparkConf
-   * @param componentName spark.driver / spark.worker
-   * @param resources the resources found by worker/driver on the host
-   * @param pid the process id of worker/driver to acquire resources.
-   * @return allocated resources for the worker/driver or throws exception if can't
-   *         meet worker/driver's requirement
-   */
-  def acquireResources(
-      conf: SparkConf,
-      componentName: String,
-      resources: Map[String, ResourceInformation],
-      pid: Int)
-    : Map[String, ResourceInformation] = {
-    if (!needCoordinate(conf)) {
-      return resources
-    }
-    val resourceRequirements = parseResourceRequirements(conf, componentName)
-    if (resourceRequirements.isEmpty) {
-      return Map.empty
-    }
-    val lock = acquireLock(conf)
-    try {
-      val resourcesFile = new File(getOrCreateResourcesDir(conf), ALLOCATED_RESOURCES_FILE)
-      // all allocated resources in ALLOCATED_RESOURCES_FILE, can be updated if any allocations'
-      // related processes detected to be terminated while checking pids below.
-      var origAllocation = Seq.empty[StandaloneResourceAllocation]
-      // Map[pid -> Map[resourceName -> Addresses[]]]
-      var allocated = {
-        if (resourcesFile.exists()) {
-          origAllocation = allocatedStandaloneResources(resourcesFile.getPath)
-          val allocations = origAllocation.map { resource =>
-            val resourceMap = {
-              resource.allocations.map { allocation =>
-                allocation.id.resourceName -> allocation.addresses.toArray
-              }.toMap
-            }
-            resource.pid -> resourceMap
-          }.toMap
-          allocations
-        } else {
-          Map.empty[Int, Map[String, Array[String]]]
-        }
-      }
-
-      // new allocated resources for worker or driver,
-      // map from resource name to its allocated addresses.
-      var newAssignments: Map[String, Array[String]] = null
-      // Whether we've checked process status and we'll only do the check at most once.
-      // Do the check iff the available resources can't meet the requirements at the first time.
-      var checked = false
-      // Whether we need to keep allocating for the worker/driver and we'll only go through
-      // the loop at most twice.
-      var keepAllocating = true
-      while (keepAllocating) {
-        keepAllocating = false
-        // store the pid whose related allocated resources conflict with
-        // discovered resources passed in.
-        val pidsToCheck = mutable.Set[Int]()
-        newAssignments = resourceRequirements.map { req =>
-          val rName = req.resourceName
-          val amount = req.amount
-          // initially, we must have available.length >= amount as we've done pre-check previously
-          var available = resources(rName).addresses
-          // gets available resource addresses by excluding all
-          // allocated resource addresses from discovered resources
-          allocated.foreach { a =>
-            val thePid = a._1
-            val resourceMap = a._2
-            val assigned = resourceMap.getOrElse(rName, Array.empty)
-            val retained = available.diff(assigned)
-            // if len(retained) < len(available) after differ to assigned, then, there must be
-            // some conflicting resources addresses between available and assigned. So, we should
-            // store its pid here to check whether it's alive in case we don't find enough
-            // resources after traversal all allocated resources.
-            if (retained.length < available.length && !checked) {
-              pidsToCheck += thePid
-            }
-            if (retained.length >= amount) {
-              available = retained
-            } else if (checked) {
-              keepAllocating = false
-              throw new SparkException(s"No more resources available since they've already" +
-                s" assigned to other workers/drivers.")
-            } else {
-              keepAllocating = true
-            }
-          }
-          val assigned = {
-            if (keepAllocating) { // can't meet the requirement
-              // excludes the allocation whose related process has already been terminated.
-              val (invalid, valid) = allocated.partition { a =>
-                pidsToCheck(a._1) && !(Utils.isTesting || Utils.isProcessRunning(a._1))}
-              allocated = valid
-              origAllocation = origAllocation.filter(
-                allocation => !invalid.contains(allocation.pid))
-              checked = true
-              // note this is a meaningless return value, just to avoid creating any new object
-              available
-            } else {
-              available.take(amount)
-            }
-          }
-          rName -> assigned
-        }.toMap
-      }
-      val newAllocation = {
-        val allocations = newAssignments.map { case (rName, addresses) =>
-          ResourceAllocation(new ResourceID(componentName, rName), addresses)
-        }.toSeq
-        StandaloneResourceAllocation(pid, allocations)
-      }
-      writeResourceAllocationJson(
-        componentName, origAllocation ++ Seq(newAllocation), resourcesFile)
-      newAllocation.toResourceInformationMap
-    } finally {
-      releaseLock(lock)
-    }
-  }
-
-  /**
-   * Frees (if coordinate needed) all the resources a worker/driver (pid) has in one shot
-   * to make those resources be available for other workers/drivers on the same host.
-   * @param conf SparkConf
-   * @param componentName spark.driver / spark.worker
-   * @param toRelease the resources expected to release
-   * @param pid the process id of worker/driver to release resources.
-   */
-  def releaseResources(
-      conf: SparkConf,
-      componentName: String,
-      toRelease: Map[String, ResourceInformation],
-      pid: Int)
-    : Unit = {
-    if (!needCoordinate(conf)) {
-      return
-    }
-    if (toRelease != null && toRelease.nonEmpty) {
-      val lock = acquireLock(conf)
-      try {
-        val resourcesFile = new File(getOrCreateResourcesDir(conf), ALLOCATED_RESOURCES_FILE)
-        if (resourcesFile.exists()) {
-          val (target, others) =
-            allocatedStandaloneResources(resourcesFile.getPath).partition(_.pid == pid)
-          if (target.nonEmpty) {
-            if (others.isEmpty) {
-              if (!resourcesFile.delete()) {
-                logError(s"Failed to delete $ALLOCATED_RESOURCES_FILE.")
-              }
-            } else {
-              writeResourceAllocationJson(componentName, others, resourcesFile)
-            }
-            logDebug(s"$componentName(pid=$pid) released resources: ${toRelease.mkString("\n")}")
-          } else {
-            logWarning(s"$componentName(pid=$pid) has already released its resources.")
-          }
-        }
-      } finally {
-        releaseLock(lock)
-      }
-    }
-  }
-
-  private def acquireLock(conf: SparkConf): FileLock = {
-    val resourcesDir = getOrCreateResourcesDir(conf)
-    val lockFile = new File(resourcesDir, RESOURCES_LOCK_FILE)
-    val lockFileChannel = new RandomAccessFile(lockFile, "rw").getChannel
-    var keepTry = true
-    var lock: FileLock = null
-    while (keepTry) {
-      try {
-        lock = lockFileChannel.lock()
-        logInfo(s"Acquired lock on $RESOURCES_LOCK_FILE.")
-        keepTry = false
-      } catch {
-        case e: OverlappingFileLockException =>
-          // This exception throws when we're in LocalSparkCluster mode. FileLock is designed
-          // to be used across JVMs, but our LocalSparkCluster is designed to launch multiple
-          // workers in the same JVM. As a result, when an worker in LocalSparkCluster try to
-          // acquire the lock on `resources.lock` which already locked by other worker, we'll
-          // hit this exception. So, we should manually control it.
-          keepTry = true
-          // there may be multiple workers race for the lock,
-          // so, sleep for a random time to avoid possible conflict
-          val duration = Random.nextInt(1000) + 1000
-          Thread.sleep(duration)
-      }
-    }
-    assert(lock != null, s"Acquired null lock on $RESOURCES_LOCK_FILE.")
-    lock
-  }
-
-  private def releaseLock(lock: FileLock): Unit = {
-    try {
-      lock.release()
-      lock.channel().close()
-      logInfo(s"Released lock on $RESOURCES_LOCK_FILE.")
-    } catch {
-      case e: Exception =>
-        logError(s"Error while releasing lock on $RESOURCES_LOCK_FILE.", e)
-    }
-  }
-
-  private def getOrCreateResourcesDir(conf: SparkConf): File = {
-    val coordinateDir = new File(conf.get(SPARK_RESOURCES_DIR).getOrElse {
-      val sparkHome = if (Utils.isTesting) {
-        assert(sys.props.contains("spark.test.home") ||
-          sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-        sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-      } else {
-        sys.env.getOrElse("SPARK_HOME", ".")
-      }
-      sparkHome
-    })
-    val resourceDir = new File(coordinateDir, SPARK_RESOURCES_COORDINATE_DIR)
-    if (!resourceDir.exists()) {
-      Utils.createDirectory(resourceDir)
-    }
-    resourceDir
-  }
-
-  private def allocatedStandaloneResources(resourcesFile: String)
-  : Seq[StandaloneResourceAllocation] = {
-    withResourcesJson[StandaloneResourceAllocation](resourcesFile) { json =>
-      implicit val formats = DefaultFormats
-      parse(json).extract[Seq[StandaloneResourceAllocation]]
-    }
-  }
-
   /**
    * Save the allocated resources of driver(cluster only) or executor into a JSON formatted
    * resources file. Used in Standalone only.
@@ -372,11 +120,6 @@ private[spark] object StandaloneResourceUtils extends Logging {
     Files.write(jsonFile.toPath, compact(render(allocationJson)).getBytes())
   }
 
-  /** Whether needs to coordinate resources among workers and drivers for user */
-  def needCoordinate(conf: SparkConf): Boolean = {
-    conf.get(SPARK_RESOURCES_COORDINATE)
-  }
-
   def toMutable(immutableResources: Map[String, ResourceInformation])
     : Map[String, MutableResourceInfo] = {
     immutableResources.map { case (rName, rInfo) =>
@@ -406,11 +149,11 @@ private[spark] object StandaloneResourceUtils extends Logging {
 
   // used for UI
   def formatResourcesUsed(
-      resourcesTotal: Map[String, ResourceInformation],
-      resourcesUsed: Map[String, ResourceInformation]): String = {
-    resourcesTotal.map { case (rName, rInfo) =>
-      val used = resourcesUsed(rName).addresses.length
-      val total = rInfo.addresses.length
+      resourcesTotal: Map[String, Int],
+      resourcesUsed: Map[String, Int]): String = {
+    resourcesTotal.map { case (rName, totalSize) =>
+      val used = resourcesUsed(rName)
+      val total = totalSize
       s"$used / $total $rName"
     }.mkString(", ")
   }
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
index 9f63a6441a838..b4771c80a175f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
@@ -116,7 +116,7 @@ object EventLogFileReader {
 
   def apply(fs: FileSystem, status: FileStatus): Option[EventLogFileReader] = {
     if (isSingleEventLog(status)) {
-      Some(new SingleFileEventLogFileReader(fs, status.getPath))
+      Some(new SingleFileEventLogFileReader(fs, status.getPath, Option(status)))
     } else if (isRollingEventLogs(status)) {
       Some(new RollingEventLogFilesFileReader(fs, status.getPath))
     } else {
@@ -164,10 +164,13 @@ object EventLogFileReader {
  * FileNotFoundException could occur if the log file is renamed before getting the
  * status of log file.
  */
-class SingleFileEventLogFileReader(
+private[history] class SingleFileEventLogFileReader(
     fs: FileSystem,
-    path: Path) extends EventLogFileReader(fs, path) {
-  private lazy val status = fileSystem.getFileStatus(rootPath)
+    path: Path,
+    maybeStatus: Option[FileStatus]) extends EventLogFileReader(fs, path) {
+  private lazy val status = maybeStatus.getOrElse(fileSystem.getFileStatus(rootPath))
+
+  def this(fs: FileSystem, path: Path) = this(fs, path, None)
 
   override def lastIndex: Option[Long] = None
 
@@ -203,7 +206,7 @@ class SingleFileEventLogFileReader(
  * This reader lists the files only once; if caller would like to play with updated list,
  * it needs to create another reader instance.
  */
-class RollingEventLogFilesFileReader(
+private[history] class RollingEventLogFilesFileReader(
     fs: FileSystem,
     path: Path) extends EventLogFileReader(fs, path) {
   import RollingEventLogFilesWriter._
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
index 1d58d054b7825..7d44cbd9f64f9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
@@ -166,7 +166,8 @@ object EventLogFileWriter {
   val IN_PROGRESS = ".inprogress"
   val COMPACTED = ".compact"
 
-  val LOG_FILE_PERMISSIONS = new FsPermission(Integer.parseInt("770", 8).toShort)
+  val LOG_FILE_PERMISSIONS = new FsPermission(Integer.parseInt("660", 8).toShort)
+  val LOG_FOLDER_PERMISSIONS = new FsPermission(Integer.parseInt("770", 8).toShort)
 
   def apply(
       appId: String,
@@ -317,7 +318,8 @@ class RollingEventLogFilesWriter(
       throw new IOException(s"Target log directory already exists ($logDirForAppPath)")
     }
 
-    fileSystem.mkdirs(logDirForAppPath, EventLogFileWriter.LOG_FILE_PERMISSIONS)
+    // SPARK-30860: use the class method to avoid the umask causing permission issues
+    FileSystem.mkdirs(fileSystem, logDirForAppPath, EventLogFileWriter.LOG_FOLDER_PERMISSIONS)
     createAppStatusFile(inProgress = true)
     rollEventLogFile()
   }
@@ -361,7 +363,9 @@ class RollingEventLogFilesWriter(
 
   private def createAppStatusFile(inProgress: Boolean): Unit = {
     val appStatusPath = getAppStatusFilePath(logDirForAppPath, appId, appAttemptId, inProgress)
-    val outputStream = fileSystem.create(appStatusPath)
+    // SPARK-30860: use the class method to avoid the umask causing permission issues
+    val outputStream = FileSystem.create(fileSystem, appStatusPath,
+      EventLogFileWriter.LOG_FILE_PERMISSIONS)
     // we intentionally create zero-byte file to minimize the cost
     outputStream.close()
   }
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index 99d3eceb1121a..b31333fc48218 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -27,6 +27,7 @@ import java.util.zip.ZipOutputStream
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.io.Source
+import scala.util.control.NonFatal
 import scala.xml.Node
 
 import com.fasterxml.jackson.annotation.JsonIgnore
@@ -459,9 +460,21 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
       val newLastScanTime = clock.getTimeMillis()
       logDebug(s"Scanning $logDir with lastScanTime==$lastScanTime")
 
+      // Mark entries that are processing as not stale. Such entries do not have a chance to be
+      // updated with the new 'lastProcessed' time and thus any entity that completes processing
+      // right after this check and before the check for stale entities will be identified as stale
+      // and will be deleted from the UI until the next 'checkForLogs' run.
+      val notStale = mutable.HashSet[String]()
       val updated = Option(fs.listStatus(new Path(logDir))).map(_.toSeq).getOrElse(Nil)
         .filter { entry => !isBlacklisted(entry.getPath) }
-        .filter { entry => !isProcessing(entry.getPath) }
+        .filter { entry =>
+          if (isProcessing(entry.getPath)) {
+            notStale.add(entry.getPath.toString())
+            false
+          } else {
+            true
+          }
+        }
         .flatMap { entry => EventLogFileReader(fs, entry) }
         .filter { reader =>
           try {
@@ -519,10 +532,21 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
               // If the file is currently not being tracked by the SHS, add an entry for it and try
               // to parse it. This will allow the cleaner code to detect the file as stale later on
               // if it was not possible to parse it.
-              listing.write(LogInfo(reader.rootPath.toString(), newLastScanTime, LogType.EventLogs,
-                None, None, reader.fileSizeForLastIndex, reader.lastIndex, None,
-                reader.completed))
-              reader.fileSizeForLastIndex > 0
+              try {
+                listing.write(LogInfo(reader.rootPath.toString(), newLastScanTime,
+                  LogType.EventLogs, None, None, reader.fileSizeForLastIndex, reader.lastIndex,
+                  None, reader.completed))
+                reader.fileSizeForLastIndex > 0
+              } catch {
+                case _: FileNotFoundException => false
+                case NonFatal(e) =>
+                  logWarning(s"Error while reading new log ${reader.rootPath}", e)
+                  false
+              }
+
+            case NonFatal(e) =>
+              logWarning(s"Error while filtering log ${reader.rootPath}", e)
+              false
           }
         }
         .sortWith { case (entry1, entry2) =>
@@ -550,12 +574,14 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
         .last(newLastScanTime - 1)
         .asScala
         .toList
-      stale.filterNot(isProcessing).foreach { log =>
-        log.appId.foreach { appId =>
-          cleanAppData(appId, log.attemptId, log.logPath)
-          listing.delete(classOf[LogInfo], log.logPath)
+      stale.filterNot(isProcessing)
+        .filterNot(info => notStale.contains(info.logPath))
+        .foreach { log =>
+          log.appId.foreach { appId =>
+            cleanAppData(appId, log.attemptId, log.logPath)
+            listing.delete(classOf[LogInfo], log.logPath)
+          }
         }
-      }
 
       lastScanTime.set(newLastScanTime)
     } catch {
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index 62cac261ae014..7e0d311311f30 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -69,6 +69,9 @@ class HistoryServer(
 
   private val loaderServlet = new HttpServlet {
     protected override def doGet(req: HttpServletRequest, res: HttpServletResponse): Unit = {
+
+      res.setContentType("text/html;charset=utf-8")
+
       // Parse the URI created by getAttemptURI(). It contains an app ID and an optional
       // attempt ID (separated by a slash).
       val parts = Option(req.getPathInfo()).getOrElse("").split("/")
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
index 0a1f33395ad62..fe18b03a7d603 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
@@ -75,14 +75,29 @@ private class HistoryServerDiskManager(
 
     // Go through the recorded store directories and remove any that may have been removed by
     // external code.
-    val orphans = listing.view(classOf[ApplicationStoreInfo]).asScala.filter { info =>
-      !new File(info.path).exists()
-    }.toSeq
+    val (existences, orphans) = listing
+      .view(classOf[ApplicationStoreInfo])
+      .asScala
+      .toSeq
+      .partition { info =>
+        new File(info.path).exists()
+      }
 
     orphans.foreach { info =>
       listing.delete(info.getClass(), info.path)
     }
 
+    // Reading level db would trigger table file compaction, then it may cause size of level db
+    // directory changed. When service restarts, "currentUsage" is calculated from real directory
+    // size. Update "ApplicationStoreInfo.size" to ensure "currentUsage" equals
+    // sum of "ApplicationStoreInfo.size".
+    existences.foreach { info =>
+      val fileSize = sizeOf(new File(info.path))
+      if (fileSize != info.size) {
+        listing.write(info.copy(size = fileSize))
+      }
+    }
+
     logInfo("Initialized disk manager: " +
       s"current usage = ${Utils.bytesToString(currentUsage.get())}, " +
       s"max usage = ${Utils.bytesToString(maxUsage)}")
@@ -233,7 +248,7 @@ private class HistoryServerDiskManager(
     }
   }
 
-  private def appStorePath(appId: String, attemptId: Option[String]): File = {
+  private[history] def appStorePath(appId: String, attemptId: Option[String]): File = {
     val fileName = appId + attemptId.map("_" + _).getOrElse("") + ".ldb"
     new File(appStoreDir, fileName)
   }
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/DriverInfo.scala b/core/src/main/scala/org/apache/spark/deploy/master/DriverInfo.scala
index bf68ba8e15af4..252e7048ba105 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/DriverInfo.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/DriverInfo.scala
@@ -37,7 +37,7 @@ private[deploy] class DriverInfo(
   @transient var worker: Option[WorkerInfo] = None
   // resources(e.f. gpu/fpga) allocated to this driver
   // map from resource name to ResourceInformation
-  private var _resources: Map[String, ResourceInformation] = _
+  private var _resources: Map[String, ResourceInformation] = Map.empty
 
   init()
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 8d3795cae707a..ab6cb304b59eb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -143,7 +143,7 @@ private[deploy] class Master(
     logInfo(s"Running Spark version ${org.apache.spark.SPARK_VERSION}")
     webUi = new MasterWebUI(this, webUiPort)
     webUi.bind()
-    masterWebUiUrl = s"${webUi.scheme}$masterPublicAddress:${webUi.boundPort}"
+    masterWebUiUrl = webUi.webUrl
     if (reverseProxy) {
       masterWebUiUrl = conf.get(UI_REVERSE_PROXY_URL).orElse(Some(masterWebUiUrl)).get
       webUi.addProxy()
@@ -704,7 +704,9 @@ private[deploy] class Master(
         val usableWorkers = workers.toArray.filter(_.state == WorkerState.ALIVE)
           .filter(canLaunchExecutor(_, app.desc))
           .sortBy(_.coresFree).reverse
-        if (waitingApps.length == 1 && usableWorkers.isEmpty) {
+        val appMayHang = waitingApps.length == 1 &&
+          waitingApps.head.executors.isEmpty && usableWorkers.isEmpty
+        if (appMayHang) {
           logWarning(s"App ${app.id} requires more resource than any of Workers could have.")
         }
         val assignedCores = scheduleExecutorsOnWorkers(app, usableWorkers, spreadOutApps)
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
index f64b449851d86..fcbeba9eef1a9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
@@ -76,19 +76,17 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
 
   private def formatMasterResourcesInUse(aliveWorkers: Array[WorkerInfo]): String = {
     val totalInfo = aliveWorkers.map(_.resourcesInfo)
-      .map(resources => toMutable(resources))
       .flatMap(_.toIterator)
       .groupBy(_._1) // group by resource name
       .map { case (rName, rInfoArr) =>
-        rName -> rInfoArr.map(_._2).reduce(_ + _)
-      }.map { case (k, v) => (k, v.toResourceInformation) }
+      rName -> rInfoArr.map(_._2.addresses.size).sum
+    }
     val usedInfo = aliveWorkers.map(_.resourcesInfoUsed)
-      .map (resources => toMutable(resources))
       .flatMap(_.toIterator)
       .groupBy(_._1) // group by resource name
       .map { case (rName, rInfoArr) =>
-      rName -> rInfoArr.map(_._2).reduce(_ + _)
-    }.map { case (k, v) => (k, v.toResourceInformation) }
+      rName -> rInfoArr.map(_._2.addresses.size).sum
+    }
     formatResourcesUsed(totalInfo, usedInfo)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
index 3168c763df4df..6ce195b6c7a34 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
@@ -178,7 +178,7 @@ private[spark] class HadoopDelegationTokenManager(
 
   private def scheduleRenewal(delay: Long): Unit = {
     val _delay = math.max(0, delay)
-    logInfo(s"Scheduling renewal in ${UIUtils.formatDuration(delay)}.")
+    logInfo(s"Scheduling renewal in ${UIUtils.formatDuration(_delay)}.")
 
     val renewalTask = new Runnable() {
       override def run(): Unit = {
@@ -230,6 +230,8 @@ private[spark] class HadoopDelegationTokenManager(
         val now = System.currentTimeMillis
         val ratio = sparkConf.get(CREDENTIALS_RENEWAL_INTERVAL_RATIO)
         val delay = (ratio * (nextRenewal - now)).toLong
+        logInfo(s"Calculated delay on renewal is $delay, based on next renewal $nextRenewal " +
+          s"and the ratio $ratio, and current time $now")
         scheduleRenewal(delay)
         creds
       }
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
index 4e91e72361488..cd9516b849565 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
@@ -63,7 +63,8 @@ private[deploy] class HadoopFSDelegationTokenProvider
             val identifier = token
               .decodeIdentifier()
               .asInstanceOf[AbstractDelegationTokenIdentifier]
-            identifier.getIssueDate + interval
+            val tokenKind = token.getKind.toString
+            getIssueDate(tokenKind, identifier) + interval
           }
         if (nextRenewalDates.isEmpty) None else Some(nextRenewalDates.min)
       }
@@ -126,13 +127,33 @@ private[deploy] class HadoopFSDelegationTokenProvider
       Try {
         val newExpiration = token.renew(hadoopConf)
         val identifier = token.decodeIdentifier().asInstanceOf[AbstractDelegationTokenIdentifier]
-        val interval = newExpiration - identifier.getIssueDate
-        logInfo(s"Renewal interval is $interval for token ${token.getKind.toString}")
+        val tokenKind = token.getKind.toString
+        val interval = newExpiration - getIssueDate(tokenKind, identifier)
+        logInfo(s"Renewal interval is $interval for token $tokenKind")
         interval
       }.toOption
     }
     if (renewIntervals.isEmpty) None else Some(renewIntervals.min)
   }
+
+  private def getIssueDate(kind: String, identifier: AbstractDelegationTokenIdentifier): Long = {
+    val now = System.currentTimeMillis()
+    val issueDate = identifier.getIssueDate
+    if (issueDate > now) {
+      logWarning(s"Token $kind has set up issue date later than current time. (provided: " +
+        s"$issueDate / current timestamp: $now) Please make sure clocks are in sync between " +
+        "machines. If the issue is not a clock mismatch, consult token implementor to check " +
+        "whether issue date is valid.")
+      issueDate
+    } else if (issueDate > 0L) {
+      issueDate
+    } else {
+      logWarning(s"Token $kind has not set up issue date properly. (provided: $issueDate) " +
+        s"Using current timestamp ($now) as issue date instead. Consult token implementor to fix " +
+        "the behavior.")
+      now
+    }
+  }
 }
 
 private[deploy] object HadoopFSDelegationTokenProvider {
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
index 53ec7b3a88f35..2caee1846595e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
@@ -33,9 +33,11 @@ import org.apache.spark.deploy.master.DriverState
 import org.apache.spark.deploy.master.DriverState.DriverState
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{DRIVER_RESOURCES_FILE, SPARK_DRIVER_PREFIX}
+import org.apache.spark.internal.config.UI.UI_REVERSE_PROXY
 import org.apache.spark.internal.config.Worker.WORKER_DRIVER_TERMINATE_TIMEOUT
 import org.apache.spark.resource.ResourceInformation
 import org.apache.spark.rpc.RpcEndpointRef
+import org.apache.spark.ui.UIUtils
 import org.apache.spark.util.{Clock, ShutdownHookManager, SystemClock, Utils}
 
 /**
@@ -50,6 +52,7 @@ private[deploy] class DriverRunner(
     val driverDesc: DriverDescription,
     val worker: RpcEndpointRef,
     val workerUrl: String,
+    val workerWebUiUrl: String,
     val securityManager: SecurityManager,
     val resources: Map[String, ResourceInformation] = Map.empty)
   extends Logging {
@@ -190,6 +193,14 @@ private[deploy] class DriverRunner(
     val builder = CommandUtils.buildProcessBuilder(driverDesc.command.copy(javaOpts = javaOpts),
       securityManager, driverDesc.mem, sparkHome.getAbsolutePath, substituteVariables)
 
+    // add WebUI driver log url to environment
+    val reverseProxy = conf.get(UI_REVERSE_PROXY)
+    val workerUrlRef = UIUtils.makeHref(reverseProxy, driverId, workerWebUiUrl)
+    builder.environment.put("SPARK_DRIVER_LOG_URL_STDOUT",
+      s"$workerUrlRef/logPage?driverId=$driverId&logType=stdout")
+    builder.environment.put("SPARK_DRIVER_LOG_URL_STDERR",
+      s"$workerUrlRef/logPage?driverId=$driverId&logType=stderr")
+
     runDriver(builder, driverDir, driverDesc.supervise)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index 4be495ac4f13f..08fdd0ab61aa3 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -34,7 +34,7 @@ import org.apache.spark.deploy.{Command, ExecutorDescription, ExecutorState}
 import org.apache.spark.deploy.DeployMessages._
 import org.apache.spark.deploy.ExternalShuffleService
 import org.apache.spark.deploy.StandaloneResourceUtils._
-import org.apache.spark.deploy.master.{DriverState, Master, WorkerResourceInfo}
+import org.apache.spark.deploy.master.{DriverState, Master}
 import org.apache.spark.deploy.worker.ui.WorkerWebUI
 import org.apache.spark.internal.{config, Logging}
 import org.apache.spark.internal.config.Tests.IS_TESTING
@@ -57,8 +57,7 @@ private[deploy] class Worker(
     val conf: SparkConf,
     val securityMgr: SecurityManager,
     resourceFileOpt: Option[String] = None,
-    externalShuffleServiceSupplier: Supplier[ExternalShuffleService] = null,
-    pid: Int = Utils.getProcessId)
+    externalShuffleServiceSupplier: Supplier[ExternalShuffleService] = null)
   extends ThreadSafeRpcEndpoint with Logging {
 
   private val host = rpcEnv.address.host
@@ -205,7 +204,6 @@ private[deploy] class Worker(
     logInfo("Spark home: " + sparkHome)
     createWorkDir()
     startExternalShuffleService()
-    releaseResourcesOnInterrupt()
     setupWorkerResources()
     webUi = new WorkerWebUI(this, workDir, webUiPort)
     webUi.bind()
@@ -219,26 +217,13 @@ private[deploy] class Worker(
     metricsSystem.getServletHandlers.foreach(webUi.attachHandler)
   }
 
-  /**
-   * Used to catch the TERM signal from sbin/stop-slave.sh and
-   * release resources before Worker exits
-   */
-  private def releaseResourcesOnInterrupt(): Unit = {
-    SignalUtils.register("TERM") {
-      releaseResources(conf, SPARK_WORKER_PREFIX, resources, pid)
-      false
-    }
-  }
-
   private def setupWorkerResources(): Unit = {
     try {
-      val allResources = getOrDiscoverAllResources(conf, SPARK_WORKER_PREFIX, resourceFileOpt)
-      resources = acquireResources(conf, SPARK_WORKER_PREFIX, allResources, pid)
+      resources = getOrDiscoverAllResources(conf, SPARK_WORKER_PREFIX, resourceFileOpt)
       logResourceInfo(SPARK_WORKER_PREFIX, resources)
     } catch {
       case e: Exception =>
         logError("Failed to setup worker resources: ", e)
-        releaseResources(conf, SPARK_WORKER_PREFIX, resources, pid)
         if (!Utils.isTesting) {
           System.exit(1)
         }
@@ -373,7 +358,6 @@ private[deploy] class Worker(
               TimeUnit.SECONDS))
         }
       } else {
-        releaseResources(conf, SPARK_WORKER_PREFIX, resources, pid)
         logError("All masters are unresponsive! Giving up.")
         System.exit(1)
       }
@@ -472,7 +456,6 @@ private[deploy] class Worker(
       case RegisterWorkerFailed(message) =>
         if (!registered) {
           logError("Worker registration failed: " + message)
-          releaseResources(conf, SPARK_WORKER_PREFIX, resources, pid)
           System.exit(1)
         }
 
@@ -645,6 +628,7 @@ private[deploy] class Worker(
         driverDesc.copy(command = Worker.maybeUpdateSSLSettings(driverDesc.command, conf)),
         self,
         workerUri,
+        workerWebUiUrl,
         securityMgr,
         resources_)
       drivers(driverId) = driver
@@ -738,7 +722,6 @@ private[deploy] class Worker(
   }
 
   override def onStop(): Unit = {
-    releaseResources(conf, SPARK_WORKER_PREFIX, resources, pid)
     cleanupThreadExecutor.shutdownNow()
     metricsSystem.report()
     cancelLastRegistrationRetry()
@@ -875,9 +858,8 @@ private[deploy] object Worker extends Logging {
     val securityMgr = new SecurityManager(conf)
     val rpcEnv = RpcEnv.create(systemName, host, port, conf, securityMgr)
     val masterAddresses = masterUrls.map(RpcAddress.fromSparkURL)
-    val pid = if (Utils.isTesting) workerNumber.get else Utils.getProcessId
     rpcEnv.setupEndpoint(ENDPOINT_NAME, new Worker(rpcEnv, webUiPort, cores, memory,
-      masterAddresses, ENDPOINT_NAME, workDir, conf, securityMgr, resourceFileOpt, pid = pid))
+      masterAddresses, ENDPOINT_NAME, workDir, conf, securityMgr, resourceFileOpt))
     rpcEnv
   }
 
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 8aeb16fe5d8c8..baf9e461eb8ff 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -153,11 +153,6 @@ private[spark] class Executor(
   // for fetching remote cached RDD blocks, so need to make sure it uses the right classloader too.
   env.serializerManager.setDefaultClassLoader(replClassLoader)
 
-  // Plugins need to load using a class loader that includes the executor's user classpath
-  private val plugins: Option[PluginContainer] = Utils.withContextClassLoader(replClassLoader) {
-    PluginContainer(env, resources.asJava)
-  }
-
   // Max size of direct result. If task result is bigger than this, we use the block manager
   // to send the result back.
   private val maxDirectResultSize = Math.min(
@@ -218,6 +213,27 @@ private[spark] class Executor(
 
   heartbeater.start()
 
+  private val appStartTime = conf.getLong("spark.app.startTime", 0)
+
+  // To allow users to distribute plugins and their required files
+  // specified by --jars and --files on application submission, those jars/files should be
+  // downloaded and added to the class loader via updateDependencies.
+  // This should be done before plugin initialization below
+  // because executors search plugins from the class loader and initialize them.
+  private val Seq(initialUserJars, initialUserFiles) = Seq("jar", "file").map { key =>
+    conf.getOption(s"spark.app.initial.$key.urls").map { urls =>
+      Map(urls.split(",").map(url => (url, appStartTime)): _*)
+    }.getOrElse(Map.empty)
+  }
+  updateDependencies(initialUserFiles, initialUserJars)
+
+  // Plugins need to load using a class loader that includes the executor's user classpath.
+  // Plugins also needs to be initialized after the heartbeater started
+  // to avoid blocking to send heartbeat (see SPARK-32175).
+  private val plugins: Option[PluginContainer] = Utils.withContextClassLoader(replClassLoader) {
+    PluginContainer(env, resources.asJava)
+  }
+
   metricsPoller.start()
 
   private[executor] def numRunningTasks: Int = runningTasks.size()
@@ -365,7 +381,9 @@ private[spark] class Executor(
       // Report executor runtime and JVM gc time
       Option(task).foreach(t => {
         t.metrics.setExecutorRunTime(TimeUnit.NANOSECONDS.toMillis(
-          System.nanoTime() - taskStartTimeNs))
+          // SPARK-32898: it's possible that a task is killed when taskStartTimeNs has the initial
+          // value(=0) still. In this case, the executorRunTime should be considered as 0.
+          if (taskStartTimeNs > 0) System.nanoTime() - taskStartTimeNs else 0))
         t.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime)
       })
 
diff --git a/core/src/main/scala/org/apache/spark/internal/Logging.scala b/core/src/main/scala/org/apache/spark/internal/Logging.scala
index 2e4846bec2db4..0c1d9635b6535 100644
--- a/core/src/main/scala/org/apache/spark/internal/Logging.scala
+++ b/core/src/main/scala/org/apache/spark/internal/Logging.scala
@@ -117,7 +117,7 @@ trait Logging {
   }
 
   // For testing
-  def initializeForcefully(isInterpreter: Boolean, silent: Boolean): Unit = {
+  private[spark] def initializeForcefully(isInterpreter: Boolean, silent: Boolean): Unit = {
     initializeLogging(isInterpreter, silent)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala b/core/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala
index 68e1994f0f94f..8d5959a0c8b7f 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala
@@ -129,7 +129,7 @@ private[spark] class TypedConfigBuilder[T](
   def createOptional: OptionalConfigEntry[T] = {
     val entry = new OptionalConfigEntry[T](parent.key, parent._prependedKey,
       parent._prependSeparator, parent._alternatives, converter, stringConverter, parent._doc,
-      parent._public)
+      parent._public, parent._version)
     parent._onCreate.foreach(_(entry))
     entry
   }
@@ -144,7 +144,7 @@ private[spark] class TypedConfigBuilder[T](
       val transformedDefault = converter(stringConverter(default))
       val entry = new ConfigEntryWithDefault[T](parent.key, parent._prependedKey,
         parent._prependSeparator, parent._alternatives, transformedDefault, converter,
-        stringConverter, parent._doc, parent._public)
+        stringConverter, parent._doc, parent._public, parent._version)
       parent._onCreate.foreach(_(entry))
       entry
     }
@@ -154,7 +154,7 @@ private[spark] class TypedConfigBuilder[T](
   def createWithDefaultFunction(defaultFunc: () => T): ConfigEntry[T] = {
     val entry = new ConfigEntryWithDefaultFunction[T](parent.key, parent._prependedKey,
       parent._prependSeparator, parent._alternatives, defaultFunc, converter, stringConverter,
-      parent._doc, parent._public)
+      parent._doc, parent._public, parent._version)
     parent._onCreate.foreach(_ (entry))
     entry
   }
@@ -166,7 +166,7 @@ private[spark] class TypedConfigBuilder[T](
   def createWithDefaultString(default: String): ConfigEntry[T] = {
     val entry = new ConfigEntryWithDefaultString[T](parent.key, parent._prependedKey,
       parent._prependSeparator, parent._alternatives, default, converter, stringConverter,
-      parent._doc, parent._public)
+      parent._doc, parent._public, parent._version)
     parent._onCreate.foreach(_(entry))
     entry
   }
@@ -186,6 +186,7 @@ private[spark] case class ConfigBuilder(key: String) {
   private[config] var _prependSeparator: String = ""
   private[config] var _public = true
   private[config] var _doc = ""
+  private[config] var _version = ""
   private[config] var _onCreate: Option[ConfigEntry[_] => Unit] = None
   private[config] var _alternatives = List.empty[String]
 
@@ -199,6 +200,11 @@ private[spark] case class ConfigBuilder(key: String) {
     this
   }
 
+  def version(v: String): ConfigBuilder = {
+    _version = v
+    this
+  }
+
   /**
    * Registers a callback for when the config entry is finally instantiated. Currently used by
    * SQLConf to keep track of SQL configuration entries.
@@ -255,7 +261,7 @@ private[spark] case class ConfigBuilder(key: String) {
 
   def fallbackConf[T](fallback: ConfigEntry[T]): ConfigEntry[T] = {
     val entry = new FallbackConfigEntry(key, _prependedKey, _prependSeparator, _alternatives, _doc,
-      _public, fallback)
+      _public, _version, fallback)
     _onCreate.foreach(_(entry))
     entry
   }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala b/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala
index c5df4c8820098..8c0b11d46312e 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala
@@ -39,6 +39,7 @@ package org.apache.spark.internal.config
  * @param doc the documentation for the configuration
  * @param isPublic if this configuration is public to the user. If it's `false`, this
  *                 configuration is only used internally and we should not expose it to users.
+ * @param version the spark version when the configuration was released.
  * @tparam T the value type
  */
 private[spark] abstract class ConfigEntry[T] (
@@ -49,7 +50,8 @@ private[spark] abstract class ConfigEntry[T] (
     val valueConverter: String => T,
     val stringConverter: T => String,
     val doc: String,
-    val isPublic: Boolean) {
+    val isPublic: Boolean,
+    val version: String) {
 
   import ConfigEntry._
 
@@ -74,7 +76,8 @@ private[spark] abstract class ConfigEntry[T] (
   def defaultValue: Option[T] = None
 
   override def toString: String = {
-    s"ConfigEntry(key=$key, defaultValue=$defaultValueString, doc=$doc, public=$isPublic)"
+    s"ConfigEntry(key=$key, defaultValue=$defaultValueString, doc=$doc, " +
+      s"public=$isPublic, version=$version)"
   }
 }
 
@@ -87,7 +90,8 @@ private class ConfigEntryWithDefault[T] (
     valueConverter: String => T,
     stringConverter: T => String,
     doc: String,
-    isPublic: Boolean)
+    isPublic: Boolean,
+    version: String)
   extends ConfigEntry(
     key,
     prependedKey,
@@ -96,7 +100,8 @@ private class ConfigEntryWithDefault[T] (
     valueConverter,
     stringConverter,
     doc,
-    isPublic
+    isPublic,
+    version
   ) {
 
   override def defaultValue: Option[T] = Some(_defaultValue)
@@ -117,7 +122,8 @@ private class ConfigEntryWithDefaultFunction[T] (
     valueConverter: String => T,
     stringConverter: T => String,
     doc: String,
-    isPublic: Boolean)
+    isPublic: Boolean,
+    version: String)
   extends ConfigEntry(
     key,
     prependedKey,
@@ -126,7 +132,8 @@ private class ConfigEntryWithDefaultFunction[T] (
     valueConverter,
     stringConverter,
     doc,
-    isPublic
+    isPublic,
+    version
   ) {
 
   override def defaultValue: Option[T] = Some(_defaultFunction())
@@ -147,7 +154,8 @@ private class ConfigEntryWithDefaultString[T] (
     valueConverter: String => T,
     stringConverter: T => String,
     doc: String,
-    isPublic: Boolean)
+    isPublic: Boolean,
+    version: String)
   extends ConfigEntry(
     key,
     prependedKey,
@@ -156,7 +164,8 @@ private class ConfigEntryWithDefaultString[T] (
     valueConverter,
     stringConverter,
     doc,
-    isPublic
+    isPublic,
+    version
   ) {
 
   override def defaultValue: Option[T] = Some(valueConverter(_defaultValue))
@@ -181,7 +190,8 @@ private[spark] class OptionalConfigEntry[T](
     val rawValueConverter: String => T,
     val rawStringConverter: T => String,
     doc: String,
-    isPublic: Boolean)
+    isPublic: Boolean,
+    version: String)
   extends ConfigEntry[Option[T]](
     key,
     prependedKey,
@@ -190,7 +200,8 @@ private[spark] class OptionalConfigEntry[T](
     s => Some(rawValueConverter(s)),
     v => v.map(rawStringConverter).orNull,
     doc,
-    isPublic
+    isPublic,
+    version
   ) {
 
   override def defaultValueString: String = ConfigEntry.UNDEFINED
@@ -210,6 +221,7 @@ private[spark] class FallbackConfigEntry[T] (
     alternatives: List[String],
     doc: String,
     isPublic: Boolean,
+    version: String,
     val fallback: ConfigEntry[T])
   extends ConfigEntry[T](
     key,
@@ -219,7 +231,8 @@ private[spark] class FallbackConfigEntry[T] (
     fallback.valueConverter,
     fallback.stringConverter,
     doc,
-    isPublic
+    isPublic,
+    version
   ) {
 
   override def defaultValueString: String = s"<value of ${fallback.key}>"
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala b/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
index ceab957b36634..d494c5ec019c7 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
@@ -19,48 +19,59 @@ package org.apache.spark.internal.config
 
 private[spark] object Deploy {
   val RECOVERY_MODE = ConfigBuilder("spark.deploy.recoveryMode")
+    .version("0.8.1")
     .stringConf
     .createWithDefault("NONE")
 
   val RECOVERY_MODE_FACTORY = ConfigBuilder("spark.deploy.recoveryMode.factory")
+    .version("1.2.0")
     .stringConf
     .createWithDefault("")
 
   val RECOVERY_DIRECTORY = ConfigBuilder("spark.deploy.recoveryDirectory")
+    .version("0.8.1")
     .stringConf
     .createWithDefault("")
 
   val ZOOKEEPER_URL = ConfigBuilder("spark.deploy.zookeeper.url")
     .doc(s"When `${RECOVERY_MODE.key}` is set to ZOOKEEPER, this " +
       "configuration is used to set the zookeeper URL to connect to.")
+    .version("0.8.1")
     .stringConf
     .createOptional
 
   val ZOOKEEPER_DIRECTORY = ConfigBuilder("spark.deploy.zookeeper.dir")
+    .version("0.8.1")
     .stringConf
     .createOptional
 
   val RETAINED_APPLICATIONS = ConfigBuilder("spark.deploy.retainedApplications")
+    .version("0.8.0")
     .intConf
     .createWithDefault(200)
 
   val RETAINED_DRIVERS = ConfigBuilder("spark.deploy.retainedDrivers")
+    .version("1.1.0")
     .intConf
     .createWithDefault(200)
 
   val REAPER_ITERATIONS = ConfigBuilder("spark.dead.worker.persistence")
+    .version("0.8.0")
     .intConf
     .createWithDefault(15)
 
   val MAX_EXECUTOR_RETRIES = ConfigBuilder("spark.deploy.maxExecutorRetries")
+    .version("1.6.3")
     .intConf
     .createWithDefault(10)
 
   val SPREAD_OUT_APPS = ConfigBuilder("spark.deploy.spreadOut")
+    .version("0.6.1")
     .booleanConf
     .createWithDefault(true)
 
   val DEFAULT_CORES = ConfigBuilder("spark.deploy.defaultCores")
+    .version("0.9.0")
     .intConf
     .createWithDefault(Int.MaxValue)
 
diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala b/core/src/main/scala/org/apache/spark/internal/config/History.scala
index 14fb5ff075472..581777de366ef 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/History.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala
@@ -26,46 +26,56 @@ private[spark] object History {
   val DEFAULT_LOG_DIR = "file:/tmp/spark-events"
 
   val HISTORY_LOG_DIR = ConfigBuilder("spark.history.fs.logDirectory")
+    .version("1.1.0")
     .stringConf
     .createWithDefault(DEFAULT_LOG_DIR)
 
   val SAFEMODE_CHECK_INTERVAL_S = ConfigBuilder("spark.history.fs.safemodeCheck.interval")
+    .version("1.6.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("5s")
 
   val UPDATE_INTERVAL_S = ConfigBuilder("spark.history.fs.update.interval")
+    .version("1.4.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("10s")
 
   val CLEANER_ENABLED = ConfigBuilder("spark.history.fs.cleaner.enabled")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(false)
 
   val CLEANER_INTERVAL_S = ConfigBuilder("spark.history.fs.cleaner.interval")
+    .version("1.4.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("1d")
 
   val MAX_LOG_AGE_S = ConfigBuilder("spark.history.fs.cleaner.maxAge")
+    .version("1.4.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("7d")
 
   val MAX_LOG_NUM = ConfigBuilder("spark.history.fs.cleaner.maxNum")
     .doc("The maximum number of log files in the event log directory.")
+    .version("3.0.0")
     .intConf
     .createWithDefault(Int.MaxValue)
 
   val LOCAL_STORE_DIR = ConfigBuilder("spark.history.store.path")
     .doc("Local directory where to cache application history information. By default this is " +
       "not set, meaning all history information will be kept in memory.")
+    .version("2.3.0")
     .stringConf
     .createOptional
 
   val MAX_LOCAL_DISK_USAGE = ConfigBuilder("spark.history.store.maxDiskUsage")
+    .version("2.3.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("10g")
 
   val HISTORY_SERVER_UI_PORT = ConfigBuilder("spark.history.ui.port")
     .doc("Web UI port to bind Spark History Server")
+    .version("1.0.0")
     .intConf
     .createWithDefault(18080)
 
@@ -73,6 +83,7 @@ private[spark] object History {
     ConfigBuilder("spark.history.fs.inProgressOptimization.enabled")
       .doc("Enable optimized handling of in-progress logs. This option may leave finished " +
         "applications that fail to rename their event logs listed as in-progress.")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -81,6 +92,7 @@ private[spark] object History {
       .doc("How many bytes to parse at the end of log files looking for the end event. " +
         "This is used to speed up generation of application listings by skipping unnecessary " +
         "parts of event log files. It can be disabled by setting this config to 0.")
+      .version("2.4.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("1m")
 
@@ -90,6 +102,7 @@ private[spark] object History {
         "By default, all event log files will be retained. Please set the configuration " +
         s"and ${EVENT_LOG_ROLLING_MAX_FILE_SIZE.key} accordingly if you want to control " +
         "the overall size of event log files.")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "Max event log files to retain should be higher than 0.")
       .createWithDefault(Integer.MAX_VALUE)
@@ -99,54 +112,67 @@ private[spark] object History {
       .doc("The threshold score to determine whether it's good to do the compaction or not. " +
         "The compaction score is calculated in analyzing, and being compared to this value. " +
         "Compaction will proceed only when the score is higher than the threshold value.")
+      .version("3.0.0")
       .internal()
       .doubleConf
       .createWithDefault(0.7d)
 
   val DRIVER_LOG_CLEANER_ENABLED = ConfigBuilder("spark.history.fs.driverlog.cleaner.enabled")
+    .version("3.0.0")
     .fallbackConf(CLEANER_ENABLED)
 
   val DRIVER_LOG_CLEANER_INTERVAL = ConfigBuilder("spark.history.fs.driverlog.cleaner.interval")
+    .version("3.0.0")
     .fallbackConf(CLEANER_INTERVAL_S)
 
   val MAX_DRIVER_LOG_AGE_S = ConfigBuilder("spark.history.fs.driverlog.cleaner.maxAge")
+    .version("3.0.0")
     .fallbackConf(MAX_LOG_AGE_S)
 
   val HISTORY_SERVER_UI_ACLS_ENABLE = ConfigBuilder("spark.history.ui.acls.enable")
+    .version("1.0.1")
     .booleanConf
     .createWithDefault(false)
 
   val HISTORY_SERVER_UI_ADMIN_ACLS = ConfigBuilder("spark.history.ui.admin.acls")
+    .version("2.1.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val HISTORY_SERVER_UI_ADMIN_ACLS_GROUPS = ConfigBuilder("spark.history.ui.admin.acls.groups")
+    .version("2.1.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val NUM_REPLAY_THREADS = ConfigBuilder("spark.history.fs.numReplayThreads")
+    .version("2.0.0")
     .intConf
     .createWithDefaultFunction(() => Math.ceil(Runtime.getRuntime.availableProcessors() / 4f).toInt)
 
   val RETAINED_APPLICATIONS = ConfigBuilder("spark.history.retainedApplications")
+    .version("1.0.0")
     .intConf
     .createWithDefault(50)
 
   val PROVIDER = ConfigBuilder("spark.history.provider")
+    .version("1.1.0")
     .stringConf
     .createOptional
 
   val KERBEROS_ENABLED = ConfigBuilder("spark.history.kerberos.enabled")
+    .version("1.0.1")
     .booleanConf
     .createWithDefault(false)
 
   val KERBEROS_PRINCIPAL = ConfigBuilder("spark.history.kerberos.principal")
+    .version("1.0.1")
     .stringConf
     .createOptional
 
   val KERBEROS_KEYTAB = ConfigBuilder("spark.history.kerberos.keytab")
+    .version("1.0.1")
     .stringConf
     .createOptional
 
@@ -156,15 +182,17 @@ private[spark] object History {
       "some path variables via patterns which can vary on cluster manager. Please check the " +
       "documentation for your cluster manager to see which patterns are supported, if any. " +
       "This configuration has no effect on a live application, it only affects the history server.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
   val APPLY_CUSTOM_EXECUTOR_LOG_URL_TO_INCOMPLETE_APP =
     ConfigBuilder("spark.history.custom.executor.log.url.applyIncompleteApplication")
       .doc("Whether to apply custom executor log url, as specified by " +
-        "`spark.history.custom.executor.log.url`, to incomplete application as well. " +
+        s"${CUSTOM_EXECUTOR_LOG_URL.key}, to incomplete application as well. " +
         "Even if this is true, this still only affects the behavior of the history server, " +
         "not running spark applications.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala b/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
index 717a09914a2f5..90c59b079461c 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
@@ -22,35 +22,44 @@ import org.apache.spark.network.util.ByteUnit
 private[spark] object Kryo {
 
   val KRYO_REGISTRATION_REQUIRED = ConfigBuilder("spark.kryo.registrationRequired")
+    .version("1.1.0")
     .booleanConf
     .createWithDefault(false)
 
   val KRYO_USER_REGISTRATORS = ConfigBuilder("spark.kryo.registrator")
+    .version("0.5.0")
     .stringConf
-    .createOptional
+    .toSequence
+    .createWithDefault(Nil)
 
   val KRYO_CLASSES_TO_REGISTER = ConfigBuilder("spark.kryo.classesToRegister")
+    .version("1.2.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val KRYO_USE_UNSAFE = ConfigBuilder("spark.kryo.unsafe")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
   val KRYO_USE_POOL = ConfigBuilder("spark.kryo.pool")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val KRYO_REFERENCE_TRACKING = ConfigBuilder("spark.kryo.referenceTracking")
+    .version("0.8.0")
     .booleanConf
     .createWithDefault(true)
 
   val KRYO_SERIALIZER_BUFFER_SIZE = ConfigBuilder("spark.kryoserializer.buffer")
+    .version("1.4.0")
     .bytesConf(ByteUnit.KiB)
     .createWithDefaultString("64k")
 
   val KRYO_SERIALIZER_MAX_BUFFER_SIZE = ConfigBuilder("spark.kryoserializer.buffer.max")
+    .version("1.4.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("64m")
 
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Network.scala b/core/src/main/scala/org/apache/spark/internal/config/Network.scala
index 129e31a82979f..0961d062cc04f 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Network.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Network.scala
@@ -23,71 +23,85 @@ private[spark] object Network {
 
   private[spark] val NETWORK_CRYPTO_SASL_FALLBACK =
     ConfigBuilder("spark.network.crypto.saslFallback")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val NETWORK_CRYPTO_ENABLED =
     ConfigBuilder("spark.network.crypto.enabled")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val NETWORK_REMOTE_READ_NIO_BUFFER_CONVERSION =
     ConfigBuilder("spark.network.remoteReadNioBufferConversion")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val NETWORK_TIMEOUT =
     ConfigBuilder("spark.network.timeout")
+      .version("1.3.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("120s")
 
   private[spark] val NETWORK_TIMEOUT_INTERVAL =
     ConfigBuilder("spark.network.timeoutInterval")
+      .version("1.3.2")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString(STORAGE_BLOCKMANAGER_TIMEOUTINTERVAL.defaultValueString)
 
   private[spark] val RPC_ASK_TIMEOUT =
     ConfigBuilder("spark.rpc.askTimeout")
+      .version("1.4.0")
       .stringConf
       .createOptional
 
   private[spark] val RPC_CONNECT_THREADS =
     ConfigBuilder("spark.rpc.connect.threads")
+      .version("1.6.0")
       .intConf
       .createWithDefault(64)
 
   private[spark] val RPC_IO_NUM_CONNECTIONS_PER_PEER =
     ConfigBuilder("spark.rpc.io.numConnectionsPerPeer")
+      .version("1.6.0")
       .intConf
       .createWithDefault(1)
 
   private[spark] val RPC_IO_THREADS =
     ConfigBuilder("spark.rpc.io.threads")
+      .version("1.6.0")
       .intConf
       .createOptional
 
   private[spark] val RPC_LOOKUP_TIMEOUT =
     ConfigBuilder("spark.rpc.lookupTimeout")
+      .version("1.4.0")
       .stringConf
       .createOptional
 
   private[spark] val RPC_MESSAGE_MAX_SIZE =
     ConfigBuilder("spark.rpc.message.maxSize")
+      .version("2.0.0")
       .intConf
       .createWithDefault(128)
 
   private[spark] val RPC_NETTY_DISPATCHER_NUM_THREADS =
     ConfigBuilder("spark.rpc.netty.dispatcher.numThreads")
+      .version("1.6.0")
       .intConf
       .createOptional
 
   private[spark] val RPC_NUM_RETRIES =
     ConfigBuilder("spark.rpc.numRetries")
+      .version("1.4.0")
       .intConf
       .createWithDefault(3)
 
   private[spark] val RPC_RETRY_WAIT =
     ConfigBuilder("spark.rpc.retry.wait")
+      .version("1.4.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("3s")
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Python.scala b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
index 26a0598f49411..188d884319644 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Python.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
@@ -22,26 +22,32 @@ import org.apache.spark.network.util.ByteUnit
 
 private[spark] object Python {
   val PYTHON_WORKER_REUSE = ConfigBuilder("spark.python.worker.reuse")
+    .version("1.2.0")
     .booleanConf
     .createWithDefault(true)
 
   val PYTHON_TASK_KILL_TIMEOUT = ConfigBuilder("spark.python.task.killTimeout")
+    .version("2.2.2")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("2s")
 
   val PYTHON_USE_DAEMON = ConfigBuilder("spark.python.use.daemon")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
   val PYTHON_DAEMON_MODULE = ConfigBuilder("spark.python.daemon.module")
+    .version("2.4.0")
     .stringConf
     .createOptional
 
   val PYTHON_WORKER_MODULE = ConfigBuilder("spark.python.worker.module")
+    .version("2.4.0")
     .stringConf
     .createOptional
 
   val PYSPARK_EXECUTOR_MEMORY = ConfigBuilder("spark.executor.pyspark.memory")
+    .version("2.4.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/R.scala b/core/src/main/scala/org/apache/spark/internal/config/R.scala
index 26e06a5231c42..46fc198cd4cf5 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/R.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/R.scala
@@ -19,22 +19,27 @@ package org.apache.spark.internal.config
 private[spark] object R {
 
   val R_BACKEND_CONNECTION_TIMEOUT = ConfigBuilder("spark.r.backendConnectionTimeout")
+    .version("2.1.0")
     .intConf
     .createWithDefault(6000)
 
   val R_NUM_BACKEND_THREADS = ConfigBuilder("spark.r.numRBackendThreads")
+    .version("1.4.0")
     .intConf
     .createWithDefault(2)
 
   val R_HEARTBEAT_INTERVAL = ConfigBuilder("spark.r.heartBeatInterval")
+    .version("2.1.0")
     .intConf
     .createWithDefault(100)
 
   val SPARKR_COMMAND = ConfigBuilder("spark.sparkr.r.command")
+    .version("1.5.3")
     .stringConf
     .createWithDefault("Rscript")
 
   val R_COMMAND = ConfigBuilder("spark.r.command")
+    .version("1.5.3")
     .stringConf
     .createOptional
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Status.scala b/core/src/main/scala/org/apache/spark/internal/config/Status.scala
index 3cc00a6f094cf..669fa07053cad 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Status.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Status.scala
@@ -22,36 +22,44 @@ import java.util.concurrent.TimeUnit
 private[spark] object Status {
 
   val ASYNC_TRACKING_ENABLED = ConfigBuilder("spark.appStateStore.asyncTracking.enable")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
   val LIVE_ENTITY_UPDATE_PERIOD = ConfigBuilder("spark.ui.liveUpdate.period")
+    .version("2.3.0")
     .timeConf(TimeUnit.NANOSECONDS)
     .createWithDefaultString("100ms")
 
   val LIVE_ENTITY_UPDATE_MIN_FLUSH_PERIOD = ConfigBuilder("spark.ui.liveUpdate.minFlushPeriod")
     .doc("Minimum time elapsed before stale UI data is flushed. This avoids UI staleness when " +
       "incoming task events are not fired frequently.")
+    .version("2.4.2")
     .timeConf(TimeUnit.NANOSECONDS)
     .createWithDefaultString("1s")
 
   val MAX_RETAINED_JOBS = ConfigBuilder("spark.ui.retainedJobs")
+    .version("1.2.0")
     .intConf
     .createWithDefault(1000)
 
   val MAX_RETAINED_STAGES = ConfigBuilder("spark.ui.retainedStages")
+    .version("0.9.0")
     .intConf
     .createWithDefault(1000)
 
   val MAX_RETAINED_TASKS_PER_STAGE = ConfigBuilder("spark.ui.retainedTasks")
+    .version("2.0.1")
     .intConf
     .createWithDefault(100000)
 
   val MAX_RETAINED_DEAD_EXECUTORS = ConfigBuilder("spark.ui.retainedDeadExecutors")
+    .version("2.0.0")
     .intConf
     .createWithDefault(100)
 
   val MAX_RETAINED_ROOT_NODES = ConfigBuilder("spark.ui.dagGraph.retainedRootRDDs")
+    .version("2.1.0")
     .intConf
     .createWithDefault(Int.MaxValue)
 
@@ -59,6 +67,7 @@ private[spark] object Status {
     ConfigBuilder("spark.metrics.appStatusSource.enabled")
       .doc("Whether Dropwizard/Codahale metrics " +
         "will be reported for the status of the running spark app.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Streaming.scala b/core/src/main/scala/org/apache/spark/internal/config/Streaming.scala
index 6e58c090e8126..44a467b278614 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Streaming.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Streaming.scala
@@ -23,16 +23,19 @@ private[spark] object Streaming {
 
   private[spark] val STREAMING_DYN_ALLOCATION_ENABLED =
     ConfigBuilder("spark.streaming.dynamicAllocation.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val STREAMING_DYN_ALLOCATION_TESTING =
     ConfigBuilder("spark.streaming.dynamicAllocation.testing")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val STREAMING_DYN_ALLOCATION_MIN_EXECUTORS =
     ConfigBuilder("spark.streaming.dynamicAllocation.minExecutors")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The min executor number of streaming dynamic " +
         "allocation must be positive.")
@@ -40,6 +43,7 @@ private[spark] object Streaming {
 
   private[spark] val STREAMING_DYN_ALLOCATION_MAX_EXECUTORS =
     ConfigBuilder("spark.streaming.dynamicAllocation.maxExecutors")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The max executor number of streaming dynamic " +
         "allocation must be positive.")
@@ -47,6 +51,7 @@ private[spark] object Streaming {
 
   private[spark] val STREAMING_DYN_ALLOCATION_SCALING_INTERVAL =
     ConfigBuilder("spark.streaming.dynamicAllocation.scalingInterval")
+      .version("3.0.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(_ > 0, "The scaling interval of streaming dynamic " +
         "allocation must be positive.")
@@ -54,6 +59,7 @@ private[spark] object Streaming {
 
   private[spark] val STREAMING_DYN_ALLOCATION_SCALING_UP_RATIO =
     ConfigBuilder("spark.streaming.dynamicAllocation.scalingUpRatio")
+      .version("3.0.0")
       .doubleConf
       .checkValue(_ > 0, "The scaling up ratio of streaming dynamic " +
         "allocation must be positive.")
@@ -61,6 +67,7 @@ private[spark] object Streaming {
 
   private[spark] val STREAMING_DYN_ALLOCATION_SCALING_DOWN_RATIO =
     ConfigBuilder("spark.streaming.dynamicAllocation.scalingDownRatio")
+      .version("3.0.0")
       .doubleConf
       .checkValue(_ > 0, "The scaling down ratio of streaming dynamic " +
         "allocation must be positive.")
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
index 21660ab3a9512..e328ed026bfb0 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
@@ -22,35 +22,58 @@ private[spark] object Tests {
   val TEST_USE_COMPRESSED_OOPS_KEY = "spark.test.useCompressedOops"
 
   val TEST_MEMORY = ConfigBuilder("spark.testing.memory")
+    .version("1.6.0")
     .longConf
     .createWithDefault(Runtime.getRuntime.maxMemory)
 
   val TEST_SCHEDULE_INTERVAL =
     ConfigBuilder("spark.testing.dynamicAllocation.scheduleInterval")
+      .version("2.3.0")
       .longConf
       .createWithDefault(100)
 
   val IS_TESTING = ConfigBuilder("spark.testing")
+    .version("1.0.1")
     .booleanConf
     .createOptional
 
   val TEST_NO_STAGE_RETRY = ConfigBuilder("spark.test.noStageRetry")
+    .version("1.2.0")
     .booleanConf
     .createWithDefault(false)
 
   val TEST_RESERVED_MEMORY = ConfigBuilder("spark.testing.reservedMemory")
+    .version("1.6.0")
     .longConf
     .createOptional
 
   val TEST_N_HOSTS = ConfigBuilder("spark.testing.nHosts")
+    .version("3.0.0")
     .intConf
     .createWithDefault(5)
 
   val TEST_N_EXECUTORS_HOST = ConfigBuilder("spark.testing.nExecutorsPerHost")
+    .version("3.0.0")
     .intConf
     .createWithDefault(4)
 
   val TEST_N_CORES_EXECUTOR = ConfigBuilder("spark.testing.nCoresPerExecutor")
+    .version("3.0.0")
     .intConf
     .createWithDefault(2)
+
+  val RESOURCES_WARNING_TESTING = ConfigBuilder("spark.resources.warnings.testing")
+    .version("3.0.1")
+    .booleanConf
+    .createWithDefault(false)
+
+  // This configuration is used for unit tests to allow skipping the task cpus to cores validation
+  // to allow emulating standalone mode behavior while running in local mode. Standalone mode
+  // by default doesn't specify a number of executor cores, it just uses all the ones available
+  // on the host.
+  val SKIP_VALIDATE_CORES_TESTING =
+  ConfigBuilder("spark.testing.skipValidateCores")
+    .version("3.0.1")
+    .booleanConf
+    .createWithDefault(false)
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/UI.scala b/core/src/main/scala/org/apache/spark/internal/config/UI.scala
index 60d985713d30e..231eecf086bbe 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/UI.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/UI.scala
@@ -25,31 +25,37 @@ private[spark] object UI {
 
   val UI_SHOW_CONSOLE_PROGRESS = ConfigBuilder("spark.ui.showConsoleProgress")
     .doc("When true, show the progress bar in the console.")
+    .version("1.2.1")
     .booleanConf
     .createWithDefault(false)
 
   val UI_CONSOLE_PROGRESS_UPDATE_INTERVAL =
     ConfigBuilder("spark.ui.consoleProgress.update.interval")
+      .version("2.1.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(200)
 
   val UI_ENABLED = ConfigBuilder("spark.ui.enabled")
     .doc("Whether to run the web UI for the Spark application.")
+    .version("1.1.1")
     .booleanConf
     .createWithDefault(true)
 
   val UI_PORT = ConfigBuilder("spark.ui.port")
     .doc("Port for your application's dashboard, which shows memory and workload data.")
+    .version("0.7.0")
     .intConf
     .createWithDefault(4040)
 
   val UI_FILTERS = ConfigBuilder("spark.ui.filters")
     .doc("Comma separated list of filter class names to apply to the Spark Web UI.")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val UI_ALLOW_FRAMING_FROM = ConfigBuilder("spark.ui.allowFramingFrom")
+    .version("1.6.0")
     .stringConf
     .createOptional
 
@@ -61,6 +67,7 @@ private[spark] object UI {
       "through spark master/proxy public URL. This setting affects all the workers and " +
       "application UIs running in the cluster and must be set on all the workers, drivers " +
       " and masters.")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -69,15 +76,18 @@ private[spark] object UI {
       "in front of Spark Master. This is useful when running proxy for authentication e.g. " +
       "OAuth proxy. Make sure this is a complete URL including scheme (http/https) and port to " +
       "reach your proxy.")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   val UI_KILL_ENABLED = ConfigBuilder("spark.ui.killEnabled")
     .doc("Allows jobs and stages to be killed from the web UI.")
+    .version("1.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val UI_THREAD_DUMPS_ENABLED = ConfigBuilder("spark.ui.threadDumpsEnabled")
+    .version("1.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -85,73 +95,88 @@ private[spark] object UI {
     .internal()
     .doc("Expose executor metrics at /metrics/executors/prometheus. " +
       "For master/worker/driver metrics, you need to configure `conf/metrics.properties`.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
   val UI_X_XSS_PROTECTION = ConfigBuilder("spark.ui.xXssProtection")
     .doc("Value for HTTP X-XSS-Protection response header")
+    .version("2.3.0")
     .stringConf
     .createWithDefaultString("1; mode=block")
 
   val UI_X_CONTENT_TYPE_OPTIONS = ConfigBuilder("spark.ui.xContentTypeOptions.enabled")
     .doc("Set to 'true' for setting X-Content-Type-Options HTTP response header to 'nosniff'")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
   val UI_STRICT_TRANSPORT_SECURITY = ConfigBuilder("spark.ui.strictTransportSecurity")
     .doc("Value for HTTP Strict Transport Security Response Header")
+    .version("2.3.0")
     .stringConf
     .createOptional
 
   val UI_REQUEST_HEADER_SIZE = ConfigBuilder("spark.ui.requestHeaderSize")
     .doc("Value for HTTP request header size in bytes.")
+    .version("2.2.3")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("8k")
 
   val UI_TIMELINE_TASKS_MAXIMUM = ConfigBuilder("spark.ui.timeline.tasks.maximum")
+    .version("1.4.0")
     .intConf
     .createWithDefault(1000)
 
   val ACLS_ENABLE = ConfigBuilder("spark.acls.enable")
+    .version("1.1.0")
     .booleanConf
     .createWithDefault(false)
 
   val UI_VIEW_ACLS = ConfigBuilder("spark.ui.view.acls")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val UI_VIEW_ACLS_GROUPS = ConfigBuilder("spark.ui.view.acls.groups")
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val ADMIN_ACLS = ConfigBuilder("spark.admin.acls")
+    .version("1.1.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val ADMIN_ACLS_GROUPS = ConfigBuilder("spark.admin.acls.groups")
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val MODIFY_ACLS = ConfigBuilder("spark.modify.acls")
+    .version("1.1.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val MODIFY_ACLS_GROUPS = ConfigBuilder("spark.modify.acls.groups")
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val USER_GROUPS_MAPPING = ConfigBuilder("spark.user.groups.mapping")
+    .version("2.0.0")
     .stringConf
     .createWithDefault("org.apache.spark.security.ShellBasedGroupsMappingProvider")
 
   val PROXY_REDIRECT_URI = ConfigBuilder("spark.ui.proxyRedirectUri")
     .doc("Proxy address to use when responding with HTTP redirects.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
@@ -163,6 +188,7 @@ private[spark] object UI {
       "This configuration replaces original log urls in event log, which will be also effective " +
       "when accessing the application on history server. The new log urls must be permanent, " +
       "otherwise you might have dead link for executor log urls.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
index f1eaae29f18df..a8072712c46ce 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
@@ -28,47 +28,58 @@ private[spark] object Worker {
     .doc("Path to a file containing the resources allocated to the worker. " +
       "The file should be formatted as a JSON array of ResourceAllocation objects. " +
       "Only used internally in standalone mode.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
   val WORKER_TIMEOUT = ConfigBuilder("spark.worker.timeout")
+    .version("0.6.2")
     .longConf
     .createWithDefault(60)
 
   val WORKER_DRIVER_TERMINATE_TIMEOUT = ConfigBuilder("spark.worker.driverTerminateTimeout")
+    .version("2.1.2")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("10s")
 
   val WORKER_CLEANUP_ENABLED = ConfigBuilder("spark.worker.cleanup.enabled")
+    .version("1.0.0")
     .booleanConf
     .createWithDefault(false)
 
   val WORKER_CLEANUP_INTERVAL = ConfigBuilder("spark.worker.cleanup.interval")
+    .version("1.0.0")
     .longConf
     .createWithDefault(60 * 30)
 
   val APP_DATA_RETENTION = ConfigBuilder("spark.worker.cleanup.appDataTtl")
+    .version("1.0.0")
     .longConf
     .createWithDefault(7 * 24 * 3600)
 
   val PREFER_CONFIGURED_MASTER_ADDRESS = ConfigBuilder("spark.worker.preferConfiguredMasterAddress")
+    .version("2.2.1")
     .booleanConf
     .createWithDefault(false)
 
   val WORKER_UI_PORT = ConfigBuilder("spark.worker.ui.port")
+    .version("1.1.0")
     .intConf
     .createOptional
 
   val WORKER_UI_RETAINED_EXECUTORS = ConfigBuilder("spark.worker.ui.retainedExecutors")
+    .version("1.5.0")
     .intConf
     .createWithDefault(1000)
 
   val WORKER_UI_RETAINED_DRIVERS = ConfigBuilder("spark.worker.ui.retainedDrivers")
+    .version("1.5.0")
     .intConf
     .createWithDefault(1000)
 
   val UNCOMPRESSED_LOG_FILE_LENGTH_CACHE_SIZE_CONF =
     ConfigBuilder("spark.worker.ui.compressedLogFileLengthCacheSize")
-    .intConf
-    .createWithDefault(100)
+      .version("2.0.2")
+      .intConf
+      .createWithDefault(100)
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index f91f31be2f1ad..0afbb5278b1da 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -37,31 +37,15 @@ package object config {
   private[spark] val SPARK_TASK_PREFIX = "spark.task"
   private[spark] val LISTENER_BUS_EVENT_QUEUE_PREFIX = "spark.scheduler.listenerbus.eventqueue"
 
-  private[spark] val SPARK_RESOURCES_COORDINATE =
-    ConfigBuilder("spark.resources.coordinate.enable")
-      .doc("Whether to coordinate resources automatically among workers/drivers(client only) " +
-        "in Standalone. If false, the user is responsible for configuring different resources " +
-        "for workers/drivers that run on the same host.")
-      .booleanConf
-      .createWithDefault(true)
-
-  private[spark] val SPARK_RESOURCES_DIR =
-    ConfigBuilder("spark.resources.dir")
-      .doc("Directory used to coordinate resources among workers/drivers(client only) in " +
-        "Standalone. Default is SPARK_HOME. Make sure to use the same directory for worker " +
-        "and drivers in client mode that run on the same host. Don't clean up this directory " +
-        "while workers/drivers are still alive to avoid the most likely resources conflict. ")
-      .stringConf
-      .createOptional
-
   private[spark] val RESOURCES_DISCOVERY_PLUGIN =
-    ConfigBuilder("spark.resources.discovery.plugin")
+    ConfigBuilder("spark.resources.discoveryPlugin")
       .doc("Comma-separated list of class names implementing" +
         "org.apache.spark.api.resource.ResourceDiscoveryPlugin to load into the application." +
         "This is for advanced users to replace the resource discovery class with a " +
         "custom implementation. Spark will try each class specified until one of them " +
         "returns the resource information for that resource. It tries the discovery " +
         "script last if none of the plugins return information for that resource.")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
@@ -72,96 +56,121 @@ package object config {
       .doc("Path to a file containing the resources allocated to the driver. " +
         "The file should be formatted as a JSON array of ResourceAllocation objects. " +
         "Only used internally in standalone mode.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_CLASS_PATH =
-    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_CLASSPATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_CLASSPATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val DRIVER_JAVA_OPTIONS =
     ConfigBuilder(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)
       .withPrepended(SparkLauncher.DRIVER_DEFAULT_JAVA_OPTIONS)
+      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_LIBRARY_PATH =
-    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val DRIVER_USER_CLASS_PATH_FIRST =
-    ConfigBuilder("spark.driver.userClassPathFirst").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.driver.userClassPathFirst")
+      .version("1.3.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val DRIVER_CORES = ConfigBuilder("spark.driver.cores")
     .doc("Number of cores to use for the driver process, only in cluster mode.")
+    .version("1.3.0")
     .intConf
     .createWithDefault(1)
 
   private[spark] val DRIVER_MEMORY = ConfigBuilder(SparkLauncher.DRIVER_MEMORY)
     .doc("Amount of memory to use for the driver process, in MiB unless otherwise specified.")
+    .version("1.1.1")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("1g")
 
   private[spark] val DRIVER_MEMORY_OVERHEAD = ConfigBuilder("spark.driver.memoryOverhead")
     .doc("The amount of non-heap memory to be allocated per driver in cluster mode, " +
       "in MiB unless otherwise specified.")
+    .version("2.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 
   private[spark] val DRIVER_LOG_DFS_DIR =
-    ConfigBuilder("spark.driver.log.dfsDir").stringConf.createOptional
+    ConfigBuilder("spark.driver.log.dfsDir").version("3.0.0").stringConf.createOptional
 
   private[spark] val DRIVER_LOG_LAYOUT =
     ConfigBuilder("spark.driver.log.layout")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_LOG_PERSISTTODFS =
     ConfigBuilder("spark.driver.log.persistToDfs.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val DRIVER_LOG_ALLOW_EC =
     ConfigBuilder("spark.driver.log.allowErasureCoding")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ENABLED = ConfigBuilder("spark.eventLog.enabled")
+    .version("1.0.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val EVENT_LOG_DIR = ConfigBuilder("spark.eventLog.dir")
+    .version("1.0.0")
     .stringConf
     .createWithDefault(EventLoggingListener.DEFAULT_LOG_DIR)
 
   private[spark] val EVENT_LOG_COMPRESS =
     ConfigBuilder("spark.eventLog.compress")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_BLOCK_UPDATES =
     ConfigBuilder("spark.eventLog.logBlockUpdates.enabled")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ALLOW_EC =
     ConfigBuilder("spark.eventLog.erasureCoding.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_TESTING =
     ConfigBuilder("spark.eventLog.testing")
       .internal()
+      .version("1.0.1")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_OUTPUT_BUFFER_SIZE = ConfigBuilder("spark.eventLog.buffer.kb")
     .doc("Buffer size to use when writing to output streams, in KiB unless otherwise specified.")
+    .version("1.0.0")
     .bytesConf(ByteUnit.KiB)
     .createWithDefaultString("100k")
 
   private[spark] val EVENT_LOG_STAGE_EXECUTOR_METRICS =
-    ConfigBuilder("spark.eventLog.logStageExecutorMetrics.enabled")
+    ConfigBuilder("spark.eventLog.logStageExecutorMetrics")
       .doc("Whether to write per-stage peaks of executor metrics (for each executor) " +
         "to the event log.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -170,6 +179,7 @@ package object config {
       .doc("Names of supported young generation garbage collector. A name usually is " +
         " the return of GarbageCollectorMXBean.getName. The built-in young generation garbage " +
         s"collectors are ${GarbageCollectionMetrics.YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS}")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(GarbageCollectionMetrics.YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS)
@@ -179,55 +189,75 @@ package object config {
       .doc("Names of supported old generation garbage collector. A name usually is " +
         "the return of GarbageCollectorMXBean.getName. The built-in old generation garbage " +
         s"collectors are ${GarbageCollectionMetrics.OLD_GENERATION_BUILTIN_GARBAGE_COLLECTORS}")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(GarbageCollectionMetrics.OLD_GENERATION_BUILTIN_GARBAGE_COLLECTORS)
 
   private[spark] val EVENT_LOG_OVERWRITE =
-    ConfigBuilder("spark.eventLog.overwrite").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.eventLog.overwrite")
+      .version("1.0.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val EVENT_LOG_CALLSITE_LONG_FORM =
-    ConfigBuilder("spark.eventLog.longForm.enabled").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.eventLog.longForm.enabled")
+      .version("2.4.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ENABLE_ROLLING =
     ConfigBuilder("spark.eventLog.rolling.enabled")
-      .doc("Whether rolling over event log files is enabled.  If set to true, it cuts down " +
+      .doc("Whether rolling over event log files is enabled. If set to true, it cuts down " +
         "each event log file to the configured size.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ROLLING_MAX_FILE_SIZE =
     ConfigBuilder("spark.eventLog.rolling.maxFileSize")
-      .doc("The max size of event log file to be rolled over.")
+      .doc(s"When ${EVENT_LOG_ENABLE_ROLLING.key}=true, specifies the max size of event log file" +
+        " to be rolled over.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(_ >= ByteUnit.MiB.toBytes(10), "Max file size of event log should be " +
         "configured to be at least 10 MiB.")
       .createWithDefaultString("128m")
 
   private[spark] val EXECUTOR_ID =
-    ConfigBuilder("spark.executor.id").stringConf.createOptional
+    ConfigBuilder("spark.executor.id").version("1.2.0").stringConf.createOptional
 
   private[spark] val EXECUTOR_CLASS_PATH =
-    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val EXECUTOR_HEARTBEAT_DROP_ZERO_ACCUMULATOR_UPDATES =
     ConfigBuilder("spark.executor.heartbeat.dropZeroAccumulatorUpdates")
       .internal()
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val EXECUTOR_HEARTBEAT_INTERVAL =
     ConfigBuilder("spark.executor.heartbeatInterval")
+      .version("1.1.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10s")
 
   private[spark] val EXECUTOR_HEARTBEAT_MAX_FAILURES =
-    ConfigBuilder("spark.executor.heartbeat.maxFailures").internal().intConf.createWithDefault(60)
+    ConfigBuilder("spark.executor.heartbeat.maxFailures")
+      .internal()
+      .version("1.6.2")
+      .intConf
+      .createWithDefault(60)
 
   private[spark] val EXECUTOR_PROCESS_TREE_METRICS_ENABLED =
     ConfigBuilder("spark.executor.processTreeMetrics.enabled")
       .doc("Whether to collect process tree metrics (from the /proc filesystem) when collecting " +
         "executor metrics.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -236,33 +266,44 @@ package object config {
       .doc("How often to collect executor metrics (in milliseconds). " +
         "If 0, the polling is done on executor heartbeats. " +
         "If positive, the polling is done at this interval.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("0")
 
   private[spark] val EXECUTOR_JAVA_OPTIONS =
     ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS)
       .withPrepended(SparkLauncher.EXECUTOR_DEFAULT_JAVA_OPTIONS)
+      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val EXECUTOR_LIBRARY_PATH =
-    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val EXECUTOR_USER_CLASS_PATH_FIRST =
-    ConfigBuilder("spark.executor.userClassPathFirst").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.executor.userClassPathFirst")
+      .version("1.3.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val EXECUTOR_CORES = ConfigBuilder(SparkLauncher.EXECUTOR_CORES)
+    .version("1.0.0")
     .intConf
     .createWithDefault(1)
 
   private[spark] val EXECUTOR_MEMORY = ConfigBuilder(SparkLauncher.EXECUTOR_MEMORY)
     .doc("Amount of memory to use per executor process, in MiB unless otherwise specified.")
+    .version("0.7.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("1g")
 
   private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead")
-    .doc("The amount of non-heap memory to be allocated per executor in cluster mode, " +
-      "in MiB unless otherwise specified.")
+    .doc("The amount of non-heap memory to be allocated per executor, in MiB unless otherwise" +
+      " specified.")
+    .version("2.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 
@@ -272,12 +313,14 @@ package object config {
       "the cluster (not from each machine). If not set, the default will be " +
       "`spark.deploy.defaultCores` on Spark's standalone cluster manager, or infinite " +
       "(all available cores) on Mesos.")
+    .version("0.6.0")
     .intConf
     .createOptional
 
   private[spark] val MEMORY_OFFHEAP_ENABLED = ConfigBuilder("spark.memory.offHeap.enabled")
     .doc("If true, Spark will attempt to use off-heap memory for certain operations. " +
       "If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive.")
+    .version("1.6.0")
     .withAlternative("spark.unsafe.offHeap")
     .booleanConf
     .createWithDefault(false)
@@ -288,6 +331,7 @@ package object config {
       "This setting has no impact on heap memory usage, so if your executors' total memory " +
       "consumption must fit within some hard limit then be sure to shrink your JVM heap size " +
       "accordingly. This must be set to a positive value when spark.memory.offHeap.enabled=true.")
+    .version("1.6.0")
     .bytesConf(ByteUnit.BYTE)
     .checkValue(_ >= 0, "The off-heap memory size must not be negative")
     .createWithDefault(0)
@@ -297,6 +341,7 @@ package object config {
       "size of the region set aside by spark.memory.fraction. The higher this is, the " +
       "less working memory may be available to execution and tasks may spill to disk more " +
       "often. Leaving this at the default value is recommended. ")
+    .version("1.6.0")
     .doubleConf
     .checkValue(v => v >= 0.0 && v < 1.0, "Storage fraction must be in [0,1)")
     .createWithDefault(0.5)
@@ -307,16 +352,19 @@ package object config {
       "The purpose of this config is to set aside memory for internal metadata, " +
       "user data structures, and imprecise size estimation in the case of sparse, " +
       "unusually large records. Leaving this at the default value is recommended.  ")
+    .version("1.6.0")
     .doubleConf
     .createWithDefault(0.6)
 
   private[spark] val STORAGE_SAFETY_FRACTION = ConfigBuilder("spark.storage.safetyFraction")
+    .version("1.1.0")
     .doubleConf
     .createWithDefault(0.9)
 
   private[spark] val STORAGE_UNROLL_MEMORY_THRESHOLD =
     ConfigBuilder("spark.storage.unrollMemoryThreshold")
       .doc("Initial memory to request before unrolling any block")
+      .version("1.1.0")
       .longConf
       .createWithDefault(1024 * 1024)
 
@@ -326,6 +374,7 @@ package object config {
         "Cached RDD block replicas lost due to executor failures are replenished " +
         "if there are any existing available replicas. This tries to " +
         "get the replication level of the block to the initial number")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -336,48 +385,62 @@ package object config {
         "This prevents Spark from memory mapping very small blocks. " +
         "In general, memory mapping has high overhead for blocks close to or below " +
         "the page size of the operating system.")
+      .version("0.9.2")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("2m")
 
   private[spark] val STORAGE_REPLICATION_POLICY =
     ConfigBuilder("spark.storage.replication.policy")
+      .version("2.1.0")
       .stringConf
       .createWithDefaultString(classOf[RandomBlockReplicationPolicy].getName)
 
   private[spark] val STORAGE_REPLICATION_TOPOLOGY_MAPPER =
     ConfigBuilder("spark.storage.replication.topologyMapper")
+      .version("2.1.0")
       .stringConf
       .createWithDefaultString(classOf[DefaultTopologyMapper].getName)
 
   private[spark] val STORAGE_CACHED_PEERS_TTL = ConfigBuilder("spark.storage.cachedPeersTtl")
-    .intConf.createWithDefault(60 * 1000)
+    .version("1.1.1")
+    .intConf
+    .createWithDefault(60 * 1000)
 
   private[spark] val STORAGE_MAX_REPLICATION_FAILURE =
     ConfigBuilder("spark.storage.maxReplicationFailures")
-      .intConf.createWithDefault(1)
+      .version("1.1.1")
+      .intConf
+      .createWithDefault(1)
 
   private[spark] val STORAGE_REPLICATION_TOPOLOGY_FILE =
-    ConfigBuilder("spark.storage.replication.topologyFile").stringConf.createOptional
+    ConfigBuilder("spark.storage.replication.topologyFile")
+      .version("2.1.0")
+      .stringConf
+      .createOptional
 
   private[spark] val STORAGE_EXCEPTION_PIN_LEAK =
     ConfigBuilder("spark.storage.exceptionOnPinLeak")
+      .version("1.6.2")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val STORAGE_BLOCKMANAGER_TIMEOUTINTERVAL =
     ConfigBuilder("spark.storage.blockManagerTimeoutIntervalMs")
+      .version("0.7.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("60s")
 
   private[spark] val STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT =
     ConfigBuilder("spark.storage.blockManagerSlaveTimeoutMs")
+      .version("0.7.0")
       .timeConf(TimeUnit.MILLISECONDS)
-      .createWithDefaultString(Network.NETWORK_TIMEOUT.defaultValueString)
+      .createOptional
 
   private[spark] val STORAGE_CLEANUP_FILES_AFTER_EXECUTOR_EXIT =
     ConfigBuilder("spark.storage.cleanupFilesAfterExecutorExit")
       .doc("Whether or not cleanup the files not served by the external shuffle service " +
         "on executor exits.")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -385,6 +448,7 @@ package object config {
     ConfigBuilder("spark.diskStore.subDirectories")
       .doc("Number of subdirectories inside each path listed in spark.local.dir for " +
         "hashing Block files into.")
+      .version("0.6.0")
       .intConf
       .checkValue(_ > 0, "The number of subdirectories must be positive.")
       .createWithDefault(64)
@@ -393,71 +457,102 @@ package object config {
     ConfigBuilder("spark.block.failures.beforeLocationRefresh")
       .doc("Max number of failures before this block manager refreshes " +
         "the block locations from the driver.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(5)
 
-  private[spark] val IS_PYTHON_APP = ConfigBuilder("spark.yarn.isPython").internal()
-    .booleanConf.createWithDefault(false)
+  private[spark] val IS_PYTHON_APP =
+    ConfigBuilder("spark.yarn.isPython")
+      .internal()
+      .version("1.5.0")
+      .booleanConf
+      .createWithDefault(false)
 
-  private[spark] val CPUS_PER_TASK = ConfigBuilder("spark.task.cpus").intConf.createWithDefault(1)
+  private[spark] val CPUS_PER_TASK =
+    ConfigBuilder("spark.task.cpus").version("0.5.0").intConf.createWithDefault(1)
 
   private[spark] val DYN_ALLOCATION_ENABLED =
-    ConfigBuilder("spark.dynamicAllocation.enabled").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.dynamicAllocation.enabled")
+      .version("1.2.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_TESTING =
-    ConfigBuilder("spark.dynamicAllocation.testing").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.dynamicAllocation.testing")
+      .version("1.2.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_MIN_EXECUTORS =
-    ConfigBuilder("spark.dynamicAllocation.minExecutors").intConf.createWithDefault(0)
+    ConfigBuilder("spark.dynamicAllocation.minExecutors")
+      .version("1.2.0")
+      .intConf
+      .createWithDefault(0)
 
   private[spark] val DYN_ALLOCATION_INITIAL_EXECUTORS =
     ConfigBuilder("spark.dynamicAllocation.initialExecutors")
+      .version("1.3.0")
       .fallbackConf(DYN_ALLOCATION_MIN_EXECUTORS)
 
   private[spark] val DYN_ALLOCATION_MAX_EXECUTORS =
-    ConfigBuilder("spark.dynamicAllocation.maxExecutors").intConf.createWithDefault(Int.MaxValue)
+    ConfigBuilder("spark.dynamicAllocation.maxExecutors")
+      .version("1.2.0")
+      .intConf
+      .createWithDefault(Int.MaxValue)
 
   private[spark] val DYN_ALLOCATION_EXECUTOR_ALLOCATION_RATIO =
     ConfigBuilder("spark.dynamicAllocation.executorAllocationRatio")
-      .doubleConf.createWithDefault(1.0)
+      .version("2.4.0")
+      .doubleConf
+      .createWithDefault(1.0)
 
   private[spark] val DYN_ALLOCATION_CACHED_EXECUTOR_IDLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.cachedExecutorIdleTimeout")
+      .version("1.4.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(Integer.MAX_VALUE)
 
   private[spark] val DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.executorIdleTimeout")
+      .version("1.2.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(60)
 
-  private[spark] val DYN_ALLOCATION_SHUFFLE_TRACKING =
+  private[spark] val DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED =
     ConfigBuilder("spark.dynamicAllocation.shuffleTracking.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
-  private[spark] val DYN_ALLOCATION_SHUFFLE_TIMEOUT =
-    ConfigBuilder("spark.dynamicAllocation.shuffleTimeout")
+  private[spark] val DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT =
+    ConfigBuilder("spark.dynamicAllocation.shuffleTracking.timeout")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(Long.MaxValue)
 
   private[spark] val DYN_ALLOCATION_SCHEDULER_BACKLOG_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.schedulerBacklogTimeout")
+      .version("1.2.0")
       .timeConf(TimeUnit.SECONDS).createWithDefault(1)
 
   private[spark] val DYN_ALLOCATION_SUSTAINED_SCHEDULER_BACKLOG_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.sustainedSchedulerBacklogTimeout")
+      .version("1.2.0")
       .fallbackConf(DYN_ALLOCATION_SCHEDULER_BACKLOG_TIMEOUT)
 
   private[spark] val LOCALITY_WAIT = ConfigBuilder("spark.locality.wait")
+    .version("0.5.0")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("3s")
 
   private[spark] val SHUFFLE_SERVICE_ENABLED =
-    ConfigBuilder("spark.shuffle.service.enabled").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.shuffle.service.enabled")
+      .version("1.2.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val SHUFFLE_SERVICE_FETCH_RDD_ENABLED =
     ConfigBuilder(Constants.SHUFFLE_SERVICE_FETCH_RDD_ENABLED)
@@ -465,6 +560,7 @@ package object config {
         "In case of dynamic allocation if this feature is enabled executors having only disk " +
         "persisted blocks are considered idle after " +
         "'spark.dynamicAllocation.executorIdleTimeout' and will be released accordingly.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -472,21 +568,26 @@ package object config {
     ConfigBuilder("spark.shuffle.service.db.enabled")
       .doc("Whether to use db in ExternalShuffleService. Note that this only affects " +
         "standalone mode.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val SHUFFLE_SERVICE_PORT =
-    ConfigBuilder("spark.shuffle.service.port").intConf.createWithDefault(7337)
+    ConfigBuilder("spark.shuffle.service.port").version("1.2.0").intConf.createWithDefault(7337)
 
   private[spark] val KEYTAB = ConfigBuilder("spark.kerberos.keytab")
     .doc("Location of user's keytab.")
+    .version("3.0.0")
     .stringConf.createOptional
 
   private[spark] val PRINCIPAL = ConfigBuilder("spark.kerberos.principal")
     .doc("Name of the Kerberos principal.")
-    .stringConf.createOptional
+    .version("3.0.0")
+    .stringConf
+    .createOptional
 
   private[spark] val KERBEROS_RELOGIN_PERIOD = ConfigBuilder("spark.kerberos.relogin.period")
+    .version("3.0.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("1m")
 
@@ -496,6 +597,7 @@ package object config {
         "Which credentials to use when renewing delegation tokens for executors. Can be either " +
         "'keytab', the default, which requires a keytab to be provided, or 'ccache', which uses " +
         "the local credentials cache.")
+      .version("3.0.0")
       .stringConf
       .checkValues(Set("keytab", "ccache"))
       .createWithDefault("keytab")
@@ -504,104 +606,124 @@ package object config {
     ConfigBuilder("spark.kerberos.access.hadoopFileSystems")
     .doc("Extra Hadoop filesystem URLs for which to request delegation tokens. The filesystem " +
       "that hosts fs.defaultFS does not need to be listed here.")
+    .version("3.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val EXECUTOR_INSTANCES = ConfigBuilder("spark.executor.instances")
+    .version("1.0.0")
     .intConf
     .createOptional
 
   private[spark] val PY_FILES = ConfigBuilder("spark.yarn.dist.pyFiles")
     .internal()
+    .version("2.2.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val TASK_MAX_DIRECT_RESULT_SIZE =
     ConfigBuilder("spark.task.maxDirectResultSize")
+      .version("2.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(1L << 20)
 
   private[spark] val TASK_MAX_FAILURES =
     ConfigBuilder("spark.task.maxFailures")
+      .version("0.8.0")
       .intConf
       .createWithDefault(4)
 
   private[spark] val TASK_REAPER_ENABLED =
     ConfigBuilder("spark.task.reaper.enabled")
+      .version("2.0.3")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val TASK_REAPER_KILL_TIMEOUT =
     ConfigBuilder("spark.task.reaper.killTimeout")
+      .version("2.0.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(-1)
 
   private[spark] val TASK_REAPER_POLLING_INTERVAL =
     ConfigBuilder("spark.task.reaper.pollingInterval")
+      .version("2.0.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10s")
 
   private[spark] val TASK_REAPER_THREAD_DUMP =
     ConfigBuilder("spark.task.reaper.threadDump")
+      .version("2.0.3")
       .booleanConf
       .createWithDefault(true)
 
   // Blacklist confs
   private[spark] val BLACKLIST_ENABLED =
     ConfigBuilder("spark.blacklist.enabled")
+      .version("2.1.0")
       .booleanConf
       .createOptional
 
   private[spark] val MAX_TASK_ATTEMPTS_PER_EXECUTOR =
     ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerExecutor")
+      .version("2.1.0")
       .intConf
       .createWithDefault(1)
 
   private[spark] val MAX_TASK_ATTEMPTS_PER_NODE =
     ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerNode")
+      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILURES_PER_EXEC =
     ConfigBuilder("spark.blacklist.application.maxFailedTasksPerExecutor")
+      .version("2.2.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILURES_PER_EXEC_STAGE =
     ConfigBuilder("spark.blacklist.stage.maxFailedTasksPerExecutor")
+      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILED_EXEC_PER_NODE =
     ConfigBuilder("spark.blacklist.application.maxFailedExecutorsPerNode")
+      .version("2.2.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILED_EXEC_PER_NODE_STAGE =
     ConfigBuilder("spark.blacklist.stage.maxFailedExecutorsPerNode")
+      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val BLACKLIST_TIMEOUT_CONF =
     ConfigBuilder("spark.blacklist.timeout")
+      .version("2.1.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val BLACKLIST_KILL_ENABLED =
     ConfigBuilder("spark.blacklist.killBlacklistedExecutors")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val BLACKLIST_LEGACY_TIMEOUT_CONF =
     ConfigBuilder("spark.scheduler.executorTaskBlacklistTime")
       .internal()
+      .version("1.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val BLACKLIST_FETCH_FAILURE_ENABLED =
     ConfigBuilder("spark.blacklist.application.fetchFailure.enabled")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
   // End blacklist confs
@@ -611,6 +733,7 @@ package object config {
       .doc("Whether to un-register all the outputs on the host in condition that we receive " +
         " a FetchFailure. This is set default to false, which means, we only un-register the " +
         " outputs related to the exact executor(instead of the host) on a FetchFailure.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -620,6 +743,7 @@ package object config {
         "an event queue using capacity specified by `spark.scheduler.listenerbus" +
         ".eventqueue.queueName.capacity` first. If it's not configured, Spark will " +
         "use the default capacity specified by this config.")
+      .version("2.3.0")
       .intConf
       .checkValue(_ > 0, "The capacity of listener bus event queue must be positive")
       .createWithDefault(10000)
@@ -627,15 +751,17 @@ package object config {
   private[spark] val LISTENER_BUS_METRICS_MAX_LISTENER_CLASSES_TIMED =
     ConfigBuilder("spark.scheduler.listenerbus.metrics.maxListenerClassesTimed")
       .internal()
+      .version("2.3.0")
       .intConf
       .createWithDefault(128)
 
   private[spark] val LISTENER_BUS_LOG_SLOW_EVENT_ENABLED =
-    ConfigBuilder("spark.scheduler.listenerbus.logSlowEvent.enabled")
+    ConfigBuilder("spark.scheduler.listenerbus.logSlowEvent")
       .internal()
       .doc("When enabled, log the event that takes too much time to process. This helps us " +
         "discover the event types that cause performance bottlenecks. The time threshold is " +
         "controlled by spark.scheduler.listenerbus.logSlowEvent.threshold.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -643,53 +769,66 @@ package object config {
     ConfigBuilder("spark.scheduler.listenerbus.logSlowEvent.threshold")
       .internal()
       .doc("The time threshold of whether a event is considered to be taking too much time to " +
-        "process. Log the event if spark.scheduler.listenerbus.logSlowEvent.enabled is true.")
+        s"process. Log the event if ${LISTENER_BUS_LOG_SLOW_EVENT_ENABLED.key} is true.")
+      .version("3.0.0")
       .timeConf(TimeUnit.NANOSECONDS)
       .createWithDefaultString("1s")
 
   // This property sets the root namespace for metrics reporting
   private[spark] val METRICS_NAMESPACE = ConfigBuilder("spark.metrics.namespace")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   private[spark] val METRICS_CONF = ConfigBuilder("spark.metrics.conf")
+    .version("0.8.0")
     .stringConf
     .createOptional
 
   private[spark] val METRICS_EXECUTORMETRICS_SOURCE_ENABLED =
     ConfigBuilder("spark.metrics.executorMetricsSource.enabled")
       .doc("Whether to register the ExecutorMetrics source with the metrics system.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val METRICS_STATIC_SOURCES_ENABLED =
     ConfigBuilder("spark.metrics.staticSources.enabled")
       .doc("Whether to register static sources with the metrics system.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val PYSPARK_DRIVER_PYTHON = ConfigBuilder("spark.pyspark.driver.python")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   private[spark] val PYSPARK_PYTHON = ConfigBuilder("spark.pyspark.python")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   // To limit how many applications are shown in the History Server summary ui
   private[spark] val HISTORY_UI_MAX_APPS =
-    ConfigBuilder("spark.history.ui.maxApplications").intConf.createWithDefault(Integer.MAX_VALUE)
+    ConfigBuilder("spark.history.ui.maxApplications")
+      .version("2.0.1")
+      .intConf
+      .createWithDefault(Integer.MAX_VALUE)
 
   private[spark] val IO_ENCRYPTION_ENABLED = ConfigBuilder("spark.io.encryption.enabled")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val IO_ENCRYPTION_KEYGEN_ALGORITHM =
     ConfigBuilder("spark.io.encryption.keygen.algorithm")
+      .version("2.1.0")
       .stringConf
       .createWithDefault("HmacSHA1")
 
   private[spark] val IO_ENCRYPTION_KEY_SIZE_BITS = ConfigBuilder("spark.io.encryption.keySizeBits")
+    .version("2.1.0")
     .intConf
     .checkValues(Set(128, 192, 256))
     .createWithDefault(128)
@@ -697,57 +836,68 @@ package object config {
   private[spark] val IO_CRYPTO_CIPHER_TRANSFORMATION =
     ConfigBuilder("spark.io.crypto.cipher.transformation")
       .internal()
+      .version("2.1.0")
       .stringConf
       .createWithDefaultString("AES/CTR/NoPadding")
 
   private[spark] val DRIVER_HOST_ADDRESS = ConfigBuilder("spark.driver.host")
     .doc("Address of driver endpoints.")
+    .version("0.7.0")
     .stringConf
     .createWithDefault(Utils.localCanonicalHostName())
 
   private[spark] val DRIVER_PORT = ConfigBuilder("spark.driver.port")
     .doc("Port of driver endpoints.")
+    .version("0.7.0")
     .intConf
     .createWithDefault(0)
 
   private[spark] val DRIVER_SUPERVISE = ConfigBuilder("spark.driver.supervise")
     .doc("If true, restarts the driver automatically if it fails with a non-zero exit status. " +
       "Only has effect in Spark standalone mode or Mesos cluster deploy mode.")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val DRIVER_BIND_ADDRESS = ConfigBuilder("spark.driver.bindAddress")
     .doc("Address where to bind network listen sockets on the driver.")
+    .version("2.1.0")
     .fallbackConf(DRIVER_HOST_ADDRESS)
 
   private[spark] val BLOCK_MANAGER_PORT = ConfigBuilder("spark.blockManager.port")
     .doc("Port to use for the block manager when a more specific setting is not provided.")
+    .version("1.1.0")
     .intConf
     .createWithDefault(0)
 
   private[spark] val DRIVER_BLOCK_MANAGER_PORT = ConfigBuilder("spark.driver.blockManager.port")
     .doc("Port to use for the block manager on the driver.")
+    .version("2.1.0")
     .fallbackConf(BLOCK_MANAGER_PORT)
 
   private[spark] val IGNORE_CORRUPT_FILES = ConfigBuilder("spark.files.ignoreCorruptFiles")
     .doc("Whether to ignore corrupt files. If true, the Spark jobs will continue to run when " +
       "encountering corrupted or non-existing files and contents that have been read will still " +
       "be returned.")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val IGNORE_MISSING_FILES = ConfigBuilder("spark.files.ignoreMissingFiles")
     .doc("Whether to ignore missing files. If true, the Spark jobs will continue to run when " +
       "encountering missing files and the contents that have been read will still be returned.")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val APP_CALLER_CONTEXT = ConfigBuilder("spark.log.callerContext")
+    .version("2.2.0")
     .stringConf
     .createOptional
 
   private[spark] val FILES_MAX_PARTITION_BYTES = ConfigBuilder("spark.files.maxPartitionBytes")
     .doc("The maximum number of bytes to pack into a single partition when reading files.")
+    .version("2.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(128 * 1024 * 1024)
 
@@ -756,6 +906,7 @@ package object config {
       " the same time. This is used when putting multiple files into a partition. It's better to" +
       " over estimate, then the partitions with small files will be faster than partitions with" +
       " bigger files.")
+    .version("2.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(4 * 1024 * 1024)
 
@@ -763,6 +914,7 @@ package object config {
     ConfigBuilder("spark.hadoopRDD.ignoreEmptySplits")
       .internal()
       .doc("When true, HadoopRDD/NewHadoopRDD will not create partitions for empty input splits.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -772,6 +924,7 @@ package object config {
         "driver and executor environments contain sensitive information. When this regex matches " +
         "a property key or value, the value is redacted from the environment UI and various logs " +
         "like YARN and event logs.")
+      .version("2.1.2")
       .regexConf
       .createWithDefault("(?i)secret|password|token".r)
 
@@ -780,26 +933,31 @@ package object config {
       .doc("Regex to decide which parts of strings produced by Spark contain sensitive " +
         "information. When this regex matches a string part, that string part is replaced by a " +
         "dummy value. This is currently used to redact the output of SQL explain commands.")
+      .version("2.2.0")
       .regexConf
       .createOptional
 
   private[spark] val AUTH_SECRET =
     ConfigBuilder("spark.authenticate.secret")
+      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val AUTH_SECRET_BIT_LENGTH =
     ConfigBuilder("spark.authenticate.secretBitLength")
+      .version("1.6.0")
       .intConf
       .createWithDefault(256)
 
   private[spark] val NETWORK_AUTH_ENABLED =
     ConfigBuilder("spark.authenticate")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SASL_ENCRYPTION_ENABLED =
     ConfigBuilder("spark.authenticate.enableSaslEncryption")
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -809,6 +967,7 @@ package object config {
         "loaded from this path on both the driver and the executors if overrides are not set for " +
         "either entity (see below). File-based secret keys are only allowed when using " +
         "Kubernetes.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -821,6 +980,7 @@ package object config {
         "be specified for the executors. The fallback configuration allows the same path to be " +
         "used for both the driver and the executors when running in cluster mode. File-based " +
         "secret keys are only allowed when using Kubernetes.")
+      .version("3.0.0")
       .fallbackConf(AUTH_SECRET_FILE)
 
   private[spark] val AUTH_SECRET_FILE_EXECUTOR =
@@ -832,12 +992,14 @@ package object config {
         "specified for the executors. The fallback configuration allows the same path to be " +
         "used for both the driver and the executors when running in cluster mode. File-based " +
         "secret keys are only allowed when using Kubernetes.")
+      .version("3.0.0")
       .fallbackConf(AUTH_SECRET_FILE)
 
   private[spark] val BUFFER_WRITE_CHUNK_SIZE =
     ConfigBuilder("spark.buffer.write.chunkSize")
       .internal()
       .doc("The chunk size in bytes during writing out the bytes of ChunkedByteBuffer.")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(_ <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
         "The chunk size during writing out the bytes of ChunkedByteBuffer should" +
@@ -848,6 +1010,7 @@ package object config {
     ConfigBuilder("spark.checkpoint.compress")
       .doc("Whether to compress RDD checkpoints. Generally a good idea. Compression will use " +
         "spark.io.compression.codec.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -858,6 +1021,7 @@ package object config {
         "Caching preferred locations can relieve query loading to DFS and save the query " +
         "time. The drawback is that the cached locations can be possibly outdated and " +
         "lose data locality. If this config is not specified, it will not cache.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MINUTES)
       .checkValue(_ > 0, "The expire time for caching preferred locations cannot be non-positive.")
       .createOptional
@@ -867,12 +1031,14 @@ package object config {
       .doc("Threshold in bytes above which the size of shuffle blocks in " +
         "HighlyCompressedMapStatus is accurately recorded. This helps to prevent OOM " +
         "by avoiding underestimating shuffle block size when fetch shuffle blocks.")
+      .version("2.2.1")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(100 * 1024 * 1024)
 
   private[spark] val SHUFFLE_REGISTRATION_TIMEOUT =
     ConfigBuilder("spark.shuffle.registration.timeout")
       .doc("Timeout in milliseconds for registration to the external shuffle service.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(5000)
 
@@ -880,6 +1046,7 @@ package object config {
     ConfigBuilder("spark.shuffle.registration.maxAttempts")
       .doc("When we fail to register to the external shuffle service, we will " +
         "retry for maxAttempts times.")
+      .version("2.3.0")
       .intConf
       .createWithDefault(3)
 
@@ -890,17 +1057,19 @@ package object config {
         "address in a single fetch or simultaneously, this could crash the serving executor or " +
         "Node Manager. This is especially useful to reduce the load on the Node Manager when " +
         "external shuffle is enabled. You can mitigate the issue by setting it to a lower value.")
+      .version("2.2.1")
       .intConf
       .checkValue(_ > 0, "The max no. of blocks in flight cannot be non-positive.")
       .createWithDefault(Int.MaxValue)
 
   private[spark] val MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM =
-    ConfigBuilder("spark.maxRemoteBlockSizeFetchToMem")
+    ConfigBuilder("spark.network.maxRemoteBlockSizeFetchToMem")
       .doc("Remote block will be fetched to disk when size of the block is above this threshold " +
         "in bytes. This is to avoid a giant request takes too much memory. Note this " +
         "configuration will affect both shuffle fetch and block manager remote block fetch. " +
         "For users who enabled external shuffle service, this feature can only work when " +
         "external shuffle service is at least 2.3.0.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       // fetch-to-mem is guaranteed to fail if the message is bigger than 2 GB, so we might
       // as well use fetch-to-disk in that case.  The message includes some metadata in addition
@@ -916,12 +1085,14 @@ package object config {
       .doc("Enable tracking of updatedBlockStatuses in the TaskMetrics. Off by default since " +
         "tracking the block statuses can use a lot of memory and its not used anywhere within " +
         "spark.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_IO_PLUGIN_CLASS =
     ConfigBuilder("spark.shuffle.sort.io.plugin.class")
       .doc("Name of the class to use for shuffle IO.")
+      .version("3.0.0")
       .stringConf
       .createWithDefault(classOf[LocalDiskShuffleDataIO].getName)
 
@@ -930,6 +1101,7 @@ package object config {
       .doc("Size of the in-memory buffer for each shuffle file output stream, in KiB unless " +
         "otherwise specified. These buffers reduce the number of disk seeks and system calls " +
         "made in creating intermediate shuffle files.")
+      .version("1.4.0")
       .bytesConf(ByteUnit.KiB)
       .checkValue(v => v > 0 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH / 1024,
         s"The file buffer size must be positive and less than or equal to" +
@@ -940,6 +1112,7 @@ package object config {
     ConfigBuilder("spark.shuffle.unsafe.file.output.buffer")
       .doc("The file system for this buffer size after each partition " +
         "is written in unsafe shuffle writer. In KiB unless otherwise specified.")
+      .version("2.3.0")
       .bytesConf(ByteUnit.KiB)
       .checkValue(v => v > 0 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH / 1024,
         s"The buffer size must be positive and less than or equal to" +
@@ -949,6 +1122,7 @@ package object config {
   private[spark] val SHUFFLE_DISK_WRITE_BUFFER_SIZE =
     ConfigBuilder("spark.shuffle.spill.diskWriteBufferSize")
       .doc("The buffer size, in bytes, to use when writing the sorted records to an on-disk file.")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v > 12 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
         s"The buffer size must be greater than 12 and less than or equal to " +
@@ -960,6 +1134,7 @@ package object config {
       .internal()
       .doc("The memory check period is used to determine how often we should check whether "
         + "there is a need to request more memory when we try to unroll the given block in memory.")
+      .version("2.3.0")
       .longConf
       .createWithDefault(16)
 
@@ -967,6 +1142,7 @@ package object config {
     ConfigBuilder("spark.storage.unrollMemoryGrowthFactor")
       .internal()
       .doc("Memory to request as a multiple of the size that used to unroll the block.")
+      .version("2.3.0")
       .doubleConf
       .createWithDefault(1.5)
 
@@ -977,12 +1153,14 @@ package object config {
         "where the YARN service does not support schemes that are supported by Spark, like http, " +
         "https and ftp, or jars required to be in the local YARN client's classpath. Wildcard " +
         "'*' is denoted to download resources for all the schemes.")
+      .version("2.3.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
 
   private[spark] val EXTRA_LISTENERS = ConfigBuilder("spark.extraListeners")
     .doc("Class names of listeners to add to SparkContext during initialization.")
+    .version("1.3.0")
     .stringConf
     .toSequence
     .createOptional
@@ -994,6 +1172,7 @@ package object config {
         "By default it's Integer.MAX_VALUE, which means we never force the sorter to spill, " +
         "until we reach some limitations, like the max page size limitation for the pointer " +
         "array in the sorter.")
+      .version("1.6.0")
       .intConf
       .createWithDefault(Integer.MAX_VALUE)
 
@@ -1003,30 +1182,35 @@ package object config {
       .doc("Multi-thread is used when the number of mappers * shuffle partitions is greater than " +
         "or equal to this threshold. Note that the actual parallelism is calculated by number of " +
         "mappers * shuffle partitions / this threshold + 1, so this threshold should be positive.")
+      .version("2.3.0")
       .intConf
       .checkValue(v => v > 0, "The threshold should be positive.")
       .createWithDefault(10000000)
 
   private[spark] val MAX_RESULT_SIZE = ConfigBuilder("spark.driver.maxResultSize")
     .doc("Size limit for results.")
+    .version("1.2.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("1g")
 
   private[spark] val CREDENTIALS_RENEWAL_INTERVAL_RATIO =
     ConfigBuilder("spark.security.credentials.renewalRatio")
       .doc("Ratio of the credential's expiration time when Spark should fetch new credentials.")
+      .version("2.4.0")
       .doubleConf
       .createWithDefault(0.75d)
 
   private[spark] val CREDENTIALS_RENEWAL_RETRY_WAIT =
     ConfigBuilder("spark.security.credentials.retryWait")
       .doc("How long to wait before retrying to fetch new credentials after a failure.")
+      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("1h")
 
   private[spark] val SHUFFLE_SORT_INIT_BUFFER_SIZE =
     ConfigBuilder("spark.shuffle.sort.initialBufferSize")
       .internal()
+      .version("2.1.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v > 0 && v <= Int.MaxValue,
         s"The buffer size must be greater than 0 and less than or equal to ${Int.MaxValue}.")
@@ -1036,6 +1220,7 @@ package object config {
     ConfigBuilder("spark.shuffle.compress")
       .doc("Whether to compress shuffle output. Compression will use " +
         "spark.io.compression.codec.")
+      .version("0.6.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1043,6 +1228,7 @@ package object config {
     ConfigBuilder("spark.shuffle.spill.compress")
       .doc("Whether to compress data spilled during shuffles. Compression will use " +
         "spark.io.compression.codec.")
+      .version("0.9.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1052,6 +1238,7 @@ package object config {
       .doc("The codec used to compress MapStatus, which is generated by ShuffleMapTask. " +
         "By default, Spark provides four codecs: lz4, lzf, snappy, and zstd. You can also " +
         "use fully qualified class names to specify the codec.")
+      .version("3.0.0")
       .stringConf
       .createWithDefault("zstd")
 
@@ -1060,6 +1247,7 @@ package object config {
       .internal()
       .doc("Initial threshold for the size of a collection before we start tracking its " +
         "memory usage.")
+      .version("1.1.1")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(5 * 1024 * 1024)
 
@@ -1067,6 +1255,7 @@ package object config {
     ConfigBuilder("spark.shuffle.spill.batchSize")
       .internal()
       .doc("Size of object batches when reading/writing from serializers.")
+      .version("0.9.0")
       .longConf
       .createWithDefault(10000)
 
@@ -1074,34 +1263,40 @@ package object config {
     ConfigBuilder("spark.shuffle.sort.bypassMergeThreshold")
       .doc("In the sort-based shuffle manager, avoid merge-sorting data if there is no " +
         "map-side aggregation and there are at most this many reduce partitions")
+      .version("1.1.1")
       .intConf
       .createWithDefault(200)
 
   private[spark] val SHUFFLE_MANAGER =
     ConfigBuilder("spark.shuffle.manager")
+      .version("1.1.0")
       .stringConf
       .createWithDefault("sort")
 
   private[spark] val SHUFFLE_REDUCE_LOCALITY_ENABLE =
     ConfigBuilder("spark.shuffle.reduceLocality.enabled")
       .doc("Whether to compute locality preferences for reduce tasks")
+      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val SHUFFLE_MAPOUTPUT_MIN_SIZE_FOR_BROADCAST =
     ConfigBuilder("spark.shuffle.mapOutput.minSizeForBroadcast")
       .doc("The size at which we use Broadcast to send the map output statuses to the executors.")
+      .version("2.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("512k")
 
   private[spark] val SHUFFLE_MAPOUTPUT_DISPATCHER_NUM_THREADS =
     ConfigBuilder("spark.shuffle.mapOutput.dispatcher.numThreads")
+      .version("2.0.0")
       .intConf
       .createWithDefault(8)
 
   private[spark] val SHUFFLE_DETECT_CORRUPT =
     ConfigBuilder("spark.shuffle.detectCorrupt")
       .doc("Whether to detect any corruption in fetched blocks.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1111,28 +1306,21 @@ package object config {
         "by using extra memory to detect early corruption. Any IOException thrown will cause " +
         "the task to be retried once and if it fails again with same exception, then " +
         "FetchFailedException will be thrown to retry previous stage")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
-  private[spark] val STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE =
-    ConfigBuilder("spark.storage.localDiskByExecutors.cacheSize")
-      .doc("The max number of executors for which the local dirs are stored. This size is " +
-        "both applied for the driver and both for the executors side to avoid having an " +
-        "unbounded store. This cache will be used to avoid the network in case of fetching disk " +
-        "persisted RDD blocks or shuffle blocks (when `spark.shuffle.readHostLocalDisk.enabled` " +
-        "is set) from the same host.")
-      .intConf
-      .createWithDefault(1000)
-
   private[spark] val SHUFFLE_SYNC =
     ConfigBuilder("spark.shuffle.sync")
       .doc("Whether to force outstanding writes to disk.")
+      .version("0.8.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_UNSAFE_FAST_MERGE_ENABLE =
     ConfigBuilder("spark.shuffle.unsafe.fastMergeEnabled")
       .doc("Whether to perform a fast spill merge.")
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1140,6 +1328,7 @@ package object config {
     ConfigBuilder("spark.shuffle.sort.useRadixSort")
       .doc("Whether to use radix sort for sorting in-memory partition ids. Radix sort is much " +
         "faster, but requires additional memory to be reserved memory as pointers are added.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1147,6 +1336,7 @@ package object config {
     ConfigBuilder("spark.shuffle.minNumPartitionsToHighlyCompress")
       .internal()
       .doc("Number of partitions to determine if MapStatus should use HighlyCompressedMapStatus")
+      .version("2.4.0")
       .intConf
       .checkValue(v => v > 0, "The value should be a positive integer.")
       .createWithDefault(2000)
@@ -1156,21 +1346,36 @@ package object config {
       .doc("Whether to use the old protocol while doing the shuffle block fetching. " +
         "It is only enabled while we need the compatibility in the scenario of new Spark " +
         "version job fetching shuffle blocks from old version external shuffle service.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED =
-    ConfigBuilder("spark.shuffle.readHostLocalDisk.enabled")
-      .doc(s"If enabled (and `${SHUFFLE_USE_OLD_FETCH_PROTOCOL.key}` is disabled), shuffle " +
+    ConfigBuilder("spark.shuffle.readHostLocalDisk")
+      .doc(s"If enabled (and `${SHUFFLE_USE_OLD_FETCH_PROTOCOL.key}` is disabled and external " +
+        s"shuffle `${SHUFFLE_SERVICE_ENABLED.key}` is enabled), shuffle " +
         "blocks requested from those block managers which are running on the same host are read " +
         "from the disk directly instead of being fetched as remote blocks over the network.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
+  private[spark] val STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE =
+    ConfigBuilder("spark.storage.localDiskByExecutors.cacheSize")
+      .doc("The max number of executors for which the local dirs are stored. This size is " +
+        "both applied for the driver and both for the executors side to avoid having an " +
+        "unbounded store. This cache will be used to avoid the network in case of fetching disk " +
+        s"persisted RDD blocks or shuffle blocks " +
+        s"(when `${SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED.key}` is set) from the same host.")
+      .version("3.0.0")
+      .intConf
+      .createWithDefault(1000)
+
   private[spark] val MEMORY_MAP_LIMIT_FOR_TESTS =
     ConfigBuilder("spark.storage.memoryMapLimitForTests")
       .internal()
       .doc("For testing only, controls the size of chunks when memory mapping a file")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1180,6 +1385,7 @@ package object config {
         "coordinator didn't receive all the sync messages from barrier tasks within the " +
         "configured time, throw a SparkException to fail all the tasks. The default value is set " +
         "to 31536000(3600 * 24 * 365) so the barrier() call shall wait for one year.")
+      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(v => v > 0, "The value should be a positive time value.")
       .createWithDefaultString("365d")
@@ -1188,6 +1394,7 @@ package object config {
     ConfigBuilder("spark.scheduler.blacklist.unschedulableTaskSetTimeout")
       .doc("The timeout in seconds to wait to acquire a new executor and schedule a task " +
         "before aborting a TaskSet which is unschedulable because of being completely blacklisted.")
+      .version("2.4.1")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(v => v >= 0, "The value should be a non negative time value.")
       .createWithDefault(120)
@@ -1202,6 +1409,7 @@ package object config {
         "configured max failure times for a job then fail current job submission. Note this " +
         "config only applies to jobs that contain one or more barrier stages, we won't perform " +
         "the check on non-barrier jobs.")
+      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("15s")
 
@@ -1215,6 +1423,7 @@ package object config {
         "max failure times for a job then fail current job submission. Note this config only " +
         "applies to jobs that contain one or more barrier stages, we won't perform the check on " +
         "non-barrier jobs.")
+      .version("2.4.0")
       .intConf
       .checkValue(v => v > 0, "The max failures should be a positive value.")
       .createWithDefault(40)
@@ -1222,18 +1431,21 @@ package object config {
   private[spark] val UNSAFE_EXCEPTION_ON_MEMORY_LEAK =
     ConfigBuilder("spark.unsafe.exceptionOnMemoryLeak")
       .internal()
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val UNSAFE_SORTER_SPILL_READ_AHEAD_ENABLED =
     ConfigBuilder("spark.unsafe.sorter.spill.read.ahead.enabled")
       .internal()
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val UNSAFE_SORTER_SPILL_READER_BUFFER_SIZE =
     ConfigBuilder("spark.unsafe.sorter.spill.reader.buffer.size")
       .internal()
+      .version("2.1.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => 1024 * 1024 <= v && v <= MAX_BUFFER_SIZE_BYTES,
         s"The value must be in allowed range [1,048,576, ${MAX_BUFFER_SIZE_BYTES}].")
@@ -1246,63 +1458,83 @@ package object config {
       .withPrepended(DEFAULT_PLUGINS_LIST, separator = ",")
       .doc("Comma-separated list of class names implementing " +
         "org.apache.spark.api.plugin.SparkPlugin to load into the application.")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
 
   private[spark] val CLEANER_PERIODIC_GC_INTERVAL =
     ConfigBuilder("spark.cleaner.periodicGC.interval")
+      .version("1.6.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("30min")
 
   private[spark] val CLEANER_REFERENCE_TRACKING =
     ConfigBuilder("spark.cleaner.referenceTracking")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_BLOCKING =
     ConfigBuilder("spark.cleaner.referenceTracking.blocking")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_BLOCKING_SHUFFLE =
     ConfigBuilder("spark.cleaner.referenceTracking.blocking.shuffle")
+      .version("1.1.1")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_CLEAN_CHECKPOINTS =
     ConfigBuilder("spark.cleaner.referenceTracking.cleanCheckpoints")
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_STRATEGY =
-    ConfigBuilder("spark.executor.logs.rolling.strategy").stringConf.createWithDefault("")
+    ConfigBuilder("spark.executor.logs.rolling.strategy")
+      .version("1.1.0")
+      .stringConf
+      .createWithDefault("")
 
   private[spark] val EXECUTOR_LOGS_ROLLING_TIME_INTERVAL =
-    ConfigBuilder("spark.executor.logs.rolling.time.interval").stringConf.createWithDefault("daily")
+    ConfigBuilder("spark.executor.logs.rolling.time.interval")
+      .version("1.1.0")
+      .stringConf
+      .createWithDefault("daily")
 
   private[spark] val EXECUTOR_LOGS_ROLLING_MAX_SIZE =
     ConfigBuilder("spark.executor.logs.rolling.maxSize")
+      .version("1.4.0")
       .stringConf
       .createWithDefault((1024 * 1024).toString)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_MAX_RETAINED_FILES =
-    ConfigBuilder("spark.executor.logs.rolling.maxRetainedFiles").intConf.createWithDefault(-1)
+    ConfigBuilder("spark.executor.logs.rolling.maxRetainedFiles")
+      .version("1.1.0")
+      .intConf
+      .createWithDefault(-1)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_ENABLE_COMPRESSION =
     ConfigBuilder("spark.executor.logs.rolling.enableCompression")
+      .version("2.0.2")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val MASTER_REST_SERVER_ENABLED = ConfigBuilder("spark.master.rest.enabled")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val MASTER_REST_SERVER_PORT = ConfigBuilder("spark.master.rest.port")
+    .version("1.3.0")
     .intConf
     .createWithDefault(6066)
 
   private[spark] val MASTER_UI_PORT = ConfigBuilder("spark.master.ui.port")
+    .version("1.1.0")
     .intConf
     .createWithDefault(8080)
 
@@ -1311,6 +1543,7 @@ package object config {
       .doc("Block size in bytes used in Snappy compression, in the case when " +
         "Snappy compression codec is used. Lowering this block size " +
         "will also lower shuffle memory usage when Snappy is used")
+      .version("1.4.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1319,6 +1552,7 @@ package object config {
       .doc("Block size in bytes used in LZ4 compression, in the case when LZ4 compression" +
         "codec is used. Lowering this block size will also lower shuffle memory " +
         "usage when LZ4 is used.")
+      .version("1.4.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1328,6 +1562,7 @@ package object config {
         "broadcast variables and shuffle outputs. By default, Spark provides four codecs: " +
         "lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify " +
         "the codec")
+      .version("0.8.0")
       .stringConf
       .createWithDefaultString("lz4")
 
@@ -1337,6 +1572,7 @@ package object config {
         "compression codec is used. Lowering this size will lower the shuffle " +
         "memory usage when Zstd is used, but it might increase the compression " +
         "cost because of excessive JNI call overhead")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1344,6 +1580,7 @@ package object config {
     ConfigBuilder("spark.io.compression.zstd.level")
       .doc("Compression level for Zstd compression codec. Increasing the compression " +
         "level will result in better compression at the expense of more CPU and memory")
+      .version("2.3.0")
       .intConf
       .createWithDefault(1)
 
@@ -1352,6 +1589,7 @@ package object config {
       .internal()
       .doc("If the size in bytes of a file loaded by Spark exceeds this threshold, " +
         "a warning is logged with the possible reasons.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(1024 * 1024 * 1024)
 
@@ -1360,28 +1598,34 @@ package object config {
       .doc("The codec used to compress event log. By default, Spark provides four codecs: " +
         "lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify " +
         "the codec. If this is not given, spark.io.compression.codec will be used.")
+      .version("3.0.0")
       .fallbackConf(IO_COMPRESSION_CODEC)
 
   private[spark] val BUFFER_SIZE =
     ConfigBuilder("spark.buffer.size")
+      .version("0.5.0")
       .intConf
       .checkValue(_ >= 0, "The buffer size must not be negative")
       .createWithDefault(65536)
 
   private[spark] val LOCALITY_WAIT_PROCESS = ConfigBuilder("spark.locality.wait.process")
+    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
   private[spark] val LOCALITY_WAIT_NODE = ConfigBuilder("spark.locality.wait.node")
+    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
   private[spark] val LOCALITY_WAIT_RACK = ConfigBuilder("spark.locality.wait.rack")
+    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
-    private[spark] val REDUCER_MAX_SIZE_IN_FLIGHT = ConfigBuilder("spark.reducer.maxSizeInFlight")
+  private[spark] val REDUCER_MAX_SIZE_IN_FLIGHT = ConfigBuilder("spark.reducer.maxSizeInFlight")
     .doc("Maximum size of map outputs to fetch simultaneously from each reduce task, " +
       "in MiB unless otherwise specified. Since each output requires us to create a " +
       "buffer to receive it, this represents a fixed memory overhead per reduce task, " +
       "so keep it small unless you have a large amount of memory")
+    .version("1.4.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("48m")
 
@@ -1391,12 +1635,14 @@ package object config {
       "it might lead to very large number of inbound connections to one or more nodes, " +
       "causing the workers to fail under load. By allowing it to limit the number of " +
       "fetch requests, this scenario can be mitigated")
+    .version("2.0.0")
     .intConf
     .createWithDefault(Int.MaxValue)
 
   private[spark] val BROADCAST_COMPRESS = ConfigBuilder("spark.broadcast.compress")
     .doc("Whether to compress broadcast variables before sending them. " +
       "Generally a good idea. Compression will use spark.io.compression.codec")
+    .version("0.6.0")
     .booleanConf.createWithDefault(true)
 
   private[spark] val BROADCAST_BLOCKSIZE = ConfigBuilder("spark.broadcast.blockSize")
@@ -1404,6 +1650,7 @@ package object config {
       "KiB unless otherwise specified. Too large a value decreases " +
       "parallelism during broadcast (makes it slower); however, " +
       "if it is too small, BlockManager might take a performance hit")
+    .version("0.5.0")
     .bytesConf(ByteUnit.KiB)
     .createWithDefaultString("4m")
 
@@ -1413,12 +1660,15 @@ package object config {
       "corrupted blocks, at the cost of computing and sending a little " +
       "more data. It's possible to disable it if the network has other " +
       "mechanisms to guarantee data won't be corrupted during broadcast")
-    .booleanConf.createWithDefault(true)
+    .version("2.1.1")
+    .booleanConf
+    .createWithDefault(true)
 
   private[spark] val BROADCAST_FOR_UDF_COMPRESSION_THRESHOLD =
     ConfigBuilder("spark.broadcast.UDFCompressionThreshold")
       .doc("The threshold at which user-defined functions (UDFs) and Python RDD commands " +
         "are compressed by broadcast in bytes unless otherwise specified")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v >= 0, "The threshold should be non-negative.")
       .createWithDefault(1L * 1024 * 1024)
@@ -1429,92 +1679,112 @@ package object config {
       "or StorageLevel.MEMORY_ONLY in Python). Can save substantial " +
       "space at the cost of some extra CPU time. " +
       "Compression will use spark.io.compression.codec")
-    .booleanConf.createWithDefault(false)
+    .version("0.6.0")
+    .booleanConf
+    .createWithDefault(false)
 
   private[spark] val RDD_PARALLEL_LISTING_THRESHOLD =
     ConfigBuilder("spark.rdd.parallelListingThreshold")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
   private[spark] val RDD_LIMIT_SCALE_UP_FACTOR =
     ConfigBuilder("spark.rdd.limit.scaleUpFactor")
+      .version("2.1.0")
       .intConf
       .createWithDefault(4)
 
   private[spark] val SERIALIZER = ConfigBuilder("spark.serializer")
+    .version("0.5.0")
     .stringConf
     .createWithDefault("org.apache.spark.serializer.JavaSerializer")
 
   private[spark] val SERIALIZER_OBJECT_STREAM_RESET =
     ConfigBuilder("spark.serializer.objectStreamReset")
+      .version("1.0.0")
       .intConf
       .createWithDefault(100)
 
   private[spark] val SERIALIZER_EXTRA_DEBUG_INFO = ConfigBuilder("spark.serializer.extraDebugInfo")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(true)
 
   private[spark] val JARS = ConfigBuilder("spark.jars")
+    .version("0.9.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val FILES = ConfigBuilder("spark.files")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val SUBMIT_DEPLOY_MODE = ConfigBuilder("spark.submit.deployMode")
+    .version("1.5.0")
     .stringConf
     .createWithDefault("client")
 
   private[spark] val SUBMIT_PYTHON_FILES = ConfigBuilder("spark.submit.pyFiles")
+    .version("1.0.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val SCHEDULER_ALLOCATION_FILE =
     ConfigBuilder("spark.scheduler.allocation.file")
+      .version("0.8.1")
       .stringConf
       .createOptional
 
   private[spark] val SCHEDULER_MIN_REGISTERED_RESOURCES_RATIO =
     ConfigBuilder("spark.scheduler.minRegisteredResourcesRatio")
+      .version("1.1.1")
       .doubleConf
       .createOptional
 
   private[spark] val SCHEDULER_MAX_REGISTERED_RESOURCE_WAITING_TIME =
     ConfigBuilder("spark.scheduler.maxRegisteredResourcesWaitingTime")
+      .version("1.1.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("30s")
 
   private[spark] val SCHEDULER_MODE =
     ConfigBuilder("spark.scheduler.mode")
+      .version("0.8.0")
       .stringConf
       .createWithDefault(SchedulingMode.FIFO.toString)
 
   private[spark] val SCHEDULER_REVIVE_INTERVAL =
     ConfigBuilder("spark.scheduler.revive.interval")
+      .version("0.8.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val SPECULATION_ENABLED =
     ConfigBuilder("spark.speculation")
+      .version("0.6.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SPECULATION_INTERVAL =
     ConfigBuilder("spark.speculation.interval")
+      .version("0.6.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(100)
 
   private[spark] val SPECULATION_MULTIPLIER =
     ConfigBuilder("spark.speculation.multiplier")
+      .version("0.6.0")
       .doubleConf
       .createWithDefault(1.5)
 
   private[spark] val SPECULATION_QUANTILE =
     ConfigBuilder("spark.speculation.quantile")
+      .version("0.6.0")
       .doubleConf
       .createWithDefault(0.75)
 
@@ -1528,17 +1798,26 @@ package object config {
         "large enough. E.g. tasks might be re-launched if there are enough successful runs " +
         "even though the threshold hasn't been reached. The number of slots is computed based " +
         "on the conf values of spark.executor.cores and spark.task.cpus minimum 1.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val STAGING_DIR = ConfigBuilder("spark.yarn.stagingDir")
     .doc("Staging directory used while submitting applications.")
+    .version("2.0.0")
     .stringConf
     .createOptional
 
   private[spark] val BUFFER_PAGESIZE = ConfigBuilder("spark.buffer.pageSize")
     .doc("The amount of memory used per page in bytes")
+    .version("1.5.0")
     .bytesConf(ByteUnit.BYTE)
     .createOptional
 
+  private[spark] val EXECUTOR_ALLOW_SPARK_CONTEXT =
+    ConfigBuilder("spark.executor.allowSparkContext")
+      .doc("If set to true, SparkContext can be created in executors.")
+      .version("3.0.1")
+      .booleanConf
+      .createWithDefault(true)
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
index a619f10bbf064..a5d2c5c88ae03 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.internal.io
 
 import java.text.NumberFormat
-import java.util.{Date, Locale}
+import java.util.{Date, Locale, UUID}
 
 import scala.reflect.ClassTag
 
@@ -70,6 +70,11 @@ object SparkHadoopWriter extends Logging {
     // Assert the output format/key/value class is set in JobConf.
     config.assertConf(jobContext, rdd.conf)
 
+    // propagate the description UUID into the jobs, so that committers
+    // get an ID guaranteed to be unique.
+    jobContext.getConfiguration.set("spark.sql.sources.writeJobUUID",
+      UUID.randomUUID.toString)
+
     val committer = config.createCommitter(commitJobId)
     committer.setupJob(jobContext)
 
diff --git a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala
index de828a6d6156e..657842c620f30 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala
@@ -20,7 +20,7 @@ package org.apache.spark.internal.io
 import java.text.SimpleDateFormat
 import java.util.{Date, Locale}
 
-import scala.util.DynamicVariable
+import scala.util.{DynamicVariable, Random}
 
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.mapred.{JobConf, JobID}
@@ -37,14 +37,35 @@ private[spark]
 object SparkHadoopWriterUtils {
 
   private val RECORDS_BETWEEN_BYTES_WRITTEN_METRIC_UPDATES = 256
+  private val RAND = new Random()
 
+  /**
+   * Create a job ID.
+   *
+   * @param time (current) time
+   * @param id job number
+   * @return a job ID
+   */
   def createJobID(time: Date, id: Int): JobID = {
+    if (id < 0) {
+      throw new IllegalArgumentException("Job number is negative")
+    }
     val jobtrackerID = createJobTrackerID(time)
     new JobID(jobtrackerID, id)
   }
 
+  /**
+   * Generate an ID for a job tracker.
+   * @param time (current) time
+   * @return a string for a job ID
+   */
   def createJobTrackerID(time: Date): String = {
-    new SimpleDateFormat("yyyyMMddHHmmss", Locale.US).format(time)
+    val base = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US).format(time)
+    var l1 = RAND.nextLong()
+    if (l1 < 0) {
+      l1 = -l1
+    }
+    base + l1
   }
 
   def createPathFromString(path: String, conf: JobConf): Path = {
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
index 7c33bce78378d..59b863b89f75a 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
@@ -24,15 +24,18 @@ import com.codahale.metrics.MetricRegistry
 import org.eclipse.jetty.servlet.ServletContextHandler
 
 import org.apache.spark.{SecurityManager, SparkConf}
+import org.apache.spark.annotation.Experimental
 import org.apache.spark.ui.JettyUtils._
 
 /**
+ * :: Experimental ::
  * This exposes the metrics of the given registry with Prometheus format.
  *
  * The output is consistent with /metrics/json result in terms of item ordering
  * and with the previous result of Spark JMX Sink + Prometheus JMX Converter combination
  * in terms of key string format.
  */
+@Experimental
 private[spark] class PrometheusServlet(
     val property: Properties,
     val registry: MetricRegistry,
@@ -53,58 +56,65 @@ private[spark] class PrometheusServlet(
   def getMetricsSnapshot(request: HttpServletRequest): String = {
     import scala.collection.JavaConverters._
 
+    val guagesLabel = """{type="gauges"}"""
+    val countersLabel = """{type="counters"}"""
+    val metersLabel = countersLabel
+    val histogramslabels = """{type="histograms"}"""
+    val timersLabels = """{type="timers"}"""
+
     val sb = new StringBuilder()
     registry.getGauges.asScala.foreach { case (k, v) =>
       if (!v.getValue.isInstanceOf[String]) {
-        sb.append(s"${normalizeKey(k)}Value ${v.getValue}\n")
+        sb.append(s"${normalizeKey(k)}Number$guagesLabel ${v.getValue}\n")
+        sb.append(s"${normalizeKey(k)}Value$guagesLabel ${v.getValue}\n")
       }
     }
     registry.getCounters.asScala.foreach { case (k, v) =>
-      sb.append(s"${normalizeKey(k)}Count ${v.getCount}\n")
+      sb.append(s"${normalizeKey(k)}Count$countersLabel ${v.getCount}\n")
     }
     registry.getHistograms.asScala.foreach { case (k, h) =>
       val snapshot = h.getSnapshot
       val prefix = normalizeKey(k)
-      sb.append(s"${prefix}Count ${h.getCount}\n")
-      sb.append(s"${prefix}Max ${snapshot.getMax}\n")
-      sb.append(s"${prefix}Mean ${snapshot.getMean}\n")
-      sb.append(s"${prefix}Min ${snapshot.getMin}\n")
-      sb.append(s"${prefix}50thPercentile ${snapshot.getMedian}\n")
-      sb.append(s"${prefix}75thPercentile ${snapshot.get75thPercentile}\n")
-      sb.append(s"${prefix}95thPercentile ${snapshot.get95thPercentile}\n")
-      sb.append(s"${prefix}98thPercentile ${snapshot.get98thPercentile}\n")
-      sb.append(s"${prefix}99thPercentile ${snapshot.get99thPercentile}\n")
-      sb.append(s"${prefix}999thPercentile ${snapshot.get999thPercentile}\n")
-      sb.append(s"${prefix}StdDev ${snapshot.getStdDev}\n")
+      sb.append(s"${prefix}Count$histogramslabels ${h.getCount}\n")
+      sb.append(s"${prefix}Max$histogramslabels ${snapshot.getMax}\n")
+      sb.append(s"${prefix}Mean$histogramslabels ${snapshot.getMean}\n")
+      sb.append(s"${prefix}Min$histogramslabels ${snapshot.getMin}\n")
+      sb.append(s"${prefix}50thPercentile$histogramslabels ${snapshot.getMedian}\n")
+      sb.append(s"${prefix}75thPercentile$histogramslabels ${snapshot.get75thPercentile}\n")
+      sb.append(s"${prefix}95thPercentile$histogramslabels ${snapshot.get95thPercentile}\n")
+      sb.append(s"${prefix}98thPercentile$histogramslabels ${snapshot.get98thPercentile}\n")
+      sb.append(s"${prefix}99thPercentile$histogramslabels ${snapshot.get99thPercentile}\n")
+      sb.append(s"${prefix}999thPercentile$histogramslabels ${snapshot.get999thPercentile}\n")
+      sb.append(s"${prefix}StdDev$histogramslabels ${snapshot.getStdDev}\n")
     }
     registry.getMeters.entrySet.iterator.asScala.foreach { kv =>
       val prefix = normalizeKey(kv.getKey)
       val meter = kv.getValue
-      sb.append(s"${prefix}Count ${meter.getCount}\n")
-      sb.append(s"${prefix}MeanRate ${meter.getMeanRate}\n")
-      sb.append(s"${prefix}OneMinuteRate ${meter.getOneMinuteRate}\n")
-      sb.append(s"${prefix}FiveMinuteRate ${meter.getFiveMinuteRate}\n")
-      sb.append(s"${prefix}FifteenMinuteRate ${meter.getFifteenMinuteRate}\n")
+      sb.append(s"${prefix}Count$metersLabel ${meter.getCount}\n")
+      sb.append(s"${prefix}MeanRate$metersLabel ${meter.getMeanRate}\n")
+      sb.append(s"${prefix}OneMinuteRate$metersLabel ${meter.getOneMinuteRate}\n")
+      sb.append(s"${prefix}FiveMinuteRate$metersLabel ${meter.getFiveMinuteRate}\n")
+      sb.append(s"${prefix}FifteenMinuteRate$metersLabel ${meter.getFifteenMinuteRate}\n")
     }
     registry.getTimers.entrySet.iterator.asScala.foreach { kv =>
       val prefix = normalizeKey(kv.getKey)
       val timer = kv.getValue
       val snapshot = timer.getSnapshot
-      sb.append(s"${prefix}Count ${timer.getCount}\n")
-      sb.append(s"${prefix}Max ${snapshot.getMax}\n")
-      sb.append(s"${prefix}Mean ${snapshot.getMax}\n")
-      sb.append(s"${prefix}Min ${snapshot.getMin}\n")
-      sb.append(s"${prefix}50thPercentile ${snapshot.getMedian}\n")
-      sb.append(s"${prefix}75thPercentile ${snapshot.get75thPercentile}\n")
-      sb.append(s"${prefix}95thPercentile ${snapshot.get95thPercentile}\n")
-      sb.append(s"${prefix}98thPercentile ${snapshot.get98thPercentile}\n")
-      sb.append(s"${prefix}99thPercentile ${snapshot.get99thPercentile}\n")
-      sb.append(s"${prefix}999thPercentile ${snapshot.get999thPercentile}\n")
-      sb.append(s"${prefix}StdDev ${snapshot.getStdDev}\n")
-      sb.append(s"${prefix}FifteenMinuteRate ${timer.getFifteenMinuteRate}\n")
-      sb.append(s"${prefix}FiveMinuteRate ${timer.getFiveMinuteRate}\n")
-      sb.append(s"${prefix}OneMinuteRate ${timer.getOneMinuteRate}\n")
-      sb.append(s"${prefix}MeanRate ${timer.getMeanRate}\n")
+      sb.append(s"${prefix}Count$timersLabels ${timer.getCount}\n")
+      sb.append(s"${prefix}Max$timersLabels ${snapshot.getMax}\n")
+      sb.append(s"${prefix}Mean$timersLabels ${snapshot.getMax}\n")
+      sb.append(s"${prefix}Min$timersLabels ${snapshot.getMin}\n")
+      sb.append(s"${prefix}50thPercentile$timersLabels ${snapshot.getMedian}\n")
+      sb.append(s"${prefix}75thPercentile$timersLabels ${snapshot.get75thPercentile}\n")
+      sb.append(s"${prefix}95thPercentile$timersLabels ${snapshot.get95thPercentile}\n")
+      sb.append(s"${prefix}98thPercentile$timersLabels ${snapshot.get98thPercentile}\n")
+      sb.append(s"${prefix}99thPercentile$timersLabels ${snapshot.get99thPercentile}\n")
+      sb.append(s"${prefix}999thPercentile$timersLabels ${snapshot.get999thPercentile}\n")
+      sb.append(s"${prefix}StdDev$timersLabels ${snapshot.getStdDev}\n")
+      sb.append(s"${prefix}FifteenMinuteRate$timersLabels ${timer.getFifteenMinuteRate}\n")
+      sb.append(s"${prefix}FiveMinuteRate$timersLabels ${timer.getFiveMinuteRate}\n")
+      sb.append(s"${prefix}OneMinuteRate$timersLabels ${timer.getOneMinuteRate}\n")
+      sb.append(s"${prefix}MeanRate$timersLabels ${timer.getMeanRate}\n")
     }
     sb.toString()
   }
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
index 2ac6d3c500f9d..d861e91771673 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
@@ -21,6 +21,7 @@ import java.io.File
 import java.util.Optional
 
 import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.api.resource.ResourceDiscoveryPlugin
 import org.apache.spark.internal.Logging
 import org.apache.spark.util.Utils.executeAndGetOutput
@@ -28,10 +29,13 @@ import org.apache.spark.util.Utils.executeAndGetOutput
 /**
  * The default plugin that is loaded into a Spark application to control how custom
  * resources are discovered. This executes the discovery script specified by the user
- * and gets the json output back and contructs ResourceInformation objects from that.
+ * and gets the json output back and constructs ResourceInformation objects from that.
  * If the user specifies custom plugins, this is the last one to be executed and
  * throws if the resource isn't discovered.
+ *
+ * @since 3.0.0
  */
+@DeveloperApi
 class ResourceDiscoveryScriptPlugin extends ResourceDiscoveryPlugin with Logging {
   override def discoverResource(
       request: ResourceRequest,
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala b/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala
index d5ac41b995559..be056e15b6d03 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala
@@ -33,6 +33,8 @@ import org.apache.spark.annotation.Evolving
  *
  * @param name the name of the resource
  * @param addresses an array of strings describing the addresses of the resource
+ *
+ * @since 3.0.0
  */
 @Evolving
 class ResourceInformation(
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
index 14019d27fc2e6..f3c39d9107e98 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
@@ -34,9 +34,12 @@ import org.apache.spark.internal.config.Python.PYSPARK_EXECUTOR_MEMORY
  * specify executor and task requirements for an RDD that will get applied during a
  * stage. This allows the user to change the resource requirements between stages.
  * This is meant to be immutable so user can't change it after building.
+ *
+ * This api is currently private until the rest of the pieces are in place and then it
+ * will become public.
  */
 @Evolving
-class ResourceProfile(
+private[spark] class ResourceProfile(
     val executorResources: Map[String, ExecutorResourceRequest],
     val taskResources: Map[String, TaskResourceRequest]) extends Serializable with Logging {
 
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala
index 0d55c176eeb65..db1c77d7c4d40 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala
@@ -29,9 +29,12 @@ import org.apache.spark.annotation.Evolving
  * A ResourceProfile allows the user to specify executor and task requirements for an RDD
  * that will get applied during a stage. This allows the user to change the resource
  * requirements between stages.
+ *
+ * This api is currently private until the rest of the pieces are in place and then it
+ * will become public.
  */
 @Evolving
-class ResourceProfileBuilder() {
+private[spark] class ResourceProfileBuilder() {
 
   private val _taskResources = new ConcurrentHashMap[String, TaskResourceRequest]()
   private val _executorResources = new ConcurrentHashMap[String, ExecutorResourceRequest]()
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
index 7dd7fc1b99353..16fe897827c33 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
@@ -148,8 +148,13 @@ private[spark] object ResourceUtils extends Logging {
 
   def listResourceIds(sparkConf: SparkConf, componentName: String): Seq[ResourceID] = {
     sparkConf.getAllWithPrefix(s"$componentName.$RESOURCE_PREFIX.").map { case (key, _) =>
-      key.substring(0, key.indexOf('.'))
-    }.toSet.toSeq.map(name => new ResourceID(componentName, name))
+      val index = key.indexOf('.')
+      if (index < 0) {
+        throw new SparkException(s"You must specify an amount config for resource: $key " +
+          s"config: $componentName.$RESOURCE_PREFIX.$key")
+      }
+      key.substring(0, index)
+    }.distinct.map(name => new ResourceID(componentName, name))
   }
 
   def parseAllResourceRequests(
diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala
index 49d58929a97a4..a3d27b0d09923 100644
--- a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala
@@ -108,17 +108,13 @@ private[spark] abstract class RpcEndpointRef(conf: SparkConf)
 /**
  * An exception thrown if the RPC is aborted.
  */
-class RpcAbortException(message: String) extends Exception(message)
+private[spark] class RpcAbortException(message: String) extends Exception(message)
 
 /**
  * A wrapper for [[Future]] but add abort method.
  * This is used in long run RPC and provide an approach to abort the RPC.
  */
-private[spark] class AbortableRpcFuture[T: ClassTag](
-    future: Future[T],
-    onAbort: String => Unit) {
-
-  def abort(reason: String): Unit = onAbort(reason)
-
-  def toFuture: Future[T] = future
+private[spark]
+class AbortableRpcFuture[T: ClassTag](val future: Future[T], onAbort: Throwable => Unit) {
+  def abort(t: Throwable): Unit = onAbort(t)
 }
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
index 2ed03f7430c32..472401b23fe8e 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
@@ -200,6 +200,16 @@ private[netty] class Inbox(val endpointName: String, val endpoint: RpcEndpoint)
    * Calls action closure, and calls the endpoint's onError function in the case of exceptions.
    */
   private def safelyCall(endpoint: RpcEndpoint)(action: => Unit): Unit = {
+    def dealWithFatalError(fatal: Throwable): Unit = {
+      inbox.synchronized {
+        assert(numActiveThreads > 0, "The number of active threads should be positive.")
+        // Should reduce the number of active threads before throw the error.
+        numActiveThreads -= 1
+      }
+      logError(s"An error happened while processing message in the inbox for $endpointName", fatal)
+      throw fatal
+    }
+
     try action catch {
       case NonFatal(e) =>
         try endpoint.onError(e) catch {
@@ -209,8 +219,18 @@ private[netty] class Inbox(val endpointName: String, val endpoint: RpcEndpoint)
             } else {
               logError("Ignoring error", ee)
             }
+          case fatal: Throwable =>
+            dealWithFatalError(fatal)
         }
+      case fatal: Throwable =>
+        dealWithFatalError(fatal)
     }
   }
 
+  // exposed only for testing
+  def getNumActiveThreads: Int = {
+    inbox.synchronized {
+      inbox.numActiveThreads
+    }
+  }
 }
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
index 265e158d7c5e3..9259ec7699262 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
@@ -208,6 +208,7 @@ private[netty] class NettyRpcEnv(
       message: RequestMessage, timeout: RpcTimeout): AbortableRpcFuture[T] = {
     val promise = Promise[Any]()
     val remoteAddr = message.receiver.address
+    var rpcMsg: Option[RpcOutboxMessage] = None
 
     def onFailure(e: Throwable): Unit = {
       if (!promise.tryFailure(e)) {
@@ -226,8 +227,9 @@ private[netty] class NettyRpcEnv(
         }
     }
 
-    def onAbort(reason: String): Unit = {
-      onFailure(new RpcAbortException(reason))
+    def onAbort(t: Throwable): Unit = {
+      onFailure(t)
+      rpcMsg.foreach(_.onAbort())
     }
 
     try {
@@ -242,10 +244,10 @@ private[netty] class NettyRpcEnv(
         val rpcMessage = RpcOutboxMessage(message.serialize(this),
           onFailure,
           (client, response) => onSuccess(deserialize[Any](client, response)))
+        rpcMsg = Option(rpcMessage)
         postToOutbox(message.receiver, rpcMessage)
         promise.future.failed.foreach {
           case _: TimeoutException => rpcMessage.onTimeout()
-          case _: RpcAbortException => rpcMessage.onAbort()
           case _ =>
         }(ThreadUtils.sameThread)
       }
@@ -270,7 +272,7 @@ private[netty] class NettyRpcEnv(
   }
 
   private[netty] def ask[T: ClassTag](message: RequestMessage, timeout: RpcTimeout): Future[T] = {
-    askAbortable(message, timeout).toFuture
+    askAbortable(message, timeout).future
   }
 
   private[netty] def serialize(content: Any): ByteBuffer = {
@@ -547,7 +549,7 @@ private[netty] class NettyRpcEndpointRef(
   }
 
   override def ask[T: ClassTag](message: Any, timeout: RpcTimeout): Future[T] = {
-    askAbortable(message, timeout).toFuture
+    askAbortable(message, timeout).future
   }
 
   override def send(message: Any): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala b/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
index 1bcddaceb3576..5164c30fce0a1 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
@@ -64,11 +64,14 @@ private class AsyncEventQueue(
   // processed (instead of just dequeued).
   private val eventCount = new AtomicLong()
 
-  /** A counter for dropped events. It will be reset every time we log it. */
+  /** A counter for dropped events. */
   private val droppedEventsCounter = new AtomicLong(0L)
 
+  /** A counter to keep number of dropped events last time it was logged */
+  @volatile private var lastDroppedEventsCounter: Long = 0L
+
   /** When `droppedEventsCounter` was logged last time in milliseconds. */
-  @volatile private var lastReportTimestamp = 0L
+  private val lastReportTimestamp = new AtomicLong(0L)
 
   private val logDroppedEvent = new AtomicBoolean(false)
 
@@ -167,21 +170,19 @@ private class AsyncEventQueue(
     }
     logTrace(s"Dropping event $event")
 
-    val droppedCount = droppedEventsCounter.get
-    if (droppedCount > 0) {
-      // Don't log too frequently
-      if (System.currentTimeMillis() - lastReportTimestamp >= 60 * 1000) {
-        // There may be multiple threads trying to decrease droppedEventsCounter.
-        // Use "compareAndSet" to make sure only one thread can win.
-        // And if another thread is increasing droppedEventsCounter, "compareAndSet" will fail and
-        // then that thread will update it.
-        if (droppedEventsCounter.compareAndSet(droppedCount, 0)) {
-          val prevLastReportTimestamp = lastReportTimestamp
-          lastReportTimestamp = System.currentTimeMillis()
-          val previous = new java.util.Date(prevLastReportTimestamp)
-          logWarning(s"Dropped $droppedCount events from $name since " +
-            s"${if (prevLastReportTimestamp == 0) "the application started" else s"$previous"}.")
-        }
+    val droppedEventsCount = droppedEventsCounter.get
+    val droppedCountIncreased = droppedEventsCount - lastDroppedEventsCounter
+    val lastReportTime = lastReportTimestamp.get
+    val curTime = System.currentTimeMillis()
+    // Don't log too frequently
+    if (droppedCountIncreased > 0 && curTime - lastReportTime >= LOGGING_INTERVAL) {
+      // There may be multiple threads trying to logging dropped events,
+      // Use 'compareAndSet' to make sure only one thread can win.
+      if (lastReportTimestamp.compareAndSet(lastReportTime, curTime)) {
+        val previous = new java.util.Date(lastReportTime)
+        lastDroppedEventsCounter = droppedEventsCount
+        logWarning(s"Dropped $droppedCountIncreased events from $name since " +
+          s"${if (lastReportTime == 0) "the application started" else s"$previous"}.")
       }
     }
   }
@@ -213,4 +214,5 @@ private object AsyncEventQueue {
 
   val POISON_PILL = new SparkListenerEvent() { }
 
+  val LOGGING_INTERVAL = 60 * 1000
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala b/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala
index 2274e6898adf6..043c6b90384b4 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala
@@ -60,6 +60,6 @@ private[spark] object BarrierJobAllocationFailed {
   val ERROR_MESSAGE_BARRIER_REQUIRE_MORE_SLOTS_THAN_CURRENT_TOTAL_NUMBER =
     "[SPARK-24819]: Barrier execution mode does not allow run a barrier stage that requires " +
       "more slots than the total number of slots in the cluster currently. Please init a new " +
-      "cluster with more CPU cores or repartition the input RDD(s) to reduce the number of " +
-      "slots required to run this barrier stage."
+      "cluster with more resources(e.g. CPU, GPU) or repartition the input RDD(s) to reduce " +
+      "the number of slots required to run this barrier stage."
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index 7bf363dd71c1b..b483b52662270 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -168,13 +168,34 @@ private[spark] class DAGScheduler(
    */
   private val cacheLocs = new HashMap[Int, IndexedSeq[Seq[TaskLocation]]]
 
-  // For tracking failed nodes, we use the MapOutputTracker's epoch number, which is sent with
-  // every task. When we detect a node failing, we note the current epoch number and failed
-  // executor, increment it for new tasks, and use this to ignore stray ShuffleMapTask results.
-  //
-  // TODO: Garbage collect information about failure epochs when we know there are no more
-  //       stray messages to detect.
-  private val failedEpoch = new HashMap[String, Long]
+  /**
+   * Tracks the latest epoch of a fully processed error related to the given executor. (We use
+   * the MapOutputTracker's epoch number, which is sent with every task.)
+   *
+   * When an executor fails, it can affect the results of many tasks, and we have to deal with
+   * all of them consistently. We don't simply ignore all future results from that executor,
+   * as the failures may have been transient; but we also don't want to "overreact" to follow-
+   * on errors we receive. Furthermore, we might receive notification of a task success, after
+   * we find out the executor has actually failed; we'll assume those successes are, in fact,
+   * simply delayed notifications and the results have been lost, if the tasks started in the
+   * same or an earlier epoch. In particular, we use this to control when we tell the
+   * BlockManagerMaster that the BlockManager has been lost.
+   */
+  private val executorFailureEpoch = new HashMap[String, Long]
+
+  /**
+   * Tracks the latest epoch of a fully processed error where shuffle files have been lost from
+   * the given executor.
+   *
+   * This is closely related to executorFailureEpoch. They only differ for the executor when
+   * there is an external shuffle service serving shuffle files and we haven't been notified that
+   * the entire worker has been lost. In that case, when an executor is lost, we do not update
+   * the shuffleFileLostEpoch; we wait for a fetch failure. This way, if only the executor
+   * fails, we do not unregister the shuffle data as it can still be served; but if there is
+   * a failure in the shuffle service (resulting in fetch failure), we unregister the shuffle
+   * data only once, even if we get many fetch failures.
+   */
+  private val shuffleFileLostEpoch = new HashMap[String, Long]
 
   private [scheduler] val outputCommitCoordinator = env.outputCommitCoordinator
 
@@ -432,10 +453,12 @@ private[spark] class DAGScheduler(
    * submission.
    */
   private def checkBarrierStageWithNumSlots(rdd: RDD[_]): Unit = {
-    val numPartitions = rdd.getNumPartitions
-    val maxNumConcurrentTasks = sc.maxNumConcurrentTasks
-    if (rdd.isBarrier() && numPartitions > maxNumConcurrentTasks) {
-      throw new BarrierJobSlotsNumberCheckFailed(numPartitions, maxNumConcurrentTasks)
+    if (rdd.isBarrier()) {
+      val numPartitions = rdd.getNumPartitions
+      val maxNumConcurrentTasks = sc.maxNumConcurrentTasks
+      if (numPartitions > maxNumConcurrentTasks) {
+        throw new BarrierJobSlotsNumberCheckFailed(numPartitions, maxNumConcurrentTasks)
+      }
     }
   }
 
@@ -1431,6 +1454,7 @@ private[spark] class DAGScheduler(
                   // If the whole job has finished, remove it
                   if (job.numFinished == job.numPartitions) {
                     markStageAsFinished(resultStage)
+                    cancelRunningIndependentStages(job, s"Job ${job.jobId} is finished.")
                     cleanupStateForJobAndIndependentStages(job)
                     try {
                       // killAllTaskAttempts will fail if a SchedulerBackend does not implement
@@ -1471,7 +1495,8 @@ private[spark] class DAGScheduler(
             val status = event.result.asInstanceOf[MapStatus]
             val execId = status.location.executorId
             logDebug("ShuffleMapTask finished on " + execId)
-            if (failedEpoch.contains(execId) && smt.epoch <= failedEpoch(execId)) {
+            if (executorFailureEpoch.contains(execId) &&
+                smt.epoch <= executorFailureEpoch(execId)) {
               logInfo(s"Ignoring possibly bogus $smt completion from executor $execId")
             } else {
               // The epoch of the task is acceptable (i.e., the task was launched after the most
@@ -1817,12 +1842,8 @@ private[spark] class DAGScheduler(
    * modify the scheduler's internal state. Use executorLost() to post a loss event from outside.
    *
    * We will also assume that we've lost all shuffle blocks associated with the executor if the
-   * executor serves its own blocks (i.e., we're not using external shuffle), the entire slave
-   * is lost (likely including the shuffle service), or a FetchFailed occurred, in which case we
-   * presume all shuffle data related to this executor to be lost.
-   *
-   * Optionally the epoch during which the failure was caught can be passed to avoid allowing
-   * stray fetch failures from possibly retriggering the detection of a node as lost.
+   * executor serves its own blocks (i.e., we're not using an external shuffle service), or the
+   * entire Standalone worker is lost.
    */
   private[scheduler] def handleExecutorLost(
       execId: String,
@@ -1838,29 +1859,44 @@ private[spark] class DAGScheduler(
       maybeEpoch = None)
   }
 
+  /**
+   * Handles removing an executor from the BlockManagerMaster as well as unregistering shuffle
+   * outputs for the executor or optionally its host.
+   *
+   * @param execId executor to be removed
+   * @param fileLost If true, indicates that we assume we've lost all shuffle blocks associated
+   *   with the executor; this happens if the executor serves its own blocks (i.e., we're not
+   *   using an external shuffle service), the entire Standalone worker is lost, or a FetchFailed
+   *   occurred (in which case we presume all shuffle data related to this executor to be lost).
+   * @param hostToUnregisterOutputs (optional) executor host if we're unregistering all the
+   *   outputs on the host
+   * @param maybeEpoch (optional) the epoch during which the failure was caught (this prevents
+   *   reprocessing for follow-on fetch failures)
+   */
   private def removeExecutorAndUnregisterOutputs(
       execId: String,
       fileLost: Boolean,
       hostToUnregisterOutputs: Option[String],
       maybeEpoch: Option[Long] = None): Unit = {
     val currentEpoch = maybeEpoch.getOrElse(mapOutputTracker.getEpoch)
-    if (!failedEpoch.contains(execId) || failedEpoch(execId) < currentEpoch) {
-      failedEpoch(execId) = currentEpoch
-      logInfo("Executor lost: %s (epoch %d)".format(execId, currentEpoch))
+    logDebug(s"Considering removal of executor $execId; " +
+      s"fileLost: $fileLost, currentEpoch: $currentEpoch")
+    if (!executorFailureEpoch.contains(execId) || executorFailureEpoch(execId) < currentEpoch) {
+      executorFailureEpoch(execId) = currentEpoch
+      logInfo(s"Executor lost: $execId (epoch $currentEpoch)")
       blockManagerMaster.removeExecutor(execId)
-      if (fileLost) {
-        hostToUnregisterOutputs match {
-          case Some(host) =>
-            logInfo("Shuffle files lost for host: %s (epoch %d)".format(host, currentEpoch))
-            mapOutputTracker.removeOutputsOnHost(host)
-          case None =>
-            logInfo("Shuffle files lost for executor: %s (epoch %d)".format(execId, currentEpoch))
-            mapOutputTracker.removeOutputsOnExecutor(execId)
-        }
-        clearCacheLocs()
-
-      } else {
-        logDebug("Additional executor lost message for %s (epoch %d)".format(execId, currentEpoch))
+      clearCacheLocs()
+    }
+    if (fileLost &&
+        (!shuffleFileLostEpoch.contains(execId) || shuffleFileLostEpoch(execId) < currentEpoch)) {
+      shuffleFileLostEpoch(execId) = currentEpoch
+      hostToUnregisterOutputs match {
+        case Some(host) =>
+          logInfo(s"Shuffle files lost for host: $host (epoch $currentEpoch)")
+          mapOutputTracker.removeOutputsOnHost(host)
+        case None =>
+          logInfo(s"Shuffle files lost for executor: $execId (epoch $currentEpoch)")
+          mapOutputTracker.removeOutputsOnExecutor(execId)
       }
     }
   }
@@ -1886,11 +1922,12 @@ private[spark] class DAGScheduler(
   }
 
   private[scheduler] def handleExecutorAdded(execId: String, host: String): Unit = {
-    // remove from failedEpoch(execId) ?
-    if (failedEpoch.contains(execId)) {
+    // remove from executorFailureEpoch(execId) ?
+    if (executorFailureEpoch.contains(execId)) {
       logInfo("Host added was in lost list earlier: " + host)
-      failedEpoch -= execId
+      executorFailureEpoch -= execId
     }
+    shuffleFileLostEpoch -= execId
   }
 
   private[scheduler] def handleStageCancellation(stageId: Int, reason: Option[String]): Unit = {
@@ -1975,18 +2012,12 @@ private[spark] class DAGScheduler(
     }
   }
 
-  /** Fails a job and all stages that are only used by that job, and cleans up relevant state. */
-  private def failJobAndIndependentStages(
-      job: ActiveJob,
-      failureReason: String,
-      exception: Option[Throwable] = None): Unit = {
-    val error = new SparkException(failureReason, exception.orNull)
+  /** Cancel all independent, running stages that are only used by this job. */
+  private def cancelRunningIndependentStages(job: ActiveJob, reason: String): Boolean = {
     var ableToCancelStages = true
-
-    // Cancel all independent, running stages.
     val stages = jobIdToStageIds(job.jobId)
     if (stages.isEmpty) {
-      logError("No stages registered for job " + job.jobId)
+      logError(s"No stages registered for job ${job.jobId}")
     }
     stages.foreach { stageId =>
       val jobsForStage: Option[HashSet[Int]] = stageIdToStage.get(stageId).map(_.jobIds)
@@ -1998,12 +2029,12 @@ private[spark] class DAGScheduler(
         if (!stageIdToStage.contains(stageId)) {
           logError(s"Missing Stage for stage with id $stageId")
         } else {
-          // This is the only job that uses this stage, so fail the stage if it is running.
+          // This stage is only used by the job, so finish the stage if it is running.
           val stage = stageIdToStage(stageId)
           if (runningStages.contains(stage)) {
             try { // cancelTasks will fail if a SchedulerBackend does not implement killTask
               taskScheduler.cancelTasks(stageId, shouldInterruptTaskThread(job))
-              markStageAsFinished(stage, Some(failureReason))
+              markStageAsFinished(stage, Some(reason))
             } catch {
               case e: UnsupportedOperationException =>
                 logWarning(s"Could not cancel tasks for stage $stageId", e)
@@ -2013,11 +2044,19 @@ private[spark] class DAGScheduler(
         }
       }
     }
+    ableToCancelStages
+  }
 
-    if (ableToCancelStages) {
+  /** Fails a job and all stages that are only used by that job, and cleans up relevant state. */
+  private def failJobAndIndependentStages(
+      job: ActiveJob,
+      failureReason: String,
+      exception: Option[Throwable] = None): Unit = {
+    if (cancelRunningIndependentStages(job, failureReason)) {
       // SPARK-15783 important to cleanup state first, just for tests where we have some asserts
       // against the state.  Otherwise we have a *little* bit of flakiness in the tests.
       cleanupStateForJobAndIndependentStages(job)
+      val error = new SparkException(failureReason, exception.orNull)
       job.listener.jobFailed(error)
       listenerBus.post(SparkListenerJobEnd(job.jobId, clock.getTimeMillis(), JobFailed(error)))
     }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index 8c23388b37a3d..24e2a5e4d4a62 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -40,7 +40,7 @@ import org.apache.spark.util.{JsonProtocol, Utils}
  *   spark.eventLog.enabled - Whether event logging is enabled.
  *   spark.eventLog.dir - Path to the directory in which events are logged.
  *   spark.eventLog.logBlockUpdates.enabled - Whether to log block updates
- *   spark.eventLog.logStageExecutorMetrics.enabled - Whether to log stage executor metrics
+ *   spark.eventLog.logStageExecutorMetrics - Whether to log stage executor metrics
  *
  * Event log file writer maintains its own parameters: refer the doc of [[EventLogFileWriter]]
  * and its descendant for more details.
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala
index fd04db8c09d76..508c6cebd9fe3 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala
@@ -36,4 +36,5 @@ private[spark] class ExecutorResourceInfo(
   override protected def resourceName = this.name
   override protected def resourceAddresses = this.addresses
   override protected def slotsPerAddress: Int = numParts
+  def totalAddressAmount: Int = resourceAddresses.length * slotsPerAddress
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala b/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
index 7f8893ff3b9d8..60052402642f9 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
@@ -206,6 +206,7 @@ private[spark] class HighlyCompressedMapStatus private (
 
   override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException {
     loc = BlockManagerId(in)
+    numNonEmptyBlocks = -1 // SPARK-32436 Scala 2.13 doesn't initialize this during deserialization
     emptyBlocks = new RoaringBitmap()
     emptyBlocks.readExternal(in)
     avgSize = in.readLong()
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
index 857c89d7a98f5..15f2161fac39d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
@@ -69,7 +69,7 @@ private[spark] class ResultTask[T, U](
   with Serializable {
 
   @transient private[this] val preferredLocs: Seq[TaskLocation] = {
-    if (locs == null) Nil else locs.toSet.toSeq
+    if (locs == null) Nil else locs.distinct
   }
 
   override def runTask(context: TaskContext): U = {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala
index 9159d2a0158d5..7b76af2f489f9 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala
@@ -77,7 +77,8 @@ private[spark] trait SchedulerBackend {
   def getDriverAttributes: Option[Map[String, String]] = None
 
   /**
-   * Get the max number of tasks that can be concurrent launched currently.
+   * Get the max number of tasks that can be concurrent launched based on the resources
+   * could be used, even if some of them are being used at the moment.
    * Note that please don't cache the value returned by this method, because the number can change
    * due to add/remove executors.
    *
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
index 4c0c30a3caf67..a0ba9208ea647 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
@@ -71,7 +71,7 @@ private[spark] class ShuffleMapTask(
   }
 
   @transient private val preferredLocs: Seq[TaskLocation] = {
-    if (locs == null) Nil else locs.toSet.toSeq
+    if (locs == null) Nil else locs.distinct
   }
 
   override def runTask(context: TaskContext): MapStatus = {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index f25a36c7af22a..46641e5bf5580 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -57,6 +57,11 @@ import org.apache.spark.util.{AccumulatorV2, SystemClock, ThreadUtils, Utils}
  *   * Periodic revival of all offers from the CoarseGrainedSchedulerBackend, to accommodate delay
  *      scheduling
  *   * task-result-getter threads
+ *
+ * CAUTION: Any non fatal exception thrown within Spark RPC framework can be swallowed.
+ * Thus, throwing exception in methods like resourceOffers, statusUpdate won't fail
+ * the application, but could lead to undefined behavior. Instead, we shall use method like
+ * TaskSetManger.abort() to abort a stage and then fail the application (SPARK-31485).
  */
 private[spark] class TaskSchedulerImpl(
     val sc: SparkContext,
@@ -356,9 +361,7 @@ private[spark] class TaskSchedulerImpl(
               // addresses are the same as that we allocated in taskSet.resourceOffer() since it's
               // synchronized. We don't remove the exact addresses allocated because the current
               // approach produces the identical result with less time complexity.
-              availableResources(i).getOrElse(rName,
-                throw new SparkException(s"Try to acquire resource $rName that doesn't exist."))
-                .remove(0, rInfo.addresses.size)
+              availableResources(i)(rName).remove(0, rInfo.addresses.size)
             }
             // Only update hosts for a barrier task.
             if (taskSet.isBarrier) {
@@ -408,7 +411,7 @@ private[spark] class TaskSchedulerImpl(
         newExecAvail = true
       }
     }
-    val hosts = offers.map(_.host).toSet.toSeq
+    val hosts = offers.map(_.host).distinct
     for ((host, Some(rack)) <- hosts.zip(getRacksForHosts(hosts))) {
       hostsByRack.getOrElseUpdate(rack, new HashSet[String]()) += host
     }
@@ -430,7 +433,7 @@ private[spark] class TaskSchedulerImpl(
     val tasks = shuffledOffers.map(o => new ArrayBuffer[TaskDescription](o.cores / CPUS_PER_TASK))
     val availableResources = shuffledOffers.map(_.resources).toArray
     val availableCpus = shuffledOffers.map(o => o.cores).toArray
-    val sortedTaskSets = rootPool.getSortedTaskSetQueue
+    val sortedTaskSets = rootPool.getSortedTaskSetQueue.filterNot(_.isZombie)
     for (taskSet <- sortedTaskSets) {
       logDebug("parentName: %s, name: %s, runningTasks: %s".format(
         taskSet.parent.name, taskSet.name, taskSet.runningTasks))
@@ -443,7 +446,17 @@ private[spark] class TaskSchedulerImpl(
     // of locality levels so that it gets a chance to launch local tasks on all of them.
     // NOTE: the preferredLocality order: PROCESS_LOCAL, NODE_LOCAL, NO_PREF, RACK_LOCAL, ANY
     for (taskSet <- sortedTaskSets) {
-      val availableSlots = availableCpus.map(c => c / CPUS_PER_TASK).sum
+      // we only need to calculate available slots if using barrier scheduling, otherwise the
+      // value is -1
+      val availableSlots = if (taskSet.isBarrier) {
+        val availableResourcesAmount = availableResources.map { resourceMap =>
+          // note that the addresses here have been expanded according to the numParts
+          resourceMap.map { case (name, addresses) => (name, addresses.length) }
+        }
+        calculateAvailableSlots(this, availableCpus, availableResourcesAmount)
+      } else {
+        -1
+      }
       // Skip the barrier taskSet if the available slots are less than the number of pending tasks.
       if (taskSet.isBarrier && availableSlots < taskSet.numTasks) {
         // Skip the launch process.
@@ -516,11 +529,18 @@ private[spark] class TaskSchedulerImpl(
           // Check whether the barrier tasks are partially launched.
           // TODO SPARK-24818 handle the assert failure case (that can happen when some locality
           // requirements are not fulfilled, and we should revert the launched tasks).
-          require(addressesWithDescs.size == taskSet.numTasks,
-            s"Skip current round of resource offers for barrier stage ${taskSet.stageId} " +
-              s"because only ${addressesWithDescs.size} out of a total number of " +
-              s"${taskSet.numTasks} tasks got resource offers. The resource offers may have " +
-              "been blacklisted or cannot fulfill task locality requirements.")
+          if (addressesWithDescs.size != taskSet.numTasks) {
+            val errorMsg =
+              s"Fail resource offers for barrier stage ${taskSet.stageId} because only " +
+                s"${addressesWithDescs.size} out of a total number of ${taskSet.numTasks}" +
+                s" tasks got resource offers. This happens because barrier execution currently " +
+                s"does not work gracefully with delay scheduling. We highly recommend you to " +
+                s"disable delay scheduling by setting spark.locality.wait=0 as a workaround if " +
+                s"you see this error frequently."
+            logWarning(errorMsg)
+            taskSet.abort(errorMsg)
+            throw new SparkException(errorMsg)
+          }
 
           // materialize the barrier coordinator.
           maybeInitBarrierCoordinator()
@@ -582,8 +602,12 @@ private[spark] class TaskSchedulerImpl(
             if (state == TaskState.LOST) {
               // TaskState.LOST is only used by the deprecated Mesos fine-grained scheduling mode,
               // where each executor corresponds to a single task, so mark the executor as failed.
-              val execId = taskIdToExecutorId.getOrElse(tid, throw new IllegalStateException(
-                "taskIdToTaskSetManager.contains(tid) <=> taskIdToExecutorId.contains(tid)"))
+              val execId = taskIdToExecutorId.getOrElse(tid, {
+                val errorMsg =
+                  "taskIdToTaskSetManager.contains(tid) <=> taskIdToExecutorId.contains(tid)"
+                taskSet.abort(errorMsg)
+                throw new SparkException(errorMsg)
+              })
               if (executorIdToRunningTaskIds.contains(execId)) {
                 reason = Some(
                   SlaveLost(s"Task $tid was lost, so marking the executor as lost as well."))
@@ -919,6 +943,30 @@ private[spark] object TaskSchedulerImpl {
 
   val SCHEDULER_MODE_PROPERTY = SCHEDULER_MODE.key
 
+  /**
+   * Calculate the max available task slots given the `availableCpus` and `availableResources`
+   * from a collection of executors.
+   *
+   * @param scheduler the TaskSchedulerImpl instance
+   * @param availableCpus an Array of the amount of available cpus from the executors.
+   * @param availableResources an Array of the resources map from the executors. In the resource
+   *                           map, it maps from the resource name to its amount.
+   * @return the number of max task slots
+   */
+  def calculateAvailableSlots(
+      scheduler: TaskSchedulerImpl,
+      availableCpus: Array[Int],
+      availableResources: Array[Map[String, Int]]): Int = {
+    val cpusPerTask = scheduler.CPUS_PER_TASK
+    val resourcesReqsPerTask = scheduler.resourcesReqsPerTask
+    availableCpus.zip(availableResources).map { case (cpu, resources) =>
+      val cpuNum = cpu / cpusPerTask
+      resourcesReqsPerTask.map { req =>
+        resources.get(req.resourceName).map(_ / req.amount).getOrElse(0)
+      }.reduceOption(Math.min).map(_.min(cpuNum)).getOrElse(cpuNum)
+    }.sum
+  }
+
   /**
    * Used to balance containers across hosts.
    *
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index 3b620ec69a9ab..2ce11347ade39 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -229,6 +229,8 @@ private[spark] class TaskSetManager(
       index: Int,
       resolveRacks: Boolean = true,
       speculatable: Boolean = false): Unit = {
+    // A zombie TaskSetManager may reach here while handling failed task.
+    if (isZombie) return
     val pendingTaskSetToAddTo = if (speculatable) pendingSpeculatableTasks else pendingTasks
     for (loc <- tasks(index).preferredLocations) {
       loc match {
@@ -1082,6 +1084,8 @@ private[spark] class TaskSetManager(
   }
 
   def recomputeLocality(): Unit = {
+    // A zombie TaskSetManager may reach here while executorLost happens
+    if (isZombie) return
     val previousLocalityLevel = myLocalityLevels(currentLocalityIndex)
     myLocalityLevels = computeValidLocalityLevels()
     localityWaits = myLocalityLevels.map(getLocalityWait)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 55f4005ef1b45..2a06b1a98e91a 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -236,7 +236,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
           }
           val data = new ExecutorData(executorRef, executorAddress, hostname,
             0, cores, logUrlHandler.applyPattern(logUrls, attributes), attributes,
-            resourcesInfo, resourceProfileId)
+            resourcesInfo, resourceProfileId, registrationTs = System.currentTimeMillis())
           // This must be synchronized because variables mutated
           // in this block are read when requesting executors
           CoarseGrainedSchedulerBackend.this.synchronized {
@@ -249,10 +249,10 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
               logDebug(s"Decremented number of pending executors ($numPendingExecutors left)")
             }
           }
-          // Note: some tests expect the reply to come after we put the executor in the map
-          context.reply(true)
           listenerBus.post(
             SparkListenerExecutorAdded(System.currentTimeMillis(), executorId, data))
+          // Note: some tests expect the reply to come after we put the executor in the map
+          context.reply(true)
         }
 
       case StopDriver =>
@@ -557,16 +557,36 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
     executorDataMap.keySet.toSeq
   }
 
+  def getExecutorsWithRegistrationTs(): Map[String, Long] = synchronized {
+    executorDataMap.mapValues(v => v.registrationTs).toMap
+  }
+
   override def isExecutorActive(id: String): Boolean = synchronized {
     executorDataMap.contains(id) &&
       !executorsPendingToRemove.contains(id) &&
       !executorsPendingLossReason.contains(id)
   }
 
+  /**
+   * Get the max number of tasks that can be concurrent launched based on the resources
+   * could be used, even if some of them are being used at the moment.
+   * Note that please don't cache the value returned by this method, because the number can change
+   * due to add/remove executors.
+   *
+   * @return The max number of tasks that can be concurrent launched currently.
+   */
   override def maxNumConcurrentTasks(): Int = synchronized {
-    executorDataMap.values.map { executor =>
-      executor.totalCores / scheduler.CPUS_PER_TASK
-    }.sum
+    val (cpus, resources) = {
+      executorDataMap
+        .filter { case (id, _) => isExecutorActive(id) }
+        .values.toArray.map { executor =>
+        (
+          executor.totalCores,
+          executor.resourcesInfo.map { case (name, rInfo) => (name, rInfo.totalAddressAmount) }
+        )
+      }.unzip
+    }
+    TaskSchedulerImpl.calculateAvailableSlots(scheduler, cpus, resources)
   }
 
   // this function is for testing only
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala
index 062146174f6a8..86b44e835368c 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala
@@ -30,6 +30,7 @@ import org.apache.spark.scheduler.ExecutorResourceInfo
  * @param totalCores The total number of cores available to the executor
  * @param resourcesInfo The information of the currently available resources on the executor
  * @param resourceProfileId The id of the ResourceProfile being used by this executor
+ * @param registrationTs The registration timestamp of this executor
  */
 private[cluster] class ExecutorData(
     val executorEndpoint: RpcEndpointRef,
@@ -40,6 +41,7 @@ private[cluster] class ExecutorData(
     override val logUrlMap: Map[String, String],
     override val attributes: Map[String, String],
     override val resourcesInfo: Map[String, ExecutorResourceInfo],
-    override val resourceProfileId: Int
+    override val resourceProfileId: Int,
+    val registrationTs: Long
 ) extends ExecutorInfo(executorHost, totalCores, logUrlMap, attributes,
   resourcesInfo, resourceProfileId)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
index a9b607d8cc38c..1b8b8cb4cf852 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.scheduler.cluster
 
+import java.util.Locale
 import java.util.concurrent.Semaphore
 import java.util.concurrent.atomic.AtomicBoolean
 
@@ -216,6 +217,13 @@ private[spark] class StandaloneSchedulerBackend(
     }
   }
 
+  override def getDriverLogUrls: Option[Map[String, String]] = {
+    val prefix = "SPARK_DRIVER_LOG_URL_"
+    val driverLogUrls = sys.env.filterKeys(_.startsWith(prefix))
+      .map(e => (e._1.substring(prefix.length).toLowerCase(Locale.ROOT), e._2)).toMap
+    if (driverLogUrls.nonEmpty) Some(driverLogUrls) else None
+  }
+
   private def waitForRegistration() = {
     registrationBarrier.acquire()
   }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
index a24f1902faa31..3664d3ff4d8cf 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
@@ -45,12 +45,12 @@ private[spark] class ExecutorMonitor(
   private val storageTimeoutNs = TimeUnit.SECONDS.toNanos(
     conf.get(DYN_ALLOCATION_CACHED_EXECUTOR_IDLE_TIMEOUT))
   private val shuffleTimeoutNs = TimeUnit.MILLISECONDS.toNanos(
-    conf.get(DYN_ALLOCATION_SHUFFLE_TIMEOUT))
+    conf.get(DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT))
 
   private val fetchFromShuffleSvcEnabled = conf.get(SHUFFLE_SERVICE_ENABLED) &&
     conf.get(SHUFFLE_SERVICE_FETCH_RDD_ENABLED)
   private val shuffleTrackingEnabled = !conf.get(SHUFFLE_SERVICE_ENABLED) &&
-    conf.get(DYN_ALLOCATION_SHUFFLE_TRACKING)
+    conf.get(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED)
 
   private val executors = new ConcurrentHashMap[String, Tracker]()
   private val execResourceProfileCount = new ConcurrentHashMap[Int, Int]()
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index 55ac2c410953b..cdaab599e2a0b 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -502,7 +502,6 @@ private[serializer] object KryoSerializer {
       "org.apache.spark.ml.attribute.NumericAttribute",
 
       "org.apache.spark.ml.feature.Instance",
-      "org.apache.spark.ml.feature.InstanceBlock",
       "org.apache.spark.ml.feature.LabeledPoint",
       "org.apache.spark.ml.feature.OffsetInstance",
       "org.apache.spark.ml.linalg.DenseMatrix",
diff --git a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
index af2c82e771970..b7caf4f0aeabf 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
@@ -209,6 +209,7 @@ private[spark] class IndexShuffleBlockResolver(
         }
       }
     } finally {
+      logDebug(s"Shuffle index for mapId $mapId: ${lengths.mkString("[", ",", "]")}")
       if (indexTmp.exists() && !indexTmp.delete()) {
         logError(s"Failed to delete temporary index file at ${indexTmp.getAbsolutePath}")
       }
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index a5850fc2ac4b9..dfa4f92f8b87e 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -71,7 +71,7 @@ private[spark] class AppStatusListener(
   // causing too many writes to the underlying store, and other expensive operations).
   private val liveStages = new ConcurrentHashMap[(Int, Int), LiveStage]()
   private val liveJobs = new HashMap[Int, LiveJob]()
-  private val liveExecutors = new HashMap[String, LiveExecutor]()
+  private[spark] val liveExecutors = new HashMap[String, LiveExecutor]()
   private val deadExecutors = new HashMap[String, LiveExecutor]()
   private val liveTasks = new HashMap[Long, LiveTask]()
   private val liveRDDs = new HashMap[Int, LiveRDD]()
@@ -281,10 +281,12 @@ private[spark] class AppStatusListener(
 
     // Implicitly blacklist every available executor for the stage associated with this node
     Option(liveStages.get((event.stageId, event.stageAttemptId))).foreach { stage =>
-      val executorIds = liveExecutors.values.filter(_.host == event.hostId).map(_.executorId).toSeq
+      val executorIds = liveExecutors.values.filter(exec => exec.host == event.hostId
+        && exec.executorId != SparkContext.DRIVER_IDENTIFIER).map(_.executorId).toSeq
       setStageBlackListStatus(stage, now, executorIds: _*)
     }
-    liveExecutors.values.filter(_.hostname == event.hostId).foreach { exec =>
+    liveExecutors.values.filter(exec => exec.hostname == event.hostId
+      && exec.executorId != SparkContext.DRIVER_IDENTIFIER).foreach { exec =>
       addBlackListedStageTo(exec, event.stageId, now)
     }
   }
@@ -333,7 +335,7 @@ private[spark] class AppStatusListener(
 
     // Implicitly (un)blacklist every executor associated with the node.
     liveExecutors.values.foreach { exec =>
-      if (exec.hostname == host) {
+      if (exec.hostname == host && exec.executorId != SparkContext.DRIVER_IDENTIFIER) {
         exec.isBlacklisted = blacklisted
         liveUpdate(exec, now)
       }
@@ -772,6 +774,11 @@ private[spark] class AppStatusListener(
     event.maxOnHeapMem.foreach { _ =>
       exec.totalOnHeap = event.maxOnHeapMem.get
       exec.totalOffHeap = event.maxOffHeapMem.get
+      // SPARK-30594: whenever(first time or re-register) a BlockManager added, all blocks
+      // from this BlockManager will be reported to driver later. So, we should clean up
+      // used memory to avoid overlapped count.
+      exec.usedOnHeap = 0
+      exec.usedOffHeap = 0
     }
     exec.isActive = true
     exec.maxMemory = event.maxMem
@@ -1042,7 +1049,7 @@ private[spark] class AppStatusListener(
     }
   }
 
-  private def updateExecutorMemoryDiskInfo(
+  private[spark] def updateExecutorMemoryDiskInfo(
       exec: LiveExecutor,
       storageLevel: StorageLevel,
       memoryDelta: Long,
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 6b89812cc2bf0..a4fa256b18adb 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -22,7 +22,7 @@ import java.util.{List => JList}
 import scala.collection.JavaConverters._
 import scala.collection.mutable.HashMap
 
-import org.apache.spark.{JobExecutionStatus, SparkConf}
+import org.apache.spark.{JobExecutionStatus, SparkConf, SparkException}
 import org.apache.spark.status.api.v1
 import org.apache.spark.ui.scope._
 import org.apache.spark.util.Utils
@@ -36,7 +36,14 @@ private[spark] class AppStatusStore(
     val listener: Option[AppStatusListener] = None) {
 
   def applicationInfo(): v1.ApplicationInfo = {
-    store.view(classOf[ApplicationInfoWrapper]).max(1).iterator().next().info
+    try {
+      // The ApplicationInfo may not be available when Spark is starting up.
+      store.view(classOf[ApplicationInfoWrapper]).max(1).iterator().next().info
+    } catch {
+      case _: NoSuchElementException =>
+        throw new NoSuchElementException("Failed to get the application information. " +
+          "If you are starting up Spark, please wait a while until it's ready.")
+    }
   }
 
   def environmentInfo(): v1.ApplicationEnvironmentInfo = {
@@ -512,7 +519,13 @@ private[spark] class AppStatusStore(
   }
 
   def appSummary(): AppSummary = {
-    store.read(classOf[AppSummary], classOf[AppSummary].getName())
+    try {
+      store.read(classOf[AppSummary], classOf[AppSummary].getName())
+    } catch {
+      case _: NoSuchElementException =>
+        throw new NoSuchElementException("Failed to get the application summary. " +
+          "If you are starting up Spark, please wait a while until it's ready.")
+    }
   }
 
   def close(): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
index e3046dce34e67..2714f30de14f0 100644
--- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
+++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
@@ -245,7 +245,7 @@ private class LiveTask(
 
 }
 
-private class LiveExecutor(val executorId: String, _addTime: Long) extends LiveEntity {
+private[spark] class LiveExecutor(val executorId: String, _addTime: Long) extends LiveEntity {
 
   var hostPort: String = null
   var host: String = null
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
index f9fb78e65a3d9..9658e5e627724 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
@@ -23,15 +23,19 @@ import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
 import org.glassfish.jersey.server.ServerProperties
 import org.glassfish.jersey.servlet.ServletContainer
 
+import org.apache.spark.{SPARK_REVISION, SPARK_VERSION_SHORT}
+import org.apache.spark.annotation.Experimental
 import org.apache.spark.ui.SparkUI
 
 /**
+ * :: Experimental ::
  * This aims to expose Executor metrics like REST API which is documented in
  *
  *    https://spark.apache.org/docs/3.0.0/monitoring.html#executor-metrics
  *
  * Note that this is based on ExecutorSummary which is different from ExecutorSource.
  */
+@Experimental
 @Path("/executors")
 private[v1] class PrometheusResource extends ApiRequestContext {
   @GET
@@ -39,6 +43,7 @@ private[v1] class PrometheusResource extends ApiRequestContext {
   @Produces(Array(MediaType.TEXT_PLAIN))
   def executors(): String = {
     val sb = new StringBuilder
+    sb.append(s"""spark_info{version="$SPARK_VERSION_SHORT", revision="$SPARK_REVISION"} 1.0\n""")
     val store = uiRoot.asInstanceOf[SparkUI].store
     store.executorList(true).foreach { executor =>
       val prefix = "metrics_executor_"
@@ -47,27 +52,27 @@ private[v1] class PrometheusResource extends ApiRequestContext {
         "application_name" -> store.applicationInfo.name,
         "executor_id" -> executor.id
       ).map { case (k, v) => s"""$k="$v"""" }.mkString("{", ", ", "}")
-      sb.append(s"${prefix}rddBlocks_Count$labels ${executor.rddBlocks}\n")
-      sb.append(s"${prefix}memoryUsed_Count$labels ${executor.memoryUsed}\n")
-      sb.append(s"${prefix}diskUsed_Count$labels ${executor.diskUsed}\n")
-      sb.append(s"${prefix}totalCores_Count$labels ${executor.totalCores}\n")
-      sb.append(s"${prefix}maxTasks_Count$labels ${executor.maxTasks}\n")
-      sb.append(s"${prefix}activeTasks_Count$labels ${executor.activeTasks}\n")
-      sb.append(s"${prefix}failedTasks_Count$labels ${executor.failedTasks}\n")
-      sb.append(s"${prefix}completedTasks_Count$labels ${executor.completedTasks}\n")
-      sb.append(s"${prefix}totalTasks_Count$labels ${executor.totalTasks}\n")
-      sb.append(s"${prefix}totalDuration_Value$labels ${executor.totalDuration}\n")
-      sb.append(s"${prefix}totalGCTime_Value$labels ${executor.totalGCTime}\n")
-      sb.append(s"${prefix}totalInputBytes_Count$labels ${executor.totalInputBytes}\n")
-      sb.append(s"${prefix}totalShuffleRead_Count$labels ${executor.totalShuffleRead}\n")
-      sb.append(s"${prefix}totalShuffleWrite_Count$labels ${executor.totalShuffleWrite}\n")
-      sb.append(s"${prefix}maxMemory_Count$labels ${executor.maxMemory}\n")
+      sb.append(s"${prefix}rddBlocks$labels ${executor.rddBlocks}\n")
+      sb.append(s"${prefix}memoryUsed_bytes$labels ${executor.memoryUsed}\n")
+      sb.append(s"${prefix}diskUsed_bytes$labels ${executor.diskUsed}\n")
+      sb.append(s"${prefix}totalCores$labels ${executor.totalCores}\n")
+      sb.append(s"${prefix}maxTasks$labels ${executor.maxTasks}\n")
+      sb.append(s"${prefix}activeTasks$labels ${executor.activeTasks}\n")
+      sb.append(s"${prefix}failedTasks_total$labels ${executor.failedTasks}\n")
+      sb.append(s"${prefix}completedTasks_total$labels ${executor.completedTasks}\n")
+      sb.append(s"${prefix}totalTasks_total$labels ${executor.totalTasks}\n")
+      sb.append(s"${prefix}totalDuration_seconds_total$labels ${executor.totalDuration * 0.001}\n")
+      sb.append(s"${prefix}totalGCTime_seconds_total$labels ${executor.totalGCTime * 0.001}\n")
+      sb.append(s"${prefix}totalInputBytes_bytes_total$labels ${executor.totalInputBytes}\n")
+      sb.append(s"${prefix}totalShuffleRead_bytes_total$labels ${executor.totalShuffleRead}\n")
+      sb.append(s"${prefix}totalShuffleWrite_bytes_total$labels ${executor.totalShuffleWrite}\n")
+      sb.append(s"${prefix}maxMemory_bytes$labels ${executor.maxMemory}\n")
       executor.executorLogs.foreach { case (k, v) => }
       executor.memoryMetrics.foreach { m =>
-        sb.append(s"${prefix}usedOnHeapStorageMemory_Count$labels ${m.usedOnHeapStorageMemory}\n")
-        sb.append(s"${prefix}usedOffHeapStorageMemory_Count$labels ${m.usedOffHeapStorageMemory}\n")
-        sb.append(s"${prefix}totalOnHeapStorageMemory_Count$labels ${m.totalOnHeapStorageMemory}\n")
-        sb.append(s"${prefix}totalOffHeapStorageMemory_Count$labels " +
+        sb.append(s"${prefix}usedOnHeapStorageMemory_bytes$labels ${m.usedOnHeapStorageMemory}\n")
+        sb.append(s"${prefix}usedOffHeapStorageMemory_bytes$labels ${m.usedOffHeapStorageMemory}\n")
+        sb.append(s"${prefix}totalOnHeapStorageMemory_bytes$labels ${m.totalOnHeapStorageMemory}\n")
+        sb.append(s"${prefix}totalOffHeapStorageMemory_bytes$labels " +
           s"${m.totalOffHeapStorageMemory}\n")
       }
       executor.peakMemoryMetrics.foreach { m =>
@@ -87,14 +92,16 @@ private[v1] class PrometheusResource extends ApiRequestContext {
           "ProcessTreePythonVMemory",
           "ProcessTreePythonRSSMemory",
           "ProcessTreeOtherVMemory",
-          "ProcessTreeOtherRSSMemory",
-          "MinorGCCount",
-          "MinorGCTime",
-          "MajorGCCount",
-          "MajorGCTime"
+          "ProcessTreeOtherRSSMemory"
         )
         names.foreach { name =>
-          sb.append(s"$prefix${name}_Count$labels ${m.getMetricValue(name)}\n")
+          sb.append(s"$prefix${name}_bytes$labels ${m.getMetricValue(name)}\n")
+        }
+        Seq("MinorGCCount", "MajorGCCount").foreach { name =>
+          sb.append(s"$prefix${name}_total$labels ${m.getMetricValue(name)}\n")
+        }
+        Seq("MinorGCTime", "MajorGCTime").foreach { name =>
+          sb.append(s"$prefix${name}_seconds_total$labels ${m.getMetricValue(name) * 0.001}\n")
         }
       }
     }
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 5ec9b36393764..37db64abb9633 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -127,6 +127,10 @@ private[spark] class ExecutorMetricsJsonDeserializer
       new TypeReference[Option[Map[String, java.lang.Long]]] {})
     metricsMap.map(metrics => new ExecutorMetrics(metrics))
   }
+
+  override def getNullValue(ctxt: DeserializationContext): Option[ExecutorMetrics] = {
+    None
+  }
 }
 /** serializer for peakMemoryMetrics: convert ExecutorMetrics to map with metric name as key */
 private[spark] class ExecutorMetricsJsonSerializer
@@ -135,11 +139,15 @@ private[spark] class ExecutorMetricsJsonSerializer
       metrics: Option[ExecutorMetrics],
       jsonGenerator: JsonGenerator,
       serializerProvider: SerializerProvider): Unit = {
-    metrics.foreach { m: ExecutorMetrics =>
-      val metricsMap = ExecutorMetricType.metricToOffset.map { case (metric, _) =>
-        metric -> m.getMetricValue(metric)
+    if (metrics.isEmpty) {
+      jsonGenerator.writeNull()
+    } else {
+      metrics.foreach { m: ExecutorMetrics =>
+        val metricsMap = ExecutorMetricType.metricToOffset.map { case (metric, _) =>
+          metric -> m.getMetricValue(metric)
+        }
+        jsonGenerator.writeObject(metricsMap)
       }
-      jsonGenerator.writeObject(metricsMap)
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
index f0a94d84d8a04..c957ff75a501f 100644
--- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala
+++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
@@ -154,7 +154,7 @@ private[spark] object TaskIndexNames {
 private[spark] class TaskDataWrapper(
     // Storing this as an object actually saves memory; it's also used as the key in the in-memory
     // store, so in that case you'd save the extra copy of the value here.
-    @KVIndexParam
+    @KVIndexParam(parent = TaskIndexNames.STAGE)
     val taskId: JLong,
     @KVIndexParam(value = TaskIndexNames.TASK_INDEX, parent = TaskIndexNames.STAGE)
     val index: Int,
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
index e335eb6ddb761..e440c1ab7bcd9 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
@@ -167,10 +167,12 @@ class BlockManagerMaster(
    * amount of remaining memory.
    */
   def getMemoryStatus: Map[BlockManagerId, (Long, Long)] = {
+    if (driverEndpoint == null) return Map.empty
     driverEndpoint.askSync[Map[BlockManagerId, (Long, Long)]](GetMemoryStatus)
   }
 
   def getStorageStatus: Array[StorageStatus] = {
+    if (driverEndpoint == null) return Array.empty
     driverEndpoint.askSync[Array[StorageStatus]](GetStorageStatus)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
index 41ef1909cd4c2..d7f7eedc7f33b 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
@@ -98,8 +98,13 @@ class BlockManagerMasterEndpoint(
 
     case _updateBlockInfo @
         UpdateBlockInfo(blockManagerId, blockId, storageLevel, deserializedSize, size) =>
-      context.reply(updateBlockInfo(blockManagerId, blockId, storageLevel, deserializedSize, size))
-      listenerBus.post(SparkListenerBlockUpdated(BlockUpdatedInfo(_updateBlockInfo)))
+      val isSuccess = updateBlockInfo(blockManagerId, blockId, storageLevel, deserializedSize, size)
+      context.reply(isSuccess)
+      // SPARK-30594: we should not post `SparkListenerBlockUpdated` when updateBlockInfo
+      // returns false since the block info would be updated again later.
+      if (isSuccess) {
+        listenerBus.post(SparkListenerBlockUpdated(BlockUpdatedInfo(_updateBlockInfo)))
+      }
 
     case GetLocations(blockId) =>
       context.reply(getLocations(blockId))
diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
index ee43b76e17010..f2113947f6bf5 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
@@ -20,8 +20,6 @@ package org.apache.spark.storage
 import java.io.{File, IOException}
 import java.util.UUID
 
-import scala.util.control.NonFatal
-
 import org.apache.spark.SparkConf
 import org.apache.spark.executor.ExecutorExitCode
 import org.apache.spark.internal.{config, Logging}
@@ -119,38 +117,20 @@ private[spark] class DiskBlockManager(conf: SparkConf, deleteFilesOnStop: Boolea
 
   /** Produces a unique block id and File suitable for storing local intermediate results. */
   def createTempLocalBlock(): (TempLocalBlockId, File) = {
-    var blockId = TempLocalBlockId(UUID.randomUUID())
-    var tempLocalFile = getFile(blockId)
-    var count = 0
-    while (!canCreateFile(tempLocalFile) && count < Utils.MAX_DIR_CREATION_ATTEMPTS) {
-      blockId = TempLocalBlockId(UUID.randomUUID())
-      tempLocalFile = getFile(blockId)
-      count += 1
+    var blockId = new TempLocalBlockId(UUID.randomUUID())
+    while (getFile(blockId).exists()) {
+      blockId = new TempLocalBlockId(UUID.randomUUID())
     }
-    (blockId, tempLocalFile)
+    (blockId, getFile(blockId))
   }
 
   /** Produces a unique block id and File suitable for storing shuffled intermediate results. */
   def createTempShuffleBlock(): (TempShuffleBlockId, File) = {
-    var blockId = TempShuffleBlockId(UUID.randomUUID())
-    var tempShuffleFile = getFile(blockId)
-    var count = 0
-    while (!canCreateFile(tempShuffleFile) && count < Utils.MAX_DIR_CREATION_ATTEMPTS) {
-      blockId = TempShuffleBlockId(UUID.randomUUID())
-      tempShuffleFile = getFile(blockId)
-      count += 1
-    }
-    (blockId, tempShuffleFile)
-  }
-
-  private def canCreateFile(file: File): Boolean = {
-    try {
-      file.createNewFile()
-    } catch {
-      case NonFatal(_) =>
-        logError("Failed to create temporary block file: " + file.getAbsoluteFile)
-        false
+    var blockId = new TempShuffleBlockId(UUID.randomUUID())
+    while (getFile(blockId).exists()) {
+      blockId = new TempShuffleBlockId(UUID.randomUUID())
     }
+    (blockId, getFile(blockId))
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
index cd4c86006af5a..5efbc0703f729 100644
--- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
+++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
@@ -91,8 +91,7 @@ final class ShuffleBlockFetcherIterator(
   private val targetRemoteRequestSize = math.max(maxBytesInFlight / 5, 1L)
 
   /**
-   * Total number of blocks to fetch. This should be equal to the total number of blocks
-   * in [[blocksByAddress]] because we already filter out zero-sized blocks in [[blocksByAddress]].
+   * Total number of blocks to fetch.
    */
   private[this] var numBlocksToFetch = 0
 
@@ -290,7 +289,6 @@ final class ShuffleBlockFetcherIterator(
     var localBlockBytes = 0L
     var hostLocalBlockBytes = 0L
     var remoteBlockBytes = 0L
-    var numRemoteBlocks = 0
 
     val hostLocalDirReadingEnabled =
       blockManager.hostLocalDirManager != null && blockManager.hostLocalDirManager.isDefined
@@ -299,25 +297,31 @@ final class ShuffleBlockFetcherIterator(
       if (address.executorId == blockManager.blockManagerId.executorId) {
         checkBlockSizes(blockInfos)
         val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded(
-          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)).to[ArrayBuffer])
+          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), doBatchFetch)
+        numBlocksToFetch += mergedBlockInfos.size
         localBlocks ++= mergedBlockInfos.map(info => (info.blockId, info.mapIndex))
         localBlockBytes += mergedBlockInfos.map(_.size).sum
       } else if (hostLocalDirReadingEnabled && address.host == blockManager.blockManagerId.host) {
         checkBlockSizes(blockInfos)
         val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded(
-          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)).to[ArrayBuffer])
+          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), doBatchFetch)
+        numBlocksToFetch += mergedBlockInfos.size
         val blocksForAddress =
           mergedBlockInfos.map(info => (info.blockId, info.size, info.mapIndex))
         hostLocalBlocksByExecutor += address -> blocksForAddress
         hostLocalBlocks ++= blocksForAddress.map(info => (info._1, info._3))
         hostLocalBlockBytes += mergedBlockInfos.map(_.size).sum
       } else {
-        numRemoteBlocks += blockInfos.size
         remoteBlockBytes += blockInfos.map(_._2).sum
         collectFetchRequests(address, blockInfos, collectedRemoteRequests)
       }
     }
+    val numRemoteBlocks = collectedRemoteRequests.map(_.blocks.size).sum
     val totalBytes = localBlockBytes + remoteBlockBytes + hostLocalBlockBytes
+    assert(numBlocksToFetch == localBlocks.size + hostLocalBlocks.size + numRemoteBlocks,
+      s"The number of non-empty blocks $numBlocksToFetch doesn't equal to the number of local " +
+        s"blocks ${localBlocks.size} + the number of host-local blocks ${hostLocalBlocks.size} " +
+        s"+ the number of remote blocks ${numRemoteBlocks}.")
     logInfo(s"Getting $numBlocksToFetch (${Utils.bytesToString(totalBytes)}) non-empty blocks " +
       s"including ${localBlocks.size} (${Utils.bytesToString(localBlockBytes)}) local and " +
       s"${hostLocalBlocks.size} (${Utils.bytesToString(hostLocalBlockBytes)}) " +
@@ -325,6 +329,39 @@ final class ShuffleBlockFetcherIterator(
     collectedRemoteRequests
   }
 
+  private def createFetchRequest(
+      blocks: Seq[FetchBlockInfo],
+      address: BlockManagerId): FetchRequest = {
+    logDebug(s"Creating fetch request of ${blocks.map(_.size).sum} at $address "
+      + s"with ${blocks.size} blocks")
+    FetchRequest(address, blocks)
+  }
+
+  private def createFetchRequests(
+      curBlocks: Seq[FetchBlockInfo],
+      address: BlockManagerId,
+      isLast: Boolean,
+      collectedRemoteRequests: ArrayBuffer[FetchRequest]): Seq[FetchBlockInfo] = {
+    val mergedBlocks = mergeContinuousShuffleBlockIdsIfNeeded(curBlocks, doBatchFetch)
+    numBlocksToFetch += mergedBlocks.size
+    var retBlocks = Seq.empty[FetchBlockInfo]
+    if (mergedBlocks.length <= maxBlocksInFlightPerAddress) {
+      collectedRemoteRequests += createFetchRequest(mergedBlocks, address)
+    } else {
+      mergedBlocks.grouped(maxBlocksInFlightPerAddress).foreach { blocks =>
+        if (blocks.length == maxBlocksInFlightPerAddress || isLast) {
+          collectedRemoteRequests += createFetchRequest(blocks, address)
+        } else {
+          // The last group does not exceed `maxBlocksInFlightPerAddress`. Put it back
+          // to `curBlocks`.
+          retBlocks = blocks
+          numBlocksToFetch -= blocks.size
+        }
+      }
+    }
+    retBlocks
+  }
+
   private def collectFetchRequests(
       address: BlockManagerId,
       blockInfos: Seq[(BlockId, Long, Int)],
@@ -333,32 +370,6 @@ final class ShuffleBlockFetcherIterator(
     var curRequestSize = 0L
     var curBlocks = new ArrayBuffer[FetchBlockInfo]
 
-    def createFetchRequest(blocks: Seq[FetchBlockInfo]): Unit = {
-      collectedRemoteRequests += FetchRequest(address, blocks)
-      logDebug(s"Creating fetch request of $curRequestSize at $address "
-        + s"with ${blocks.size} blocks")
-    }
-
-    def createFetchRequests(): Unit = {
-      val mergedBlocks = mergeContinuousShuffleBlockIdsIfNeeded(curBlocks)
-      curBlocks = new ArrayBuffer[FetchBlockInfo]
-      if (mergedBlocks.length <= maxBlocksInFlightPerAddress) {
-        createFetchRequest(mergedBlocks)
-      } else {
-        mergedBlocks.grouped(maxBlocksInFlightPerAddress).foreach { blocks =>
-          if (blocks.length == maxBlocksInFlightPerAddress) {
-            createFetchRequest(blocks)
-          } else {
-            // The last group does not exceed `maxBlocksInFlightPerAddress`. Put it back
-            // to `curBlocks`.
-            curBlocks = blocks
-            numBlocksToFetch -= blocks.size
-          }
-        }
-      }
-      curRequestSize = curBlocks.map(_.size).sum
-    }
-
     while (iterator.hasNext) {
       val (blockId, size, mapIndex) = iterator.next()
       assertPositiveBlockSize(blockId, size)
@@ -367,12 +378,16 @@ final class ShuffleBlockFetcherIterator(
       // For batch fetch, the actual block in flight should count for merged block.
       val mayExceedsMaxBlocks = !doBatchFetch && curBlocks.size >= maxBlocksInFlightPerAddress
       if (curRequestSize >= targetRemoteRequestSize || mayExceedsMaxBlocks) {
-        createFetchRequests()
+        curBlocks = createFetchRequests(curBlocks, address, isLast = false,
+          collectedRemoteRequests).to[ArrayBuffer]
+        curRequestSize = curBlocks.map(_.size).sum
       }
     }
     // Add in the final request
     if (curBlocks.nonEmpty) {
-      createFetchRequests()
+      curBlocks = createFetchRequests(curBlocks, address, isLast = true,
+        collectedRemoteRequests).to[ArrayBuffer]
+      curRequestSize = curBlocks.map(_.size).sum
     }
   }
 
@@ -388,73 +403,6 @@ final class ShuffleBlockFetcherIterator(
     blockInfos.foreach { case (blockId, size, _) => assertPositiveBlockSize(blockId, size) }
   }
 
-  private[this] def mergeContinuousShuffleBlockIdsIfNeeded(
-      blocks: ArrayBuffer[FetchBlockInfo]): ArrayBuffer[FetchBlockInfo] = {
-    val result = if (doBatchFetch) {
-      var curBlocks = new ArrayBuffer[FetchBlockInfo]
-      val mergedBlockInfo = new ArrayBuffer[FetchBlockInfo]
-
-      def mergeFetchBlockInfo(toBeMerged: ArrayBuffer[FetchBlockInfo]): FetchBlockInfo = {
-        val startBlockId = toBeMerged.head.blockId.asInstanceOf[ShuffleBlockId]
-
-        // The last merged block may comes from the input, and we can merge more blocks
-        // into it, if the map id is the same.
-        def shouldMergeIntoPreviousBatchBlockId =
-          mergedBlockInfo.last.blockId.asInstanceOf[ShuffleBlockBatchId].mapId == startBlockId.mapId
-
-        val startReduceId = if (mergedBlockInfo.nonEmpty && shouldMergeIntoPreviousBatchBlockId) {
-          // Remove the previous batch block id as we will add a new one to replace it.
-          mergedBlockInfo.remove(mergedBlockInfo.length - 1).blockId
-            .asInstanceOf[ShuffleBlockBatchId].startReduceId
-        } else {
-          startBlockId.reduceId
-        }
-
-        FetchBlockInfo(
-          ShuffleBlockBatchId(
-            startBlockId.shuffleId,
-            startBlockId.mapId,
-            startReduceId,
-            toBeMerged.last.blockId.asInstanceOf[ShuffleBlockId].reduceId + 1),
-          toBeMerged.map(_.size).sum,
-          toBeMerged.head.mapIndex)
-      }
-
-      val iter = blocks.iterator
-      while (iter.hasNext) {
-        val info = iter.next()
-        // It's possible that the input block id is already a batch ID. For example, we merge some
-        // blocks, and then make fetch requests with the merged blocks according to "max blocks per
-        // request". The last fetch request may be too small, and we give up and put the remaining
-        // merged blocks back to the input list.
-        if (info.blockId.isInstanceOf[ShuffleBlockBatchId]) {
-          mergedBlockInfo += info
-        } else {
-          if (curBlocks.isEmpty) {
-            curBlocks += info
-          } else {
-            val curBlockId = info.blockId.asInstanceOf[ShuffleBlockId]
-            val currentMapId = curBlocks.head.blockId.asInstanceOf[ShuffleBlockId].mapId
-            if (curBlockId.mapId != currentMapId) {
-              mergedBlockInfo += mergeFetchBlockInfo(curBlocks)
-              curBlocks.clear()
-            }
-            curBlocks += info
-          }
-        }
-      }
-      if (curBlocks.nonEmpty) {
-        mergedBlockInfo += mergeFetchBlockInfo(curBlocks)
-      }
-      mergedBlockInfo
-    } else {
-      blocks
-    }
-    // update metrics
-    numBlocksToFetch += result.size
-    result
-  }
-
   /**
    * Fetch the local blocks while we are fetching remote blocks. This is ok because
    * `ManagedBuffer`'s memory is allocated lazily when we create the input stream, so all we
@@ -903,6 +851,86 @@ private class ShuffleFetchCompletionListener(var data: ShuffleBlockFetcherIterat
 private[storage]
 object ShuffleBlockFetcherIterator {
 
+  /**
+   * This function is used to merged blocks when doBatchFetch is true. Blocks which have the
+   * same `mapId` can be merged into one block batch. The block batch is specified by a range
+   * of reduceId, which implies the continuous shuffle blocks that we can fetch in a batch.
+   * For example, input blocks like (shuffle_0_0_0, shuffle_0_0_1, shuffle_0_1_0) can be
+   * merged into (shuffle_0_0_0_2, shuffle_0_1_0_1), and input blocks like (shuffle_0_0_0_2,
+   * shuffle_0_0_2, shuffle_0_0_3) can be merged into (shuffle_0_0_0_4).
+   *
+   * @param blocks blocks to be merged if possible. May contains already merged blocks.
+   * @param doBatchFetch whether to merge blocks.
+   * @return the input blocks if doBatchFetch=false, or the merged blocks if doBatchFetch=true.
+   */
+  def mergeContinuousShuffleBlockIdsIfNeeded(
+      blocks: Seq[FetchBlockInfo],
+      doBatchFetch: Boolean): Seq[FetchBlockInfo] = {
+    val result = if (doBatchFetch) {
+      var curBlocks = new ArrayBuffer[FetchBlockInfo]
+      val mergedBlockInfo = new ArrayBuffer[FetchBlockInfo]
+
+      def mergeFetchBlockInfo(toBeMerged: ArrayBuffer[FetchBlockInfo]): FetchBlockInfo = {
+        val startBlockId = toBeMerged.head.blockId.asInstanceOf[ShuffleBlockId]
+
+        // The last merged block may comes from the input, and we can merge more blocks
+        // into it, if the map id is the same.
+        def shouldMergeIntoPreviousBatchBlockId =
+          mergedBlockInfo.last.blockId.asInstanceOf[ShuffleBlockBatchId].mapId == startBlockId.mapId
+
+        val (startReduceId, size) =
+          if (mergedBlockInfo.nonEmpty && shouldMergeIntoPreviousBatchBlockId) {
+            // Remove the previous batch block id as we will add a new one to replace it.
+            val removed = mergedBlockInfo.remove(mergedBlockInfo.length - 1)
+            (removed.blockId.asInstanceOf[ShuffleBlockBatchId].startReduceId,
+              removed.size + toBeMerged.map(_.size).sum)
+          } else {
+            (startBlockId.reduceId, toBeMerged.map(_.size).sum)
+          }
+
+        FetchBlockInfo(
+          ShuffleBlockBatchId(
+            startBlockId.shuffleId,
+            startBlockId.mapId,
+            startReduceId,
+            toBeMerged.last.blockId.asInstanceOf[ShuffleBlockId].reduceId + 1),
+          size,
+          toBeMerged.head.mapIndex)
+      }
+
+      val iter = blocks.iterator
+      while (iter.hasNext) {
+        val info = iter.next()
+        // It's possible that the input block id is already a batch ID. For example, we merge some
+        // blocks, and then make fetch requests with the merged blocks according to "max blocks per
+        // request". The last fetch request may be too small, and we give up and put the remaining
+        // merged blocks back to the input list.
+        if (info.blockId.isInstanceOf[ShuffleBlockBatchId]) {
+          mergedBlockInfo += info
+        } else {
+          if (curBlocks.isEmpty) {
+            curBlocks += info
+          } else {
+            val curBlockId = info.blockId.asInstanceOf[ShuffleBlockId]
+            val currentMapId = curBlocks.head.blockId.asInstanceOf[ShuffleBlockId].mapId
+            if (curBlockId.mapId != currentMapId) {
+              mergedBlockInfo += mergeFetchBlockInfo(curBlocks)
+              curBlocks.clear()
+            }
+            curBlocks += info
+          }
+        }
+      }
+      if (curBlocks.nonEmpty) {
+        mergedBlockInfo += mergeFetchBlockInfo(curBlocks)
+      }
+      mergedBlockInfo
+    } else {
+      blocks
+    }
+    result
+  }
+
   /**
    * The block information to fetch used in FetchRequest.
    * @param blockId block id
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index 94c99d48e773c..3820a880b2109 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.ui
 
-import java.net.{URI, URL}
+import java.net.{URI, URL, URLDecoder}
 import java.util.EnumSet
 import javax.servlet.DispatcherType
 import javax.servlet.http._
@@ -376,8 +376,7 @@ private[spark] object JettyUtils extends Logging {
         if (baseRequest.isSecure) {
           return
         }
-        val httpsURI = createRedirectURI(scheme, baseRequest.getServerName, securePort,
-          baseRequest.getRequestURI, baseRequest.getQueryString)
+        val httpsURI = createRedirectURI(scheme, securePort, baseRequest)
         response.setContentLength(0)
         response.sendRedirect(response.encodeRedirectURL(httpsURI))
         baseRequest.setHandled(true)
@@ -401,17 +400,13 @@ private[spark] object JettyUtils extends Logging {
       uri.append(rest)
     }
 
-    val rewrittenURI = URI.create(uri.toString())
-    if (query != null) {
-      return new URI(
-          rewrittenURI.getScheme(),
-          rewrittenURI.getAuthority(),
-          rewrittenURI.getPath(),
-          query,
-          rewrittenURI.getFragment()
-        ).normalize()
+    val queryString = if (query == null) {
+      ""
+    } else {
+      s"?$query"
     }
-    rewrittenURI.normalize()
+    // SPARK-33611: use method `URI.create` to avoid percent-encoding twice on the query string.
+    URI.create(uri.toString() + queryString).normalize()
   }
 
   def createProxyLocationHeader(
@@ -439,16 +434,34 @@ private[spark] object JettyUtils extends Logging {
     handler.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType]))
   }
 
+  private def decodeURL(url: String, encoding: String): String = {
+    if (url == null) {
+      null
+    } else {
+      URLDecoder.decode(url, encoding)
+    }
+  }
+
   // Create a new URI from the arguments, handling IPv6 host encoding and default ports.
-  private def createRedirectURI(
-      scheme: String, server: String, port: Int, path: String, query: String) = {
+  private def createRedirectURI(scheme: String, port: Int, request: Request): String = {
+    val server = request.getServerName
     val redirectServer = if (server.contains(":") && !server.startsWith("[")) {
       s"[${server}]"
     } else {
       server
     }
     val authority = s"$redirectServer:$port"
-    new URI(scheme, authority, path, query, null).toString
+    val queryEncoding = if (request.getQueryEncoding != null) {
+      request.getQueryEncoding
+    } else {
+      // By default decoding the URI as "UTF-8" should be enough for SparkUI
+      "UTF-8"
+    }
+    // The request URL can be raw or encoded here. To avoid the request URL being
+    // encoded twice, let's decode it here.
+    val requestURI = decodeURL(request.getRequestURI, queryEncoding)
+    val queryString = decodeURL(request.getQueryString, queryEncoding)
+    new URI(scheme, authority, requestURI, queryString, null).toString
   }
 
   def toVirtualHosts(connectors: String*): Array[String] = connectors.map("@" + _).toArray
diff --git a/core/src/main/scala/org/apache/spark/ui/PagedTable.scala b/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
index 9a3cd53076c43..8ce2065de4cf8 100644
--- a/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
@@ -115,17 +115,18 @@ private[spark] trait PagedTable[T] {
         _dataSource.pageSize
       }
 
-      val pageNavi = pageNavigation(pageToShow, pageSize, totalPages)
+      val pageNaviTop = pageNavigation(pageToShow, pageSize, totalPages, tableId + "-top")
+      val pageNaviBottom = pageNavigation(pageToShow, pageSize, totalPages, tableId + "-bottom")
 
       <div>
-        {pageNavi}
+        {pageNaviTop}
         <table class={tableCssClass} id={tableId}>
           {headers}
           <tbody>
             {data.map(row)}
           </tbody>
         </table>
-        {pageNavi}
+        {pageNaviBottom}
       </div>
     } catch {
       case e: IndexOutOfBoundsException =>
@@ -171,7 +172,11 @@ private[spark] trait PagedTable[T] {
    * > means jumping to the next page.
    * }}}
    */
-  private[ui] def pageNavigation(page: Int, pageSize: Int, totalPages: Int): Seq[Node] = {
+  private[ui] def pageNavigation(
+      page: Int,
+      pageSize: Int,
+      totalPages: Int,
+      navigationId: String = tableId): Seq[Node] = {
     // A group includes all page numbers will be shown in the page navigation.
     // The size of group is 10 means there are 10 page numbers will be shown.
     // The first group is 1 to 10, the second is 2 to 20, and so on
@@ -214,7 +219,7 @@ private[spark] trait PagedTable[T] {
 
     <div>
       <div>
-        <form id={s"form-$tableId-page"}
+        <form id={s"form-$navigationId-page"}
               method="get"
               action={Unparsed(goButtonFormPath)}
               class="form-inline pull-right"
@@ -223,12 +228,11 @@ private[spark] trait PagedTable[T] {
           <label>{totalPages} Pages. Jump to</label>
           <input type="text"
                  name={pageNumberFormField}
-                 id={s"form-$tableId-page-no"}
+                 id={s"form-$navigationId-page-no"}
                  value={page.toString} class="span1" />
-
           <label>. Show </label>
           <input type="text"
-                 id={s"form-$tableId-page-size"}
+                 id={s"form-$navigationId-page-size"}
                  name={pageSizeFormField}
                  value={pageSize.toString}
                  class="span1" />
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 94c45215b5ff2..87ab5436fa4b5 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -224,13 +224,14 @@ private[spark] object UIUtils extends Logging {
     <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
     <link rel="stylesheet"
           href={prependBaseUri(request, "/static/bootstrap.min.css")} type="text/css"/>
-    <link rel="stylesheet" href={prependBaseUri(request, "/static/vis.min.css")} type="text/css"/>
+    <link rel="stylesheet"
+          href={prependBaseUri(request, "/static/vis-timeline-graph2d.min.css")} type="text/css"/>
     <link rel="stylesheet" href={prependBaseUri(request, "/static/webui.css")} type="text/css"/>
     <link rel="stylesheet"
           href={prependBaseUri(request, "/static/timeline-view.css")} type="text/css"/>
     <script src={prependBaseUri(request, "/static/sorttable.js")} ></script>
     <script src={prependBaseUri(request, "/static/jquery-3.4.1.min.js")}></script>
-    <script src={prependBaseUri(request, "/static/vis.min.js")}></script>
+    <script src={prependBaseUri(request, "/static/vis-timeline-graph2d.min.js")}></script>
     <script src={prependBaseUri(request, "/static/bootstrap-tooltip.js")}></script>
     <script src={prependBaseUri(request, "/static/initialize-tooltips.js")}></script>
     <script src={prependBaseUri(request, "/static/table.js")}></script>
@@ -288,6 +289,7 @@ private[spark] object UIUtils extends Logging {
     <html>
       <head>
         {commonHeaderNodes(request)}
+        <script>setAppBasePath('{activeTab.basePath}')</script>
         {if (showVisualization) vizHeaderNodes(request) else Seq.empty}
         {if (useDataTables) dataTablesHeaderNodes(request) else Seq.empty}
         <link rel="shortcut icon"
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
index f53e67ff5cc98..06762f69f2ecb 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
@@ -88,7 +88,8 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
       // The timeline library treats contents as HTML, so we have to escape them. We need to add
       // extra layers of escaping in order to embed this in a Javascript string literal.
       val escapedDesc = Utility.escape(jobDescription)
-      val jsEscapedDesc = StringEscapeUtils.escapeEcmaScript(escapedDesc)
+      val jsEscapedDescForTooltip = StringEscapeUtils.escapeEcmaScript(Utility.escape(escapedDesc))
+      val jsEscapedDescForLabel = StringEscapeUtils.escapeEcmaScript(escapedDesc)
       val jobEventJsonAsStr =
         s"""
            |{
@@ -98,7 +99,7 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
            |  'end': new Date(${completionTime}),
            |  'content': '<div class="application-timeline-content"' +
            |     'data-html="true" data-placement="top" data-toggle="tooltip"' +
-           |     'data-title="${jsEscapedDesc} (Job ${jobId})<br>' +
+           |     'data-title="${jsEscapedDescForTooltip} (Job ${jobId})<br>' +
            |     'Status: ${status}<br>' +
            |     'Submitted: ${UIUtils.formatDate(new Date(submissionTime))}' +
            |     '${
@@ -108,7 +109,7 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
                        ""
                      }
                   }">' +
-           |    '${jsEscapedDesc} (Job ${jobId})</div>'
+           |    '${jsEscapedDescForLabel} (Job ${jobId})</div>'
            |}
          """.stripMargin
       jobEventJsonAsStr
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
index 12f1aa25e8d2a..c80b6d13970a9 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
@@ -69,7 +69,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
       // The timeline library treats contents as HTML, so we have to escape them. We need to add
       // extra layers of escaping in order to embed this in a Javascript string literal.
       val escapedName = Utility.escape(name)
-      val jsEscapedName = StringEscapeUtils.escapeEcmaScript(escapedName)
+      val jsEscapedNameForTooltip = StringEscapeUtils.escapeEcmaScript(Utility.escape(escapedName))
+      val jsEscapedNameForLabel = StringEscapeUtils.escapeEcmaScript(escapedName)
       s"""
          |{
          |  'className': 'stage job-timeline-object ${status}',
@@ -78,7 +79,7 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
          |  'end': new Date(${completionTime}),
          |  'content': '<div class="job-timeline-content" data-toggle="tooltip"' +
          |   'data-placement="top" data-html="true"' +
-         |   'data-title="${jsEscapedName} (Stage ${stageId}.${attemptId})<br>' +
+         |   'data-title="${jsEscapedNameForTooltip} (Stage ${stageId}.${attemptId})<br>' +
          |   'Status: ${status.toUpperCase(Locale.ROOT)}<br>' +
          |   'Submitted: ${UIUtils.formatDate(submissionTime)}' +
          |   '${
@@ -88,7 +89,7 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
                    ""
                  }
               }">' +
-         |    '${jsEscapedName} (Stage ${stageId}.${attemptId})</div>',
+         |    '${jsEscapedNameForLabel} (Stage ${stageId}.${attemptId})</div>',
          |}
        """.stripMargin
     }
@@ -279,9 +280,9 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
 
     val pendingOrSkippedTableId =
       if (isComplete) {
-        "pending"
-      } else {
         "skipped"
+      } else {
+        "pending"
       }
 
     val activeStagesTable =
diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
index 9ace324322947..842ee7aaf49bf 100644
--- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
+++ b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
@@ -21,6 +21,7 @@ import java.util.Objects
 
 import scala.collection.mutable
 import scala.collection.mutable.{ListBuffer, StringBuilder}
+import scala.xml.Utility
 
 import org.apache.commons.text.StringEscapeUtils
 
@@ -245,8 +246,9 @@ private[spark] object RDDOperationGraph extends Logging {
     } else {
       ""
     }
-    val label = s"${node.name} [${node.id}]$isCached$isBarrier\n${node.callsite}"
-    s"""${node.id} [label="${StringEscapeUtils.escapeJava(label)}"]"""
+    val escapedCallsite = Utility.escape(node.callsite)
+    val label = s"${node.name} [${node.id}]$isCached$isBarrier<br>${escapedCallsite}"
+    s"""${node.id} [labelType="html" label="${StringEscapeUtils.escapeJava(label)}"]"""
   }
 
   /** Update the dot representation of the RDDOperationGraph in cluster to subgraph. */
diff --git a/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala b/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
index d2ad14f2a1a96..6ffd6605f75b8 100644
--- a/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
@@ -18,12 +18,15 @@
 package org.apache.spark.util
 
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
-import java.lang.invoke.SerializedLambda
+import java.lang.invoke.{MethodHandleInfo, SerializedLambda}
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable.{Map, Set, Stack}
 
-import org.apache.xbean.asm7.{ClassReader, ClassVisitor, MethodVisitor, Type}
+import org.apache.commons.lang3.ClassUtils
+import org.apache.xbean.asm7.{ClassReader, ClassVisitor, Handle, MethodVisitor, Type}
 import org.apache.xbean.asm7.Opcodes._
+import org.apache.xbean.asm7.tree.{ClassNode, MethodNode}
 
 import org.apache.spark.{SparkEnv, SparkException}
 import org.apache.spark.internal.Logging
@@ -159,39 +162,6 @@ private[spark] object ClosureCleaner extends Logging {
     clean(closure, checkSerializable, cleanTransitively, Map.empty)
   }
 
-  /**
-   * Try to get a serialized Lambda from the closure.
-   *
-   * @param closure the closure to check.
-   */
-  private def getSerializedLambda(closure: AnyRef): Option[SerializedLambda] = {
-    val isClosureCandidate =
-      closure.getClass.isSynthetic &&
-        closure
-          .getClass
-          .getInterfaces.exists(_.getName == "scala.Serializable")
-
-    if (isClosureCandidate) {
-      try {
-        Option(inspect(closure))
-      } catch {
-        case e: Exception =>
-          // no need to check if debug is enabled here the Spark
-          // logging api covers this.
-          logDebug("Closure is not a serialized lambda.", e)
-          None
-      }
-    } else {
-      None
-    }
-  }
-
-  private def inspect(closure: AnyRef): SerializedLambda = {
-    val writeReplace = closure.getClass.getDeclaredMethod("writeReplace")
-    writeReplace.setAccessible(true)
-    writeReplace.invoke(closure).asInstanceOf[java.lang.invoke.SerializedLambda]
-  }
-
   /**
    * Helper method to clean the given closure in place.
    *
@@ -239,12 +209,12 @@ private[spark] object ClosureCleaner extends Logging {
       cleanTransitively: Boolean,
       accessedFields: Map[Class[_], Set[String]]): Unit = {
 
-    // most likely to be the case with 2.12, 2.13
+    // indylambda check. Most likely to be the case with 2.12, 2.13
     // so we check first
     // non LMF-closures should be less frequent from now on
-    val lambdaFunc = getSerializedLambda(func)
+    val maybeIndylambdaProxy = IndylambdaScalaClosures.getSerializationProxy(func)
 
-    if (!isClosure(func.getClass) && lambdaFunc.isEmpty) {
+    if (!isClosure(func.getClass) && maybeIndylambdaProxy.isEmpty) {
       logDebug(s"Expected a closure; got ${func.getClass.getName}")
       return
     }
@@ -256,7 +226,7 @@ private[spark] object ClosureCleaner extends Logging {
       return
     }
 
-    if (lambdaFunc.isEmpty) {
+    if (maybeIndylambdaProxy.isEmpty) {
       logDebug(s"+++ Cleaning closure $func (${func.getClass.getName}) +++")
 
       // A list of classes that represents closures enclosed in the given one
@@ -300,7 +270,7 @@ private[spark] object ClosureCleaner extends Logging {
         }
       }
 
-      logDebug(s" + fields accessed by starting closure: " + accessedFields.size)
+      logDebug(s" + fields accessed by starting closure: ${accessedFields.size} classes")
       accessedFields.foreach { f => logDebug("     " + f) }
 
       // List of outer (class, object) pairs, ordered from outermost to innermost
@@ -372,14 +342,64 @@ private[spark] object ClosureCleaner extends Logging {
 
       logDebug(s" +++ closure $func (${func.getClass.getName}) is now cleaned +++")
     } else {
-      logDebug(s"Cleaning lambda: ${lambdaFunc.get.getImplMethodName}")
+      val lambdaProxy = maybeIndylambdaProxy.get
+      val implMethodName = lambdaProxy.getImplMethodName
+
+      logDebug(s"Cleaning indylambda closure: $implMethodName")
+
+      // capturing class is the class that declared this lambda
+      val capturingClassName = lambdaProxy.getCapturingClass.replace('/', '.')
+      val classLoader = func.getClass.getClassLoader // this is the safest option
+      // scalastyle:off classforname
+      val capturingClass = Class.forName(capturingClassName, false, classLoader)
+      // scalastyle:on classforname
 
-      val captClass = Utils.classForName(lambdaFunc.get.getCapturingClass.replace('/', '.'),
-        initialize = false, noSparkClassLoader = true)
       // Fail fast if we detect return statements in closures
-      getClassReader(captClass)
-        .accept(new ReturnStatementFinder(Some(lambdaFunc.get.getImplMethodName)), 0)
-      logDebug(s" +++ Lambda closure (${lambdaFunc.get.getImplMethodName}) is now cleaned +++")
+      val capturingClassReader = getClassReader(capturingClass)
+      capturingClassReader.accept(new ReturnStatementFinder(Option(implMethodName)), 0)
+
+      val isClosureDeclaredInScalaRepl = capturingClassName.startsWith("$line") &&
+        capturingClassName.endsWith("$iw")
+      val outerThisOpt = if (lambdaProxy.getCapturedArgCount > 0) {
+        Option(lambdaProxy.getCapturedArg(0))
+      } else {
+        None
+      }
+
+      // only need to clean when there is an enclosing "this" captured by the closure, and it
+      // should be something cleanable, i.e. a Scala REPL line object
+      val needsCleaning = isClosureDeclaredInScalaRepl &&
+        outerThisOpt.isDefined && outerThisOpt.get.getClass.getName == capturingClassName
+
+      if (needsCleaning) {
+        // indylambda closures do not reference enclosing closures via an `$outer` chain, so no
+        // transitive cleaning on the `$outer` chain is needed.
+        // Thus clean() shouldn't be recursively called with a non-empty accessedFields.
+        assert(accessedFields.isEmpty)
+
+        initAccessedFields(accessedFields, Seq(capturingClass))
+        IndylambdaScalaClosures.findAccessedFields(
+          lambdaProxy, classLoader, accessedFields, cleanTransitively)
+
+        logDebug(s" + fields accessed by starting closure: ${accessedFields.size} classes")
+        accessedFields.foreach { f => logDebug("     " + f) }
+
+        if (accessedFields(capturingClass).size < capturingClass.getDeclaredFields.length) {
+          // clone and clean the enclosing `this` only when there are fields to null out
+
+          val outerThis = outerThisOpt.get
+
+          logDebug(s" + cloning instance of REPL class $capturingClassName")
+          val clonedOuterThis = cloneAndSetFields(
+            parent = null, outerThis, capturingClass, accessedFields)
+
+          val outerField = func.getClass.getDeclaredField("arg$1")
+          outerField.setAccessible(true)
+          outerField.set(func, clonedOuterThis)
+        }
+      }
+
+      logDebug(s" +++ indylambda closure ($implMethodName) is now cleaned +++")
     }
 
     if (checkSerializable) {
@@ -414,6 +434,312 @@ private[spark] object ClosureCleaner extends Logging {
   }
 }
 
+private[spark] object IndylambdaScalaClosures extends Logging {
+  // internal name of java.lang.invoke.LambdaMetafactory
+  val LambdaMetafactoryClassName = "java/lang/invoke/LambdaMetafactory"
+  // the method that Scala indylambda use for bootstrap method
+  val LambdaMetafactoryMethodName = "altMetafactory"
+  val LambdaMetafactoryMethodDesc = "(Ljava/lang/invoke/MethodHandles$Lookup;" +
+    "Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)" +
+    "Ljava/lang/invoke/CallSite;"
+
+  /**
+   * Check if the given reference is a indylambda style Scala closure.
+   * If so (e.g. for Scala 2.12+ closures), return a non-empty serialization proxy
+   * (SerializedLambda) of the closure;
+   * otherwise (e.g. for Scala 2.11 closures) return None.
+   *
+   * @param maybeClosure the closure to check.
+   */
+  def getSerializationProxy(maybeClosure: AnyRef): Option[SerializedLambda] = {
+    def isClosureCandidate(cls: Class[_]): Boolean = {
+      // TODO: maybe lift this restriction to support other functional interfaces in the future
+      val implementedInterfaces = ClassUtils.getAllInterfaces(cls).asScala
+      implementedInterfaces.exists(_.getName.startsWith("scala.Function"))
+    }
+
+    maybeClosure.getClass match {
+      // shortcut the fast check:
+      // 1. indylambda closure classes are generated by Java's LambdaMetafactory, and they're
+      //    always synthetic.
+      // 2. We only care about Serializable closures, so let's check that as well
+      case c if !c.isSynthetic || !maybeClosure.isInstanceOf[Serializable] => None
+
+      case c if isClosureCandidate(c) =>
+        try {
+          Option(inspect(maybeClosure)).filter(isIndylambdaScalaClosure)
+        } catch {
+          case e: Exception =>
+            logDebug("The given reference is not an indylambda Scala closure.", e)
+            None
+        }
+
+      case _ => None
+    }
+  }
+
+  def isIndylambdaScalaClosure(lambdaProxy: SerializedLambda): Boolean = {
+    lambdaProxy.getImplMethodKind == MethodHandleInfo.REF_invokeStatic &&
+      lambdaProxy.getImplMethodName.contains("$anonfun$")
+  }
+
+  def inspect(closure: AnyRef): SerializedLambda = {
+    val writeReplace = closure.getClass.getDeclaredMethod("writeReplace")
+    writeReplace.setAccessible(true)
+    writeReplace.invoke(closure).asInstanceOf[SerializedLambda]
+  }
+
+  /**
+   * Check if the handle represents the LambdaMetafactory that indylambda Scala closures
+   * use for creating the lambda class and getting a closure instance.
+   */
+  def isLambdaMetafactory(bsmHandle: Handle): Boolean = {
+    bsmHandle.getOwner == LambdaMetafactoryClassName &&
+      bsmHandle.getName == LambdaMetafactoryMethodName &&
+      bsmHandle.getDesc == LambdaMetafactoryMethodDesc
+  }
+
+  /**
+   * Check if the handle represents a target method that is:
+   * - a STATIC method that implements a Scala lambda body in the indylambda style
+   * - captures the enclosing `this`, i.e. the first argument is a reference to the same type as
+   *   the owning class.
+   * Returns true if both criteria above are met.
+   */
+  def isLambdaBodyCapturingOuter(handle: Handle, ownerInternalName: String): Boolean = {
+    handle.getTag == H_INVOKESTATIC &&
+      handle.getName.contains("$anonfun$") &&
+      handle.getOwner == ownerInternalName &&
+      handle.getDesc.startsWith(s"(L$ownerInternalName;")
+  }
+
+  /**
+   * Check if the callee of a call site is a inner class constructor.
+   * - A constructor has to be invoked via INVOKESPECIAL
+   * - A constructor's internal name is "&lt;init&gt;" and the return type is "V" (void)
+   * - An inner class' first argument in the signature has to be a reference to the
+   *   enclosing "this", aka `$outer` in Scala.
+   */
+  def isInnerClassCtorCapturingOuter(
+      op: Int, owner: String, name: String, desc: String, callerInternalName: String): Boolean = {
+    op == INVOKESPECIAL && name == "<init>" && desc.startsWith(s"(L$callerInternalName;")
+  }
+
+  /**
+   * Scans an indylambda Scala closure, along with its lexically nested closures, and populate
+   * the accessed fields info on which fields on the outer object are accessed.
+   *
+   * This is equivalent to getInnerClosureClasses() + InnerClosureFinder + FieldAccessFinder fused
+   * into one for processing indylambda closures. The traversal order along the call graph is the
+   * same for all three combined, so they can be fused together easily while maintaining the same
+   * ordering as the existing implementation.
+   *
+   * Precondition: this function expects the `accessedFields` to be populated with all known
+   *               outer classes and their super classes to be in the map as keys, e.g.
+   *               initializing via ClosureCleaner.initAccessedFields.
+   */
+  // scalastyle:off line.size.limit
+  // Example: run the following code snippet in a Spark Shell w/ Scala 2.12+:
+  //   val topLevelValue = "someValue"; val closure = (j: Int) => {
+  //     class InnerFoo {
+  //       val innerClosure = (x: Int) => (1 to x).map { y => y + topLevelValue }
+  //     }
+  //     val innerFoo = new InnerFoo
+  //     (1 to j).flatMap(innerFoo.innerClosure)
+  //   }
+  //   sc.parallelize(0 to 2).map(closure).collect
+  //
+  // produces the following trace-level logs:
+  // (slightly simplified:
+  //   - omitting the "ignoring ..." lines;
+  //   - "$iw" is actually "$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw";
+  //   - "invokedynamic" lines are simplified to just show the name+desc, omitting the bsm info)
+  //   Cleaning indylambda closure: $anonfun$closure$1$adapted
+  //     scanning $iw.$anonfun$closure$1$adapted(L$iw;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
+  //       found intra class call to $iw.$anonfun$closure$1(L$iw;I)Lscala/collection/immutable/IndexedSeq;
+  //     scanning $iw.$anonfun$closure$1(L$iw;I)Lscala/collection/immutable/IndexedSeq;
+  //       found inner class $iw$InnerFoo$1
+  //         found method innerClosure()Lscala/Function1;
+  //         found method $anonfun$innerClosure$2(L$iw$InnerFoo$1;I)Ljava/lang/String;
+  //         found method $anonfun$innerClosure$1(L$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
+  //         found method <init>(L$iw;)V
+  //         found method $anonfun$innerClosure$2$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
+  //         found method $anonfun$innerClosure$1$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
+  //         found method $deserializeLambda$(Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;
+  //       found call to outer $iw$InnerFoo$1.innerClosure()Lscala/Function1;
+  //     scanning $iw$InnerFoo$1.innerClosure()Lscala/Function1;
+  //     scanning $iw$InnerFoo$1.$deserializeLambda$(Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;
+  //       invokedynamic: lambdaDeserialize(Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;, bsm...)
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$1$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
+  //       found intra class call to $iw$InnerFoo$1.$anonfun$innerClosure$1(L$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$1(L$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
+  //       invokedynamic: apply(L$iw$InnerFoo$1;)Lscala/Function1;, bsm...)
+  //       found inner closure $iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String; (6)
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
+  //       found intra class call to $iw$InnerFoo$1.$anonfun$innerClosure$2(L$iw$InnerFoo$1;I)Ljava/lang/String;
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$2(L$iw$InnerFoo$1;I)Ljava/lang/String;
+  //       found call to outer $iw.topLevelValue()Ljava/lang/String;
+  //     scanning $iw.topLevelValue()Ljava/lang/String;
+  //       found field access topLevelValue on $iw
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
+  //       found intra class call to $iw$InnerFoo$1.$anonfun$innerClosure$2(L$iw$InnerFoo$1;I)Ljava/lang/String;
+  //     scanning $iw$InnerFoo$1.<init>(L$iw;)V
+  //       invokedynamic: apply(L$iw$InnerFoo$1;)Lscala/Function1;, bsm...)
+  //       found inner closure $iw$InnerFoo$1.$anonfun$innerClosure$1$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq; (6)
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$1(L$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
+  //       invokedynamic: apply(L$iw$InnerFoo$1;)Lscala/Function1;, bsm...)
+  //       found inner closure $iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String; (6)
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$2(L$iw$InnerFoo$1;I)Ljava/lang/String;
+  //       found call to outer $iw.topLevelValue()Ljava/lang/String;
+  //     scanning $iw$InnerFoo$1.innerClosure()Lscala/Function1;
+  //    + fields accessed by starting closure: 2 classes
+  //        (class java.lang.Object,Set())
+  //        (class $iw,Set(topLevelValue))
+  //    + cloning instance of REPL class $iw
+  //    +++ indylambda closure ($anonfun$closure$1$adapted) is now cleaned +++
+  //
+  // scalastyle:on line.size.limit
+  def findAccessedFields(
+      lambdaProxy: SerializedLambda,
+      lambdaClassLoader: ClassLoader,
+      accessedFields: Map[Class[_], Set[String]],
+      findTransitively: Boolean): Unit = {
+
+    // We may need to visit the same class multiple times for different methods on it, and we'll
+    // need to lookup by name. So we use ASM's Tree API and cache the ClassNode/MethodNode.
+    val classInfoByInternalName = Map.empty[String, (Class[_], ClassNode)]
+    val methodNodeById = Map.empty[MethodIdentifier[_], MethodNode]
+    def getOrUpdateClassInfo(classInternalName: String): (Class[_], ClassNode) = {
+      val classInfo = classInfoByInternalName.getOrElseUpdate(classInternalName, {
+        val classExternalName = classInternalName.replace('/', '.')
+        // scalastyle:off classforname
+        val clazz = Class.forName(classExternalName, false, lambdaClassLoader)
+        // scalastyle:on classforname
+        val classNode = new ClassNode()
+        val classReader = ClosureCleaner.getClassReader(clazz)
+        classReader.accept(classNode, 0)
+
+        for (m <- classNode.methods.asScala) {
+          methodNodeById(MethodIdentifier(clazz, m.name, m.desc)) = m
+        }
+
+        (clazz, classNode)
+      })
+      classInfo
+    }
+
+    val implClassInternalName = lambdaProxy.getImplClass
+    val (implClass, _) = getOrUpdateClassInfo(implClassInternalName)
+
+    val implMethodId = MethodIdentifier(
+      implClass, lambdaProxy.getImplMethodName, lambdaProxy.getImplMethodSignature)
+
+    // The set internal names of classes that we would consider following the calls into.
+    // Candidates are: known outer class which happens to be the starting closure's impl class,
+    // and all inner classes discovered below.
+    // Note that code in an inner class can make calls to methods in any of its enclosing classes,
+    // e.g.
+    //   starting closure (in class T)
+    //     inner class A
+    //        inner class B
+    //          inner closure
+    // we need to track calls from "inner closure" to outer classes relative to it (class T, A, B)
+    // to better find and track field accesses.
+    val trackedClassInternalNames = Set[String](implClassInternalName)
+
+    // Depth-first search for inner closures and track the fields that were accessed in them.
+    // Start from the lambda body's implementation method, follow method invocations
+    val visited = Set.empty[MethodIdentifier[_]]
+    val stack = Stack[MethodIdentifier[_]](implMethodId)
+    def pushIfNotVisited(methodId: MethodIdentifier[_]): Unit = {
+      if (!visited.contains(methodId)) {
+        stack.push(methodId)
+      }
+    }
+
+    while (!stack.isEmpty) {
+      val currentId = stack.pop
+      visited += currentId
+
+      val currentClass = currentId.cls
+      val currentMethodNode = methodNodeById(currentId)
+      logTrace(s"  scanning ${currentId.cls.getName}.${currentId.name}${currentId.desc}")
+      currentMethodNode.accept(new MethodVisitor(ASM7) {
+        val currentClassName = currentClass.getName
+        val currentClassInternalName = currentClassName.replace('.', '/')
+
+        // Find and update the accessedFields info. Only fields on known outer classes are tracked.
+        // This is the FieldAccessFinder equivalent.
+        override def visitFieldInsn(op: Int, owner: String, name: String, desc: String): Unit = {
+          if (op == GETFIELD || op == PUTFIELD) {
+            val ownerExternalName = owner.replace('/', '.')
+            for (cl <- accessedFields.keys if cl.getName == ownerExternalName) {
+              logTrace(s"    found field access $name on $ownerExternalName")
+              accessedFields(cl) += name
+            }
+          }
+        }
+
+        override def visitMethodInsn(
+            op: Int, owner: String, name: String, desc: String, itf: Boolean): Unit = {
+          val ownerExternalName = owner.replace('/', '.')
+          if (owner == currentClassInternalName) {
+            logTrace(s"    found intra class call to $ownerExternalName.$name$desc")
+            // could be invoking a helper method or a field accessor method, just follow it.
+            pushIfNotVisited(MethodIdentifier(currentClass, name, desc))
+          } else if (isInnerClassCtorCapturingOuter(
+              op, owner, name, desc, currentClassInternalName)) {
+            // Discover inner classes.
+            // This this the InnerClassFinder equivalent for inner classes, which still use the
+            // `$outer` chain. So this is NOT controlled by the `findTransitively` flag.
+            logDebug(s"    found inner class $ownerExternalName")
+            val innerClassInfo = getOrUpdateClassInfo(owner)
+            val innerClass = innerClassInfo._1
+            val innerClassNode = innerClassInfo._2
+            trackedClassInternalNames += owner
+            // We need to visit all methods on the inner class so that we don't missing anything.
+            for (m <- innerClassNode.methods.asScala) {
+              logTrace(s"      found method ${m.name}${m.desc}")
+              pushIfNotVisited(MethodIdentifier(innerClass, m.name, m.desc))
+            }
+          } else if (findTransitively && trackedClassInternalNames.contains(owner)) {
+            logTrace(s"    found call to outer $ownerExternalName.$name$desc")
+            val (calleeClass, _) = getOrUpdateClassInfo(owner) // make sure MethodNodes are cached
+            pushIfNotVisited(MethodIdentifier(calleeClass, name, desc))
+          } else {
+            // keep the same behavior as the original ClosureCleaner
+            logTrace(s"    ignoring call to $ownerExternalName.$name$desc")
+          }
+        }
+
+        // Find the lexically nested closures
+        // This is the InnerClosureFinder equivalent for indylambda nested closures
+        override def visitInvokeDynamicInsn(
+            name: String, desc: String, bsmHandle: Handle, bsmArgs: Object*): Unit = {
+          logTrace(s"    invokedynamic: $name$desc, bsmHandle=$bsmHandle, bsmArgs=$bsmArgs")
+
+          // fast check: we only care about Scala lambda creation
+          // TODO: maybe lift this restriction and support other functional interfaces
+          if (!name.startsWith("apply")) return
+          if (!Type.getReturnType(desc).getDescriptor.startsWith("Lscala/Function")) return
+
+          if (isLambdaMetafactory(bsmHandle)) {
+            // OK we're in the right bootstrap method for serializable Java 8 style lambda creation
+            val targetHandle = bsmArgs(1).asInstanceOf[Handle]
+            if (isLambdaBodyCapturingOuter(targetHandle, currentClassInternalName)) {
+              // this is a lexically nested closure that also captures the enclosing `this`
+              logDebug(s"    found inner closure $targetHandle")
+              val calleeMethodId =
+                MethodIdentifier(currentClass, targetHandle.getName, targetHandle.getDesc)
+              pushIfNotVisited(calleeMethodId)
+            }
+          }
+        }
+      })
+    }
+  }
+}
+
 private[spark] class ReturnStatementInClosureException
   extends SparkException("Return statements aren't allowed in Spark closures")
 
@@ -422,7 +748,7 @@ private class ReturnStatementFinder(targetMethodName: Option[String] = None)
   override def visitMethod(access: Int, name: String, desc: String,
       sig: String, exceptions: Array[String]): MethodVisitor = {
 
-    // $anonfun$ covers Java 8 lambdas
+    // $anonfun$ covers indylambda closures
     if (name.contains("apply") || name.contains("$anonfun$")) {
       // A method with suffix "$adapted" will be generated in cases like
       // { _:Int => return; Seq()} but not { _:Int => return; true}
diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index 4d89c4f079f29..78fbd0cadc5c3 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -22,7 +22,7 @@ import java.util.{Properties, UUID}
 import scala.collection.JavaConverters._
 import scala.collection.Map
 
-import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST._
@@ -59,6 +59,7 @@ private[spark] object JsonProtocol {
   private implicit val format = DefaultFormats
 
   private val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
+    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
 
   /** ------------------------------------------------- *
    * JSON serialization methods for SparkListenerEvents |
@@ -350,12 +351,22 @@ private[spark] object JsonProtocol {
         case v: Long => JInt(v)
         // We only have 3 kind of internal accumulator types, so if it's not int or long, it must be
         // the blocks accumulator, whose type is `java.util.List[(BlockId, BlockStatus)]`
-        case v =>
-          JArray(v.asInstanceOf[java.util.List[(BlockId, BlockStatus)]].asScala.toList.map {
-            case (id, status) =>
-              ("Block ID" -> id.toString) ~
-              ("Status" -> blockStatusToJson(status))
+        case v: java.util.List[_] =>
+          JArray(v.asScala.toList.flatMap {
+            case (id: BlockId, status: BlockStatus) =>
+              Some(
+                ("Block ID" -> id.toString) ~
+                ("Status" -> blockStatusToJson(status))
+              )
+            case _ =>
+              // Ignore unsupported types. A user may put `METRICS_PREFIX` in the name. We should
+              // not crash.
+              None
           })
+        case _ =>
+          // Ignore unsupported types. A user may put `METRICS_PREFIX` in the name. We should not
+          // crash.
+          JNothing
       }
     } else {
       // For all external accumulators, just use strings
@@ -474,6 +485,7 @@ private[spark] object JsonProtocol {
     ("Callsite" -> rddInfo.callSite) ~
     ("Parent IDs" -> parentIds) ~
     ("Storage Level" -> storageLevel) ~
+    ("Barrier" -> rddInfo.isBarrier) ~
     ("Number of Partitions" -> rddInfo.numPartitions) ~
     ("Number of Cached Partitions" -> rddInfo.numCachedPartitions) ~
     ("Memory Size" -> rddInfo.memSize) ~
@@ -981,7 +993,14 @@ private[spark] object JsonProtocol {
         val blockManagerAddress = blockManagerIdFromJson(json \ "Block Manager Address")
         val shuffleId = (json \ "Shuffle ID").extract[Int]
         val mapId = (json \ "Map ID").extract[Long]
-        val mapIndex = (json \ "Map Index").extract[Int]
+        val mapIndex = json \ "Map Index" match {
+          case JNothing =>
+            // Note, we use the invalid value Int.MinValue here to fill the map index for backward
+            // compatibility. Otherwise, the fetch failed event will be dropped when the history
+            // server loads the event log written by the Spark version before 3.0.
+            Int.MinValue
+          case x => x.extract[Int]
+        }
         val reduceId = (json \ "Reduce ID").extract[Int]
         val message = jsonOption(json \ "Message").map(_.extract[String])
         new FetchFailed(blockManagerAddress, shuffleId, mapId, mapIndex, reduceId,
diff --git a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
index 4f1311224bb95..4db268604a3e9 100644
--- a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
+++ b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
@@ -209,9 +209,7 @@ private [util] class SparkShutdownHookManager {
 private class SparkShutdownHook(private val priority: Int, hook: () => Unit)
   extends Comparable[SparkShutdownHook] {
 
-  override def compareTo(other: SparkShutdownHook): Int = {
-    other.priority - priority
-  }
+  override def compareTo(other: SparkShutdownHook): Int = other.priority.compareTo(priority)
 
   def run(): Unit = hook()
 
diff --git a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
index de39e4b410f25..78206c51c1028 100644
--- a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.util
 
 import java.util.concurrent._
+import java.util.concurrent.{Future => JFuture}
 import java.util.concurrent.locks.ReentrantLock
 
 import scala.concurrent.{Awaitable, ExecutionContext, ExecutionContextExecutor, Future}
@@ -28,7 +29,6 @@ import scala.util.control.NonFatal
 import com.google.common.util.concurrent.ThreadFactoryBuilder
 
 import org.apache.spark.SparkException
-import org.apache.spark.rpc.RpcAbortException
 
 private[spark] object ThreadUtils {
 
@@ -298,12 +298,28 @@ private[spark] object ThreadUtils {
       // TimeoutException and RpcAbortException is thrown in the current thread, so not need to warp
       // the exception.
       case NonFatal(t)
-          if !t.isInstanceOf[TimeoutException] && !t.isInstanceOf[RpcAbortException] =>
+          if !t.isInstanceOf[TimeoutException] =>
         throw new SparkException("Exception thrown in awaitResult: ", t)
     }
   }
   // scalastyle:on awaitresult
 
+  @throws(classOf[SparkException])
+  def awaitResult[T](future: JFuture[T], atMost: Duration): T = {
+    try {
+      atMost match {
+        case Duration.Inf => future.get()
+        case _ => future.get(atMost._1, atMost._2)
+      }
+    } catch {
+      case e: SparkFatalException =>
+        throw e.throwable
+      case NonFatal(t)
+        if !t.isInstanceOf[TimeoutException] =>
+        throw new SparkException("Exception thrown in awaitResult: ", t)
+    }
+  }
+
   // scalastyle:off awaitready
   /**
    * Preferred alternative to `Await.ready()`.
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 297cc5e4cb100..867cd1983be8f 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -95,7 +95,7 @@ private[spark] object Utils extends Logging {
    */
   val DEFAULT_DRIVER_MEM_MB = JavaUtils.DEFAULT_DRIVER_MEM_MB.toInt
 
-  val MAX_DIR_CREATION_ATTEMPTS: Int = 10
+  private val MAX_DIR_CREATION_ATTEMPTS: Int = 10
   @volatile private var localRootDirs: Array[String] = null
 
   /** Scheme used for files that are locally available on worker nodes in the cluster. */
@@ -1879,7 +1879,9 @@ private[spark] object Utils extends Logging {
    * Indicates whether Spark is currently running unit tests.
    */
   def isTesting: Boolean = {
-    sys.env.contains("SPARK_TESTING") || sys.props.contains(IS_TESTING.key)
+    // Scala's `sys.env` creates a ton of garbage by constructing Scala immutable maps, so
+    // we directly use the Java APIs instead.
+    System.getenv("SPARK_TESTING") != null || System.getProperty(IS_TESTING.key) != null
   }
 
   /**
@@ -2546,28 +2548,6 @@ private[spark] object Utils extends Logging {
     new File(path.getAbsolutePath + "." + UUID.randomUUID())
   }
 
-  /**
-   * Given a process id, return true if the process is still running.
-   */
-  def isProcessRunning(pid: Int): Boolean = {
-    val process = executeCommand(Seq("kill", "-0", pid.toString))
-    process.waitFor(10, TimeUnit.SECONDS)
-    process.exitValue() == 0
-  }
-
-  /**
-   * Returns the pid of this JVM process.
-   */
-  def getProcessId: Int = {
-    val PROCESS = "(\\d+)@(.*)".r
-    val name = getProcessName()
-    name match {
-      case PROCESS(pid, _) => pid.toInt
-      case _ =>
-        throw new SparkException(s"Unexpected process name: $name, expected to be PID@hostname.")
-    }
-  }
-
   /**
    * Returns the name of this JVM process. This is OS dependent but typically (OSX, Linux, Windows),
    * this is formatted as PID@hostname.
@@ -2772,19 +2752,16 @@ private[spark] object Utils extends Logging {
     }
 
     val masterScheme = new URI(masterWithoutK8sPrefix).getScheme
-    val resolvedURL = masterScheme.toLowerCase(Locale.ROOT) match {
-      case "https" =>
+
+    val resolvedURL = Option(masterScheme).map(_.toLowerCase(Locale.ROOT)) match {
+      case Some("https") =>
         masterWithoutK8sPrefix
-      case "http" =>
+      case Some("http") =>
         logWarning("Kubernetes master URL uses HTTP instead of HTTPS.")
         masterWithoutK8sPrefix
-      case null =>
-        val resolvedURL = s"https://$masterWithoutK8sPrefix"
-        logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " +
-          s"URL is $resolvedURL.")
-        resolvedURL
       case _ =>
-        throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme)
+        throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme
+          + " found in URL: " + masterWithoutK8sPrefix)
     }
 
     s"k8s://$resolvedURL"
diff --git a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
index 6e995a3929a75..ab177cefa1cca 100644
--- a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
+++ b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
@@ -561,6 +561,8 @@ public void spillInIterator() throws IOException {
         iter2.next();
       }
       assertFalse(iter2.hasNext());
+      // calls hasNext twice deliberately, make sure it's idempotent
+      assertFalse(iter2.hasNext());
     } finally {
       map.free();
       for (File spillFile : spillFilesCreated) {
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
index 43977717f6c97..9904a0a113fdd 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
@@ -23,7 +23,6 @@
 import java.util.LinkedList;
 import java.util.UUID;
 
-import org.hamcrest.Matchers;
 import scala.Tuple2$;
 
 import org.junit.After;
@@ -38,7 +37,6 @@
 import org.apache.spark.executor.TaskMetrics;
 import org.apache.spark.internal.config.package$;
 import org.apache.spark.memory.TestMemoryManager;
-import org.apache.spark.memory.SparkOutOfMemoryError;
 import org.apache.spark.memory.TaskMemoryManager;
 import org.apache.spark.serializer.JavaSerializer;
 import org.apache.spark.serializer.SerializerInstance;
@@ -359,6 +357,39 @@ public void forcedSpillingWithReadIterator() throws Exception {
     assertSpillFilesWereCleanedUp();
   }
 
+  @Test
+  public void forcedSpillingNullsWithReadIterator() throws Exception {
+    final UnsafeExternalSorter sorter = newSorter();
+    long[] record = new long[100];
+    final int recordSize = record.length * 8;
+    final int n = (int) pageSizeBytes / recordSize * 3;
+    for (int i = 0; i < n; i++) {
+      boolean isNull = i % 2 == 0;
+      sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, isNull);
+    }
+    assertTrue(sorter.getNumberOfAllocatedPages() >= 2);
+
+    UnsafeExternalSorter.SpillableIterator iter =
+            (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator();
+    final int numRecordsToReadBeforeSpilling = n / 3;
+    for (int i = 0; i < numRecordsToReadBeforeSpilling; i++) {
+      assertTrue(iter.hasNext());
+      iter.loadNext();
+    }
+
+    assertTrue(iter.spill() > 0);
+    assertEquals(0, iter.spill());
+
+    for (int i = numRecordsToReadBeforeSpilling; i < n; i++) {
+      assertTrue(iter.hasNext());
+      iter.loadNext();
+    }
+    assertFalse(iter.hasNext());
+
+    sorter.cleanupResources();
+    assertSpillFilesWereCleanedUp();
+  }
+
   @Test
   public void forcedSpillingWithNotReadIterator() throws Exception {
     final UnsafeExternalSorter sorter = newSorter();
@@ -518,40 +549,28 @@ public void testGetIterator() throws Exception {
   }
 
   @Test
-  public void testOOMDuringSpill() throws Exception {
+  public void testNoOOMDuringSpill() throws Exception {
     final UnsafeExternalSorter sorter = newSorter();
-    // we assume that given default configuration,
-    // the size of the data we insert to the sorter (ints)
-    // and assuming we shouldn't spill before pointers array is exhausted
-    // (memory manager is not configured to throw at this point)
-    // - so this loop runs a reasonable number of iterations (<2000).
-    // test indeed completed within <30ms (on a quad i7 laptop).
-    for (int i = 0; sorter.hasSpaceForAnotherRecord(); ++i) {
+    for (int i = 0; i < 100; i++) {
       insertNumber(sorter, i);
     }
-    // we expect the next insert to attempt growing the pointerssArray first
-    // allocation is expected to fail, then a spill is triggered which
-    // attempts another allocation which also fails and we expect to see this
-    // OOM here.  the original code messed with a released array within the
-    // spill code and ended up with a failed assertion.  we also expect the
-    // location of the OOM to be
-    // org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset
-    memoryManager.markconsequentOOM(2);
-    try {
-      insertNumber(sorter, 1024);
-      fail("expected OutOfMmoryError but it seems operation surprisingly succeeded");
-    }
-    // we expect an SparkOutOfMemoryError here, anything else (i.e the original NPE is a failure)
-    catch (SparkOutOfMemoryError oom){
-      String oomStackTrace = Utils.exceptionString(oom);
-      assertThat("expected SparkOutOfMemoryError in " +
-        "org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset",
-        oomStackTrace,
-        Matchers.containsString(
-          "org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset"));
+
+    // Check that spilling still succeeds when the task is starved for memory.
+    memoryManager.markconsequentOOM(Integer.MAX_VALUE);
+    sorter.spill();
+    memoryManager.resetConsequentOOM();
+
+    // Ensure that records can be appended after spilling, i.e. check that the sorter will allocate
+    // the new pointer array that it could not allocate while spilling.
+    for (int i = 0; i < 100; ++i) {
+      insertNumber(sorter, i);
     }
+
+    sorter.cleanupResources();
+    assertSpillFilesWereCleanedUp();
   }
 
+
   private void verifyIntIterator(UnsafeSorterIterator iter, int start, int end)
       throws IOException {
     for (int i = start; i < end; i++) {
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
index 2b8a0602730e1..9d4909ddce792 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
@@ -20,6 +20,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
+import org.apache.spark.unsafe.array.LongArray;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -27,7 +28,6 @@
 import org.apache.spark.SparkConf;
 import org.apache.spark.memory.TestMemoryConsumer;
 import org.apache.spark.memory.TestMemoryManager;
-import org.apache.spark.memory.SparkOutOfMemoryError;
 import org.apache.spark.memory.TaskMemoryManager;
 import org.apache.spark.unsafe.Platform;
 import org.apache.spark.unsafe.memory.MemoryBlock;
@@ -37,7 +37,6 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.isIn;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 
 public class UnsafeInMemorySorterSuite {
@@ -147,7 +146,7 @@ public int compare(
   }
 
   @Test
-  public void freeAfterOOM() {
+  public void testNoOOMDuringReset() {
     final SparkConf sparkConf = new SparkConf();
     sparkConf.set(package$.MODULE$.MEMORY_OFFHEAP_ENABLED(), false);
 
@@ -156,12 +155,7 @@ public void freeAfterOOM() {
     final TaskMemoryManager memoryManager = new TaskMemoryManager(
             testMemoryManager, 0);
     final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager);
-    final MemoryBlock dataPage = memoryManager.allocatePage(2048, consumer);
-    final Object baseObject = dataPage.getBaseObject();
-    // Write the records into the data page:
-    long position = dataPage.getBaseOffset();
 
-    final HashPartitioner hashPartitioner = new HashPartitioner(4);
     // Use integer comparison for comparing prefixes (which are partition ids, in this case)
     final PrefixComparator prefixComparator = PrefixComparators.LONG;
     final RecordComparator recordComparator = new RecordComparator() {
@@ -179,18 +173,24 @@ public int compare(
     UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer, memoryManager,
             recordComparator, prefixComparator, 100, shouldUseRadixSort());
 
-    testMemoryManager.markExecutionAsOutOfMemoryOnce();
-    try {
-      sorter.reset();
-      fail("expected SparkOutOfMemoryError but it seems operation surprisingly succeeded");
-    } catch (SparkOutOfMemoryError oom) {
-      // as expected
-    }
-    // [SPARK-21907] this failed on NPE at
-    // org.apache.spark.memory.MemoryConsumer.freeArray(MemoryConsumer.java:108)
-    sorter.free();
-    // simulate a 'back to back' free.
-    sorter.free();
+    // Ensure that the sorter does not OOM while freeing its memory.
+    testMemoryManager.markconsequentOOM(Integer.MAX_VALUE);
+    sorter.freeMemory();
+    testMemoryManager.resetConsequentOOM();
+    Assert.assertFalse(sorter.hasSpaceForAnotherRecord());
+
+    // Get the sorter in an usable state again by allocating a new pointer array.
+    LongArray array = consumer.allocateArray(1000);
+    sorter.expandPointerArray(array);
+
+    // Ensure that it is safe to call freeMemory() multiple times.
+    testMemoryManager.markconsequentOOM(Integer.MAX_VALUE);
+    sorter.freeMemory();
+    sorter.freeMemory();
+    testMemoryManager.resetConsequentOOM();
+    Assert.assertFalse(sorter.hasSpaceForAnotherRecord());
+
+    assertEquals(0L, memoryManager.cleanUpAllAllocatedMemory());
   }
 
 }
diff --git a/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java b/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java
index 0f489fb219010..b188ee16b97d0 100644
--- a/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java
+++ b/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java
@@ -28,6 +28,7 @@
 
 import org.apache.spark.api.java.*;
 import org.apache.spark.*;
+import org.apache.spark.util.Utils;
 
 /**
  * Java apps can use both Java-friendly JavaSparkContext and Scala SparkContext.
@@ -35,14 +36,16 @@
 public class JavaSparkContextSuite implements Serializable {
 
   @Test
-  public void javaSparkContext() {
+  public void javaSparkContext() throws IOException {
+    File tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark");
+    String dummyJarFile = File.createTempFile(tempDir.toString(), "jarFile").toString();
     String[] jars = new String[] {};
     java.util.Map<String, String> environment = new java.util.HashMap<>();
 
     new JavaSparkContext(new SparkConf().setMaster("local").setAppName("name")).stop();
     new JavaSparkContext("local", "name", new SparkConf()).stop();
     new JavaSparkContext("local", "name").stop();
-    new JavaSparkContext("local", "name", "sparkHome", "jarFile").stop();
+    new JavaSparkContext("local", "name", "sparkHome", dummyJarFile).stop();
     new JavaSparkContext("local", "name", "sparkHome", jars).stop();
     new JavaSparkContext("local", "name", "sparkHome", jars, environment).stop();
   }
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index 7c3f77d8c10cf..388e89ceff054 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -16,7 +16,7 @@
   "totalInputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
-  "isBlacklisted" : true,
+  "isBlacklisted" : false,
   "maxMemory" : 908381388,
   "addTime" : "2016-11-16T22:33:31.477GMT",
   "executorLogs" : { },
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
index 0986e85f16b3e..d030d47b2e8cc 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
@@ -16,7 +16,7 @@
   "totalInputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
-  "isBlacklisted" : true,
+  "isBlacklisted" : false,
   "maxMemory" : 908381388,
   "addTime" : "2016-11-16T22:33:31.477GMT",
   "executorLogs" : { },
diff --git a/core/src/test/resources/spark-events/application_1553914137147_0018 b/core/src/test/resources/spark-events/application_1553914137147_0018
index 8c34e7265f8da..03ea3040de1db 100644
--- a/core/src/test/resources/spark-events/application_1553914137147_0018
+++ b/core/src/test/resources/spark-events/application_1553914137147_0018
@@ -1,6 +1,6 @@
 {"Event":"SparkListenerLogStart","Spark Version":"3.0.0-SNAPSHOT"}
 {"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"test-1.vpc.company.com","Port":44768},"Maximum Memory":956615884,"Timestamp":1554755989747,"Maximum Onheap Memory":956615884,"Maximum Offheap Memory":0}
-{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/java/jdk1.8.0_144/jre","Java Version":"1.8.0_144 (Oracle Corporation)","Scala Version":"version 2.12.8"},"Spark Properties":{"spark.lineage.log.dir":"/var/log/spark2/lineage","spark.serializer":"org.apache.spark.serializer.KryoSerializer","spark.yarn.jars":"local:/opt/cloudera/parcels/SPARK2/lib/spark2/jars/*","spark.executor.extraJavaOptions":"-Djava.security.egd=file:///dev/urandom","spark.driver.host":"test-1.vpc.company.com","spark.eventLog.enabled":"true","spark.executor.heartbeatInterval":"1000","spark.executor.memoryOverhead":"1024","spark.driver.port":"34194","spark.shuffle.service.enabled":"false","spark.driver.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.lineage.enabled":"false","spark.jars":"file:/tmp/__spark_test__/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar","spark.executor.metrics.pollingInterval":"100","spark.yarn.historyServer.address":"http://test-1.vpc.company.com:18089","spark.ui.enabled":"true","spark.app.name":"LargeBlocks","spark.ui.killEnabled":"true","spark.sql.hive.metastore.jars":"${env:HADOOP_COMMON_HOME}/../hive/lib/*:${env:HADOOP_COMMON_HOME}/client/*","spark.locality.wait.process":"0","spark.dynamicAllocation.schedulerBacklogTimeout":"1","spark.yarn.am.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.scheduler.mode":"FIFO","spark.eventLog.logStageExecutorMetrics.enabled":"true","spark.driver.memory":"2g","spark.executor.instances":"3","spark.submit.pyFiles":"","spark.yarn.config.gatewayPath":"/opt/cloudera/parcels","spark.executor.id":"driver","spark.yarn.config.replacementPath":"{{HADOOP_COMMON_HOME}}/../../..","spark.driver.extraJavaOptions":"-Djava.security.egd=file:///dev/urandom","spark.eventLog.logStageExecutorProcessTreeMetrics.enabled":"true","spark.submit.deployMode":"client","spark.shuffle.service.port":"7337","spark.yarn.maxAppAttempts":"1","spark.master":"yarn","spark.authenticate":"false","spark.ui.filters":"org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter","spark.network.crypto.enabled":"false","spark.executor.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.executor.memory":"7g","spark.io.encryption.enabled":"false","spark.eventLog.dir":"hdfs://test-1.vpc.company.com:8020/user/spark/spark2ApplicationHistory","spark.dynamicAllocation.enabled":"false","spark.sql.catalogImplementation":"hive","spark.executor.cores":"1","spark.driver.appUIAddress":"http://test-1.vpc.company.com:4040","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_HOSTS":"test-1.vpc.company.com","spark.dynamicAllocation.minExecutors":"0","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES":"http://test-1.vpc.company.com:8088/proxy/application_1553914137147_0018","spark.dynamicAllocation.executorIdleTimeout":"60","spark.app.id":"application_1553914137147_0018","spark.sql.hive.metastore.version":"1.1.0"},"Hadoop Properties":{"yarn.resourcemanager.amlauncher.thread-count":"50","dfs.namenode.resource.check.interval":"5000","fs.s3a.connection.maximum":"100","mapreduce.jobtracker.jobhistory.task.numberprogresssplits":"12","dfs.data.transfer.server.tcpnodelay":"true","mapreduce.tasktracker.healthchecker.script.timeout":"600000","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"yarn","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","dfs.cachereport.intervalMsec":"10000","dfs.namenode.checkpoint.txns":"1000000","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","mapreduce.tasktracker.local.dir.minspacekill":"0","hadoop.security.groups.cache.background.reload.threads":"3","dfs.namenode.lease-recheck-interval-ms":"2000","fs.s3.block.size":"67108864","dfs.client.domain.socket.data.traffic":"false","dfs.ha.zkfc.nn.http.timeout.ms":"20000","hadoop.registry.secure":"false","hadoop.hdfs.configuration.version":"1","dfs.bytes-per-checksum":"512","fs.s3.buffer.dir":"${hadoop.tmp.dir}/s3","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25","mapreduce.jobhistory.loadedjobs.cache.size":"5","mapreduce.jobtracker.persist.jobstatus.hours":"1","fs.s3a.s3guard.ddb.table.create":"false","dfs.datanode.slow.io.warning.threshold.ms":"300","dfs.namenode.handler.count":"10","dfs.namenode.list.reencryption.status.num.responses":"100","mapreduce.input.fileinputformat.split.minsize":"0","dfs.datanode.failed.volumes.tolerated":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.amliveliness-monitor.interval-ms":"1000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","mapreduce.tasktracker.http.threads":"40","dfs.namenode.retrycache.expirytime.millis":"600000","dfs.namenode.backup.address":"0.0.0.0:50100","dfs.datanode.data.dir":"file://${hadoop.tmp.dir}/dfs/data","dfs.datanode.shared.file.descriptor.paths":"/dev/shm,/tmp","dfs.replication":"3","mapreduce.jobtracker.jobhistory.block.size":"3145728","dfs.encrypt.data.transfer.cipher.key.bitlength":"128","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","dfs.secondary.namenode.kerberos.internal.spnego.principal":"${dfs.web.authentication.kerberos.principal}","mapreduce.task.profile.maps":"0-2","dfs.datanode.block-pinning.enabled":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","mapreduce.jobtracker.retiredjobs.cache.size":"1000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"test-1.vpc.company.com:19890","fs.trash.checkpoint.interval":"0","dfs.namenode.checkpoint.check.period":"60","yarn.nodemanager.container-monitor.interval-ms":"3000","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","hadoop.proxyuser.HTTP.groups":"*","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"10000","dfs.namenode.reencrypt.edek.threads":"10","yarn.resourcemanager.system-metrics-publisher.enabled":"false","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","dfs.namenode.fs-limits.max-component-length":"255","mapreduce.tasktracker.taskcontroller":"org.apache.hadoop.mapred.DefaultTaskController","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","dfs.namenode.top.window.num.buckets":"10","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","dfs.datanode.block.id.layout.upgrade.threads":"12","mapreduce.jobtracker.tasktracker.maxblacklists":"4","yarn.nodemanager.docker-container-executor.exec-name":"/usr/bin/docker","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"0.23.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","dfs.namenode.decommission.max.concurrent.tracked.nodes":"100","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","dfs.client.read.shortcircuit.skip.checksum":"false","mapreduce.task.profile.reduces":"0-2","dfs.datanode.address":"0.0.0.0:50010","dfs.https.server.keystore.resource":"ssl-server.xml","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"test-1.vpc.company.com:8030","mapreduce.task.timeout":"600000","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"hdfs://test-1.vpc.company.com:8020","fs.har.impl.disable.cache":"true","io.compression.codec.bzip2.library":"system-native","dfs.namenode.audit.loggers":"default","dfs.block.access.key.update.interval":"600","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"test-1.vpc.company.com:8090","dfs.namenode.max.objects":"0","mapreduce.jobhistory.address":"test-1.vpc.company.com:10020","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","dfs.namenode.max.full.block.report.leases":"6","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","dfs.client.mmap.cache.timeout.ms":"3600000","dfs.mover.max-no-move-interval":"60000","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","dfs.client.datanode-restart.timeout":"30","dfs.datanode.drop.cache.behind.reads":"false","ipc.server.log.slow.rpc":"false","dfs.namenode.read-lock-reporting-threshold-ms":"5000","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","dfs.namenode.checkpoint.edits.dir":"${dfs.namenode.checkpoint.dir}","yarn.app.attempt.diagnostics.limit.kc":"64","dfs.balancer.block-move.timeout":"0","dfs.client.block.write.replace-datanode-on-failure.enable":"true","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"8192","io.compression.codecs":"org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","dfs.heartbeat.interval":"3","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","dfs.ha.tail-edits.period":"60","dfs.datanode.max.locked.memory":"0","dfs.datanode.scan.period.hours":"504","mapreduce.jobtracker.expire.trackers.interval":"600000","yarn.resourcemanager.nodemanager-connect-retries":"10","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","dfs.namenode.edits.noeditlogchannelflush":"false","mapreduce.task.io.sort.factor":"64","mapreduce.tasktracker.outofband.heartbeat":"false","ha.failover-controller.new-active.rpc-timeout.ms":"60000","dfs.webhdfs.ugi.expire.after.access":"600000","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","dfs.namenode.https-address":"test-1.vpc.company.com:20102","yarn.am.blacklisting.enabled":"true","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","dfs.image.transfer-bootstrap-standby.bandwidthPerSec":"0","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.proxyuser.hive.groups":"*","ha.zookeeper.session-timeout.ms":"5000","mapreduce.map.java.opts":"-Djava.net.preferIPv4Stack=true","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","dfs.ha.automatic-failover.enabled":"false","mapreduce.job.counters.groups.max":"50","dfs.namenode.decommission.interval":"30","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","dfs.datanode.handler.count":"10","dfs.namenode.xattrs.enabled":"true","dfs.namenode.safemode.threshold-pct":"0.999f","dfs.client.socket.send.buffer.size":"0","mapreduce.map.sort.spill.percent":"0.8","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","dfs.datanode.sync.behind.writes":"false","dfs.namenode.stale.datanode.interval":"30000","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","dfs.datanode.transfer.socket.send.buffer.size":"0","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","dfs.namenode.reencrypt.throttle.limit.handler.ratio":"1.0","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler","yarn.app.mapreduce.am.command-opts":"-Djava.net.preferIPv4Stack=true -Xmx825955249","dfs.journalnode.https-address":"0.0.0.0:8481","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","hadoop.proxyuser.hue.hosts":"*","io.mapfile.bloom.error.rate":"0.005","dfs.user.home.dir.prefix":"/user","hadoop.proxyuser.hue.groups":"*","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","mapreduce.jobtracker.persist.jobstatus.dir":"/jobtracker/jobsInfo","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","dfs.namenode.snapshot.capture.openfiles":"false","dfs.datanode.directoryscan.interval":"21600","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","dfs.client.read.shortcircuit.streams.cache.expiry.ms":"300000","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"10","dfs.namenode.write.stale.datanode.ratio":"0.5f","hadoop.registry.zk.session.timeout.ms":"60000","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","mapreduce.jobtracker.taskcache.levels":"2","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","mapreduce.job.emit-timeline-data":"false","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","dfs.domain.socket.path":"/var/run/hdfs-sockets/dn","yarn.resourcemanager.admin.client.thread-count":"1","mapreduce.jobtracker.persist.jobstatus.active":"true","dfs.namenode.reencrypt.sleep.interval":"1m","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"test-1.vpc.company.com:10033","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","mapreduce.tasktracker.report.address":"127.0.0.1:0","dfs.namenode.edit.log.autoroll.multiplier.threshold":"2.0","io.seqfile.lazydecompress":"true","ftp.blocksize":"67108864","dfs.namenode.backup.http-address":"0.0.0.0:50105","dfs.disk.balancer.max.disk.throughputInMBperSec":"10","mapreduce.jobtracker.instrumentation":"org.apache.hadoop.mapred.JobTrackerMetricsInst","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.delete.debug-delay-sec":"0","mapreduce.jobtracker.http.address":"0.0.0.0:50030","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","hadoop.security.groups.cache.secs":"300","yarn.resourcemanager.zk-retry-interval-ms":"1000","yarn.scheduler.increment-allocation-mb":"512","nfs.mountd.port":"4242","mapreduce.shuffle.max.threads":"0","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","mapreduce.jobhistory.http.policy":"HTTP_ONLY","dfs.datanode.dns.interface":"default","mapreduce.reduce.java.opts":"-Djava.net.preferIPv4Stack=true","s3native.replication":"3","hadoop.security.group.mapping.ldap.ssl":"false","dfs.namenode.fs-limits.max-xattrs-per-inode":"32","yarn.client.application-client-protocol.poll-interval-ms":"200","hadoop.proxyuser.flume.groups":"*","dfs.namenode.fs-limits.max-xattr-size":"16384","dfs.namenode.maintenance.replication.min":"1","dfs.client.write.exclude.nodes.cache.expiry.interval.millis":"600000","ha.zookeeper.parent-znode":"/hadoop-ha","dfs.namenode.safemode.extension":"30000","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","dfs.blocksize":"134217728","dfs.namenode.servicerpc-address":"test-1.vpc.company.com:8022","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"test-1.vpc.company.com:8032","ipc.client.ping":"true","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","dfs.namenode.resource.checked.volumes.minimum":"1","hadoop.proxyuser.HTTP.hosts":"*","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","dfs.namenode.quota.init-threads":"4","dfs.journalnode.http-address":"0.0.0.0:8480","dfs.block.scanner.volume.bytes.per.second":"1048576","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","dfs.storage.policy.enabled":"true","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","dfs.namenode.edits.dir":"${dfs.namenode.name.dir}","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","dfs.namenode.support.allow.format":"true","dfs.ha.tail-edits.rolledits.timeout":"60","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","mapreduce.cluster.temp.dir":"${hadoop.tmp.dir}/mapred/temp","s3.replication":"3","dfs.client.failover.connection.retries":"0","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","dfs.namenode.secondary.http-address":"0.0.0.0:50090","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","nfs.wtmax":"1048576","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","nfs.dump.dir":"/tmp/.hdfs-nfs","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","mapreduce.client.submit.file.replication":"3","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","dfs.namenode.service.handler.count":"10","dfs.datanode.data.dir.perm":"700","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","dfs.namenode.name.dir":"file:///dataroot/dataroot/dfs/nn","yarn.resourcemanager.zk-acl":"world:anyone:rwcda","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","fs.ftp.host.port":"21","ipc.ping.interval":"60000","dfs.namenode.num.checkpoints.retained":"2","dfs.namenode.kerberos.internal.spnego.principal":"${dfs.web.authentication.kerberos.principal}","yarn.resourcemanager.admin.address":"test-1.vpc.company.com:8033","file.client-write-packet-size":"65536","hadoop.treat.subject.external":"false","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"false","dfs.disk.balancer.plan.threshold.percent":"10","mapreduce.local.clientfactory.class.name":"org.apache.hadoop.mapred.LocalClientFactory","dfs.client.use.legacy.blockreader":"false","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"256","yarn.nodemanager.localizer.client.thread-count":"5","dfs.namenode.checkpoint.max-retries":"3","dfs.namenode.reject-unresolved-dn-topology-mapping":"false","dfs.namenode.delegation.token.max-lifetime":"*********(redacted)","dfs.client.block.write.replace-datanode-on-failure.min-replication":"0","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"2000","dfs.namenode.num.extra.edits.retained":"1000000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.scheduler.increment-allocation-vcores":"1","fs.df.interval":"60000","fs.s3.sleepTimeSeconds":"10","fs.s3a.multiobjectdelete.enable":"true","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"true","io.file.buffer.size":"65536","hadoop.work.around.non.threadsafe.getpwuid":"false","dfs.permissions.superusergroup":"supergroup","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","mapreduce.tasktracker.dns.interface":"default","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","dfs.permissions.enabled":"true","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/user","mapreduce.reduce.shuffle.read.timeout":"180000","yarn.app.mapreduce.am.admin.user.env":"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH","dfs.datanode.https.address":"0.0.0.0:50475","dfs.namenode.hosts.provider.classname":"org.apache.hadoop.hdfs.server.blockmanagement.HostFileManager","dfs.datanode.transfer.socket.recv.buffer.size":"0","fs.s3a.connection.establish.timeout":"5000","dfs.namenode.fslock.fair":"true","mapreduce.job.running.map.limit":"0","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","hadoop.fuse.connection.timeout":"300","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"5000","yarn.nodemanager.vmem-pmem-ratio":"2.1","dfs.client.slow.io.warning.threshold.ms":"30000","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","s3native.stream-buffer-size":"4096","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","fs.s3a.s3guard.cli.prune.age":"86400000","dfs.client.read.shortcircuit.streams.cache.size":"256","dfs.client.use.legacy.blockreader.local":"false","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3n.multipart.uploads.enabled":"false","dfs.namenode.path.based.cache.retry.interval.ms":"30000","hadoop.security.crypto.buffer.size":"8192","yarn.client.failover-retries-on-socket-timeouts":"0","dfs.balancer.keytab.enabled":"false","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","dfs.datanode.balance.bandwidthPerSec":"10485760","dfs.namenode.name.dir.restore":"false","hadoop.registry.jaas.context":"Client","dfs.client.failover.sleep.max.millis":"15000","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","s3.blocksize":"67108864","yarn.am.blacklisting.disable-failure-threshold":"0.8f","io.map.index.interval":"128","mapreduce.job.counters.max":"120","dfs.namenode.max-lock-hold-to-release-lease-ms":"25","dfs.namenode.datanode.registration.ip-hostname-check":"true","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","dfs.namenode.resource.du.reserved":"104857600","dfs.datanode.bp-ready.timeout":"20","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","dfs.namenode.full.block.report.lease.length.ms":"300000","mapreduce.tasktracker.instrumentation":"org.apache.hadoop.mapred.TaskTrackerMetricsInst","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","s3.stream-buffer-size":"4096","yarn.resourcemanager.nm.liveness-monitor.interval-ms":"1000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","dfs.namenode.secondary.https-address":"0.0.0.0:50091","s3native.bytes-per-checksum":"512","dfs.namenode.fs-limits.max-directory-items":"1048576","nfs.server.port":"2049","dfs.namenode.delegation.token.renew-interval":"*********(redacted)","mapreduce.jobtracker.address":"local","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","dfs.namenode.list.openfiles.num.responses":"1000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","dfs.namenode.blocks.per.postponedblocks.rescan":"10000","dfs.namenode.checkpoint.period":"3600","hadoop.security.groups.cache.background.reload":"false","yarn.resourcemanager.amlauncher.log.command":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","dfs.namenode.snapshot.skip.capture.accesstime-only-change":"false","dfs.webhdfs.user.provider.user.pattern":"^[A-Za-z_][A-Za-z0-9._-]*[$]?$","dfs.webhdfs.acl.provider.permission.pattern":"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"false","mapreduce.map.skip.maxrecords":"0","mapreduce.jobtracker.handler.count":"10","hadoop.http.authentication.type":"simple","mapreduce.job.jvm.numtasks":"1","hadoop.proxyuser.flume.hosts":"*","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","fs.s3n.block.size":"67108864","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","mapreduce.jobtracker.staging.root.dir":"${hadoop.tmp.dir}/mapred/staging","dfs.namenode.http-address":"test-1.vpc.company.com:20101","mapreduce.jobtracker.jobhistory.lru.cache.size":"5","dfs.datanode.directoryscan.threads":"1","dfs.datanode.fsdatasetcache.max.threads.per.volume":"4","dfs.namenode.fs-limits.max-blocks-per-file":"1048576","dfs.disk.balancer.enabled":"false","mapreduce.shuffle.listen.queue.size":"128","mapreduce.tasktracker.local.dir.minspacestart":"0","mapreduce.map.cpu.vcores":"1","hadoop.user.group.static.mapping.overrides":"dr.who=;","dfs.datanode.cache.revocation.timeout.ms":"900000","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","dfs.client.mmap.cache.size":"256","dfs.ha.log-roll.period":"120","dfs.client.failover.sleep.base.millis":"500","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","dfs.namenode.accesstime.precision":"3600000","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","hadoop.registry.zk.retry.interval.ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","dfs.client.context":"default","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","hadoop.ssl.server.conf":"ssl-server.xml","dfs.http.policy":"HTTP_ONLY","dfs.client.https.keystore.resource":"ssl-client.xml","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.acl.enable":"true","dfs.domain.socket.disable.interval.seconds":"1","dfs.image.transfer.chunksize":"65536","dfs.balancer.max-no-move-interval":"60000","mapreduce.tasktracker.map.tasks.maximum":"2","dfs.namenode.edits.journal-plugin.qjournal":"org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager","mapreduce.task.profile":"false","dfs.webhdfs.enabled":"true","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","dfs.namenode.list.encryption.zones.num.responses":"100","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","dfs.namenode.top.num.users":"10","dfs.disk.balancer.block.tolerance.percent":"10","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","io.seqfile.sorter.recordlimit":"1000000","hadoop.security.auth_to_local":"DEFAULT","dfs.blockreport.initialDelay":"0","fs.automatic.close":"true","dfs.client.block.write.replace-datanode-on-failure.best-effort":"false","dfs.namenode.replication.min":"1","dfs.balancer.address":"0.0.0.0:0","fs.s3n.multipart.copy.block.size":"5368709120","yarn.nodemanager.hostname":"0.0.0.0","nfs.rtmax":"1048576","yarn.resourcemanager.zk-timeout-ms":"10000","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","dfs.datanode.directoryscan.throttle.limit.ms.per.sec":"1000","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","dfs.namenode.replication.work.multiplier.per.iteration":"2","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","dfs.namenode.reencrypt.throttle.limit.updater.ratio":"1.0","dfs.namenode.avoid.write.stale.datanode":"false","dfs.short.circuit.shared.memory.watcher.interrupt.check.ms":"60000","dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction":"0.75f","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","dfs.client.mmap.enabled":"true","mapreduce.reduce.cpu.vcores":"1","hadoop.proxyuser.oozie.groups":"*","fs.client.resolve.remote.symlinks":"true","dfs.image.compression.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.jobtracker.restart.recover":"false","dfs.namenode.decommission.blocks.per.interval":"500000","mapreduce.tasktracker.reduce.tasks.maximum":"2","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","dfs.namenode.safemode.min.datanodes":"0","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.userlog.retain.hours":"24","yarn.scheduler.maximum-allocation-vcores":"4","yarn.nodemanager.log-aggregation.compression-type":"none","dfs.namenode.enable.retrycache":"true","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","dfs.namenode.startup.delay.block.deletion.sec":"0","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","dfs.datanode.readahead.bytes":"4194304","mapreduce.jobtracker.heartbeats.in.second":"100","mapreduce.job.running.reduce.limit":"0","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","mapreduce.task.tmp.dir":"./tmp","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:mapred@hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","dfs.namenode.delegation.key.update-interval":"86400000","fs.s3a.max.total.tasks":"5","dfs.client.file-block-storage-locations.num-threads":"10","mapreduce.tasktracker.healthchecker.interval":"60000","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.fast.upload":"false","fs.s3a.attempts.maximum":"20","dfs.namenode.avoid.read.stale.datanode":"false","hadoop.registry.zk.connection.timeout.ms":"15000","dfs.https.port":"20102","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","mapreduce.map.log.level":"INFO","mapreduce.output.fileoutputformat.compress.type":"BLOCK","hadoop.registry.rm.enabled":"false","mapreduce.ifile.readahead.bytes":"4194304","mapreduce.tasktracker.tasks.sleeptimebeforesigkill":"5000","yarn.resourcemanager.fs.state-store.retry-policy-spec":"2000, 500","dfs.namenode.posix.acl.inheritance.enabled":"false","dfs.blockreport.intervalMsec":"21600000","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","dfs.namenode.path.based.cache.refresh.interval.ms":"30000","dfs.namenode.edekcacheloader.interval.ms":"1000","file.stream-buffer-size":"4096","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.SnappyCodec","mapreduce.map.speculative":"false","dfs.disk.balancer.max.disk.errors":"5","dfs.datanode.use.datanode.hostname":"false","mapreduce.job.speculative.retry-after-speculate":"15000","hadoop.proxyuser.hdfs.hosts":"*","dfs.namenode.fs-limits.min-block-size":"1048576","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","mapreduce.job.reduce.slowstart.completedmaps":"0.8","dfs.client.read.shortcircuit":"false","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","hadoop.proxyuser.yarn.groups":"*","dfs.client.cached.conn.retry":"3","dfs.namenode.invalidate.work.pct.per.iteration":"0.32f","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.nodemanager.logaggregation.threadpool-size-max":"100","dfs.replication.max":"512","dfs.namenode.inotify.max.events.per.rpc":"1000","yarn.resourcemanager.hostname":"0.0.0.0","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"0","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","ipc.client.rpc-timeout.ms":"0","fs.s3.maxRetries":"4","dfs.default.chunk.view.size":"32768","mapreduce.input.lineinputformat.linespermap":"1","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","dfs.client.mmap.retry.timeout.ms":"300000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","dfs.namenode.list.cache.directives.num.responses":"100","fs.s3a.socket.recv.buffer":"8192","dfs.image.compress":"false","dfs.namenode.kerberos.principal.pattern":"*","yarn.application.classpath":"$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*","fs.s3n.multipart.uploads.block.size":"67108864","mapreduce.tasktracker.http.address":"0.0.0.0:50060","yarn.resourcemanager.resource-tracker.address":"test-1.vpc.company.com:8031","hadoop.fuse.timer.period":"5","mapreduce.job.heap.memory-mb.ratio":"0.8","dfs.datanode.hdfs-blocks-metadata.enabled":"true","dfs.namenode.checkpoint.dir":"file://${hadoop.tmp.dir}/dfs/namesecondary","dfs.datanode.max.transfer.threads":"4096","dfs.namenode.edits.asynclogging":"true","nfs.allow.insecure.ports":"true","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"0","s3native.client-write-packet-size":"65536","mapreduce.admin.user.env":"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","hadoop.proxyuser.mapred.hosts":"*","hadoop.proxyuser.oozie.hosts":"*","yarn.nodemanager.log.retain-seconds":"10800","hadoop.proxyuser.mapred.groups":"*","yarn.resourcemanager.keytab":"/etc/krb5.keytab","mapreduce.reduce.merge.inmem.threshold":"1000","dfs.client.https.need-auth":"false","dfs.blockreport.split.threshold":"1000000","dfs.client.block.write.replace-datanode-on-failure.policy":"DEFAULT","mapreduce.shuffle.ssl.enabled":"false","dfs.namenode.write-lock-reporting-threshold-ms":"5000","dfs.block.access.token.enable":"*********(redacted)","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","httpfs.buffer.size":"4096","dfs.client.file-block-storage-locations.timeout.millis":"1000","dfs.namenode.block-placement-policy.default.prefer-local-node":"true","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","dfs.namenode.replication.considerLoad":"true","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.proxyuser.hdfs.groups":"*","dfs.namenode.retrycache.heap.percent":"0.03f","dfs.datanode.cache.revocation.polling.ms":"500","mapreduce.jobhistory.webapp.address":"test-1.vpc.company.com:19888","dfs.namenode.path.based.cache.block.map.allocation.percent":"0.25","mapreduce.jobtracker.system.dir":"${hadoop.tmp.dir}/mapred/system","mapreduce.tasktracker.taskmemorymanager.monitoringinterval":"5000","dfs.journalnode.rpc-address":"0.0.0.0:8485","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"test-1.vpc.company.com:8088","mapreduce.jobhistory.recovery.enable":"false","dfs.client.short.circuit.replica.stale.threshold.ms":"1800000","mapreduce.reduce.shuffle.parallelcopies":"10","fs.trash.interval":"1","dfs.namenode.replication.interval":"3","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","dfs.namenode.top.enabled":"true","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","dfs.datanode.du.reserved":"0","yarn.app.mapreduce.am.resource.mb":"1024","mapreduce.input.fileinputformat.list-status.num-threads":"1","dfs.namenode.lazypersist.file.scrub.interval.sec":"300","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.nodemanager.resource.cpu-vcores":"8","mapreduce.job.reduces":"6","fs.s3a.multipart.size":"64M","yarn.scheduler.minimum-allocation-vcores":"1","dfs.namenode.reencrypt.batch.size":"1000","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","dfs.datanode.http.address":"0.0.0.0:50075","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","fs.s3a.metadatastore.authoritative":"false","ha.health-monitor.sleep-after-disconnect.ms":"1000","s3.bytes-per-checksum":"512","yarn.app.mapreduce.shuffle.log.limit.kb":"0","dfs.namenode.list.cache.pools.num.responses":"100","hadoop.security.group.mapping":"org.apache.hadoop.security.ShellBasedUnixGroupsMapping","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","dfs.encrypt.data.transfer":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","mapreduce.job.redacted-properties":"*********(redacted)","dfs.namenode.top.windows.minutes":"1,5,25","s3.client-write-packet-size":"65536","mapreduce.map.output.compress":"true","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.scheduler.maximum-allocation-mb":"12288","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.app.mapreduce.am.container.log.limit.kb":"0","s3native.blocksize":"67108864","ipc.client.connect.retry.interval":"1000","hadoop.proxyuser.httpfs.groups":"*","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","dfs.namenode.edit.log.autoroll.check.interval.ms":"300000","mapreduce.jobhistory.cleaner.enable":"true","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","hadoop.proxyuser.httpfs.hosts":"*","dfs.client.use.datanode.hostname":"false","dfs.stream-buffer-size":"4096","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","dfs.datanode.drop.cache.behind.writes":"false","mapreduce.tasktracker.dns.nameserver":"default","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","mapreduce.job.end-notification.retry.attempts":"0","hadoop.proxyuser.yarn.hosts":"*","yarn.resourcemanager.zk-num-retries":"1000","dfs.client.failover.max.attempts":"15","mapreduce.tasktracker.indexcache.mb":"10","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","dfs.namenode.snapshotdiff.allow.snap-root-descendant":"true","yarn.nodemanager.localizer.cache.target-size-mb":"10240","zlib.compress.level":"DEFAULT_COMPRESSION","ftp.client-write-packet-size":"65536","mapreduce.jobtracker.maxtasks.perjob":"-1","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.proxyuser.hive.hosts":"*","dfs.block.access.token.lifetime":"*********(redacted)","dfs.namenode.max.extra.edits.segments.retained":"10000","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","dfs.image.transfer.bandwidthPerSec":"0","io.native.lib.available":"true","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","ipc.server.listen.queue.size":"128","dfs.namenode.edekcacheloader.initial.delay.ms":"3000","map.sort.class":"org.apache.hadoop.util.QuickSort","dfs.namenode.acls.enabled":"false","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","dfs.datanode.ipc.address":"0.0.0.0:50020","yarn.nodemanager.vmem-check-enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","dfs.lock.suppress.warning.interval":"10s","dfs.client.block.write.retries":"3","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.application.classpath":"$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH","yarn.client.nodemanager-connect.retry-interval-ms":"10000","dfs.client-write-packet-size":"65536","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME","dfs.datanode.dns.nameserver":"default","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","dfs.image.transfer.timeout":"60000","yarn.resourcemanager.recovery.enabled":"false","dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold":"10737418240","dfs.client.failover.connection.retries.on.timeouts":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.security.egd":"file:///dev/urandom","java.vm.specification.version":"1.8","user.home":"/home/systest","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/java/jdk1.8.0_144/jre/lib/amd64","user.dir":"/tmp","java.library.path":":/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.144-b01","jetty.git.hash":"27208684755d94a92186989f695db2d7b21ebc51","java.endorsed.dirs":"/usr/java/jdk1.8.0_144/jre/lib/endorsed","java.runtime.version":"1.8.0_144-b01","java.vm.info":"mixed mode","java.ext.dirs":"/usr/java/jdk1.8.0_144/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"Java(TM) SE Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/java/jdk1.8.0_144/jre/lib/resources.jar:/usr/java/jdk1.8.0_144/jre/lib/rt.jar:/usr/java/jdk1.8.0_144/jre/lib/sunrsasign.jar:/usr/java/jdk1.8.0_144/jre/lib/jsse.jar:/usr/java/jdk1.8.0_144/jre/lib/jce.jar:/usr/java/jdk1.8.0_144/jre/lib/charsets.jar:/usr/java/jdk1.8.0_144/jre/lib/jfr.jar:/usr/java/jdk1.8.0_144/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Los_Angeles","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"3.10.0-514.26.2.el7.x86_64","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"systest","java.vm.name":"Java HotSpot(TM) 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master yarn --deploy-mode client --conf spark.driver.memory=2g --conf spark.executor.heartbeatInterval=1000 --conf spark.executor.metrics.pollingInterval=100 --conf spark.eventLog.logStageExecutorProcessTreeMetrics.enabled=true --conf spark.yarn.maxAppAttempts=1 --conf spark.locality.wait.process=0 --conf spark.executor.memoryOverhead=1024 --conf spark.executor.extraJavaOptions=-Djava.security.egd=file:///dev/urandom --conf spark.eventLog.logStageExecutorMetrics.enabled=true --conf spark.driver.extraJavaOptions=-Djava.security.egd=file:///dev/urandom --class com.company.spark.LargeBlocks --num-executors 3 --executor-memory 7g /tmp/__spark_test__/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar --targetBlockSizeGb 2.5 --taskSleepMillis 200 --doCache true --cacheOnDisk true --replicas 1 --concurrentReadJobs 2","java.home":"/usr/java/jdk1.8.0_144/jre","java.version":"1.8.0_144","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/opt/cloudera/parcels/CDH/jars/jackson-mapper-asl-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/joni-2.1.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-core-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-external-blockcache-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-json-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/xz-1.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-annotations-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/cglib-2.2.1-v20090111.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-procedure-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/asm-3.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xml-apis-1.3.04.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-jvm-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-dbcp-1.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-streaming_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-xml-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/minlog-1.3.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hsqldb-1.8.0.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-rdbms-3.2.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/pmml-model-1.4.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spire-macros_2.12-0.13.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/zstd-jni-1.3.2-2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/httpcore-4.2.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jta-1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-yarn_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-logging-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/activation-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xbean-asm7-shaded-4.12.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/paranamer-2.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/lib/hadoop/NOTICE.txt":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-format-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/httpclient-4.2.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/netty-3.9.9.Final.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/htrace-core-3.1.0-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.ws.rs-api-2.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-container-servlet-core-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-nativetask-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-client-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-xc-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-configuration-1.6.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-math3-3.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jsp-api-2.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-auth-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-registry-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/JavaEWAH-0.3.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-graphite-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/joda-time-2.9.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aopalliance-1.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-gridmix-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-compress-1.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javolution-5.5.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-beanutils-1.7.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/core-1.1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr-runtime-3.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-hdfs-nfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-hdfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-net-3.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-framework-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/snappy-java-1.1.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-assembly_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-examples-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/shapeless_2.12-2.3.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-hs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xercesImpl-2.9.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-server-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-lang-2.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jtransforms-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/snappy-0.2.jar":"System Classpath","/etc/spark2/conf/yarn-conf/":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-cli-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-core-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.annotation-api-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-util-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stax-api-1.0-2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javassist-3.18.1-GA.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-applicationhistoryservice-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/kafka-clients-0.9.0-kafka-2.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/guice-3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr4-runtime-4.7.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/core-3.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/opencsv-2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-datajoin-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-common-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jdo-api-3.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-webapp-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-compiler-3.0.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-annotations-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/libthrift-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/kafka_2.11-0.9.0-kafka-2.0.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-azure-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-proxy-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr-2.7.7.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jettison-1.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-core-asl-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/libfb303-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/zookeeper-3.4.5-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-resource-bundle-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jaxb-api-2.2.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-streaming-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-hadoop2-compat-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/pyrolite-4.13.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/activation-1.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aircompressor-0.10.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-ant-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jline-2.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/netty-all-4.0.23.Final.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-media-jaxb-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-servlets-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-xml_2.12-1.0.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-graphx_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-hs-plugins-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-web-proxy-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-math-2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-1.8.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-kvstore_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/objenesis-2.5.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apacheds-i18n-2.0.0-M15.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stream-2.7.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-aws-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-module-scala_2.12-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/spymemcached-2.11.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/api-util-1.0.0-M20.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-core-1.5.5-nohive.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/mockito-all-1.8.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jets3t-0.9.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-xc-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-rsgroup-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hive-metastore-1.2.1.spark2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/py4j-0.10.8.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-applications-distributedshell-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/osgi-resource-locator-1.0.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-rumen-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsch-0.1.42.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/zkclient-0.7.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xmlenc-0.52.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-io-2.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-core-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jamon-runtime-2.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.inject-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-common-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-common-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/netty-3.10.5.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-app-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/machinist_2.12-0.6.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-util-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-network-shuffle_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/chill_2.12-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/findbugs-annotations-1.3.9-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-rest-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-hadoop-compat-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/ST4-4.0.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jruby-cloudera-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-hive_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/leveldbjni-all-1.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-core_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-distcp-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-server-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/metrics-core-2.2.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/htrace-core-3.2.0-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-mapreduce-1.5.5-nohive.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jcl-over-slf4j-1.7.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-digester-1.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-http-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-jackson_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/metrics-core-2.2.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-math3-3.4.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/slf4j-log4j12-1.7.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-jobclient-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-shell-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-server-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/chill-java-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-locator-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stax-api-1.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-parser-combinators_2.12-1.1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/RoaringBitmap-0.5.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsp-api-2.1-6.1.14.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/log4j-1.2.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.inject-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jasper-compiler-5.5.23.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-security-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-shuffle-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-nfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/netty-all-4.1.30.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jaxb-api-2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/breeze-macros_2.12-0.13.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jodd-core-3.5.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/azure-data-lake-store-sdk-2.2.9.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/guava-12.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-codec-1.10.jar":"System Classpath","/usr/java/jdk1.8.0_144/lib/tools.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-nodemanager-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-sql_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hive-exec-1.2.1.spark2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jcodings-1.0.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-azure-datalake-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apacheds-kerberos-codec-2.0.0-M15.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/guava-11.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-catalyst_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-io-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-applications-unmanaged-am-launcher-2.6.0-cdh5.15.2.jar":"System Classpath","/etc/spark2/conf/":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-sslengine-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-api-jdo-3.2.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/httpclient-4.5.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-continuation-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-ast_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-jndi-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsr305-3.0.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/avro-1.7.6-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/univocity-parsers-2.7.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-annotations-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/api-asn1-api-1.0.0-M20.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jettison-1.3.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-utils-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hppc-0.7.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/logredactor-1.0.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-vector-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jaxb-impl-2.2.3-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-archives-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-plus-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-client-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hue-plugins-3.9.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/high-scale-lib-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-jaxrs-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-collections-3.2.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-library-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/java-xmlbuilder-0.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apache-log4j-extras-1.2.17.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-format-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/paranamer-2.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spire_2.12-0.13.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/disruptor-3.3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-mllib-local_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-pool-1.5.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-servlet-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arpack_combined_all-0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-annotations-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsp-2.1-6.1.14.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/log4j-1.2.17.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-repl_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/derby-10.12.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-shims-1.5.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-core-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aopalliance-repackaged-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-api-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-daemon-1.0.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/ivy-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hamcrest-core-1.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-lang3-3.8.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/spark-streaming-kafka-0-8_2.11-2.4.0.cloudera1-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-databind-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-reflect-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-examples-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/httpcore-4.4.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-memory-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/lib/hadoop/LICENSE.txt":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-compress-1.8.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-client-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/oro-2.0.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-thrift-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-ipc-1.8.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-httpclient-3.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/okhttp-2.4.0.jar":"System Classpath","spark://test-1.vpc.company.com:34194/jars/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar":"Added By User","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-compiler-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-prefix-tree-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-api-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-codec-1.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-beanutils-core-1.8.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/lz4-1.3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/janino-3.0.11.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/libthrift-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-launcher_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-core_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/lz4-java-1.5.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-jaxrs-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-network-common_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/breeze_2.12-0.13.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-unsafe_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-sls-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/snappy-java-1.0.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-guava-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/validation-api-1.1.0.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/microsoft-windowsazure-storage-sdk-0.6.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/compress-lzf-1.0.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-openstack-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-mllib_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-client-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-it-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-extras-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-container-servlet-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.servlet-api-3.1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-recipes-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-column-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/unused-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-crypto-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-databind-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-codec-1.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/htrace-core4-4.0.1-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/guava-14.0.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/aws-java-sdk-bundle-1.11.134.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/zookeeper-3.4.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-module-paranamer-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-protocol-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jasper-runtime-5.5.23.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-scalap_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-logging-1.1.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-resourcemanager-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-archive-logs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/paranamer-2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/metrics-core-3.0.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-beanutils-1.9.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jul-to-slf4j-1.7.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stringtemplate-3.2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/gson-2.2.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/okio-1.4.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/slf4j-api-1.7.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jsr305-3.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-sketch_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-el-1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-tags_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-core-3.2.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/protobuf-java-2.5.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-hadoop-bundle-1.6.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xz-1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-client-2.6.0-cdh5.15.2.jar":"System Classpath"}}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/java/jdk1.8.0_144/jre","Java Version":"1.8.0_144 (Oracle Corporation)","Scala Version":"version 2.12.8"},"Spark Properties":{"spark.lineage.log.dir":"/var/log/spark2/lineage","spark.serializer":"org.apache.spark.serializer.KryoSerializer","spark.yarn.jars":"local:/opt/cloudera/parcels/SPARK2/lib/spark2/jars/*","spark.executor.extraJavaOptions":"-Djava.security.egd=file:///dev/urandom","spark.driver.host":"test-1.vpc.company.com","spark.eventLog.enabled":"true","spark.executor.heartbeatInterval":"1000","spark.executor.memoryOverhead":"1024","spark.driver.port":"34194","spark.shuffle.service.enabled":"false","spark.driver.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.lineage.enabled":"false","spark.jars":"file:/tmp/__spark_test__/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar","spark.executor.metrics.pollingInterval":"100","spark.yarn.historyServer.address":"http://test-1.vpc.company.com:18089","spark.ui.enabled":"true","spark.app.name":"LargeBlocks","spark.ui.killEnabled":"true","spark.sql.hive.metastore.jars":"${env:HADOOP_COMMON_HOME}/../hive/lib/*:${env:HADOOP_COMMON_HOME}/client/*","spark.locality.wait.process":"0","spark.dynamicAllocation.schedulerBacklogTimeout":"1","spark.yarn.am.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.scheduler.mode":"FIFO","spark.eventLog.logStageExecutorMetrics":"true","spark.driver.memory":"2g","spark.executor.instances":"3","spark.submit.pyFiles":"","spark.yarn.config.gatewayPath":"/opt/cloudera/parcels","spark.executor.id":"driver","spark.yarn.config.replacementPath":"{{HADOOP_COMMON_HOME}}/../../..","spark.driver.extraJavaOptions":"-Djava.security.egd=file:///dev/urandom","spark.eventLog.logStageExecutorProcessTreeMetrics.enabled":"true","spark.submit.deployMode":"client","spark.shuffle.service.port":"7337","spark.yarn.maxAppAttempts":"1","spark.master":"yarn","spark.authenticate":"false","spark.ui.filters":"org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter","spark.network.crypto.enabled":"false","spark.executor.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.executor.memory":"7g","spark.io.encryption.enabled":"false","spark.eventLog.dir":"hdfs://test-1.vpc.company.com:8020/user/spark/spark2ApplicationHistory","spark.dynamicAllocation.enabled":"false","spark.sql.catalogImplementation":"hive","spark.executor.cores":"1","spark.driver.appUIAddress":"http://test-1.vpc.company.com:4040","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_HOSTS":"test-1.vpc.company.com","spark.dynamicAllocation.minExecutors":"0","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES":"http://test-1.vpc.company.com:8088/proxy/application_1553914137147_0018","spark.dynamicAllocation.executorIdleTimeout":"60","spark.app.id":"application_1553914137147_0018","spark.sql.hive.metastore.version":"1.1.0"},"Hadoop Properties":{"yarn.resourcemanager.amlauncher.thread-count":"50","dfs.namenode.resource.check.interval":"5000","fs.s3a.connection.maximum":"100","mapreduce.jobtracker.jobhistory.task.numberprogresssplits":"12","dfs.data.transfer.server.tcpnodelay":"true","mapreduce.tasktracker.healthchecker.script.timeout":"600000","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"yarn","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","dfs.cachereport.intervalMsec":"10000","dfs.namenode.checkpoint.txns":"1000000","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","mapreduce.tasktracker.local.dir.minspacekill":"0","hadoop.security.groups.cache.background.reload.threads":"3","dfs.namenode.lease-recheck-interval-ms":"2000","fs.s3.block.size":"67108864","dfs.client.domain.socket.data.traffic":"false","dfs.ha.zkfc.nn.http.timeout.ms":"20000","hadoop.registry.secure":"false","hadoop.hdfs.configuration.version":"1","dfs.bytes-per-checksum":"512","fs.s3.buffer.dir":"${hadoop.tmp.dir}/s3","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25","mapreduce.jobhistory.loadedjobs.cache.size":"5","mapreduce.jobtracker.persist.jobstatus.hours":"1","fs.s3a.s3guard.ddb.table.create":"false","dfs.datanode.slow.io.warning.threshold.ms":"300","dfs.namenode.handler.count":"10","dfs.namenode.list.reencryption.status.num.responses":"100","mapreduce.input.fileinputformat.split.minsize":"0","dfs.datanode.failed.volumes.tolerated":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.amliveliness-monitor.interval-ms":"1000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","mapreduce.tasktracker.http.threads":"40","dfs.namenode.retrycache.expirytime.millis":"600000","dfs.namenode.backup.address":"0.0.0.0:50100","dfs.datanode.data.dir":"file://${hadoop.tmp.dir}/dfs/data","dfs.datanode.shared.file.descriptor.paths":"/dev/shm,/tmp","dfs.replication":"3","mapreduce.jobtracker.jobhistory.block.size":"3145728","dfs.encrypt.data.transfer.cipher.key.bitlength":"128","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","dfs.secondary.namenode.kerberos.internal.spnego.principal":"${dfs.web.authentication.kerberos.principal}","mapreduce.task.profile.maps":"0-2","dfs.datanode.block-pinning.enabled":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","mapreduce.jobtracker.retiredjobs.cache.size":"1000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"test-1.vpc.company.com:19890","fs.trash.checkpoint.interval":"0","dfs.namenode.checkpoint.check.period":"60","yarn.nodemanager.container-monitor.interval-ms":"3000","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","hadoop.proxyuser.HTTP.groups":"*","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"10000","dfs.namenode.reencrypt.edek.threads":"10","yarn.resourcemanager.system-metrics-publisher.enabled":"false","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","dfs.namenode.fs-limits.max-component-length":"255","mapreduce.tasktracker.taskcontroller":"org.apache.hadoop.mapred.DefaultTaskController","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","dfs.namenode.top.window.num.buckets":"10","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","dfs.datanode.block.id.layout.upgrade.threads":"12","mapreduce.jobtracker.tasktracker.maxblacklists":"4","yarn.nodemanager.docker-container-executor.exec-name":"/usr/bin/docker","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"0.23.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","dfs.namenode.decommission.max.concurrent.tracked.nodes":"100","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","dfs.client.read.shortcircuit.skip.checksum":"false","mapreduce.task.profile.reduces":"0-2","dfs.datanode.address":"0.0.0.0:50010","dfs.https.server.keystore.resource":"ssl-server.xml","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"test-1.vpc.company.com:8030","mapreduce.task.timeout":"600000","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"hdfs://test-1.vpc.company.com:8020","fs.har.impl.disable.cache":"true","io.compression.codec.bzip2.library":"system-native","dfs.namenode.audit.loggers":"default","dfs.block.access.key.update.interval":"600","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"test-1.vpc.company.com:8090","dfs.namenode.max.objects":"0","mapreduce.jobhistory.address":"test-1.vpc.company.com:10020","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","dfs.namenode.max.full.block.report.leases":"6","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","dfs.client.mmap.cache.timeout.ms":"3600000","dfs.mover.max-no-move-interval":"60000","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","dfs.client.datanode-restart.timeout":"30","dfs.datanode.drop.cache.behind.reads":"false","ipc.server.log.slow.rpc":"false","dfs.namenode.read-lock-reporting-threshold-ms":"5000","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","dfs.namenode.checkpoint.edits.dir":"${dfs.namenode.checkpoint.dir}","yarn.app.attempt.diagnostics.limit.kc":"64","dfs.balancer.block-move.timeout":"0","dfs.client.block.write.replace-datanode-on-failure.enable":"true","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"8192","io.compression.codecs":"org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","dfs.heartbeat.interval":"3","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","dfs.ha.tail-edits.period":"60","dfs.datanode.max.locked.memory":"0","dfs.datanode.scan.period.hours":"504","mapreduce.jobtracker.expire.trackers.interval":"600000","yarn.resourcemanager.nodemanager-connect-retries":"10","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","dfs.namenode.edits.noeditlogchannelflush":"false","mapreduce.task.io.sort.factor":"64","mapreduce.tasktracker.outofband.heartbeat":"false","ha.failover-controller.new-active.rpc-timeout.ms":"60000","dfs.webhdfs.ugi.expire.after.access":"600000","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","dfs.namenode.https-address":"test-1.vpc.company.com:20102","yarn.am.blacklisting.enabled":"true","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","dfs.image.transfer-bootstrap-standby.bandwidthPerSec":"0","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.proxyuser.hive.groups":"*","ha.zookeeper.session-timeout.ms":"5000","mapreduce.map.java.opts":"-Djava.net.preferIPv4Stack=true","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","dfs.ha.automatic-failover.enabled":"false","mapreduce.job.counters.groups.max":"50","dfs.namenode.decommission.interval":"30","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","dfs.datanode.handler.count":"10","dfs.namenode.xattrs.enabled":"true","dfs.namenode.safemode.threshold-pct":"0.999f","dfs.client.socket.send.buffer.size":"0","mapreduce.map.sort.spill.percent":"0.8","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","dfs.datanode.sync.behind.writes":"false","dfs.namenode.stale.datanode.interval":"30000","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","dfs.datanode.transfer.socket.send.buffer.size":"0","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","dfs.namenode.reencrypt.throttle.limit.handler.ratio":"1.0","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler","yarn.app.mapreduce.am.command-opts":"-Djava.net.preferIPv4Stack=true -Xmx825955249","dfs.journalnode.https-address":"0.0.0.0:8481","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","hadoop.proxyuser.hue.hosts":"*","io.mapfile.bloom.error.rate":"0.005","dfs.user.home.dir.prefix":"/user","hadoop.proxyuser.hue.groups":"*","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","mapreduce.jobtracker.persist.jobstatus.dir":"/jobtracker/jobsInfo","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","dfs.namenode.snapshot.capture.openfiles":"false","dfs.datanode.directoryscan.interval":"21600","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","dfs.client.read.shortcircuit.streams.cache.expiry.ms":"300000","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"10","dfs.namenode.write.stale.datanode.ratio":"0.5f","hadoop.registry.zk.session.timeout.ms":"60000","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","mapreduce.jobtracker.taskcache.levels":"2","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","mapreduce.job.emit-timeline-data":"false","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","dfs.domain.socket.path":"/var/run/hdfs-sockets/dn","yarn.resourcemanager.admin.client.thread-count":"1","mapreduce.jobtracker.persist.jobstatus.active":"true","dfs.namenode.reencrypt.sleep.interval":"1m","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"test-1.vpc.company.com:10033","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","mapreduce.tasktracker.report.address":"127.0.0.1:0","dfs.namenode.edit.log.autoroll.multiplier.threshold":"2.0","io.seqfile.lazydecompress":"true","ftp.blocksize":"67108864","dfs.namenode.backup.http-address":"0.0.0.0:50105","dfs.disk.balancer.max.disk.throughputInMBperSec":"10","mapreduce.jobtracker.instrumentation":"org.apache.hadoop.mapred.JobTrackerMetricsInst","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.delete.debug-delay-sec":"0","mapreduce.jobtracker.http.address":"0.0.0.0:50030","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","hadoop.security.groups.cache.secs":"300","yarn.resourcemanager.zk-retry-interval-ms":"1000","yarn.scheduler.increment-allocation-mb":"512","nfs.mountd.port":"4242","mapreduce.shuffle.max.threads":"0","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","mapreduce.jobhistory.http.policy":"HTTP_ONLY","dfs.datanode.dns.interface":"default","mapreduce.reduce.java.opts":"-Djava.net.preferIPv4Stack=true","s3native.replication":"3","hadoop.security.group.mapping.ldap.ssl":"false","dfs.namenode.fs-limits.max-xattrs-per-inode":"32","yarn.client.application-client-protocol.poll-interval-ms":"200","hadoop.proxyuser.flume.groups":"*","dfs.namenode.fs-limits.max-xattr-size":"16384","dfs.namenode.maintenance.replication.min":"1","dfs.client.write.exclude.nodes.cache.expiry.interval.millis":"600000","ha.zookeeper.parent-znode":"/hadoop-ha","dfs.namenode.safemode.extension":"30000","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","dfs.blocksize":"134217728","dfs.namenode.servicerpc-address":"test-1.vpc.company.com:8022","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"test-1.vpc.company.com:8032","ipc.client.ping":"true","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","dfs.namenode.resource.checked.volumes.minimum":"1","hadoop.proxyuser.HTTP.hosts":"*","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","dfs.namenode.quota.init-threads":"4","dfs.journalnode.http-address":"0.0.0.0:8480","dfs.block.scanner.volume.bytes.per.second":"1048576","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","dfs.storage.policy.enabled":"true","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","dfs.namenode.edits.dir":"${dfs.namenode.name.dir}","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","dfs.namenode.support.allow.format":"true","dfs.ha.tail-edits.rolledits.timeout":"60","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","mapreduce.cluster.temp.dir":"${hadoop.tmp.dir}/mapred/temp","s3.replication":"3","dfs.client.failover.connection.retries":"0","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","dfs.namenode.secondary.http-address":"0.0.0.0:50090","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","nfs.wtmax":"1048576","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","nfs.dump.dir":"/tmp/.hdfs-nfs","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","mapreduce.client.submit.file.replication":"3","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","dfs.namenode.service.handler.count":"10","dfs.datanode.data.dir.perm":"700","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","dfs.namenode.name.dir":"file:///dataroot/dataroot/dfs/nn","yarn.resourcemanager.zk-acl":"world:anyone:rwcda","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","fs.ftp.host.port":"21","ipc.ping.interval":"60000","dfs.namenode.num.checkpoints.retained":"2","dfs.namenode.kerberos.internal.spnego.principal":"${dfs.web.authentication.kerberos.principal}","yarn.resourcemanager.admin.address":"test-1.vpc.company.com:8033","file.client-write-packet-size":"65536","hadoop.treat.subject.external":"false","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"false","dfs.disk.balancer.plan.threshold.percent":"10","mapreduce.local.clientfactory.class.name":"org.apache.hadoop.mapred.LocalClientFactory","dfs.client.use.legacy.blockreader":"false","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"256","yarn.nodemanager.localizer.client.thread-count":"5","dfs.namenode.checkpoint.max-retries":"3","dfs.namenode.reject-unresolved-dn-topology-mapping":"false","dfs.namenode.delegation.token.max-lifetime":"*********(redacted)","dfs.client.block.write.replace-datanode-on-failure.min-replication":"0","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"2000","dfs.namenode.num.extra.edits.retained":"1000000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.scheduler.increment-allocation-vcores":"1","fs.df.interval":"60000","fs.s3.sleepTimeSeconds":"10","fs.s3a.multiobjectdelete.enable":"true","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"true","io.file.buffer.size":"65536","hadoop.work.around.non.threadsafe.getpwuid":"false","dfs.permissions.superusergroup":"supergroup","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","mapreduce.tasktracker.dns.interface":"default","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","dfs.permissions.enabled":"true","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/user","mapreduce.reduce.shuffle.read.timeout":"180000","yarn.app.mapreduce.am.admin.user.env":"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH","dfs.datanode.https.address":"0.0.0.0:50475","dfs.namenode.hosts.provider.classname":"org.apache.hadoop.hdfs.server.blockmanagement.HostFileManager","dfs.datanode.transfer.socket.recv.buffer.size":"0","fs.s3a.connection.establish.timeout":"5000","dfs.namenode.fslock.fair":"true","mapreduce.job.running.map.limit":"0","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","hadoop.fuse.connection.timeout":"300","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"5000","yarn.nodemanager.vmem-pmem-ratio":"2.1","dfs.client.slow.io.warning.threshold.ms":"30000","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","s3native.stream-buffer-size":"4096","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","fs.s3a.s3guard.cli.prune.age":"86400000","dfs.client.read.shortcircuit.streams.cache.size":"256","dfs.client.use.legacy.blockreader.local":"false","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3n.multipart.uploads.enabled":"false","dfs.namenode.path.based.cache.retry.interval.ms":"30000","hadoop.security.crypto.buffer.size":"8192","yarn.client.failover-retries-on-socket-timeouts":"0","dfs.balancer.keytab.enabled":"false","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","dfs.datanode.balance.bandwidthPerSec":"10485760","dfs.namenode.name.dir.restore":"false","hadoop.registry.jaas.context":"Client","dfs.client.failover.sleep.max.millis":"15000","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","s3.blocksize":"67108864","yarn.am.blacklisting.disable-failure-threshold":"0.8f","io.map.index.interval":"128","mapreduce.job.counters.max":"120","dfs.namenode.max-lock-hold-to-release-lease-ms":"25","dfs.namenode.datanode.registration.ip-hostname-check":"true","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","dfs.namenode.resource.du.reserved":"104857600","dfs.datanode.bp-ready.timeout":"20","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","dfs.namenode.full.block.report.lease.length.ms":"300000","mapreduce.tasktracker.instrumentation":"org.apache.hadoop.mapred.TaskTrackerMetricsInst","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","s3.stream-buffer-size":"4096","yarn.resourcemanager.nm.liveness-monitor.interval-ms":"1000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","dfs.namenode.secondary.https-address":"0.0.0.0:50091","s3native.bytes-per-checksum":"512","dfs.namenode.fs-limits.max-directory-items":"1048576","nfs.server.port":"2049","dfs.namenode.delegation.token.renew-interval":"*********(redacted)","mapreduce.jobtracker.address":"local","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","dfs.namenode.list.openfiles.num.responses":"1000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","dfs.namenode.blocks.per.postponedblocks.rescan":"10000","dfs.namenode.checkpoint.period":"3600","hadoop.security.groups.cache.background.reload":"false","yarn.resourcemanager.amlauncher.log.command":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","dfs.namenode.snapshot.skip.capture.accesstime-only-change":"false","dfs.webhdfs.user.provider.user.pattern":"^[A-Za-z_][A-Za-z0-9._-]*[$]?$","dfs.webhdfs.acl.provider.permission.pattern":"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"false","mapreduce.map.skip.maxrecords":"0","mapreduce.jobtracker.handler.count":"10","hadoop.http.authentication.type":"simple","mapreduce.job.jvm.numtasks":"1","hadoop.proxyuser.flume.hosts":"*","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","fs.s3n.block.size":"67108864","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","mapreduce.jobtracker.staging.root.dir":"${hadoop.tmp.dir}/mapred/staging","dfs.namenode.http-address":"test-1.vpc.company.com:20101","mapreduce.jobtracker.jobhistory.lru.cache.size":"5","dfs.datanode.directoryscan.threads":"1","dfs.datanode.fsdatasetcache.max.threads.per.volume":"4","dfs.namenode.fs-limits.max-blocks-per-file":"1048576","dfs.disk.balancer.enabled":"false","mapreduce.shuffle.listen.queue.size":"128","mapreduce.tasktracker.local.dir.minspacestart":"0","mapreduce.map.cpu.vcores":"1","hadoop.user.group.static.mapping.overrides":"dr.who=;","dfs.datanode.cache.revocation.timeout.ms":"900000","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","dfs.client.mmap.cache.size":"256","dfs.ha.log-roll.period":"120","dfs.client.failover.sleep.base.millis":"500","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","dfs.namenode.accesstime.precision":"3600000","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","hadoop.registry.zk.retry.interval.ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","dfs.client.context":"default","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","hadoop.ssl.server.conf":"ssl-server.xml","dfs.http.policy":"HTTP_ONLY","dfs.client.https.keystore.resource":"ssl-client.xml","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.acl.enable":"true","dfs.domain.socket.disable.interval.seconds":"1","dfs.image.transfer.chunksize":"65536","dfs.balancer.max-no-move-interval":"60000","mapreduce.tasktracker.map.tasks.maximum":"2","dfs.namenode.edits.journal-plugin.qjournal":"org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager","mapreduce.task.profile":"false","dfs.webhdfs.enabled":"true","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","dfs.namenode.list.encryption.zones.num.responses":"100","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","dfs.namenode.top.num.users":"10","dfs.disk.balancer.block.tolerance.percent":"10","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","io.seqfile.sorter.recordlimit":"1000000","hadoop.security.auth_to_local":"DEFAULT","dfs.blockreport.initialDelay":"0","fs.automatic.close":"true","dfs.client.block.write.replace-datanode-on-failure.best-effort":"false","dfs.namenode.replication.min":"1","dfs.balancer.address":"0.0.0.0:0","fs.s3n.multipart.copy.block.size":"5368709120","yarn.nodemanager.hostname":"0.0.0.0","nfs.rtmax":"1048576","yarn.resourcemanager.zk-timeout-ms":"10000","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","dfs.datanode.directoryscan.throttle.limit.ms.per.sec":"1000","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","dfs.namenode.replication.work.multiplier.per.iteration":"2","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","dfs.namenode.reencrypt.throttle.limit.updater.ratio":"1.0","dfs.namenode.avoid.write.stale.datanode":"false","dfs.short.circuit.shared.memory.watcher.interrupt.check.ms":"60000","dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction":"0.75f","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","dfs.client.mmap.enabled":"true","mapreduce.reduce.cpu.vcores":"1","hadoop.proxyuser.oozie.groups":"*","fs.client.resolve.remote.symlinks":"true","dfs.image.compression.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.jobtracker.restart.recover":"false","dfs.namenode.decommission.blocks.per.interval":"500000","mapreduce.tasktracker.reduce.tasks.maximum":"2","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","dfs.namenode.safemode.min.datanodes":"0","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.userlog.retain.hours":"24","yarn.scheduler.maximum-allocation-vcores":"4","yarn.nodemanager.log-aggregation.compression-type":"none","dfs.namenode.enable.retrycache":"true","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","dfs.namenode.startup.delay.block.deletion.sec":"0","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","dfs.datanode.readahead.bytes":"4194304","mapreduce.jobtracker.heartbeats.in.second":"100","mapreduce.job.running.reduce.limit":"0","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","mapreduce.task.tmp.dir":"./tmp","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:mapred@hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","dfs.namenode.delegation.key.update-interval":"86400000","fs.s3a.max.total.tasks":"5","dfs.client.file-block-storage-locations.num-threads":"10","mapreduce.tasktracker.healthchecker.interval":"60000","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.fast.upload":"false","fs.s3a.attempts.maximum":"20","dfs.namenode.avoid.read.stale.datanode":"false","hadoop.registry.zk.connection.timeout.ms":"15000","dfs.https.port":"20102","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","mapreduce.map.log.level":"INFO","mapreduce.output.fileoutputformat.compress.type":"BLOCK","hadoop.registry.rm.enabled":"false","mapreduce.ifile.readahead.bytes":"4194304","mapreduce.tasktracker.tasks.sleeptimebeforesigkill":"5000","yarn.resourcemanager.fs.state-store.retry-policy-spec":"2000, 500","dfs.namenode.posix.acl.inheritance.enabled":"false","dfs.blockreport.intervalMsec":"21600000","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","dfs.namenode.path.based.cache.refresh.interval.ms":"30000","dfs.namenode.edekcacheloader.interval.ms":"1000","file.stream-buffer-size":"4096","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.SnappyCodec","mapreduce.map.speculative":"false","dfs.disk.balancer.max.disk.errors":"5","dfs.datanode.use.datanode.hostname":"false","mapreduce.job.speculative.retry-after-speculate":"15000","hadoop.proxyuser.hdfs.hosts":"*","dfs.namenode.fs-limits.min-block-size":"1048576","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","mapreduce.job.reduce.slowstart.completedmaps":"0.8","dfs.client.read.shortcircuit":"false","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","hadoop.proxyuser.yarn.groups":"*","dfs.client.cached.conn.retry":"3","dfs.namenode.invalidate.work.pct.per.iteration":"0.32f","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.nodemanager.logaggregation.threadpool-size-max":"100","dfs.replication.max":"512","dfs.namenode.inotify.max.events.per.rpc":"1000","yarn.resourcemanager.hostname":"0.0.0.0","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"0","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","ipc.client.rpc-timeout.ms":"0","fs.s3.maxRetries":"4","dfs.default.chunk.view.size":"32768","mapreduce.input.lineinputformat.linespermap":"1","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","dfs.client.mmap.retry.timeout.ms":"300000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","dfs.namenode.list.cache.directives.num.responses":"100","fs.s3a.socket.recv.buffer":"8192","dfs.image.compress":"false","dfs.namenode.kerberos.principal.pattern":"*","yarn.application.classpath":"$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*","fs.s3n.multipart.uploads.block.size":"67108864","mapreduce.tasktracker.http.address":"0.0.0.0:50060","yarn.resourcemanager.resource-tracker.address":"test-1.vpc.company.com:8031","hadoop.fuse.timer.period":"5","mapreduce.job.heap.memory-mb.ratio":"0.8","dfs.datanode.hdfs-blocks-metadata.enabled":"true","dfs.namenode.checkpoint.dir":"file://${hadoop.tmp.dir}/dfs/namesecondary","dfs.datanode.max.transfer.threads":"4096","dfs.namenode.edits.asynclogging":"true","nfs.allow.insecure.ports":"true","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"0","s3native.client-write-packet-size":"65536","mapreduce.admin.user.env":"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","hadoop.proxyuser.mapred.hosts":"*","hadoop.proxyuser.oozie.hosts":"*","yarn.nodemanager.log.retain-seconds":"10800","hadoop.proxyuser.mapred.groups":"*","yarn.resourcemanager.keytab":"/etc/krb5.keytab","mapreduce.reduce.merge.inmem.threshold":"1000","dfs.client.https.need-auth":"false","dfs.blockreport.split.threshold":"1000000","dfs.client.block.write.replace-datanode-on-failure.policy":"DEFAULT","mapreduce.shuffle.ssl.enabled":"false","dfs.namenode.write-lock-reporting-threshold-ms":"5000","dfs.block.access.token.enable":"*********(redacted)","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","httpfs.buffer.size":"4096","dfs.client.file-block-storage-locations.timeout.millis":"1000","dfs.namenode.block-placement-policy.default.prefer-local-node":"true","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","dfs.namenode.replication.considerLoad":"true","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.proxyuser.hdfs.groups":"*","dfs.namenode.retrycache.heap.percent":"0.03f","dfs.datanode.cache.revocation.polling.ms":"500","mapreduce.jobhistory.webapp.address":"test-1.vpc.company.com:19888","dfs.namenode.path.based.cache.block.map.allocation.percent":"0.25","mapreduce.jobtracker.system.dir":"${hadoop.tmp.dir}/mapred/system","mapreduce.tasktracker.taskmemorymanager.monitoringinterval":"5000","dfs.journalnode.rpc-address":"0.0.0.0:8485","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"test-1.vpc.company.com:8088","mapreduce.jobhistory.recovery.enable":"false","dfs.client.short.circuit.replica.stale.threshold.ms":"1800000","mapreduce.reduce.shuffle.parallelcopies":"10","fs.trash.interval":"1","dfs.namenode.replication.interval":"3","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","dfs.namenode.top.enabled":"true","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","dfs.datanode.du.reserved":"0","yarn.app.mapreduce.am.resource.mb":"1024","mapreduce.input.fileinputformat.list-status.num-threads":"1","dfs.namenode.lazypersist.file.scrub.interval.sec":"300","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.nodemanager.resource.cpu-vcores":"8","mapreduce.job.reduces":"6","fs.s3a.multipart.size":"64M","yarn.scheduler.minimum-allocation-vcores":"1","dfs.namenode.reencrypt.batch.size":"1000","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","dfs.datanode.http.address":"0.0.0.0:50075","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","fs.s3a.metadatastore.authoritative":"false","ha.health-monitor.sleep-after-disconnect.ms":"1000","s3.bytes-per-checksum":"512","yarn.app.mapreduce.shuffle.log.limit.kb":"0","dfs.namenode.list.cache.pools.num.responses":"100","hadoop.security.group.mapping":"org.apache.hadoop.security.ShellBasedUnixGroupsMapping","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","dfs.encrypt.data.transfer":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","mapreduce.job.redacted-properties":"*********(redacted)","dfs.namenode.top.windows.minutes":"1,5,25","s3.client-write-packet-size":"65536","mapreduce.map.output.compress":"true","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.scheduler.maximum-allocation-mb":"12288","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.app.mapreduce.am.container.log.limit.kb":"0","s3native.blocksize":"67108864","ipc.client.connect.retry.interval":"1000","hadoop.proxyuser.httpfs.groups":"*","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","dfs.namenode.edit.log.autoroll.check.interval.ms":"300000","mapreduce.jobhistory.cleaner.enable":"true","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","hadoop.proxyuser.httpfs.hosts":"*","dfs.client.use.datanode.hostname":"false","dfs.stream-buffer-size":"4096","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","dfs.datanode.drop.cache.behind.writes":"false","mapreduce.tasktracker.dns.nameserver":"default","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","mapreduce.job.end-notification.retry.attempts":"0","hadoop.proxyuser.yarn.hosts":"*","yarn.resourcemanager.zk-num-retries":"1000","dfs.client.failover.max.attempts":"15","mapreduce.tasktracker.indexcache.mb":"10","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","dfs.namenode.snapshotdiff.allow.snap-root-descendant":"true","yarn.nodemanager.localizer.cache.target-size-mb":"10240","zlib.compress.level":"DEFAULT_COMPRESSION","ftp.client-write-packet-size":"65536","mapreduce.jobtracker.maxtasks.perjob":"-1","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.proxyuser.hive.hosts":"*","dfs.block.access.token.lifetime":"*********(redacted)","dfs.namenode.max.extra.edits.segments.retained":"10000","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","dfs.image.transfer.bandwidthPerSec":"0","io.native.lib.available":"true","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","ipc.server.listen.queue.size":"128","dfs.namenode.edekcacheloader.initial.delay.ms":"3000","map.sort.class":"org.apache.hadoop.util.QuickSort","dfs.namenode.acls.enabled":"false","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","dfs.datanode.ipc.address":"0.0.0.0:50020","yarn.nodemanager.vmem-check-enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","dfs.lock.suppress.warning.interval":"10s","dfs.client.block.write.retries":"3","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.application.classpath":"$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH","yarn.client.nodemanager-connect.retry-interval-ms":"10000","dfs.client-write-packet-size":"65536","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME","dfs.datanode.dns.nameserver":"default","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","dfs.image.transfer.timeout":"60000","yarn.resourcemanager.recovery.enabled":"false","dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold":"10737418240","dfs.client.failover.connection.retries.on.timeouts":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.security.egd":"file:///dev/urandom","java.vm.specification.version":"1.8","user.home":"/home/systest","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/java/jdk1.8.0_144/jre/lib/amd64","user.dir":"/tmp","java.library.path":":/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.144-b01","jetty.git.hash":"27208684755d94a92186989f695db2d7b21ebc51","java.endorsed.dirs":"/usr/java/jdk1.8.0_144/jre/lib/endorsed","java.runtime.version":"1.8.0_144-b01","java.vm.info":"mixed mode","java.ext.dirs":"/usr/java/jdk1.8.0_144/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"Java(TM) SE Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/java/jdk1.8.0_144/jre/lib/resources.jar:/usr/java/jdk1.8.0_144/jre/lib/rt.jar:/usr/java/jdk1.8.0_144/jre/lib/sunrsasign.jar:/usr/java/jdk1.8.0_144/jre/lib/jsse.jar:/usr/java/jdk1.8.0_144/jre/lib/jce.jar:/usr/java/jdk1.8.0_144/jre/lib/charsets.jar:/usr/java/jdk1.8.0_144/jre/lib/jfr.jar:/usr/java/jdk1.8.0_144/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Los_Angeles","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"3.10.0-514.26.2.el7.x86_64","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"systest","java.vm.name":"Java HotSpot(TM) 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master yarn --deploy-mode client --conf spark.driver.memory=2g --conf spark.executor.heartbeatInterval=1000 --conf spark.executor.metrics.pollingInterval=100 --conf spark.eventLog.logStageExecutorProcessTreeMetrics.enabled=true --conf spark.yarn.maxAppAttempts=1 --conf spark.locality.wait.process=0 --conf spark.executor.memoryOverhead=1024 --conf spark.executor.extraJavaOptions=-Djava.security.egd=file:///dev/urandom --conf spark.eventLog.logStageExecutorMetrics=true --conf spark.driver.extraJavaOptions=-Djava.security.egd=file:///dev/urandom --class com.company.spark.LargeBlocks --num-executors 3 --executor-memory 7g /tmp/__spark_test__/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar --targetBlockSizeGb 2.5 --taskSleepMillis 200 --doCache true --cacheOnDisk true --replicas 1 --concurrentReadJobs 2","java.home":"/usr/java/jdk1.8.0_144/jre","java.version":"1.8.0_144","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/opt/cloudera/parcels/CDH/jars/jackson-mapper-asl-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/joni-2.1.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-core-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-external-blockcache-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-json-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/xz-1.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-annotations-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/cglib-2.2.1-v20090111.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-procedure-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/asm-3.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xml-apis-1.3.04.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-jvm-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-dbcp-1.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-streaming_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-xml-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/minlog-1.3.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hsqldb-1.8.0.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-rdbms-3.2.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/pmml-model-1.4.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spire-macros_2.12-0.13.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/zstd-jni-1.3.2-2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/httpcore-4.2.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jta-1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-yarn_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-logging-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/activation-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xbean-asm7-shaded-4.12.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/paranamer-2.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/lib/hadoop/NOTICE.txt":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-format-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/httpclient-4.2.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/netty-3.9.9.Final.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/htrace-core-3.1.0-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.ws.rs-api-2.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-container-servlet-core-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-nativetask-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-client-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-xc-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-configuration-1.6.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-math3-3.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jsp-api-2.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-auth-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-registry-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/JavaEWAH-0.3.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-graphite-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/joda-time-2.9.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aopalliance-1.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-gridmix-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-compress-1.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javolution-5.5.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-beanutils-1.7.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/core-1.1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr-runtime-3.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-hdfs-nfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-hdfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-net-3.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-framework-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/snappy-java-1.1.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-assembly_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-examples-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/shapeless_2.12-2.3.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-hs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xercesImpl-2.9.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-server-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-lang-2.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jtransforms-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/snappy-0.2.jar":"System Classpath","/etc/spark2/conf/yarn-conf/":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-cli-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-core-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.annotation-api-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-util-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stax-api-1.0-2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javassist-3.18.1-GA.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-applicationhistoryservice-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/kafka-clients-0.9.0-kafka-2.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/guice-3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr4-runtime-4.7.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/core-3.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/opencsv-2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-datajoin-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-common-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jdo-api-3.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-webapp-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-compiler-3.0.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-annotations-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/libthrift-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/kafka_2.11-0.9.0-kafka-2.0.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-azure-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-proxy-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr-2.7.7.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jettison-1.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-core-asl-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/libfb303-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/zookeeper-3.4.5-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-resource-bundle-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jaxb-api-2.2.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-streaming-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-hadoop2-compat-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/pyrolite-4.13.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/activation-1.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aircompressor-0.10.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-ant-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jline-2.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/netty-all-4.0.23.Final.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-media-jaxb-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-servlets-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-xml_2.12-1.0.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-graphx_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-hs-plugins-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-web-proxy-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-math-2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-1.8.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-kvstore_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/objenesis-2.5.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apacheds-i18n-2.0.0-M15.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stream-2.7.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-aws-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-module-scala_2.12-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/spymemcached-2.11.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/api-util-1.0.0-M20.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-core-1.5.5-nohive.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/mockito-all-1.8.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jets3t-0.9.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-xc-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-rsgroup-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hive-metastore-1.2.1.spark2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/py4j-0.10.8.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-applications-distributedshell-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/osgi-resource-locator-1.0.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-rumen-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsch-0.1.42.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/zkclient-0.7.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xmlenc-0.52.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-io-2.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-core-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jamon-runtime-2.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.inject-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-common-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-common-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/netty-3.10.5.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-app-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/machinist_2.12-0.6.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-util-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-network-shuffle_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/chill_2.12-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/findbugs-annotations-1.3.9-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-rest-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-hadoop-compat-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/ST4-4.0.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jruby-cloudera-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-hive_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/leveldbjni-all-1.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-core_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-distcp-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-server-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/metrics-core-2.2.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/htrace-core-3.2.0-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-mapreduce-1.5.5-nohive.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jcl-over-slf4j-1.7.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-digester-1.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-http-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-jackson_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/metrics-core-2.2.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-math3-3.4.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/slf4j-log4j12-1.7.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-jobclient-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-shell-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-server-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/chill-java-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-locator-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stax-api-1.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-parser-combinators_2.12-1.1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/RoaringBitmap-0.5.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsp-api-2.1-6.1.14.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/log4j-1.2.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.inject-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jasper-compiler-5.5.23.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-security-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-shuffle-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-nfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/netty-all-4.1.30.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jaxb-api-2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/breeze-macros_2.12-0.13.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jodd-core-3.5.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/azure-data-lake-store-sdk-2.2.9.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/guava-12.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-codec-1.10.jar":"System Classpath","/usr/java/jdk1.8.0_144/lib/tools.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-nodemanager-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-sql_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hive-exec-1.2.1.spark2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jcodings-1.0.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-azure-datalake-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apacheds-kerberos-codec-2.0.0-M15.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/guava-11.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-catalyst_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-io-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-applications-unmanaged-am-launcher-2.6.0-cdh5.15.2.jar":"System Classpath","/etc/spark2/conf/":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-sslengine-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-api-jdo-3.2.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/httpclient-4.5.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-continuation-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-ast_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-jndi-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsr305-3.0.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/avro-1.7.6-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/univocity-parsers-2.7.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-annotations-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/api-asn1-api-1.0.0-M20.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jettison-1.3.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-utils-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hppc-0.7.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/logredactor-1.0.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-vector-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jaxb-impl-2.2.3-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-archives-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-plus-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-client-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hue-plugins-3.9.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/high-scale-lib-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-jaxrs-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-collections-3.2.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-library-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/java-xmlbuilder-0.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apache-log4j-extras-1.2.17.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-format-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/paranamer-2.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spire_2.12-0.13.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/disruptor-3.3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-mllib-local_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-pool-1.5.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-servlet-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arpack_combined_all-0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-annotations-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsp-2.1-6.1.14.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/log4j-1.2.17.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-repl_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/derby-10.12.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-shims-1.5.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-core-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aopalliance-repackaged-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-api-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-daemon-1.0.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/ivy-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hamcrest-core-1.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-lang3-3.8.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/spark-streaming-kafka-0-8_2.11-2.4.0.cloudera1-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-databind-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-reflect-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-examples-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/httpcore-4.4.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-memory-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/lib/hadoop/LICENSE.txt":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-compress-1.8.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-client-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/oro-2.0.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-thrift-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-ipc-1.8.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-httpclient-3.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/okhttp-2.4.0.jar":"System Classpath","spark://test-1.vpc.company.com:34194/jars/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar":"Added By User","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-compiler-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-prefix-tree-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-api-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-codec-1.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-beanutils-core-1.8.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/lz4-1.3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/janino-3.0.11.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/libthrift-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-launcher_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-core_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/lz4-java-1.5.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-jaxrs-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-network-common_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/breeze_2.12-0.13.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-unsafe_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-sls-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/snappy-java-1.0.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-guava-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/validation-api-1.1.0.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/microsoft-windowsazure-storage-sdk-0.6.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/compress-lzf-1.0.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-openstack-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-mllib_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-client-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-it-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-extras-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-container-servlet-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.servlet-api-3.1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-recipes-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-column-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/unused-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-crypto-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-databind-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-codec-1.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/htrace-core4-4.0.1-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/guava-14.0.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/aws-java-sdk-bundle-1.11.134.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/zookeeper-3.4.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-module-paranamer-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-protocol-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jasper-runtime-5.5.23.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-scalap_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-logging-1.1.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-resourcemanager-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-archive-logs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/paranamer-2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/metrics-core-3.0.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-beanutils-1.9.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jul-to-slf4j-1.7.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stringtemplate-3.2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/gson-2.2.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/okio-1.4.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/slf4j-api-1.7.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jsr305-3.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-sketch_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-el-1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-tags_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-core-3.2.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/protobuf-java-2.5.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-hadoop-bundle-1.6.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xz-1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-client-2.6.0-cdh5.15.2.jar":"System Classpath"}}
 {"Event":"SparkListenerApplicationStart","App Name":"LargeBlocks","App ID":"application_1553914137147_0018","Timestamp":1554755984286,"User":"systest"}
 {"Event":"SparkListenerExecutorAdded","Timestamp":1554755994596,"Executor ID":"1","Executor Info":{"Host":"test-2.vpc.company.com","Total Cores":1,"Log Urls":{"stdout":"http://test-2.vpc.company.com:8042/node/containerlogs/container_1553914137147_0018_01_000002/systest/stdout?start=-4096","stderr":"http://test-2.vpc.company.com:8042/node/containerlogs/container_1553914137147_0018_01_000002/systest/stderr?start=-4096"},"Attributes":{"NM_HTTP_ADDRESS":"test-2.vpc.company.com:8042","USER":"systest","LOG_FILES":"stderr,stdout","NM_HTTP_PORT":"8042","CLUSTER_ID":"","NM_PORT":"8041","HTTP_SCHEME":"http://","NM_HOST":"test-2.vpc.company.com","CONTAINER_ID":"container_1553914137147_0018_01_000002"}}}
 {"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"test-2.vpc.company.com","Port":43764},"Maximum Memory":3820172083,"Timestamp":1554755994649,"Maximum Onheap Memory":3820172083,"Maximum Offheap Memory":0}
diff --git a/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala b/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
index 435b927068e60..7052d1a2028bb 100644
--- a/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
+++ b/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
@@ -19,9 +19,12 @@ package org.apache.spark
 
 import scala.concurrent.duration._
 
+import org.apache.spark.TestUtils.createTempScriptWithExpectedOutput
 import org.apache.spark.internal.config._
 import org.apache.spark.rdd.{PartitionPruningRDD, RDD}
+import org.apache.spark.resource.TestResourceIDs.{EXECUTOR_GPU_ID, TASK_GPU_ID, WORKER_GPU_ID}
 import org.apache.spark.scheduler.BarrierJobAllocationFailed._
+import org.apache.spark.scheduler.BarrierJobSlotsNumberCheckFailed
 import org.apache.spark.util.ThreadUtils
 
 /**
@@ -259,4 +262,37 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     testSubmitJob(sc, rdd,
       message = ERROR_MESSAGE_BARRIER_REQUIRE_MORE_SLOTS_THAN_CURRENT_TOTAL_NUMBER)
   }
+
+  test("SPARK-32518: CoarseGrainedSchedulerBackend.maxNumConcurrentTasks should " +
+    "consider all kinds of resources for the barrier stage") {
+    withTempDir { dir =>
+      val discoveryScript = createTempScriptWithExpectedOutput(
+        dir, "gpuDiscoveryScript", """{"name": "gpu","addresses":["0"]}""")
+
+      val conf = new SparkConf()
+        .setMaster("local-cluster[1, 2, 1024]")
+        .setAppName("test-cluster")
+        .set(WORKER_GPU_ID.amountConf, "1")
+        .set(WORKER_GPU_ID.discoveryScriptConf, discoveryScript)
+        .set(EXECUTOR_GPU_ID.amountConf, "1")
+        .set(TASK_GPU_ID.amountConf, "1")
+        // disable barrier stage retry to fail the application as soon as possible
+        .set(BARRIER_MAX_CONCURRENT_TASKS_CHECK_MAX_FAILURES, 1)
+        // disable the check to simulate the behavior of Standalone in order to
+        // reproduce the issue.
+        .set(Tests.SKIP_VALIDATE_CORES_TESTING, true)
+      sc = new SparkContext(conf)
+      // setup an executor which will have 2 CPUs and 1 GPU
+      TestUtils.waitUntilExecutorsUp(sc, 1, 60000)
+
+      val exception = intercept[BarrierJobSlotsNumberCheckFailed] {
+        sc.parallelize(Range(1, 10), 2)
+          .barrier()
+          .mapPartitions { iter => iter }
+          .collect()
+      }
+      assert(exception.getMessage.contains("[SPARK-24819]: Barrier execution " +
+        "mode does not allow run a barrier stage that requires more slots"))
+    }
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/DistributedSuite.scala b/core/src/test/scala/org/apache/spark/DistributedSuite.scala
index 3f309819065be..4d157b9607000 100644
--- a/core/src/test/scala/org/apache/spark/DistributedSuite.scala
+++ b/core/src/test/scala/org/apache/spark/DistributedSuite.scala
@@ -174,7 +174,7 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex
 
   private def testCaching(conf: SparkConf, storageLevel: StorageLevel): Unit = {
     sc = new SparkContext(conf.setMaster(clusterUrl).setAppName("test"))
-    TestUtils.waitUntilExecutorsUp(sc, 2, 30000)
+    TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
     val data = sc.parallelize(1 to 1000, 10)
     val cachedData = data.persist(storageLevel)
     assert(cachedData.count === 1000)
diff --git a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
index 8d958494d52be..0b19146713966 100644
--- a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
@@ -1142,7 +1142,7 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
       .set(config.DYN_ALLOCATION_TESTING, true)
       // SPARK-22864: effectively disable the allocation schedule by setting the period to a
       // really long value.
-      .set(TEST_SCHEDULE_INTERVAL, 10000L)
+      .set(TEST_SCHEDULE_INTERVAL, 30000L)
   }
 
   private def createManager(
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
index 3bc2061c4f2ad..72e7ee0214187 100644
--- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
@@ -221,7 +221,7 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst
     conf.registerKryoClasses(Array(classOf[Class1]))
     assert(conf.get(KRYO_CLASSES_TO_REGISTER).toSet === Seq(classOf[Class1].getName).toSet)
 
-    conf.set(KRYO_USER_REGISTRATORS, classOf[CustomRegistrator].getName)
+    conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[CustomRegistrator].getName))
 
     // Kryo doesn't expose a way to discover registered classes, but at least make sure this doesn't
     // blow up.
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index df9c7c5eaa368..dc1c0451c628d 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -36,6 +36,7 @@ import org.scalatest.concurrent.Eventually
 
 import org.apache.spark.TestUtils._
 import org.apache.spark.internal.config._
+import org.apache.spark.internal.config.Tests.RESOURCES_WARNING_TESTING
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.resource.ResourceAllocation
 import org.apache.spark.resource.ResourceUtils._
@@ -841,7 +842,6 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
         .setAppName("test-cluster")
         .set(DRIVER_GPU_ID.amountConf, "3")
         .set(DRIVER_GPU_ID.discoveryScriptConf, scriptPath)
-        .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
 
@@ -891,6 +891,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
       .setAppName("test-cluster")
     conf.set(TASK_GPU_ID.amountConf, "2")
     conf.set(EXECUTOR_GPU_ID.amountConf, "4")
+    conf.set(RESOURCES_WARNING_TESTING, true)
 
     var error = intercept[SparkException] {
       sc = new SparkContext(conf)
@@ -924,7 +925,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
     assume(!(Utils.isWindows))
     withTempDir { dir =>
       val discoveryScript = createTempScriptWithExpectedOutput(dir, "resourceDiscoveryScript",
-        """{"name": "gpu","addresses":["0", "1", "2", "3", "4", "5", "6", "7", "8"]}""")
+        """{"name": "gpu","addresses":["0", "1", "2"]}""")
 
       val conf = new SparkConf()
         .setMaster("local-cluster[3, 1, 1024]")
@@ -933,7 +934,6 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
         .set(WORKER_GPU_ID.discoveryScriptConf, discoveryScript)
         .set(TASK_GPU_ID.amountConf, "3")
         .set(EXECUTOR_GPU_ID.amountConf, "3")
-        .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
 
@@ -945,13 +945,34 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
         context.resources().get(GPU).get.addresses.iterator
       }
       val gpus = rdd.collect()
-      assert(gpus.sorted === Seq("0", "1", "2", "3", "4", "5", "6", "7", "8"))
+      assert(gpus.sorted === Seq("0", "0", "0", "1", "1", "1", "2", "2", "2"))
 
       eventually(timeout(10.seconds)) {
         assert(sc.statusTracker.getExecutorInfos.map(_.numRunningTasks()).sum == 0)
       }
     }
   }
+
+  test("SPARK-32160: Disallow to create SparkContext in executors if the config is set") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+
+    val error = intercept[SparkException] {
+      sc.range(0, 1).foreach { _ =>
+        new SparkContext(new SparkConf().setAppName("test").setMaster("local")
+          .set(EXECUTOR_ALLOW_SPARK_CONTEXT, false))
+      }
+    }.getMessage()
+
+    assert(error.contains("SparkContext should only be created and accessed on the driver."))
+  }
+
+  test("SPARK-32160: Allow to create SparkContext in executors") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+
+    sc.range(0, 1).foreach { _ =>
+      new SparkContext(new SparkConf().setAppName("test").setMaster("local")).stop()
+    }
+  }
 }
 
 object SparkContextSuite {
diff --git a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
index cf4400e080e37..d40207423b7ae 100644
--- a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark
 
 // scalastyle:off
 import java.io.File
+import java.util.{Locale, TimeZone}
 
 import org.apache.log4j.spi.LoggingEvent
 
@@ -63,6 +64,17 @@ abstract class SparkFunSuite
   with Logging {
 // scalastyle:on
 
+  // Initialize the logger forcibly to let the logger log timestamp
+  // based on the local time zone depending on environments.
+  // The default time zone will be set to America/Los_Angeles later
+  // so this initialization is necessary here.
+  log
+
+  // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
+  TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
+  // Add Locale setting
+  Locale.setDefault(Locale.US)
+
   protected val enableAutoThreadAudit = true
 
   protected override def beforeAll(): Unit = {
diff --git a/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala b/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
index 9629f5ab1a3dd..022fcbb25b0af 100644
--- a/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
+++ b/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
@@ -26,7 +26,6 @@ import scala.util.Try
 
 import org.apache.commons.io.output.TeeOutputStream
 import org.apache.commons.lang3.SystemUtils
-import org.scalatest.Assertions._
 
 import org.apache.spark.util.Utils
 
diff --git a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
index 31f065ec55749..b182b11a0e85e 100644
--- a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
@@ -102,6 +102,7 @@ private[deploy] object DeployTestUtils {
       createDriverDesc(),
       null,
       "spark://worker",
+      "http://publicAddress:80",
       new SecurityManager(conf))
   }
 
diff --git a/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala
new file mode 100644
index 0000000000000..d681c13337e0d
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
+import org.apache.spark.internal.config.{SHUFFLE_SERVICE_DB_ENABLED, SHUFFLE_SERVICE_ENABLED}
+import org.apache.spark.util.Utils
+
+class ExternalShuffleServiceMetricsSuite extends SparkFunSuite {
+
+  var sparkConf: SparkConf = _
+  var externalShuffleService: ExternalShuffleService = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    sparkConf = new SparkConf()
+    sparkConf.set(SHUFFLE_SERVICE_ENABLED, true)
+    sparkConf.set(SHUFFLE_SERVICE_DB_ENABLED, false)
+    sparkConf.set("spark.local.dir", System.getProperty("java.io.tmpdir"))
+    Utils.loadDefaultSparkProperties(sparkConf, null)
+    val securityManager = new SecurityManager(sparkConf)
+    externalShuffleService = new ExternalShuffleService(sparkConf, securityManager)
+    externalShuffleService.start()
+  }
+
+  override def afterAll(): Unit = {
+    if (externalShuffleService != null) {
+      externalShuffleService.stop()
+    }
+    super.afterAll()
+  }
+
+  test("SPARK-31646: metrics should be registered") {
+    val sourceRef = classOf[ExternalShuffleService].getDeclaredField("shuffleServiceSource")
+    sourceRef.setAccessible(true)
+    val source = sourceRef.get(externalShuffleService).asInstanceOf[ExternalShuffleServiceSource]
+    assert(source.metricRegistry.getMetrics.keySet().asScala ==
+      Set(
+        "blockTransferRateBytes",
+        "numActiveConnections",
+        "numCaughtExceptions",
+        "numRegisteredConnections",
+        "openBlockRequestLatencyMillis",
+        "registeredExecutorsSize",
+        "registerExecutorRequestLatencyMillis",
+        "shuffle-server.usedDirectMemory",
+        "shuffle-server.usedHeapMemory")
+    )
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 9d4736825618e..fb2a65e8c07cf 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, FSDataInputStream, Path}
 import org.scalatest.{BeforeAndAfterEach, Matchers}
 import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits}
+import org.scalatest.time.Span
 import org.scalatest.time.SpanSugar._
 
 import org.apache.spark._
@@ -565,7 +566,8 @@ class SparkSubmitSuite
       }
     }
 
-    val clArgs2 = Seq("--class", "org.SomeClass", "thejar.jar")
+    val dummyJarFile = TestUtils.createJarWithClasses(Seq.empty)
+    val clArgs2 = Seq("--class", "org.SomeClass", dummyJarFile.toString)
     val appArgs2 = new SparkSubmitArguments(clArgs2)
     val (_, _, conf2, _) = submit.prepareSubmitEnvironment(appArgs2)
     assert(!conf2.contains(UI_SHOW_CONSOLE_PROGRESS))
@@ -1215,6 +1217,86 @@ class SparkSubmitSuite
     testRemoteResources(enableHttpFs = true, blacklistSchemes = Seq("*"))
   }
 
+  test("SPARK-32119: Jars and files should be loaded when Executors launch for plugins") {
+    val tempDir = Utils.createTempDir()
+    val tempFileName = "test.txt"
+    val tempFile = new File(tempDir, tempFileName)
+
+    // scalastyle:off println
+    Utils.tryWithResource {
+      new PrintWriter(tempFile)
+    } { writer =>
+      writer.println("SparkPluginTest")
+    }
+    // scalastyle:on println
+
+    val sparkPluginCodeBody =
+      """
+        |@Override
+        |public org.apache.spark.api.plugin.ExecutorPlugin executorPlugin() {
+        |  return new TestExecutorPlugin();
+        |}
+        |
+        |@Override
+        |public org.apache.spark.api.plugin.DriverPlugin driverPlugin() { return null; }
+      """.stripMargin
+    val executorPluginCodeBody =
+      s"""
+        |@Override
+        |public void init(
+        |    org.apache.spark.api.plugin.PluginContext ctx,
+        |    java.util.Map<String, String> extraConf) {
+        |  String str = null;
+        |  try (java.io.BufferedReader reader =
+        |    new java.io.BufferedReader(new java.io.InputStreamReader(
+        |      new java.io.FileInputStream("$tempFileName")))) {
+        |    str = reader.readLine();
+        |  } catch (java.io.IOException e) {
+        |    throw new RuntimeException(e);
+        |  } finally {
+        |    assert str == "SparkPluginTest";
+        |  }
+        |}
+      """.stripMargin
+
+    val compiledExecutorPlugin = TestUtils.createCompiledClass(
+      "TestExecutorPlugin",
+      tempDir,
+      "",
+      null,
+      Seq.empty,
+      Seq("org.apache.spark.api.plugin.ExecutorPlugin"),
+      executorPluginCodeBody)
+
+    val thisClassPath =
+      sys.props("java.class.path").split(File.pathSeparator).map(p => new File(p).toURI.toURL)
+    val compiledSparkPlugin = TestUtils.createCompiledClass(
+      "TestSparkPlugin",
+      tempDir,
+      "",
+      null,
+      Seq(tempDir.toURI.toURL) ++ thisClassPath,
+      Seq("org.apache.spark.api.plugin.SparkPlugin"),
+      sparkPluginCodeBody)
+
+    val jarUrl = TestUtils.createJar(
+      Seq(compiledSparkPlugin, compiledExecutorPlugin),
+      new File(tempDir, "testplugin.jar"))
+
+    val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
+    val unusedFile = Files.createTempFile(tempDir.toPath, "unused", null)
+    val args = Seq(
+      "--class", SimpleApplicationTest.getClass.getName.stripSuffix("$"),
+      "--name", "testApp",
+      "--master", "local-cluster[1,1,1024]",
+      "--conf", "spark.plugins=TestSparkPlugin",
+      "--conf", "spark.ui.enabled=false",
+      "--jars", jarUrl.toString + "," + unusedJar.toString,
+      "--files", tempFile.toString + "," + unusedFile.toString,
+      unusedJar.toString)
+    runSparkSubmit(args)
+  }
+
   private def testRemoteResources(
       enableHttpFs: Boolean,
       blacklistSchemes: Seq[String] = Nil): Unit = {
@@ -1417,7 +1499,7 @@ object SparkSubmitSuite extends SparkFunSuite with TimeLimits {
   implicit val defaultSignaler: Signaler = ThreadSignaler
 
   // NOTE: This is an expensive operation in terms of time (10 seconds+). Use sparingly.
-  def runSparkSubmit(args: Seq[String], root: String = ".."): Unit = {
+  def runSparkSubmit(args: Seq[String], root: String = "..", timeout: Span = 1.minute): Unit = {
     val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!"))
     val sparkSubmitFile = if (Utils.isWindows) {
       new File(s"$root\\bin\\spark-submit.cmd")
@@ -1430,7 +1512,7 @@ object SparkSubmitSuite extends SparkFunSuite with TimeLimits {
       Map("SPARK_TESTING" -> "1", "SPARK_HOME" -> sparkHome))
 
     try {
-      val exitCode = failAfter(1.minute) { process.waitFor() }
+      val exitCode = failAfter(timeout) { process.waitFor() }
       if (exitCode != 0) {
         fail(s"Process returned with exit code $exitCode. See the log4j logs for more detail.")
       }
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
index 060b878fb8ef2..e9b739ce7a4c6 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
@@ -213,7 +213,7 @@ class SingleEventLogFileWriterSuite extends EventLogFileWritersSuite {
       compressionCodecShortName)
 
     val finalLogPath = new Path(logPath)
-    assert(fileSystem.exists(finalLogPath) && fileSystem.isFile(finalLogPath))
+    assert(fileSystem.exists(finalLogPath) && fileSystem.getFileStatus(finalLogPath).isFile)
     assert(expectedLines === readLinesFromEventLogFile(finalLogPath, fileSystem))
   }
 }
@@ -357,10 +357,10 @@ class RollingEventLogFilesWriterSuite extends EventLogFileWritersSuite {
       expectedLines: Seq[String]): Unit = {
     val logDirPath = getAppEventLogDirPath(logBaseDir, appId, appAttemptId)
 
-    assert(fileSystem.exists(logDirPath) && fileSystem.isDirectory(logDirPath))
+    assert(fileSystem.exists(logDirPath) && fileSystem.getFileStatus(logDirPath).isDirectory)
 
     val appStatusFile = getAppStatusFilePath(logDirPath, appId, appAttemptId, inProgress = false)
-    assert(fileSystem.exists(appStatusFile) && fileSystem.isFile(appStatusFile))
+    assert(fileSystem.exists(appStatusFile) && fileSystem.getFileStatus(appStatusFile).isFile)
 
     val eventLogFiles = listEventLogFiles(logDirPath)
     val allLines = mutable.ArrayBuffer[String]()
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index c2f34fc3a95ed..f3beb35f1f011 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -1470,6 +1470,55 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     }
   }
 
+  test("SPARK-33146: don't let one bad rolling log folder prevent loading other applications") {
+    withTempDir { dir =>
+      val conf = createTestConf(true)
+      conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+      val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
+      val fs = new Path(dir.getAbsolutePath).getFileSystem(hadoopConf)
+
+      val provider = new FsHistoryProvider(conf)
+
+      val writer = new RollingEventLogFilesWriter("app", None, dir.toURI, conf, hadoopConf)
+      writer.start()
+
+      writeEventsToRollingWriter(writer, Seq(
+        SparkListenerApplicationStart("app", Some("app"), 0, "user", None),
+        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(dir.listFiles().size === 1)
+      assert(provider.getListing.length === 1)
+
+      // Manually delete the appstatus file to make an invalid rolling event log
+      val appStatusPath = RollingEventLogFilesWriter.getAppStatusFilePath(new Path(writer.logPath),
+        "app", None, true)
+      fs.delete(appStatusPath, false)
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(provider.getListing.length === 0)
+
+      // Create a new application
+      val writer2 = new RollingEventLogFilesWriter("app2", None, dir.toURI, conf, hadoopConf)
+      writer2.start()
+      writeEventsToRollingWriter(writer2, Seq(
+        SparkListenerApplicationStart("app2", Some("app2"), 0, "user", None),
+        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
+
+      // Both folders exist but only one application found
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(provider.getListing.length === 1)
+      assert(dir.listFiles().size === 2)
+
+      // Make sure a new provider sees the valid application
+      provider.stop()
+      val newProvider = new FsHistoryProvider(conf)
+      newProvider.checkForLogs()
+      assert(newProvider.getListing.length === 1)
+    }
+  }
+
   /**
    * Asks the provider to check for logs and calls a function to perform checks on the updated
    * app list. Example:
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
index f78469e132490..9004e86323691 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
@@ -158,4 +158,56 @@ class HistoryServerDiskManagerSuite extends SparkFunSuite with BeforeAndAfter {
     assert(manager.approximateSize(50L, true) > 50L)
   }
 
+  test("SPARK-32024: update ApplicationStoreInfo.size during initializing") {
+    val manager = mockManager()
+    val leaseA = manager.lease(2)
+    doReturn(3L).when(manager).sizeOf(meq(leaseA.tmpPath))
+    val dstPathA = manager.appStorePath("app1", None)
+    doReturn(3L).when(manager).sizeOf(meq(dstPathA))
+    val dstA = leaseA.commit("app1", None)
+    assert(manager.free() === 0)
+    assert(manager.committed() === 3)
+    // Listing store tracks dstA now.
+    assert(store.read(classOf[ApplicationStoreInfo], dstA.getAbsolutePath).size === 3)
+
+    // Simulate: service restarts, new disk manager (manager1) is initialized.
+    val manager1 = mockManager()
+    // Simulate: event KVstore compaction before restart, directory size reduces.
+    doReturn(2L).when(manager1).sizeOf(meq(dstA))
+    doReturn(2L).when(manager1).sizeOf(meq(new File(testDir, "apps")))
+    manager1.initialize()
+    // "ApplicationStoreInfo.size" is updated for dstA.
+    assert(store.read(classOf[ApplicationStoreInfo], dstA.getAbsolutePath).size === 2)
+    assert(manager1.free() === 1)
+    // If "ApplicationStoreInfo.size" is not correctly updated, "IllegalStateException"
+    // would be thrown.
+    val leaseB = manager1.lease(2)
+    assert(manager1.free() === 1)
+    doReturn(2L).when(manager1).sizeOf(meq(leaseB.tmpPath))
+    val dstPathB = manager.appStorePath("app2", None)
+    doReturn(2L).when(manager1).sizeOf(meq(dstPathB))
+    val dstB = leaseB.commit("app2", None)
+    assert(manager1.committed() === 2)
+    // Listing store tracks dstB only, dstA is evicted by "makeRoom()".
+    assert(store.read(classOf[ApplicationStoreInfo], dstB.getAbsolutePath).size === 2)
+
+    val manager2 = mockManager()
+    // Simulate: cache entities are written after replaying, directory size increases.
+    doReturn(3L).when(manager2).sizeOf(meq(dstB))
+    doReturn(3L).when(manager2).sizeOf(meq(new File(testDir, "apps")))
+    manager2.initialize()
+    // "ApplicationStoreInfo.size" is updated for dstB.
+    assert(store.read(classOf[ApplicationStoreInfo], dstB.getAbsolutePath).size === 3)
+    assert(manager2.free() === 0)
+    val leaseC = manager2.lease(2)
+    doReturn(2L).when(manager2).sizeOf(meq(leaseC.tmpPath))
+    val dstPathC = manager.appStorePath("app3", None)
+    doReturn(2L).when(manager2).sizeOf(meq(dstPathC))
+    val dstC = leaseC.commit("app3", None)
+    assert(manager2.free() === 1)
+    assert(manager2.committed() === 2)
+    // Listing store tracks dstC only, dstB is evicted by "makeRoom()".
+    assert(store.read(classOf[ApplicationStoreInfo], dstC.getAbsolutePath).size === 2)
+  }
+
 }
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 206db0feb5716..c55b29b15051d 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -314,7 +314,8 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
     all (directSiteRelativeLinks) should not startWith (knoxBaseUrl)
   }
 
-  test("static relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
+  // TODO (SPARK-31723): re-enable it
+  ignore("static relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
     val uiRoot = Option(System.getenv("APPLICATION_WEB_PROXY_BASE")).getOrElse("/testwebproxybase")
     val page = new HistoryPage(server)
     val request = mock[HttpServletRequest]
@@ -693,6 +694,17 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
     out.close()
   }
 
+  test("SPARK-31697: HistoryServer should set Content-Type") {
+    val port = server.boundPort
+    val nonExistenceAppId = "local-non-existence"
+    val url = new URL(s"http://localhost:$port/history/$nonExistenceAppId")
+    val conn = url.openConnection().asInstanceOf[HttpURLConnection]
+    conn.setRequestMethod("GET")
+    conn.connect()
+    val expectedContentType = "text/html;charset=utf-8"
+    val actualContentType = conn.getContentType
+    assert(actualContentType === expectedContentType)
+  }
 }
 
 object HistoryServerSuite {
diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
index 0cf573c2490b3..994eb6f14e411 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
@@ -689,7 +689,16 @@ class MasterSuite extends SparkFunSuite
     val master = makeAliveMaster()
     var worker: MockExecutorLaunchFailWorker = null
     try {
-      worker = new MockExecutorLaunchFailWorker(master)
+      val conf = new SparkConf()
+      // SPARK-32250: When running test on Github Action machine, the available processors in JVM
+      // is only 2, while on Jenkins it's 32. For this specific test, 2 available processors, which
+      // also decides number of threads in Dispatcher, is not enough to consume the messages. In
+      // the worst situation, MockExecutorLaunchFailWorker would occupy these 2 threads for
+      // handling messages LaunchDriver, LaunchExecutor at the same time but leave no thread for
+      // the driver to handle the message RegisteredApplication. At the end, it results in the dead
+      // lock situation. Therefore, we need to set more threads to avoid the dead lock.
+      conf.set(Network.RPC_NETTY_DISPATCHER_NUM_THREADS, 6)
+      worker = new MockExecutorLaunchFailWorker(master, conf)
       worker.rpcEnv.setupEndpoint("worker", worker)
       val workerRegMsg = RegisterWorker(
         worker.id,
diff --git a/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
index 275bca3459855..d9d559509f4fb 100644
--- a/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
@@ -19,10 +19,14 @@ package org.apache.spark.deploy.security
 
 import java.security.PrivilegedExceptionAction
 
+import scala.util.control.NonFatal
+
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION
 import org.apache.hadoop.minikdc.MiniKdc
 import org.apache.hadoop.security.{Credentials, UserGroupInformation}
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.SparkHadoopUtil
@@ -88,8 +92,30 @@ class HadoopDelegationTokenManagerSuite extends SparkFunSuite {
       // krb5.conf. MiniKdc sets "java.security.krb5.conf" in start and removes it when stop called.
       val kdcDir = Utils.createTempDir()
       val kdcConf = MiniKdc.createConf()
-      kdc = new MiniKdc(kdcConf, kdcDir)
-      kdc.start()
+      // The port for MiniKdc service gets selected in the constructor, but will be bound
+      // to it later in MiniKdc.start() -> MiniKdc.initKDCServer() -> KdcServer.start().
+      // In meantime, when some other service might capture the port during this progress, and
+      // cause BindException.
+      // This makes our tests which have dedicated JVMs and rely on MiniKDC being flaky
+      //
+      // https://issues.apache.org/jira/browse/HADOOP-12656 get fixed in Hadoop 2.8.0.
+      //
+      // The workaround here is to periodically repeat this process with a timeout , since we are
+      // using Hadoop 2.7.4 as default.
+      // https://issues.apache.org/jira/browse/SPARK-31631
+      eventually(timeout(60.seconds), interval(1.second)) {
+        try {
+          kdc = new MiniKdc(kdcConf, kdcDir)
+          kdc.start()
+        } catch {
+          case NonFatal(e) =>
+            if (kdc != null) {
+              kdc.stop()
+              kdc = null
+            }
+            throw e
+        }
+      }
 
       val krbConf = new Configuration()
       krbConf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos")
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala b/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala
index c3b580e7ccac4..e429ddfd570de 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala
@@ -40,7 +40,8 @@ class DriverRunnerTest extends SparkFunSuite {
     val worker = mock(classOf[RpcEndpointRef])
     doNothing().when(worker).send(any())
     spy(new DriverRunner(conf, "driverId", new File("workDir"), new File("sparkHome"),
-      driverDescription, worker, "spark://1.2.3.4/worker/", new SecurityManager(conf)))
+      driverDescription, worker, "spark://1.2.3.4/worker/", "http://publicAddress:80",
+      new SecurityManager(conf)))
   }
 
   private def createProcessBuilderAndProcess(): (ProcessBuilderLike, Process) = {
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
index bb541b4cad8bd..2d3d0afe3f80c 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
@@ -36,7 +36,6 @@ import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
 import org.apache.spark.TestUtils.{createTempJsonFile, createTempScriptWithExpectedOutput}
 import org.apache.spark.deploy.{Command, ExecutorState, ExternalShuffleService}
 import org.apache.spark.deploy.DeployMessages.{DriverStateChanged, ExecutorStateChanged, WorkDirCleanup}
-import org.apache.spark.deploy.StandaloneResourceUtils.{ALLOCATED_RESOURCES_FILE, SPARK_RESOURCES_COORDINATE_DIR}
 import org.apache.spark.deploy.master.DriverState
 import org.apache.spark.internal.config
 import org.apache.spark.internal.config.Worker._
@@ -64,7 +63,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
   private def makeWorker(
       conf: SparkConf = new SparkConf(),
       shuffleServiceSupplier: Supplier[ExternalShuffleService] = null,
-      pid: Int = Utils.getProcessId,
       local: Boolean = false): Worker = {
     assert(_worker === null, "Some Worker's RpcEnv is leaked in tests")
     val securityMgr = new SecurityManager(conf)
@@ -72,7 +70,7 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
     val resourcesFile = conf.get(SPARK_WORKER_RESOURCE_FILE)
     val localWorker = new Worker(rpcEnv, 50000, 20, 1234 * 5,
       Array.fill(1)(RpcAddress("1.2.3.4", 1234)), "Worker", "/tmp",
-      conf, securityMgr, resourcesFile, shuffleServiceSupplier, pid)
+      conf, securityMgr, resourcesFile, shuffleServiceSupplier)
     if (local) {
       localWorker
     } else {
@@ -81,14 +79,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
     }
   }
 
-  private def assertResourcesFileDeleted(): Unit = {
-    assert(sys.props.contains("spark.test.home"))
-    val sparkHome = sys.props.get("spark.test.home")
-    val resourceFile = new File(sparkHome + "/" + SPARK_RESOURCES_COORDINATE_DIR,
-      ALLOCATED_RESOURCES_FILE)
-    assert(!resourceFile.exists())
-  }
-
   before {
     MockitoAnnotations.initMocks(this)
   }
@@ -251,7 +241,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
       worker.rpcEnv.shutdown()
       worker.rpcEnv.awaitTermination()
     }
-    assertResourcesFileDeleted()
   }
 
   test("worker could load resources from resources file while launching") {
@@ -273,7 +262,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
         worker.rpcEnv.shutdown()
         worker.rpcEnv.awaitTermination()
       }
-      assertResourcesFileDeleted()
     }
   }
 
@@ -292,7 +280,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
         worker.rpcEnv.shutdown()
         worker.rpcEnv.awaitTermination()
       }
-      assertResourcesFileDeleted()
     }
   }
 
@@ -316,65 +303,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
         worker.rpcEnv.shutdown()
         worker.rpcEnv.awaitTermination()
       }
-      assertResourcesFileDeleted()
-    }
-  }
-
-  test("Workers run on the same host should avoid resources conflict when coordinate is on") {
-    val conf = new SparkConf()
-    withTempDir { dir =>
-      val scriptPath = createTempScriptWithExpectedOutput(dir, "fpgaDiscoverScript",
-        """{"name": "fpga","addresses":["f1", "f2", "f3", "f4", "f5"]}""")
-      conf.set(WORKER_FPGA_ID.discoveryScriptConf, scriptPath)
-      conf.set(WORKER_FPGA_ID.amountConf, "2")
-      val workers = (0 until 3).map(id => makeWorker(conf, pid = id, local = true))
-      workers.zipWithIndex.foreach{case (w, i) => w.rpcEnv.setupEndpoint(s"worker$i", w)}
-      eventually(timeout(20.seconds)) {
-        val (empty, nonEmpty) = workers.partition(_.resources.isEmpty)
-        assert(empty.length === 1)
-        assert(nonEmpty.length === 2)
-        val totalResources = nonEmpty.flatMap(_.resources(FPGA).addresses).toSet.toSeq.sorted
-        assert(totalResources === Seq("f1", "f2", "f3", "f4"))
-        workers.foreach(_.rpcEnv.shutdown())
-        workers.foreach(_.rpcEnv.awaitTermination())
-      }
-      assertResourcesFileDeleted()
-    }
-  }
-
-  test("Workers run on the same host should load resources naively when coordinate is off") {
-    val conf = new SparkConf()
-    // disable coordination
-    conf.set(config.SPARK_RESOURCES_COORDINATE, false)
-    withTempDir { dir =>
-      val gpuArgs = ResourceAllocation(WORKER_GPU_ID, Seq("g0", "g1"))
-      val ja = Extraction.decompose(Seq(gpuArgs))
-      val resourcesPath = createTempJsonFile(dir, "resources", ja)
-      val scriptPath = createTempScriptWithExpectedOutput(dir, "fpgaDiscoverScript",
-        """{"name": "fpga","addresses":["f1", "f2", "f3", "f4", "f5"]}""")
-      conf.set(SPARK_WORKER_RESOURCE_FILE.key, resourcesPath)
-      conf.set(WORKER_GPU_ID.amountConf, "2")
-      conf.set(WORKER_FPGA_ID.discoveryScriptConf, scriptPath)
-      conf.set(WORKER_FPGA_ID.amountConf, "2")
-      val workers = (0 until 3).map(id => makeWorker(conf, pid = id, local = true))
-      workers.zipWithIndex.foreach{case (w, i) => w.rpcEnv.setupEndpoint(s"worker$i", w)}
-      eventually(timeout(20.seconds)) {
-        val (empty, nonEmpty) = workers.partition(_.resources.isEmpty)
-        assert(empty.length === 0)
-        assert(nonEmpty.length === 3)
-        // Each Worker should get the same resources from resources file and discovery script
-        // without coordination. Note that, normally, we must config different resources
-        // for workers run on the same host when coordinate config is off. Test here is used
-        // to validate the different behaviour comparing to the above test when coordinate config
-        // is on, so we admit the resources collision here.
-        nonEmpty.foreach { worker =>
-          assert(worker.resources === Map(GPU -> gpuArgs.toResourceInformation,
-            FPGA -> new ResourceInformation(FPGA, Array("f1", "f2", "f3", "f4", "f5"))))
-        }
-        workers.foreach(_.rpcEnv.shutdown())
-        workers.foreach(_.rpcEnv.awaitTermination())
-      }
-      assertResourcesFileDeleted()
     }
   }
 
diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
index 31049d104e63d..6b3df6d0c9970 100644
--- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
@@ -17,8 +17,9 @@
 
 package org.apache.spark.executor
 
-import java.io.{Externalizable, ObjectInput, ObjectOutput}
+import java.io.{Externalizable, File, ObjectInput, ObjectOutput}
 import java.lang.Thread.UncaughtExceptionHandler
+import java.net.URL
 import java.nio.ByteBuffer
 import java.util.Properties
 import java.util.concurrent.{ConcurrentHashMap, CountDownLatch, TimeUnit}
@@ -41,6 +42,7 @@ import org.scalatestplus.mockito.MockitoSugar
 import org.apache.spark._
 import org.apache.spark.TaskState.TaskState
 import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.deploy.{SimpleApplicationTest, SparkSubmitSuite}
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.memory.TestMemoryManager
@@ -52,7 +54,7 @@ import org.apache.spark.scheduler.{DirectTaskResult, FakeTask, ResultTask, Task,
 import org.apache.spark.serializer.{JavaSerializer, SerializerInstance, SerializerManager}
 import org.apache.spark.shuffle.FetchFailedException
 import org.apache.spark.storage.{BlockManager, BlockManagerId}
-import org.apache.spark.util.{LongAccumulator, UninterruptibleThread}
+import org.apache.spark.util.{LongAccumulator, SparkUncaughtExceptionHandler, UninterruptibleThread, Utils}
 
 class ExecutorSuite extends SparkFunSuite
     with LocalSparkContext with MockitoSugar with Eventually with PrivateMethodTester {
@@ -63,6 +65,33 @@ class ExecutorSuite extends SparkFunSuite
     super.afterEach()
   }
 
+  /**
+   * Creates an Executor with the provided arguments, is then passed to `f`
+   * and will be stopped after `f` returns.
+   */
+  def withExecutor(
+      executorId: String,
+      executorHostname: String,
+      env: SparkEnv,
+      userClassPath: Seq[URL] = Nil,
+      isLocal: Boolean = true,
+      uncaughtExceptionHandler: UncaughtExceptionHandler
+        = new SparkUncaughtExceptionHandler,
+      resources: immutable.Map[String, ResourceInformation]
+        = immutable.Map.empty[String, ResourceInformation])(f: Executor => Unit): Unit = {
+    var executor: Executor = null
+    try {
+      executor = new Executor(executorId, executorHostname, env, userClassPath, isLocal,
+        uncaughtExceptionHandler, resources)
+
+      f(executor)
+    } finally {
+      if (executor != null) {
+        executor.stop()
+      }
+    }
+  }
+
   test("SPARK-15963: Catch `TaskKilledException` correctly in Executor.TaskRunner") {
     // mock some objects to make Executor.launchTask() happy
     val conf = new SparkConf
@@ -115,10 +144,8 @@ class ExecutorSuite extends SparkFunSuite
         }
       })
 
-    var executor: Executor = null
-    try {
-      executor = new Executor("id", "localhost", env, userClassPath = Nil, isLocal = true,
-        resources = immutable.Map.empty[String, ResourceInformation])
+    withExecutor("id", "localhost", env) { executor =>
+
       // the task will be launched in a dedicated worker thread
       executor.launchTask(mockExecutorBackend, taskDescription)
 
@@ -138,11 +165,6 @@ class ExecutorSuite extends SparkFunSuite
       assert(executorSuiteHelper.testFailedReason.toErrorString === "TaskKilled (test)")
       assert(executorSuiteHelper.taskState === TaskState.KILLED)
     }
-    finally {
-      if (executor != null) {
-        executor.stop()
-      }
-    }
   }
 
   test("SPARK-19276: Handle FetchFailedExceptions that are hidden by user exceptions") {
@@ -254,25 +276,24 @@ class ExecutorSuite extends SparkFunSuite
     confs.foreach { case (k, v) => conf.set(k, v) }
     val serializer = new JavaSerializer(conf)
     val env = createMockEnv(conf, serializer)
-    val executor =
-      new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true,
-        resources = immutable.Map.empty[String, ResourceInformation])
-    val executorClass = classOf[Executor]
-
-    // Save all heartbeats sent into an ArrayBuffer for verification
-    val heartbeats = ArrayBuffer[Heartbeat]()
-    val mockReceiver = mock[RpcEndpointRef]
-    when(mockReceiver.askSync(any[Heartbeat], any[RpcTimeout])(any))
-      .thenAnswer((invocation: InvocationOnMock) => {
-        val args = invocation.getArguments()
-        heartbeats += args(0).asInstanceOf[Heartbeat]
-        HeartbeatResponse(false)
-      })
-    val receiverRef = executorClass.getDeclaredField("heartbeatReceiverRef")
-    receiverRef.setAccessible(true)
-    receiverRef.set(executor, mockReceiver)
+    withExecutor("id", "localhost", SparkEnv.get) { executor =>
+      val executorClass = classOf[Executor]
 
-    f(executor, heartbeats)
+      // Save all heartbeats sent into an ArrayBuffer for verification
+      val heartbeats = ArrayBuffer[Heartbeat]()
+      val mockReceiver = mock[RpcEndpointRef]
+      when(mockReceiver.askSync(any[Heartbeat], any[RpcTimeout])(any))
+        .thenAnswer((invocation: InvocationOnMock) => {
+          val args = invocation.getArguments()
+          heartbeats += args(0).asInstanceOf[Heartbeat]
+          HeartbeatResponse(false)
+        })
+      val receiverRef = executorClass.getDeclaredField("heartbeatReceiverRef")
+      receiverRef.setAccessible(true)
+      receiverRef.set(executor, mockReceiver)
+
+      f(executor, heartbeats)
+    }
   }
 
   private def heartbeatZeroAccumulatorUpdateTest(dropZeroMetrics: Boolean): Unit = {
@@ -353,10 +374,7 @@ class ExecutorSuite extends SparkFunSuite
     val taskDescription = createResultTaskDescription(serializer, taskBinary, rdd, 0)
 
     val mockBackend = mock[ExecutorBackend]
-    var executor: Executor = null
-    try {
-      executor = new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true,
-        resources = immutable.Map.empty[String, ResourceInformation])
+    withExecutor("id", "localhost", SparkEnv.get) { executor =>
       executor.launchTask(mockBackend, taskDescription)
 
       // Ensure that the executor's metricsPoller is polled so that values are recorded for
@@ -367,10 +385,6 @@ class ExecutorSuite extends SparkFunSuite
       eventually(timeout(5.seconds), interval(10.milliseconds)) {
         assert(executor.numRunningTasks === 0)
       }
-    } finally {
-      if (executor != null) {
-        executor.stop()
-      }
     }
 
     // Verify that peak values for task metrics get sent in the TaskResult
@@ -465,12 +479,11 @@ class ExecutorSuite extends SparkFunSuite
       poll: Boolean = false): (TaskFailedReason, UncaughtExceptionHandler) = {
     val mockBackend = mock[ExecutorBackend]
     val mockUncaughtExceptionHandler = mock[UncaughtExceptionHandler]
-    var executor: Executor = null
     val timedOut = new AtomicBoolean(false)
-    try {
-      executor = new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true,
-        uncaughtExceptionHandler = mockUncaughtExceptionHandler,
-        resources = immutable.Map.empty[String, ResourceInformation])
+
+    withExecutor("id", "localhost", SparkEnv.get,
+        uncaughtExceptionHandler = mockUncaughtExceptionHandler) { executor =>
+
       // the task will be launched in a dedicated worker thread
       executor.launchTask(mockBackend, taskDescription)
       if (killTask) {
@@ -503,11 +516,8 @@ class ExecutorSuite extends SparkFunSuite
         assert(executor.numRunningTasks === 0)
       }
       assert(!timedOut.get(), "timed out waiting to be ready to kill tasks")
-    } finally {
-      if (executor != null) {
-        executor.stop()
-      }
     }
+
     val orderedMock = inOrder(mockBackend)
     val statusCaptor = ArgumentCaptor.forClass(classOf[ByteBuffer])
     orderedMock.verify(mockBackend)
diff --git a/core/src/test/scala/org/apache/spark/internal/io/SparkHadoopWriterUtilsSuite.scala b/core/src/test/scala/org/apache/spark/internal/io/SparkHadoopWriterUtilsSuite.scala
new file mode 100644
index 0000000000000..33b58ec9e6665
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/internal/io/SparkHadoopWriterUtilsSuite.scala
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.internal.io
+
+import java.util.Date
+
+import org.apache.hadoop.mapreduce.JobID
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.internal.io.SparkHadoopWriterUtils.createJobID
+
+/**
+ * Unit tests for functions in SparkHadoopWriterUtils.
+ */
+class SparkHadoopWriterUtilsSuite extends SparkFunSuite {
+
+  /**
+   * Core test of JobID generation:
+   * They are created.
+   * The job number is converted to the job ID.
+   * They round trip to string and back
+   * (which implies that the full string matches the regexp
+   * in the JobID class).
+   */
+  test("JobID Generation") {
+    val jobNumber = 1010
+    val j1 = createJobID(new Date(), jobNumber)
+    assert(jobNumber == j1.getId,
+      s"Job number mismatch in $j1")
+
+    val jobStr = j1.toString
+    // the string value begins with job_
+    assert(jobStr.startsWith("job_"),
+      s"wrong prefix of $jobStr")
+    // and the hadoop code can parse it
+    val j2 = roundTrip(j1)
+    assert(j1.getId == j2.getId, "Job ID mismatch")
+    assert(j1.getJtIdentifier == j2.getJtIdentifier, "Job identifier mismatch")
+  }
+
+  /**
+   * This is the problem surfacing in situations where committers expect
+   * Job IDs to be unique: if the timestamp is (exclusively) used
+   * then there will conflict in directories created.
+   */
+  test("JobIDs generated at same time are different") {
+    val now = new Date()
+    val j1 = createJobID(now, 1)
+    val j2 = createJobID(now, 1)
+    assert(j1.toString != j2.toString)
+  }
+
+  /**
+   * There's nothing explicitly in the Hadoop classes to stop
+   * job numbers being negative.
+   * There's some big assumptions in the FileOutputCommitter about attempt IDs
+   * being positive during any recovery operations; for safety the ID
+   * job number is validated.
+   */
+  test("JobIDs with negative job number") {
+    intercept[IllegalArgumentException] {
+      createJobID(new Date(), -1)
+    }
+  }
+
+  /**
+   * If someone ever does reinstate use of timestamps,
+   * make sure that the case of timestamp == 0 is handled.
+   */
+  test("JobIDs on Epoch are different") {
+    val j1 = createJobID(new Date(0), 0)
+    val j2 = createJobID(new Date(0), 0)
+    assert (j1.toString != j2.toString)
+  }
+
+  /**
+   * Do a round trip as a string and back again.
+   * This uses the JobID parser.
+   * @param jobID job ID
+   * @return the returned jobID
+   */
+  private def roundTrip(jobID: JobID): JobID = {
+    val parsedJobId = JobID.forName(jobID.toString)
+    assert(jobID == parsedJobId, "Round trip was inconsistent")
+    parsedJobId
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
index cf2d9293ef822..7888796dd55e6 100644
--- a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
@@ -139,7 +139,7 @@ class PluginContainerSuite extends SparkFunSuite with BeforeAndAfterEach with Lo
       .set(NonLocalModeSparkPlugin.TEST_PATH_CONF, path.getAbsolutePath())
 
     sc = new SparkContext(conf)
-    TestUtils.waitUntilExecutorsUp(sc, 2, 10000)
+    TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
     eventually(timeout(10.seconds), interval(100.millis)) {
       val children = path.listFiles()
@@ -169,7 +169,7 @@ class PluginContainerSuite extends SparkFunSuite with BeforeAndAfterEach with Lo
       sc = new SparkContext(conf)
 
       // Ensure all executors has started
-      TestUtils.waitUntilExecutorsUp(sc, 1, 10000)
+      TestUtils.waitUntilExecutorsUp(sc, 1, 60000)
 
       var children = Array.empty[File]
       eventually(timeout(10.seconds), interval(100.millis)) {
diff --git a/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala b/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala
index 60f67699f81be..987f383c9c4fa 100644
--- a/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala
+++ b/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala
@@ -119,6 +119,14 @@ class TestMemoryManager(conf: SparkConf)
     consequentOOM += n
   }
 
+  /**
+   * Undos the effects of [[markExecutionAsOutOfMemoryOnce]] and [[markconsequentOOM]] and lets
+   * calls to [[acquireExecutionMemory()]] (if there is enough memory available).
+   */
+  def resetConsequentOOM(): Unit = synchronized {
+    consequentOOM = 0
+  }
+
   def limit(avail: Long): Unit = synchronized {
     require(avail >= 0)
     available = avail
diff --git a/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala b/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala
index 0e21a36071c42..3d4b8c868d6fc 100644
--- a/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala
+++ b/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala
@@ -35,12 +35,27 @@ class StatsdSinkSuite extends SparkFunSuite {
     STATSD_KEY_UNIT -> "seconds",
     STATSD_KEY_HOST -> "127.0.0.1"
   )
-  private val socketTimeout = 30000 // milliseconds
-  private val socketBufferSize = 8192
+  // The maximum size of a single datagram packet payload. Payloads
+  // larger than this will be truncated.
+  private val maxPayloadSize = 256 // bytes
+
+  // The receive buffer must be large enough to hold all inflight
+  // packets. This includes any kernel and protocol overhead.
+  // This value was determined experimentally and should be
+  // increased if timeouts are seen.
+  private val socketMinRecvBufferSize = 16384 // bytes
+  private val socketTimeout = 30000           // milliseconds
 
   private def withSocketAndSink(testCode: (DatagramSocket, StatsdSink) => Any): Unit = {
     val socket = new DatagramSocket
-    socket.setReceiveBufferSize(socketBufferSize)
+
+    // Leave the receive buffer size untouched unless it is too
+    // small. If the receive buffer is too small packets will be
+    // silently dropped and receive operations will timeout.
+    if (socket.getReceiveBufferSize() < socketMinRecvBufferSize) {
+      socket.setReceiveBufferSize(socketMinRecvBufferSize)
+    }
+
     socket.setSoTimeout(socketTimeout)
     val props = new Properties
     defaultProps.foreach(e => props.put(e._1, e._2))
@@ -61,7 +76,7 @@ class StatsdSinkSuite extends SparkFunSuite {
       sink.registry.register("counter", counter)
       sink.report()
 
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       socket.receive(p)
 
       val result = new String(p.getData, 0, p.getLength, UTF_8)
@@ -77,7 +92,7 @@ class StatsdSinkSuite extends SparkFunSuite {
       sink.registry.register("gauge", gauge)
       sink.report()
 
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       socket.receive(p)
 
       val result = new String(p.getData, 0, p.getLength, UTF_8)
@@ -87,7 +102,7 @@ class StatsdSinkSuite extends SparkFunSuite {
 
   test("metrics StatsD sink with Histogram") {
     withSocketAndSink { (socket, sink) =>
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       val histogram = new Histogram(new UniformReservoir)
       histogram.update(10)
       histogram.update(20)
@@ -121,7 +136,7 @@ class StatsdSinkSuite extends SparkFunSuite {
 
   test("metrics StatsD sink with Timer") {
     withSocketAndSink { (socket, sink) =>
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       val timer = new Timer()
       timer.update(1, SECONDS)
       timer.update(2, SECONDS)
diff --git a/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala b/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala
index 7a05daa2ad715..ff7d680352177 100644
--- a/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala
+++ b/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala
@@ -50,13 +50,12 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(WORKER_GPU_ID.amountConf, "2")
         .set(TASK_GPU_ID.amountConf, "1")
         .set(EXECUTOR_GPU_ID.amountConf, "1")
-        .set(SPARK_RESOURCES_DIR, dir.getName())
         .set(WORKER_FPGA_ID.amountConf, "2")
         .set(TASK_FPGA_ID.amountConf, "1")
         .set(EXECUTOR_FPGA_ID.amountConf, "1")
 
       sc = new SparkContext(conf)
-      TestUtils.waitUntilExecutorsUp(sc, 2, 10000)
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
       eventually(timeout(10.seconds), interval(100.millis)) {
         val children = dir.listFiles()
@@ -81,10 +80,9 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(WORKER_GPU_ID.amountConf, "2")
         .set(TASK_GPU_ID.amountConf, "1")
         .set(EXECUTOR_GPU_ID.amountConf, "1")
-        .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
-      TestUtils.waitUntilExecutorsUp(sc, 2, 10000)
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
       eventually(timeout(10.seconds), interval(100.millis)) {
         val children = dir.listFiles()
@@ -108,10 +106,9 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(WORKER_GPU_ID.amountConf, "2")
         .set(TASK_GPU_ID.amountConf, "1")
         .set(EXECUTOR_GPU_ID.amountConf, "1")
-        .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
-      TestUtils.waitUntilExecutorsUp(sc, 2, 10000)
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
       eventually(timeout(10.seconds), interval(100.millis)) {
         val children = dir.listFiles()
@@ -134,10 +131,9 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(RESOURCES_DISCOVERY_PLUGIN, Seq(classOf[TestResourceDiscoveryPluginEmpty].getName()))
         .set(DRIVER_GPU_ID.discoveryScriptConf, scriptPath)
         .set(DRIVER_GPU_ID.amountConf, "2")
-        .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
-      TestUtils.waitUntilExecutorsUp(sc, 2, 10000)
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
       assert(sc.resources.size === 1)
       assert(sc.resources.get(GPU).get.addresses === Array("5", "6"))
diff --git a/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala b/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
index c10f2c244e133..01c67b3856df4 100644
--- a/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
@@ -209,7 +209,7 @@ abstract class RpcEnvSuite extends SparkFunSuite with BeforeAndAfterAll {
     // Use anotherEnv to find out the RpcEndpointRef
     val rpcEndpointRef = anotherEnv.setupEndpointRef(env.address, "ask-abort")
     try {
-      val e = intercept[RpcAbortException] {
+      val e = intercept[SparkException] {
         val timeout = new RpcTimeout(10.seconds, shortProp)
         val abortableRpcFuture = rpcEndpointRef.askAbortable[String](
           "hello", timeout)
@@ -217,15 +217,15 @@ abstract class RpcEnvSuite extends SparkFunSuite with BeforeAndAfterAll {
         new Thread {
           override def run: Unit = {
             Thread.sleep(100)
-            abortableRpcFuture.abort("TestAbort")
+            abortableRpcFuture.abort(new RuntimeException("TestAbort"))
           }
         }.start()
 
-        timeout.awaitResult(abortableRpcFuture.toFuture)
+        timeout.awaitResult(abortableRpcFuture.future)
       }
-      // The SparkException cause should be a RpcAbortException with "TestAbort" message
-      assert(e.isInstanceOf[RpcAbortException])
-      assert(e.getMessage.contains("TestAbort"))
+      // The SparkException cause should be a RuntimeException with "TestAbort" message
+      assert(e.getCause.isInstanceOf[RuntimeException])
+      assert(e.getCause.getMessage.contains("TestAbort"))
     } finally {
       anotherEnv.shutdown()
       anotherEnv.awaitTermination()
diff --git a/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala b/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala
index c74c728b3e3f3..8b1c602cd8e58 100644
--- a/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala
@@ -136,4 +136,17 @@ class InboxSuite extends SparkFunSuite {
 
     endpoint.verifySingleOnNetworkErrorMessage(cause, remoteAddress)
   }
+
+  test("SPARK-32738: should reduce the number of active threads when fatal error happens") {
+    val endpoint = mock(classOf[TestRpcEndpoint])
+    when(endpoint.receive).thenThrow(new OutOfMemoryError())
+
+    val dispatcher = mock(classOf[Dispatcher])
+    val inbox = new Inbox("name", endpoint)
+    inbox.post(OneWayMessage(null, "hi"))
+    intercept[OutOfMemoryError] {
+      inbox.process(dispatcher)
+    }
+    assert(inbox.getNumActiveThreads == 0)
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index fc8ac38479932..9b214afabdd37 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -21,19 +21,23 @@ import java.io.File
 
 import scala.util.Random
 
+import org.scalatest.concurrent.Eventually
+import org.scalatest.time.SpanSugar._
+
 import org.apache.spark._
 import org.apache.spark.internal.config.Tests.TEST_NO_STAGE_RETRY
 
-class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
+class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with Eventually {
 
-  def initLocalClusterSparkContext(): Unit = {
+  def initLocalClusterSparkContext(numWorker: Int = 4): Unit = {
     val conf = new SparkConf()
       // Init local cluster here so each barrier task runs in a separated process, thus `barrier()`
       // call is actually useful.
-      .setMaster("local-cluster[4, 1, 1024]")
+      .setMaster(s"local-cluster[$numWorker, 1, 1024]")
       .setAppName("test-cluster")
       .set(TEST_NO_STAGE_RETRY, true)
     sc = new SparkContext(conf)
+    TestUtils.waitUntilExecutorsUp(sc, numWorker, 60000)
   }
 
   test("global sync by barrier() call") {
@@ -52,6 +56,70 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
     assert(times.max - times.min <= 1000)
   }
 
+  test("share messages with allGather() call") {
+    initLocalClusterSparkContext()
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      // Pass partitionId message in
+      val message: String = context.partitionId().toString
+      val messages: Array[String] = context.allGather(message)
+      Iterator.single(messages.toList)
+    }
+    val messages = rdd2.collect()
+    // All the task partitionIds are shared across all tasks
+    assert(messages.length === 4)
+    assert(messages.forall(_ == List("0", "1", "2", "3")))
+  }
+
+  test("throw exception if we attempt to synchronize with different blocking calls") {
+    initLocalClusterSparkContext()
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      val partitionId = context.partitionId
+      if (partitionId == 0) {
+        context.barrier()
+      } else {
+        context.allGather(partitionId.toString)
+      }
+      Seq(null).iterator
+    }
+    val error = intercept[SparkException] {
+      rdd2.collect()
+    }.getMessage
+    assert(error.contains("Different barrier sync types found"))
+  }
+
+  test("successively sync with allGather and barrier") {
+    initLocalClusterSparkContext()
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      context.barrier()
+      val time1 = System.currentTimeMillis()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      // Pass partitionId message in
+      val message = context.partitionId().toString
+      val messages = context.allGather(message)
+      val time2 = System.currentTimeMillis()
+      Seq((time1, time2)).iterator
+    }
+    val times = rdd2.collect()
+    // All the tasks shall finish the first round of global sync within a short time slot.
+    val times1 = times.map(_._1)
+    assert(times1.max - times1.min <= 1000)
+
+    // All the tasks shall finish the second round of global sync within a short time slot.
+    val times2 = times.map(_._2)
+    assert(times2.max - times2.min <= 1000)
+  }
+
   test("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
@@ -148,12 +216,14 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
 
   def testBarrierTaskKilled(interruptOnKill: Boolean): Unit = {
     withTempDir { dir =>
+      val runningFlagFile = "barrier.task.running"
       val killedFlagFile = "barrier.task.killed"
       val rdd = sc.makeRDD(Seq(0, 1), 2)
       val rdd2 = rdd.barrier().mapPartitions { it =>
         val context = BarrierTaskContext.get()
         if (context.partitionId() == 0) {
           try {
+            new File(dir, runningFlagFile).createNewFile()
             context.barrier()
           } catch {
             case _: TaskKilledException =>
@@ -172,8 +242,10 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
           if (partitionId == 0) {
             new Thread {
               override def run: Unit = {
-                Thread.sleep(1000)
-                sc.killTaskAttempt(taskStart.taskInfo.taskId, interruptThread = interruptOnKill)
+                eventually(timeout(10.seconds)) {
+                  assert(new File(dir, runningFlagFile).exists())
+                  sc.killTaskAttempt(taskStart.taskInfo.taskId, interruptThread = interruptOnKill)
+                }
               }
             }.start()
           }
@@ -200,4 +272,21 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
     initLocalClusterSparkContext()
     testBarrierTaskKilled(interruptOnKill = true)
   }
+
+  test("SPARK-31485: barrier stage should fail if only partial tasks are launched") {
+    initLocalClusterSparkContext(2)
+    val id = sc.getExecutorIds().head
+    val rdd0 = sc.parallelize(Seq(0, 1, 2, 3), 2)
+    val dep = new OneToOneDependency[Int](rdd0)
+    // set up a barrier stage with 2 tasks and both tasks prefer the same executor (only 1 core) for
+    // scheduling. So, one of tasks won't be scheduled in one round of resource offer.
+    val rdd = new MyRDD(sc, 2, List(dep), Seq(Seq(s"executor_h_$id"), Seq(s"executor_h_$id")))
+    val errorMsg = intercept[SparkException] {
+      rdd.barrier().mapPartitions { iter =>
+        BarrierTaskContext.get().barrier()
+        iter
+      }.collect()
+    }.getMessage
+    assert(errorMsg.contains("Fail resource offers for barrier stage"))
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
index c063301673598..fad971fd4ddec 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
@@ -251,6 +251,9 @@ class CoarseGrainedSchedulerBackendSuite extends SparkFunSuite with LocalSparkCo
       assert(execResources(GPU).assignedAddrs === Array("0"))
     }
 
+    // To avoid allocating any resources immediately after releasing the resource from the task to
+    // make sure that `availableAddrs` below won't change
+    when(ts.resourceOffers(any[IndexedSeq[WorkerOffer]])).thenReturn(Seq.empty)
     backend.driverEndpoint.send(
       StatusUpdate("1", 1, TaskState.FINISHED, buffer, taskResources))
 
diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 101e60c73e9f8..d92bd5bcba5ad 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -25,6 +25,9 @@ import scala.annotation.meta.param
 import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Map}
 import scala.util.control.NonFatal
 
+import org.mockito.Mockito.spy
+import org.mockito.Mockito.times
+import org.mockito.Mockito.verify
 import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits}
 import org.scalatest.exceptions.TestFailedException
 import org.scalatest.time.SpanSugar._
@@ -232,6 +235,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
   var sparkListener: EventInfoRecordingListener = null
 
+  var blockManagerMaster: BlockManagerMaster = null
   var mapOutputTracker: MapOutputTrackerMaster = null
   var broadcastManager: BroadcastManager = null
   var securityMgr: SecurityManager = null
@@ -245,17 +249,18 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
    */
   val cacheLocations = new HashMap[(Int, Int), Seq[BlockManagerId]]
   // stub out BlockManagerMaster.getLocations to use our cacheLocations
-  val blockManagerMaster = new BlockManagerMaster(null, null, conf, true) {
-      override def getLocations(blockIds: Array[BlockId]): IndexedSeq[Seq[BlockManagerId]] = {
-        blockIds.map {
-          _.asRDDId.map(id => (id.rddId -> id.splitIndex)).flatMap(key => cacheLocations.get(key)).
-            getOrElse(Seq())
-        }.toIndexedSeq
-      }
-      override def removeExecutor(execId: String): Unit = {
-        // don't need to propagate to the driver, which we don't have
-      }
+  class MyBlockManagerMaster(conf: SparkConf) extends BlockManagerMaster(null, null, conf, true) {
+    override def getLocations(blockIds: Array[BlockId]): IndexedSeq[Seq[BlockManagerId]] = {
+      blockIds.map {
+        _.asRDDId.map { id => (id.rddId -> id.splitIndex)
+        }.flatMap { key => cacheLocations.get(key)
+        }.getOrElse(Seq())
+      }.toIndexedSeq
     }
+    override def removeExecutor(execId: String): Unit = {
+      // don't need to propagate to the driver, which we don't have
+    }
+  }
 
   /** The list of results that DAGScheduler has collected. */
   val results = new HashMap[Int, Any]()
@@ -273,6 +278,16 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     override def jobFailed(exception: Exception): Unit = { failure = exception }
   }
 
+  class MyMapOutputTrackerMaster(
+      conf: SparkConf,
+      broadcastManager: BroadcastManager)
+    extends MapOutputTrackerMaster(conf, broadcastManager, true) {
+
+    override def sendTracker(message: Any): Unit = {
+      // no-op, just so we can stop this to avoid leaking threads
+    }
+  }
+
   override def beforeEach(): Unit = {
     super.beforeEach()
     init(new SparkConf())
@@ -290,11 +305,8 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     results.clear()
     securityMgr = new SecurityManager(conf)
     broadcastManager = new BroadcastManager(true, conf, securityMgr)
-    mapOutputTracker = new MapOutputTrackerMaster(conf, broadcastManager, true) {
-      override def sendTracker(message: Any): Unit = {
-        // no-op, just so we can stop this to avoid leaking threads
-      }
-    }
+    mapOutputTracker = spy(new MyMapOutputTrackerMaster(conf, broadcastManager))
+    blockManagerMaster = spy(new MyBlockManagerMaster(conf))
     scheduler = new DAGScheduler(
       sc,
       taskScheduler,
@@ -537,6 +549,59 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assert(mapStatus2(2).location.host === "hostB")
   }
 
+  test("SPARK-32003: All shuffle files for executor should be cleaned up on fetch failure") {
+    // reset the test context with the right shuffle service config
+    afterEach()
+    val conf = new SparkConf()
+    conf.set(config.SHUFFLE_SERVICE_ENABLED.key, "true")
+    init(conf)
+
+    val shuffleMapRdd = new MyRDD(sc, 3, Nil)
+    val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(3))
+    val shuffleId = shuffleDep.shuffleId
+    val reduceRdd = new MyRDD(sc, 3, List(shuffleDep), tracker = mapOutputTracker)
+
+    submit(reduceRdd, Array(0, 1, 2))
+    // Map stage completes successfully,
+    // two tasks are run on an executor on hostA and one on an executor on hostB
+    complete(taskSets(0), Seq(
+      (Success, makeMapStatus("hostA", 3)),
+      (Success, makeMapStatus("hostA", 3)),
+      (Success, makeMapStatus("hostB", 3))))
+    // Now the executor on hostA is lost
+    runEvent(ExecutorLost("exec-hostA", ExecutorExited(-100, false, "Container marked as failed")))
+    // Executor is removed but shuffle files are not unregistered
+    verify(blockManagerMaster, times(1)).removeExecutor("exec-hostA")
+    verify(mapOutputTracker, times(0)).removeOutputsOnExecutor("exec-hostA")
+
+    // The MapOutputTracker has all the shuffle files
+    val mapStatuses = mapOutputTracker.shuffleStatuses(shuffleId).mapStatuses
+    assert(mapStatuses.count(_ != null) === 3)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "exec-hostA") === 2)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "exec-hostB") === 1)
+
+    // Now a fetch failure from the lost executor occurs
+    complete(taskSets(1), Seq(
+      (FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 0, 0, "ignored"), null)
+    ))
+    // blockManagerMaster.removeExecutor is not called again
+    // but shuffle files are unregistered
+    verify(blockManagerMaster, times(1)).removeExecutor("exec-hostA")
+    verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("exec-hostA")
+
+    // Shuffle files for exec-hostA should be lost
+    assert(mapStatuses.count(_ != null) === 1)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "exec-hostA") === 0)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "exec-hostB") === 1)
+
+    // Additional fetch failure from the executor does not result in further call to
+    // mapOutputTracker.removeOutputsOnExecutor
+    complete(taskSets(1), Seq(
+      (FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 1, 0, "ignored"), null)
+    ))
+    verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("exec-hostA")
+  }
+
   test("zero split job") {
     var numResults = 0
     var failureReason: Option[Exception] = None
@@ -762,8 +827,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     complete(taskSets(1), Seq(
       (Success, 42),
       (FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 0, 0, "ignored"), null)))
-    // this will get called
-    // blockManagerMaster.removeExecutor("exec-hostA")
+    verify(blockManagerMaster, times(1)).removeExecutor("exec-hostA")
     // ask the scheduler to try it again
     scheduler.resubmitFailedStages()
     // have the 2nd attempt pass
@@ -806,11 +870,14 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
         (Success, makeMapStatus("hostA", 1)),
         (Success, makeMapStatus("hostB", 1))))
       runEvent(ExecutorLost("exec-hostA", event))
+      verify(blockManagerMaster, times(1)).removeExecutor("exec-hostA")
       if (expectFileLoss) {
+        verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("exec-hostA")
         intercept[MetadataFetchFailedException] {
           mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0)
         }
       } else {
+        verify(mapOutputTracker, times(0)).removeOutputsOnExecutor("exec-hostA")
         assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1).toSet ===
           HashSet(makeBlockManagerId("hostA"), makeBlockManagerId("hostB")))
       }
@@ -1931,6 +1998,53 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assertDataStructuresEmpty()
   }
 
+  test("SPARK-30388: shuffle fetch failed on speculative task, but original task succeed") {
+    var completedStage: List[Int] = Nil
+    val listener = new SparkListener() {
+      override def onStageCompleted(event: SparkListenerStageCompleted): Unit = {
+        completedStage = completedStage :+ event.stageInfo.stageId
+      }
+    }
+    sc.addSparkListener(listener)
+
+    val shuffleMapRdd = new MyRDD(sc, 2, Nil)
+    val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(2))
+    val reduceRdd = new MyRDD(sc, 2, List(shuffleDep))
+    submit(reduceRdd, Array(0, 1))
+    completeShuffleMapStageSuccessfully(0, 0, 2)
+    sc.listenerBus.waitUntilEmpty()
+    assert(completedStage === List(0))
+
+    // result task 0.0 succeed
+    runEvent(makeCompletionEvent(taskSets(1).tasks(0), Success, 42))
+    // speculative result task 1.1 fetch failed
+    val info = new TaskInfo(4, index = 1, attemptNumber = 1, 0L, "", "", TaskLocality.ANY, true)
+    runEvent(makeCompletionEvent(
+        taskSets(1).tasks(1),
+        FetchFailed(makeBlockManagerId("hostA"), shuffleDep.shuffleId, 0L, 0, 1, "ignored"),
+        null,
+        Seq.empty,
+        Array.empty,
+        info
+      )
+    )
+    sc.listenerBus.waitUntilEmpty()
+    assert(completedStage === List(0, 1))
+
+    Thread.sleep(DAGScheduler.RESUBMIT_TIMEOUT * 2)
+    // map stage resubmitted
+    assert(scheduler.runningStages.size === 1)
+    val mapStage = scheduler.runningStages.head
+    assert(mapStage.id === 0)
+    assert(mapStage.latestInfo.failureReason.isEmpty)
+
+    // original result task 1.0 succeed
+    runEvent(makeCompletionEvent(taskSets(1).tasks(1), Success, 42))
+    sc.listenerBus.waitUntilEmpty()
+    assert(completedStage === List(0, 1, 1, 0))
+    assert(scheduler.activeJobs.isEmpty)
+  }
+
   test("misbehaved accumulator should not crash DAGScheduler and SparkContext") {
     val acc = new LongAccumulator {
       override def add(v: java.lang.Long): Unit = throw new DAGSchedulerSuiteDummyException
diff --git a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
index 286924001e920..046564d65b767 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
@@ -36,6 +36,7 @@ import org.apache.spark.deploy.history.{EventLogFileReader, SingleEventLogFileWr
 import org.apache.spark.deploy.history.EventLogTestHelper._
 import org.apache.spark.executor.{ExecutorMetrics, TaskMetrics}
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.{EVENT_LOG_DIR, EVENT_LOG_ENABLED}
 import org.apache.spark.io._
 import org.apache.spark.metrics.{ExecutorMetricType, MetricsSystem}
 import org.apache.spark.scheduler.cluster.ExecutorInfo
@@ -99,6 +100,49 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit
     testStageExecutorMetricsEventLogging()
   }
 
+  test("SPARK-31764: isBarrier should be logged in event log") {
+    val conf = new SparkConf()
+    conf.set(EVENT_LOG_ENABLED, true)
+    conf.set(EVENT_LOG_DIR, testDirPath.toString)
+    val sc = new SparkContext("local", "test-SPARK-31764", conf)
+    val appId = sc.applicationId
+
+    sc.parallelize(1 to 10)
+      .barrier()
+      .mapPartitions(_.map(elem => (elem, elem)))
+      .filter(elem => elem._1 % 2 == 0)
+      .reduceByKey(_ + _)
+      .collect
+    sc.stop()
+
+    val eventLogStream = EventLogFileReader.openEventLog(new Path(testDirPath, appId), fileSystem)
+    val events = readLines(eventLogStream).map(line => JsonProtocol.sparkEventFromJson(parse(line)))
+    val jobStartEvents = events
+      .filter(event => event.isInstanceOf[SparkListenerJobStart])
+      .map(_.asInstanceOf[SparkListenerJobStart])
+
+    assert(jobStartEvents.size === 1)
+    val stageInfos = jobStartEvents.head.stageInfos
+    assert(stageInfos.size === 2)
+
+    val stage0 = stageInfos(0)
+    val rddInfosInStage0 = stage0.rddInfos
+    assert(rddInfosInStage0.size === 3)
+    val sortedRddInfosInStage0 = rddInfosInStage0.sortBy(_.scope.get.name)
+    assert(sortedRddInfosInStage0(0).scope.get.name === "filter")
+    assert(sortedRddInfosInStage0(0).isBarrier === true)
+    assert(sortedRddInfosInStage0(1).scope.get.name === "mapPartitions")
+    assert(sortedRddInfosInStage0(1).isBarrier === true)
+    assert(sortedRddInfosInStage0(2).scope.get.name === "parallelize")
+    assert(sortedRddInfosInStage0(2).isBarrier === false)
+
+    val stage1 = stageInfos(1)
+    val rddInfosInStage1 = stage1.rddInfos
+    assert(rddInfosInStage1.size === 1)
+    assert(rddInfosInStage1(0).scope.get.name === "reduceByKey")
+    assert(rddInfosInStage1(0).isBarrier === false) // reduceByKey
+  }
+
   /* ----------------- *
    * Actual test logic *
    * ----------------- */
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala
index e7ecf847ff4f4..a083cdb95b20f 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala
@@ -758,7 +758,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
         // that are explicitly blacklisted, plus those that have *any* executors blacklisted.
         val nodesForBlacklistedExecutors = offers.filter { offer =>
           execBlacklist.contains(offer.executorId)
-        }.map(_.host).toSet.toSeq
+        }.map(_.host).distinct
         val nodesWithAnyBlacklisting = (nodeBlacklist ++ nodesForBlacklistedExecutors).toSet
         // Similarly, figure out which executors have any blacklisting.  This means all executors
         // that are explicitly blacklisted, plus all executors on nodes that are blacklisted.
diff --git a/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
index 615389ae5c2d4..c5f73150f5db5 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
@@ -42,7 +42,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
   private val conf = new SparkConf()
     .set(DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT.key, "60s")
     .set(DYN_ALLOCATION_CACHED_EXECUTOR_IDLE_TIMEOUT.key, "120s")
-    .set(DYN_ALLOCATION_SHUFFLE_TIMEOUT.key, "240s")
+    .set(DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT.key, "240s")
     .set(SHUFFLE_SERVICE_ENABLED, true)
 
   private var monitor: ExecutorMonitor = _
@@ -287,7 +287,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
 
   test("shuffle block tracking") {
     val bus = mockListenerBus()
-    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING, true).set(SHUFFLE_SERVICE_ENABLED, false)
+    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true).set(SHUFFLE_SERVICE_ENABLED, false)
     monitor = new ExecutorMonitor(conf, client, bus, clock)
 
     // 3 jobs: 2 and 3 share a shuffle, 1 has a separate shuffle.
@@ -355,7 +355,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
 
   test("SPARK-28839: Avoids NPE in context cleaner when shuffle service is on") {
     val bus = mockListenerBus()
-    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING, true).set(SHUFFLE_SERVICE_ENABLED, true)
+    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true).set(SHUFFLE_SERVICE_ENABLED, true)
     monitor = new ExecutorMonitor(conf, client, bus, clock) {
       override def onOtherEvent(event: SparkListenerEvent): Unit = {
         throw new IllegalStateException("No event should be sent.")
@@ -367,7 +367,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
 
   test("shuffle tracking with multiple executors and concurrent jobs") {
     val bus = mockListenerBus()
-    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING, true).set(SHUFFLE_SERVICE_ENABLED, false)
+    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true).set(SHUFFLE_SERVICE_ENABLED, false)
     monitor = new ExecutorMonitor(conf, client, bus, clock)
 
     monitor.onExecutorAdded(SparkListenerExecutorAdded(clock.getTimeMillis(), "1", execInfo))
@@ -410,8 +410,8 @@ class ExecutorMonitorSuite extends SparkFunSuite {
 
   test("SPARK-28455: avoid overflow in timeout calculation") {
     conf
-      .set(DYN_ALLOCATION_SHUFFLE_TIMEOUT, Long.MaxValue)
-      .set(DYN_ALLOCATION_SHUFFLE_TRACKING, true)
+      .set(DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT, Long.MaxValue)
+      .set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true)
       .set(SHUFFLE_SERVICE_ENABLED, false)
     monitor = new ExecutorMonitor(conf, client, null, clock)
 
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala b/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala
index fd228cded783a..525e682dd5d42 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala
@@ -125,7 +125,7 @@ object KryoBenchmark extends BenchmarkBase {
   def createSerializer(useUnsafe: Boolean): SerializerInstance = {
     val conf = new SparkConf()
     conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-    conf.set(KRYO_USER_REGISTRATORS, classOf[MyRegistrator].getName)
+    conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[MyRegistrator].getName))
     conf.set(KRYO_USE_UNSAFE, useUnsafe)
 
     new KryoSerializer(conf).newInstance()
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
index 953b651c72a83..dde0c98704d00 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
@@ -76,7 +76,7 @@ object KryoSerializerBenchmark extends BenchmarkBase {
     conf.set(EXECUTOR_EXTRA_JAVA_OPTIONS,
       "-XX:+UseParallelGC -XX:-UseDynamicNumberOfGCThreads")
     conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-    conf.set(KRYO_USER_REGISTRATORS, classOf[MyRegistrator].getName)
+    conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[MyRegistrator].getName))
     conf.set(KRYO_USE_POOL, usePool)
 
     if (sc != null) {
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala
index d4fafab4a5d64..397fdce8ae6e3 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala
@@ -29,7 +29,7 @@ class KryoSerializerDistributedSuite extends SparkFunSuite with LocalSparkContex
   test("kryo objects are serialised consistently in different processes") {
     val conf = new SparkConf(false)
       .set(config.SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-      .set(config.Kryo.KRYO_USER_REGISTRATORS, classOf[AppJarRegistrator].getName)
+      .set(config.Kryo.KRYO_USER_REGISTRATORS, Seq(classOf[AppJarRegistrator].getName))
       .set(config.TASK_MAX_FAILURES, 1)
       .set(config.BLACKLIST_ENABLED, false)
 
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index 4c47a67ee9ffc..229ef69973775 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -42,7 +42,7 @@ import org.apache.spark.util.ThreadUtils
 
 class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
   conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-  conf.set(KRYO_USER_REGISTRATORS, classOf[MyRegistrator].getName)
+  conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[MyRegistrator].getName))
   conf.set(KRYO_USE_UNSAFE, false)
 
   test("SPARK-7392 configuration limits") {
@@ -313,7 +313,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
     import org.apache.spark.SparkException
 
     val conf = new SparkConf(false)
-    conf.set(KRYO_USER_REGISTRATORS, "this.class.does.not.exist")
+    conf.set(KRYO_USER_REGISTRATORS, Seq("this.class.does.not.exist"))
 
     val thrown = intercept[SparkException](new KryoSerializer(conf).newInstance().serialize(1))
     assert(thrown.getMessage.contains("Failed to register classes with Kryo"))
@@ -412,7 +412,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
     val ser = new KryoSerializer(new SparkConf).newInstance().asInstanceOf[KryoSerializerInstance]
     assert(ser.getAutoReset)
     val conf = new SparkConf().set(KRYO_USER_REGISTRATORS,
-      classOf[RegistratorWithoutAutoReset].getName)
+      Seq(classOf[RegistratorWithoutAutoReset].getName))
     val ser2 = new KryoSerializer(conf).newInstance().asInstanceOf[KryoSerializerInstance]
     assert(!ser2.getAutoReset)
   }
@@ -443,7 +443,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
       .set(KRYO_REFERENCE_TRACKING, referenceTracking)
       .set(KRYO_USE_POOL, usePool)
     if (!autoReset) {
-      conf.set(KRYO_USER_REGISTRATORS, classOf[RegistratorWithoutAutoReset].getName)
+      conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[RegistratorWithoutAutoReset].getName))
     }
     val ser = new KryoSerializer(conf)
     val serInstance = ser.newInstance().asInstanceOf[KryoSerializerInstance]
@@ -530,7 +530,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
 
 class KryoSerializerAutoResetDisabledSuite extends SparkFunSuite with SharedSparkContext {
   conf.set(SERIALIZER, classOf[KryoSerializer].getName)
-  conf.set(KRYO_USER_REGISTRATORS, classOf[RegistratorWithoutAutoReset].getName)
+  conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[RegistratorWithoutAutoReset].getName))
   conf.set(KRYO_REFERENCE_TRACKING, true)
   conf.set(SHUFFLE_MANAGER, "sort")
   conf.set(SHUFFLE_SORT_BYPASS_MERGE_THRESHOLD, 200)
diff --git a/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala b/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala
index dad080c5fc161..9747f5780dd1e 100644
--- a/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala
@@ -52,7 +52,7 @@ class SerializerPropertiesSuite extends SparkFunSuite {
 
   test("KryoSerializer does not support relocation when auto-reset is disabled") {
     val conf = new SparkConf().set(KRYO_USER_REGISTRATORS,
-      classOf[RegistratorWithoutAutoReset].getName)
+      Seq(classOf[RegistratorWithoutAutoReset].getName))
     val ser = new KryoSerializer(conf)
     assert(!ser.newInstance().asInstanceOf[KryoSerializerInstance].getAutoReset())
     testSupportsRelocationOfSerializedObjects(ser, generateRandomItem)
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
index e7eed7bf4c879..255f91866ef58 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
@@ -1657,6 +1657,30 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
     }
   }
 
+  test("clean up used memory when BlockManager added") {
+    val listener = new AppStatusListener(store, conf, true)
+    // Add block manager at the first time
+    val driver = BlockManagerId(SparkContext.DRIVER_IDENTIFIER, "localhost", 42)
+    listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(
+      time, driver, 42L, Some(43L), Some(44L)))
+    // Update the memory metrics
+    listener.updateExecutorMemoryDiskInfo(
+      listener.liveExecutors(SparkContext.DRIVER_IDENTIFIER),
+      StorageLevel.MEMORY_AND_DISK,
+      10L,
+      10L
+    )
+    // Re-add the same block manager again
+    listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(
+      time, driver, 42L, Some(43L), Some(44L)))
+
+    check[ExecutorSummaryWrapper](SparkContext.DRIVER_IDENTIFIER) { d =>
+      val memoryMetrics = d.info.memoryMetrics.get
+      assert(memoryMetrics.usedOffHeapStorageMemory == 0)
+      assert(memoryMetrics.usedOnHeapStorageMemory == 0)
+    }
+  }
+
 
   private def key(stage: StageInfo): Array[Int] = Array(stage.stageId, stage.attemptNumber)
 
diff --git a/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala b/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala
new file mode 100644
index 0000000000000..2723af75227e4
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.status.api.v1
+
+import java.util.Date
+
+import com.fasterxml.jackson.core.`type`.TypeReference
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+
+import org.apache.spark.SparkFunSuite
+
+class ExecutorSummarySuite extends SparkFunSuite {
+
+  test("Check ExecutorSummary serialize and deserialize with empty peakMemoryMetrics") {
+    val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
+    val executorSummary = new ExecutorSummary("id", "host:port", true, 1,
+      10, 10, 1, 1, 1,
+      0, 0, 1, 100,
+      1, 100, 100,
+      10, false, 20, new Date(1600984336352L),
+      Option.empty, Option.empty, Map(), Option.empty, Set(), Option.empty, Map(), Map())
+    val expectedJson = "{\"id\":\"id\",\"hostPort\":\"host:port\",\"isActive\":true," +
+      "\"rddBlocks\":1,\"memoryUsed\":10,\"diskUsed\":10,\"totalCores\":1,\"maxTasks\":1," +
+      "\"activeTasks\":1,\"failedTasks\":0,\"completedTasks\":0,\"totalTasks\":1," +
+      "\"totalDuration\":100,\"totalGCTime\":1,\"totalInputBytes\":100," +
+      "\"totalShuffleRead\":100,\"totalShuffleWrite\":10,\"isBlacklisted\":false," +
+      "\"maxMemory\":20,\"addTime\":1600984336352,\"removeTime\":null,\"removeReason\":null," +
+      "\"executorLogs\":{},\"memoryMetrics\":null,\"blacklistedInStages\":[]," +
+      "\"peakMemoryMetrics\":null,\"attributes\":{},\"resources\":{}}"
+    val json = mapper.writeValueAsString(executorSummary)
+    assert(expectedJson.equals(json))
+    val deserializeExecutorSummary = mapper.readValue(json, new TypeReference[ExecutorSummary] {})
+    assert(deserializeExecutorSummary.peakMemoryMetrics == None)
+  }
+
+}
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala
new file mode 100644
index 0000000000000..0d54726af7ee8
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import org.junit.Assert.assertTrue
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+
+class BlockManagerMasterSuite extends SparkFunSuite {
+
+  test("SPARK-31422: getMemoryStatus should not fail after BlockManagerMaster stops") {
+    val bmm = new BlockManagerMaster(null, null, new SparkConf, true)
+    assertTrue(bmm.getMemoryStatus.isEmpty)
+  }
+
+  test("SPARK-31422: getStorageStatus should not fail after BlockManagerMaster stops") {
+    val bmm = new BlockManagerMaster(null, null, new SparkConf, true)
+    assertTrue(bmm.getStorageStatus.isEmpty)
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
index 59ace850d0bd2..6238d862d62df 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
@@ -92,8 +92,6 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
     conf.set(MEMORY_STORAGE_FRACTION, 0.999)
     conf.set(STORAGE_UNROLL_MEMORY_THRESHOLD, 512L)
 
-    // to make a replication attempt to inactive store fail fast
-    conf.set("spark.core.connection.ack.wait.timeout", "1s")
     // to make cached peers refresh frequently
     conf.set(STORAGE_CACHED_PEERS_TTL, 10)
 
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
index 89f00b5a9d902..9cf531383ce31 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -50,7 +50,7 @@ import org.apache.spark.network.server.{NoOpRpcHandler, TransportServer, Transpo
 import org.apache.spark.network.shuffle.{BlockFetchingListener, DownloadFileManager, ExecutorDiskUtils, ExternalBlockStoreClient}
 import org.apache.spark.network.shuffle.protocol.{BlockTransferMessage, RegisterExecutor}
 import org.apache.spark.rpc.RpcEnv
-import org.apache.spark.scheduler.LiveListenerBus
+import org.apache.spark.scheduler.{LiveListenerBus, SparkListenerBlockUpdated}
 import org.apache.spark.security.{CryptoStreamUtils, EncryptionFunSuite}
 import org.apache.spark.serializer.{JavaSerializer, KryoSerializer, SerializerManager}
 import org.apache.spark.shuffle.sort.SortShuffleManager
@@ -71,6 +71,7 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
   val allStores = ArrayBuffer[BlockManager]()
   var rpcEnv: RpcEnv = null
   var master: BlockManagerMaster = null
+  var liveListenerBus: LiveListenerBus = null
   val securityMgr = new SecurityManager(new SparkConf(false))
   val bcastManager = new BroadcastManager(true, new SparkConf(false), securityMgr)
   val mapOutputTracker = new MapOutputTrackerMaster(new SparkConf(false), bcastManager, true)
@@ -129,10 +130,28 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     blockManager
   }
 
+  // Save modified system properties so that we can restore them after tests.
+  val originalArch = System.getProperty("os.arch")
+  val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY)
+
+  def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = {
+    def set(k: String, v: String): Unit = {
+      if (v == null) {
+        System.clearProperty(k)
+      } else {
+        System.setProperty(k, v)
+      }
+    }
+    set("os.arch", arch)
+    set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops)
+    val initialize = PrivateMethod[Unit](Symbol("initialize"))
+    SizeEstimator invokePrivate initialize()
+  }
+
   override def beforeEach(): Unit = {
     super.beforeEach()
     // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case
-    System.setProperty("os.arch", "amd64")
+    reinitializeSizeEstimator("amd64", "true")
     conf = new SparkConf(false)
     init(conf)
 
@@ -145,17 +164,18 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     when(sc.conf).thenReturn(conf)
 
     val blockManagerInfo = new mutable.HashMap[BlockManagerId, BlockManagerInfo]()
+    liveListenerBus = spy(new LiveListenerBus(conf))
     master = spy(new BlockManagerMaster(rpcEnv.setupEndpoint("blockmanager",
       new BlockManagerMasterEndpoint(rpcEnv, true, conf,
-        new LiveListenerBus(conf), None, blockManagerInfo)),
+        liveListenerBus, None, blockManagerInfo)),
       rpcEnv.setupEndpoint("blockmanagerHeartbeat",
       new BlockManagerMasterHeartbeatEndpoint(rpcEnv, true, blockManagerInfo)), conf, true))
-
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
   }
 
   override def afterEach(): Unit = {
+    // Restore system properties and SizeEstimator to their original states.
+    reinitializeSizeEstimator(originalArch, originalCompressedOops)
+
     try {
       conf = null
       allStores.foreach(_.stop())
@@ -164,6 +184,7 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
       rpcEnv.awaitTermination()
       rpcEnv = null
       master = null
+      liveListenerBus = null
     } finally {
       super.afterEach()
     }
@@ -1693,6 +1714,16 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     assert(locs(blockIds(0)) == expectedLocs)
   }
 
+  test("SPARK-30594: Do not post SparkListenerBlockUpdated when updateBlockInfo returns false") {
+    // update block info for non-existent block manager
+    val updateInfo = UpdateBlockInfo(BlockManagerId("1", "host1", 100),
+      BlockId("test_1"), StorageLevel.MEMORY_ONLY, 1, 1)
+    val result = master.driverEndpoint.askSync[Boolean](updateInfo)
+
+    assert(!result)
+    verify(liveListenerBus, never()).post(SparkListenerBlockUpdated(BlockUpdatedInfo(updateInfo)))
+  }
+
   class MockBlockTransferService(val maxFailures: Int) extends BlockTransferService {
     var numCalls = 0
     var tempFileManager: DownloadFileManager = null
diff --git a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala
index ccc525e854838..c757dee43808d 100644
--- a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala
@@ -51,7 +51,7 @@ class DiskBlockManagerSuite extends SparkFunSuite with BeforeAndAfterEach with B
   override def beforeEach(): Unit = {
     super.beforeEach()
     val conf = testConf.clone
-    conf.set("spark.local.dir", rootDirs).set("spark.diskStore.subDirectories", "1")
+    conf.set("spark.local.dir", rootDirs)
     diskBlockManager = new DiskBlockManager(conf, deleteFilesOnStop = true)
   }
 
@@ -90,45 +90,4 @@ class DiskBlockManagerSuite extends SparkFunSuite with BeforeAndAfterEach with B
     for (i <- 0 until numBytes) writer.write(i)
     writer.close()
   }
-
-  test("temporary shuffle/local file should be able to handle disk failures") {
-    try {
-      // the following two lines pre-create subdirectories under each root dir of block manager
-      diskBlockManager.getFile("1")
-      diskBlockManager.getFile("2")
-
-      val tempShuffleFile1 = diskBlockManager.createTempShuffleBlock()._2
-      val tempLocalFile1 = diskBlockManager.createTempLocalBlock()._2
-      assert(tempShuffleFile1.exists(), "There are no bad disks, so temp shuffle file exists")
-      assert(tempLocalFile1.exists(), "There are no bad disks, so temp local file exists")
-
-      // partial disks damaged
-      rootDir0.setExecutable(false)
-      val tempShuffleFile2 = diskBlockManager.createTempShuffleBlock()._2
-      val tempLocalFile2 = diskBlockManager.createTempLocalBlock()._2
-      // It's possible that after 10 retries we still not able to find the healthy disk. we need to
-      // remove the flakiness of these two asserts
-      if (tempShuffleFile2.getParentFile.getParentFile.getParent === rootDir1.getAbsolutePath) {
-        assert(tempShuffleFile2.exists(),
-          "There is only one bad disk, so temp shuffle file should be created")
-      }
-      if (tempLocalFile2.getParentFile.getParentFile.getParent === rootDir1.getAbsolutePath) {
-        assert(tempLocalFile2.exists(),
-          "There is only one bad disk, so temp local file should be created")
-      }
-
-      // all disks damaged
-      rootDir1.setExecutable(false)
-      val tempShuffleFile3 = diskBlockManager.createTempShuffleBlock()._2
-      val tempLocalFile3 = diskBlockManager.createTempLocalBlock()._2
-      assert(!tempShuffleFile3.exists(),
-        "All disks are broken, so there should be no temp shuffle file created")
-      assert(!tempLocalFile3.exists(),
-        "All disks are broken, so there should be no temp local file created")
-    } finally {
-      rootDir0.setExecutable(true)
-      rootDir1.setExecutable(true)
-    }
-
-  }
 }
diff --git a/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala b/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala
index ccd7e4b62ad9e..c46ab2d199f0b 100644
--- a/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala
@@ -26,6 +26,7 @@ import org.scalatest._
 
 import org.apache.spark._
 import org.apache.spark.internal.config._
+import org.apache.spark.internal.config.Tests.TEST_USE_COMPRESSED_OOPS_KEY
 import org.apache.spark.memory.{MemoryMode, UnifiedMemoryManager}
 import org.apache.spark.serializer.{KryoSerializer, SerializerManager}
 import org.apache.spark.storage.memory.{BlockEvictionHandler, MemoryStore, PartiallySerializedBlock, PartiallyUnrolledIterator}
@@ -51,12 +52,34 @@ class MemoryStoreSuite
   implicit def StringToBlockId(value: String): BlockId = new TestBlockId(value)
   def rdd(rddId: Int, splitId: Int): RDDBlockId = RDDBlockId(rddId, splitId)
 
+  // Save modified system properties so that we can restore them after tests.
+  val originalArch = System.getProperty("os.arch")
+  val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY)
+
+  def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = {
+    def set(k: String, v: String): Unit = {
+      if (v == null) {
+        System.clearProperty(k)
+      } else {
+        System.setProperty(k, v)
+      }
+    }
+    set("os.arch", arch)
+    set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops)
+    val initialize = PrivateMethod[Unit](Symbol("initialize"))
+    SizeEstimator invokePrivate initialize()
+  }
+
   override def beforeEach(): Unit = {
     super.beforeEach()
     // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case
-    System.setProperty("os.arch", "amd64")
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
+    reinitializeSizeEstimator("amd64", "true")
+  }
+
+  override def afterEach(): Unit = {
+    super.afterEach()
+    // Restore system properties and SizeEstimator to their original states.
+    reinitializeSizeEstimator(originalArch, originalCompressedOops)
   }
 
   def makeMemoryStore(maxMem: Long): (MemoryStore, BlockInfoManager) = {
diff --git a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
index 45f47c7c49bca..43917a5b83bb0 100644
--- a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
@@ -36,6 +36,7 @@ import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer}
 import org.apache.spark.network.shuffle.{BlockFetchingListener, DownloadFileManager, ExternalBlockStoreClient}
 import org.apache.spark.network.util.LimitedInputStream
 import org.apache.spark.shuffle.FetchFailedException
+import org.apache.spark.storage.ShuffleBlockFetcherIterator.FetchBlockInfo
 import org.apache.spark.util.Utils
 
 
@@ -254,6 +255,98 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
     intercept[FetchFailedException] { iterator.next() }
   }
 
+  test("Hit maxBytesInFlight limitation before maxBlocksInFlightPerAddress") {
+    val blockManager = mock(classOf[BlockManager])
+    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
+    doReturn(localBmId).when(blockManager).blockManagerId
+
+    val remoteBmId1 = BlockManagerId("test-remote-client-1", "test-remote-host1", 1)
+    val remoteBmId2 = BlockManagerId("test-remote-client-2", "test-remote-host2", 2)
+    val blockId1 = ShuffleBlockId(0, 1, 0)
+    val blockId2 = ShuffleBlockId(1, 1, 0)
+    val blocksByAddress = Seq(
+      (remoteBmId1, Seq((blockId1, 1000L, 0))),
+      (remoteBmId2, Seq((blockId2, 1000L, 0)))).toIterator
+    val transfer = createMockTransfer(Map(
+      blockId1 -> createMockManagedBuffer(1000),
+      blockId2 -> createMockManagedBuffer(1000)))
+    val taskContext = TaskContext.empty()
+    val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
+    val iterator = new ShuffleBlockFetcherIterator(
+      taskContext,
+      transfer,
+      blockManager,
+      blocksByAddress,
+      (_, in) => in,
+      1000L, // allow 1 FetchRequests at most at the same time
+      Int.MaxValue,
+      Int.MaxValue, // set maxBlocksInFlightPerAddress to Int.MaxValue
+      Int.MaxValue,
+      true,
+      false,
+      metrics,
+      false)
+    // After initialize() we'll have 2 FetchRequests and each is 1000 bytes. So only the
+    // first FetchRequests can be sent, and the second one will hit maxBytesInFlight so
+    // it won't be sent.
+    verify(transfer, times(1)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    assert(iterator.hasNext)
+    // next() will trigger off sending deferred request
+    iterator.next()
+    // the second FetchRequest should be sent at this time
+    verify(transfer, times(2)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    assert(iterator.hasNext)
+    iterator.next()
+    assert(!iterator.hasNext)
+  }
+
+  test("Hit maxBlocksInFlightPerAddress limitation before maxBytesInFlight") {
+    val blockManager = mock(classOf[BlockManager])
+    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
+    doReturn(localBmId).when(blockManager).blockManagerId
+
+    val remoteBmId = BlockManagerId("test-remote-client-1", "test-remote-host", 2)
+    val blockId1 = ShuffleBlockId(0, 1, 0)
+    val blockId2 = ShuffleBlockId(0, 2, 0)
+    val blockId3 = ShuffleBlockId(0, 3, 0)
+    val blocksByAddress = Seq((remoteBmId,
+      Seq((blockId1, 1000L, 0), (blockId2, 1000L, 0), (blockId3, 1000L, 0)))).toIterator
+    val transfer = createMockTransfer(Map(
+      blockId1 -> createMockManagedBuffer(),
+      blockId2 -> createMockManagedBuffer(),
+      blockId3 -> createMockManagedBuffer()))
+    val taskContext = TaskContext.empty()
+    val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
+    val iterator = new ShuffleBlockFetcherIterator(
+      taskContext,
+      transfer,
+      blockManager,
+      blocksByAddress,
+      (_, in) => in,
+      Int.MaxValue, // set maxBytesInFlight to Int.MaxValue
+      Int.MaxValue,
+      2, // set maxBlocksInFlightPerAddress to 2
+      Int.MaxValue,
+      true,
+      false,
+      metrics,
+      false)
+    // After initialize(), we'll have 2 FetchRequests that one has 2 blocks inside and another one
+    // has only one block. So only the first FetchRequest can be sent. The second FetchRequest will
+    // hit maxBlocksInFlightPerAddress so it won't be sent.
+    verify(transfer, times(1)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    // the first request packaged 2 blocks, so we also need to
+    // call next() for 2 times to exhaust the iterator.
+    assert(iterator.hasNext)
+    iterator.next()
+    assert(iterator.hasNext)
+    iterator.next()
+    verify(transfer, times(2)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    assert(iterator.hasNext)
+    iterator.next()
+    assert(!iterator.hasNext)
+  }
+
   test("fetch continuous blocks in batch successful 3 local + 4 host local + 2 remote reads") {
     val blockManager = mock(classOf[BlockManager])
     val localBmId = BlockManagerId("test-client", "test-local-host", 1)
@@ -341,32 +434,86 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
     assert(blockManager.hostLocalDirManager.get.getCachedHostLocalDirs().size === 1)
   }
 
-  test("fetch continuous blocks in batch respects maxSize and maxBlocks") {
+  test("fetch continuous blocks in batch should respect maxBytesInFlight") {
     val blockManager = mock(classOf[BlockManager])
     val localBmId = BlockManagerId("test-client", "test-local-host", 1)
     doReturn(localBmId).when(blockManager).blockManagerId
 
     // Make sure remote blocks would return the merged block
-    val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2)
-    val remoteBlocks = Seq[BlockId](
+    val remoteBmId1 = BlockManagerId("test-client-1", "test-client-1", 1)
+    val remoteBmId2 = BlockManagerId("test-client-2", "test-client-2", 2)
+    val remoteBlocks1 = (0 until 15).map(ShuffleBlockId(0, 3, _))
+    val remoteBlocks2 = Seq[BlockId](ShuffleBlockId(0, 4, 0), ShuffleBlockId(0, 4, 1))
+    val mergedRemoteBlocks = Map[BlockId, ManagedBuffer](
+      ShuffleBlockBatchId(0, 3, 0, 3) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 3, 3, 6) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 3, 6, 9) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 3, 9, 12) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 3, 12, 15) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 4, 0, 2) -> createMockManagedBuffer())
+    val transfer = createMockTransfer(mergedRemoteBlocks)
+
+    val blocksByAddress = Seq[(BlockManagerId, Seq[(BlockId, Long, Int)])](
+      (remoteBmId1, remoteBlocks1.map(blockId => (blockId, 100L, 1))),
+      (remoteBmId2, remoteBlocks2.map(blockId => (blockId, 100L, 1)))).toIterator
+
+    val taskContext = TaskContext.empty()
+    val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
+    val iterator = new ShuffleBlockFetcherIterator(
+      taskContext,
+      transfer,
+      blockManager,
+      blocksByAddress,
+      (_, in) => in,
+      1500,
+      Int.MaxValue,
+      Int.MaxValue,
+      Int.MaxValue,
+      true,
+      false,
+      metrics,
+      true)
+
+    var numResults = 0
+    // After initialize(), there will be 6 FetchRequests. And each of the first 5 requests
+    // includes 1 merged block which is merged from 3 shuffle blocks. The last request has 1 merged
+    // block which merged from 2 shuffle blocks. So, only the first 5 requests(5 * 3 * 100 >= 1500)
+    // can be sent. The 6th FetchRequest will hit maxBlocksInFlightPerAddress so it won't
+    // be sent.
+    verify(transfer, times(5)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    while (iterator.hasNext) {
+      val (blockId, inputStream) = iterator.next()
+      // Make sure we release buffers when a wrapped input stream is closed.
+      val mockBuf = mergedRemoteBlocks(blockId)
+      verifyBufferRelease(mockBuf, inputStream)
+      numResults += 1
+    }
+    // The 6th request will be sent after next() is called.
+    verify(transfer, times(6)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    assert(numResults == 6)
+  }
+
+  test("fetch continuous blocks in batch should respect maxBlocksInFlightPerAddress") {
+    val blockManager = mock(classOf[BlockManager])
+    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
+    doReturn(localBmId).when(blockManager).blockManagerId
+
+    // Make sure remote blocks would return the merged block
+    val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 1)
+    val remoteBlocks = Seq(
       ShuffleBlockId(0, 3, 0),
       ShuffleBlockId(0, 3, 1),
-      ShuffleBlockId(0, 3, 2),
       ShuffleBlockId(0, 4, 0),
       ShuffleBlockId(0, 4, 1),
-      ShuffleBlockId(0, 5, 0),
-      ShuffleBlockId(0, 5, 1),
-      ShuffleBlockId(0, 5, 2))
+      ShuffleBlockId(0, 5, 0))
     val mergedRemoteBlocks = Map[BlockId, ManagedBuffer](
-      ShuffleBlockBatchId(0, 3, 0, 3) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 3, 0, 2) -> createMockManagedBuffer(),
       ShuffleBlockBatchId(0, 4, 0, 2) -> createMockManagedBuffer(),
-      ShuffleBlockBatchId(0, 5, 0, 3) -> createMockManagedBuffer())
-    val transfer = createMockTransfer(mergedRemoteBlocks)
+      ShuffleBlockBatchId(0, 5, 0, 1) -> createMockManagedBuffer())
 
+    val transfer = createMockTransfer(mergedRemoteBlocks)
     val blocksByAddress = Seq[(BlockManagerId, Seq[(BlockId, Long, Int)])](
-      (remoteBmId, remoteBlocks.map(blockId => (blockId, 1L, 1)))
-    ).toIterator
-
+      (remoteBmId, remoteBlocks.map(blockId => (blockId, 100L, 1)))).toIterator
     val taskContext = TaskContext.empty()
     val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
     val iterator = new ShuffleBlockFetcherIterator(
@@ -375,7 +522,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
       blockManager,
       blocksByAddress,
       (_, in) => in,
-      35,
+      Int.MaxValue,
       Int.MaxValue,
       2,
       Int.MaxValue,
@@ -383,8 +530,12 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
       false,
       metrics,
       true)
-
     var numResults = 0
+    // After initialize(), there will be 2 FetchRequests. First one has 2 merged blocks and each
+    // of them is merged from 2 shuffle blocks, second one has 1 merged block which is merged from
+    // 1 shuffle block. So only the first FetchRequest can be sent. The second FetchRequest will
+    // hit maxBlocksInFlightPerAddress so it won't be sent.
+    verify(transfer, times(1)).fetchBlocks(any(), any(), any(), any(), any(), any())
     while (iterator.hasNext) {
       val (blockId, inputStream) = iterator.next()
       // Make sure we release buffers when a wrapped input stream is closed.
@@ -392,8 +543,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
       verifyBufferRelease(mockBuf, inputStream)
       numResults += 1
     }
-    // The first 2 batch block ids are in the same fetch request as they don't exceed the max size
-    // and max blocks, so 2 requests in total.
+    // The second request will be sent after next() is called.
     verify(transfer, times(2)).fetchBlocks(any(), any(), any(), any(), any(), any())
     assert(numResults == 3)
   }
@@ -922,4 +1072,23 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
     val e = intercept[FetchFailedException] { iterator.next() }
     assert(e.getMessage.contains("Received a zero-size buffer"))
   }
+
+  test("SPARK-31521: correct the fetch size when merging blocks into a merged block") {
+    val bId1 = ShuffleBlockBatchId(0, 0, 0, 5)
+    val bId2 = ShuffleBlockId(0, 0, 6)
+    val bId3 = ShuffleBlockId(0, 0, 7)
+    val block1 = FetchBlockInfo(bId1, 40, 0)
+    val block2 = FetchBlockInfo(bId2, 50, 0)
+    val block3 = FetchBlockInfo(bId3, 60, 0)
+    val inputBlocks = Seq(block1, block2, block3)
+
+    val mergedBlocks = ShuffleBlockFetcherIterator.
+      mergeContinuousShuffleBlockIdsIfNeeded(inputBlocks, true)
+    assert(mergedBlocks.size === 1)
+    val mergedBlock = mergedBlocks.head
+    val mergedBlockId = mergedBlock.blockId.asInstanceOf[ShuffleBlockBatchId]
+    assert(mergedBlockId.startReduceId === bId1.startReduceId)
+    assert(mergedBlockId.endReduceId === bId3.reduceId + 1)
+    assert(mergedBlock.size === inputBlocks.map(_.size).sum)
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
new file mode 100644
index 0000000000000..9ba705c4abd75
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+import org.openqa.selenium.WebDriver
+import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions}
+
+import org.apache.spark.tags.ChromeUITest
+
+/**
+ * Selenium tests for the Spark Web UI with Chrome.
+ */
+@ChromeUITest
+class ChromeUISeleniumSuite extends RealBrowserUISeleniumSuite("webdriver.chrome.driver") {
+
+  override var webDriver: WebDriver = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    val chromeOptions = new ChromeOptions
+    chromeOptions.addArguments("--headless", "--disable-gpu")
+    webDriver = new ChromeDriver(chromeOptions)
+  }
+
+  override def afterAll(): Unit = {
+    try {
+      if (webDriver != null) {
+        webDriver.quit()
+      }
+    } finally {
+      super.afterAll()
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/ui/PagedTableSuite.scala b/core/src/test/scala/org/apache/spark/ui/PagedTableSuite.scala
index d18f55474bdb3..4125436907007 100644
--- a/core/src/test/scala/org/apache/spark/ui/PagedTableSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/PagedTableSuite.scala
@@ -85,6 +85,35 @@ class PagedTableSuite extends SparkFunSuite {
     assert((pagedTable.pageNavigation(93, 10, 97).head \\ "li").map(_.text.trim) ===
       Seq("<<", "<") ++ (91 to 97).map(_.toString) ++ Seq(">"))
   }
+
+  test("pageNavigation with different id") {
+    val pagedTable = new PagedTable[Int] {
+      override def tableId: String = "testTable"
+
+      override def tableCssClass: String = ""
+
+      override def dataSource: PagedDataSource[Int] = null
+
+      override def pageLink(page: Int): String = ""
+
+      override def headers: Seq[Node] = Nil
+
+      override def row(t: Int): Seq[Node] = Nil
+
+      override def pageSizeFormField: String = ""
+
+      override def pageNumberFormField: String = ""
+
+      override def goButtonFormPath: String = ""
+    }
+
+    val defaultIdNavigation = pagedTable.pageNavigation(1, 10, 2).head \\ "form"
+    assert(defaultIdNavigation \@ "id" === "form-testTable-page")
+
+    val customIdNavigation = pagedTable.pageNavigation(1, 10, 2, "customIdTable").head \\ "form"
+    assert(customIdNavigation \@ "id" === "form-customIdTable-page")
+    assert(defaultIdNavigation !== customIdNavigation)
+  }
 }
 
 private[spark] class SeqPagedDataSource[T](seq: Seq[T], pageSize: Int)
diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
new file mode 100644
index 0000000000000..4b018f69b1660
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+import org.openqa.selenium.{By, WebDriver}
+import org.scalatest._
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.time.SpanSugar._
+import org.scalatestplus.selenium.WebBrowser
+
+import org.apache.spark._
+import org.apache.spark.LocalSparkContext.withSpark
+import org.apache.spark.internal.config.MEMORY_OFFHEAP_SIZE
+import org.apache.spark.internal.config.UI.{UI_ENABLED, UI_KILL_ENABLED, UI_PORT}
+import org.apache.spark.util.CallSite
+
+/**
+ * Selenium tests for the Spark Web UI with real web browsers.
+ */
+abstract class RealBrowserUISeleniumSuite(val driverProp: String)
+  extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll {
+
+  implicit var webDriver: WebDriver
+  private val driverPropPrefix = "spark.test."
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    assume(
+      sys.props(driverPropPrefix + driverProp) !== null,
+      "System property " + driverPropPrefix + driverProp +
+        " should be set to the corresponding driver path.")
+    sys.props(driverProp) = sys.props(driverPropPrefix + driverProp)
+  }
+
+  override def afterAll(): Unit = {
+    sys.props.remove(driverProp)
+    super.afterAll()
+  }
+
+  test("SPARK-31534: text for tooltip should be escaped") {
+    withSpark(newSparkContext()) { sc =>
+      sc.setLocalProperty(CallSite.LONG_FORM, "collect at <console>:25")
+      sc.setLocalProperty(CallSite.SHORT_FORM, "collect at <console>:25")
+      sc.parallelize(1 to 10).collect
+
+      eventually(timeout(10.seconds), interval(50.milliseconds)) {
+        goToUi(sc, "/jobs")
+
+        val jobDesc =
+          webDriver.findElement(By.cssSelector("div[class='application-timeline-content']"))
+        jobDesc.getAttribute("data-title") should include  ("collect at &lt;console&gt;:25")
+
+        goToUi(sc, "/jobs/job/?id=0")
+        webDriver.get(sc.ui.get.webUrl.stripSuffix("/") + "/jobs/job/?id=0")
+        val stageDesc = webDriver.findElement(By.cssSelector("div[class='job-timeline-content']"))
+        stageDesc.getAttribute("data-title") should include ("collect at &lt;console&gt;:25")
+
+        // Open DAG Viz.
+        webDriver.findElement(By.id("job-dag-viz")).click()
+        val nodeDesc = webDriver.findElement(By.cssSelector("g[class='node_0 node']"))
+        nodeDesc.getAttribute("name") should include ("collect at &lt;console&gt;:25")
+      }
+    }
+  }
+
+  test("SPARK-31882: Link URL for Stage DAGs should not depend on paged table.") {
+    withSpark(newSparkContext()) { sc =>
+      sc.parallelize(1 to 100).map(v => (v, v)).repartition(10).reduceByKey(_ + _).collect
+
+      eventually(timeout(10.seconds), interval(50.microseconds)) {
+        val pathWithPagedTable =
+          "/jobs/job/?id=0&completedStage.page=2&completedStage.sort=Stage+Id&" +
+            "completedStage.desc=true&completedStage.pageSize=1#completed"
+        goToUi(sc, pathWithPagedTable)
+
+        // Open DAG Viz.
+        webDriver.findElement(By.id("job-dag-viz")).click()
+        val stages = webDriver.findElements(By.cssSelector("svg[class='job'] > a"))
+        stages.size() should be (3)
+
+        stages.get(0).getAttribute("href") should include ("/stages/stage/?id=0&attempt=0")
+        stages.get(1).getAttribute("href") should include ("/stages/stage/?id=1&attempt=0")
+        stages.get(2).getAttribute("href") should include ("/stages/stage/?id=2&attempt=0")
+      }
+    }
+  }
+
+  test("SPARK-31886: Color barrier execution mode RDD correctly") {
+    withSpark(newSparkContext()) { sc =>
+      sc.parallelize(1 to 10).barrier.mapPartitions(identity).repartition(1).collect()
+
+      eventually(timeout(10.seconds), interval(50.milliseconds)) {
+        goToUi(sc, "/jobs/job/?id=0")
+        webDriver.findElement(By.id("job-dag-viz")).click()
+
+        val stage0 = webDriver.findElement(By.cssSelector("g[id='graph_0']"))
+        val stage1 = webDriver.findElement(By.cssSelector("g[id='graph_1']"))
+        val barrieredOps = webDriver.findElements(By.className("barrier-rdd")).iterator()
+
+        while (barrieredOps.hasNext) {
+          val barrieredOpId = barrieredOps.next().getAttribute("innerHTML")
+          val foundInStage0 =
+            stage0.findElements(
+              By.cssSelector("g.barrier.cluster.cluster_" + barrieredOpId))
+          assert(foundInStage0.size === 1)
+
+          val foundInStage1 =
+            stage1.findElements(
+              By.cssSelector("g.barrier.cluster.cluster_" + barrieredOpId))
+          assert(foundInStage1.size === 0)
+        }
+      }
+    }
+  }
+
+  /**
+   * Create a test SparkContext with the SparkUI enabled.
+   * It is safe to `get` the SparkUI directly from the SparkContext returned here.
+   */
+  private def newSparkContext(
+      killEnabled: Boolean = true,
+      master: String = "local",
+      additionalConfs: Map[String, String] = Map.empty): SparkContext = {
+    val conf = new SparkConf()
+      .setMaster(master)
+      .setAppName("test")
+      .set(UI_ENABLED, true)
+      .set(UI_PORT, 0)
+      .set(UI_KILL_ENABLED, killEnabled)
+      .set(MEMORY_OFFHEAP_SIZE.key, "64m")
+    additionalConfs.foreach { case (k, v) => conf.set(k, v) }
+    val sc = new SparkContext(conf)
+    assert(sc.ui.isDefined)
+    sc
+  }
+
+  def goToUi(sc: SparkContext, path: String): Unit = {
+    goToUi(sc.ui.get, path)
+  }
+
+  def goToUi(ui: SparkUI, path: String): Unit = {
+    go to (ui.webUrl.stripSuffix("/") + path)
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 9f0cdeac9ca39..909056eab8c5a 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -44,10 +44,11 @@ import org.apache.spark.internal.config.Status._
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.shuffle.FetchFailedException
 import org.apache.spark.status.api.v1.{JacksonMessageWriter, RDDDataDistribution, StageStatus}
+import org.apache.spark.util.CallSite
 
 private[spark] class SparkUICssErrorHandler extends DefaultCssErrorHandler {
 
-  private val cssWhiteList = List("bootstrap.min.css", "vis.min.css")
+  private val cssWhiteList = List("bootstrap.min.css", "vis-timeline-graph2d.min.css")
 
   private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith)
 
@@ -687,29 +688,29 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
         assert(stage0.contains("digraph G {\n  subgraph clusterstage_0 {\n    " +
           "label=&quot;Stage 0&quot;;\n    subgraph "))
         assert(stage0.contains("{\n      label=&quot;parallelize&quot;;\n      " +
-          "0 [label=&quot;ParallelCollectionRDD [0]"))
+          "0 [labelType=&quot;html&quot; label=&quot;ParallelCollectionRDD [0]"))
         assert(stage0.contains("{\n      label=&quot;map&quot;;\n      " +
-          "1 [label=&quot;MapPartitionsRDD [1]"))
+          "1 [labelType=&quot;html&quot; label=&quot;MapPartitionsRDD [1]"))
         assert(stage0.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-          "2 [label=&quot;MapPartitionsRDD [2]"))
+          "2 [labelType=&quot;html&quot; label=&quot;MapPartitionsRDD [2]"))
 
         val stage1 = Source.fromURL(sc.ui.get.webUrl +
           "/stages/stage/?id=1&attempt=0&expandDagViz=true").mkString
         assert(stage1.contains("digraph G {\n  subgraph clusterstage_1 {\n    " +
           "label=&quot;Stage 1&quot;;\n    subgraph "))
         assert(stage1.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-          "3 [label=&quot;ShuffledRDD [3]"))
+          "3 [labelType=&quot;html&quot; label=&quot;ShuffledRDD [3]"))
         assert(stage1.contains("{\n      label=&quot;map&quot;;\n      " +
-          "4 [label=&quot;MapPartitionsRDD [4]"))
+          "4 [labelType=&quot;html&quot; label=&quot;MapPartitionsRDD [4]"))
         assert(stage1.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-          "5 [label=&quot;MapPartitionsRDD [5]"))
+          "5 [labelType=&quot;html&quot; label=&quot;MapPartitionsRDD [5]"))
 
         val stage2 = Source.fromURL(sc.ui.get.webUrl +
           "/stages/stage/?id=2&attempt=0&expandDagViz=true").mkString
         assert(stage2.contains("digraph G {\n  subgraph clusterstage_2 {\n    " +
           "label=&quot;Stage 2&quot;;\n    subgraph "))
         assert(stage2.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-          "6 [label=&quot;ShuffledRDD [6]"))
+          "6 [labelType=&quot;html&quot; label=&quot;ShuffledRDD [6]"))
       }
     }
   }
diff --git a/core/src/test/scala/org/apache/spark/ui/UISuite.scala b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
index 2ad4a634cd9a7..c7e1dfe71d563 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
@@ -216,6 +216,15 @@ class UISuite extends SparkFunSuite {
     assert(rewrittenURI === null)
   }
 
+  test("SPARK-33611: Avoid encoding twice on the query parameter of proxy rewrittenURI") {
+    val prefix = "/worker-id"
+    val target = "http://localhost:8081"
+    val path = "/worker-id/json"
+    val rewrittenURI =
+      JettyUtils.createProxyURI(prefix, target, path, "order%5B0%5D%5Bcolumn%5D=0")
+    assert(rewrittenURI.toString === "http://localhost:8081/json?order%5B0%5D%5Bcolumn%5D=0")
+  }
+
   test("verify rewriting location header for reverse proxy") {
     val clientRequest = mock(classOf[HttpServletRequest])
     var headerValue = "http://localhost:4040/jobs"
@@ -262,6 +271,27 @@ class UISuite extends SparkFunSuite {
     }
   }
 
+  test("SPARK-32467: Avoid encoding URL twice on https redirect") {
+    val (conf, securityMgr, sslOptions) = sslEnabledConf()
+    val serverInfo = JettyUtils.startJettyServer("0.0.0.0", 0, sslOptions, conf)
+    try {
+      val serverAddr = s"http://localhost:${serverInfo.boundPort}"
+
+      val (_, ctx) = newContext("/ctx1")
+      serverInfo.addHandler(ctx, securityMgr)
+
+      TestUtils.withHttpConnection(new URL(s"$serverAddr/ctx%281%29?a%5B0%5D=b")) { conn =>
+        assert(conn.getResponseCode() === HttpServletResponse.SC_FOUND)
+        val location = Option(conn.getHeaderFields().get("Location"))
+          .map(_.get(0)).orNull
+        val expectedLocation = s"https://localhost:${serverInfo.securePort.get}/ctx(1)?a[0]=b"
+        assert(location == expectedLocation)
+      }
+    } finally {
+      stopServer(serverInfo)
+    }
+  }
+
   test("http -> https redirect applies to all URIs") {
     val (conf, securityMgr, sslOptions) = sslEnabledConf()
     val serverInfo = JettyUtils.startJettyServer("0.0.0.0", 0, sslOptions, conf)
diff --git a/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala b/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
index 21e69550785a4..12d97573ff6ee 100644
--- a/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
@@ -337,7 +337,7 @@ class FileAppenderSuite extends SparkFunSuite with BeforeAndAfter with Logging {
     assert(generatedFiles.size > 1)
     if (isCompressed) {
       assert(
-        generatedFiles.filter(_.getName.endsWith(RollingFileAppender.GZIP_LOG_SUFFIX)).size > 0)
+        generatedFiles.exists(_.getName.endsWith(RollingFileAppender.GZIP_LOG_SUFFIX)))
     }
     val allText = generatedFiles.map { file =>
       if (file.getName.endsWith(RollingFileAppender.GZIP_LOG_SUFFIX)) {
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index a2a4b3aa974fc..b77cd810a0e61 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -305,6 +305,17 @@ class JsonProtocolSuite extends SparkFunSuite {
     assert(expectedFetchFailed === JsonProtocol.taskEndReasonFromJson(oldEvent))
   }
 
+  test("SPARK-32124: FetchFailed Map Index backwards compatibility") {
+    // FetchFailed in Spark 2.4.0 does not have "Map Index" property.
+    val fetchFailed = FetchFailed(BlockManagerId("With or", "without you", 15), 17, 16L, 18, 19,
+      "ignored")
+    val oldEvent = JsonProtocol.taskEndReasonToJson(fetchFailed)
+      .removeField({ _._1 == "Map Index" })
+    val expectedFetchFailed = FetchFailed(BlockManagerId("With or", "without you", 15), 17, 16L,
+      Int.MinValue, 19, "ignored")
+    assert(expectedFetchFailed === JsonProtocol.taskEndReasonFromJson(oldEvent))
+  }
+
   test("ShuffleReadMetrics: Local bytes read backwards compatibility") {
     // Metrics about local shuffle bytes read were added in 1.3.1.
     val metrics = makeTaskMetrics(1L, 2L, 3L, 4L, 5, 6,
@@ -481,6 +492,76 @@ class JsonProtocolSuite extends SparkFunSuite {
     testAccumValue(Some("anything"), blocks, JString(blocks.toString))
     testAccumValue(Some("anything"), 123, JString("123"))
   }
+
+  /** Create an AccumulableInfo and verify we can serialize and deserialize it. */
+  private def testAccumulableInfo(
+      name: String,
+      value: Option[Any],
+      expectedValue: Option[Any]): Unit = {
+    val isInternal = name.startsWith(InternalAccumulator.METRICS_PREFIX)
+    val accum = AccumulableInfo(
+      123L,
+      Some(name),
+      update = value,
+      value = value,
+      internal = isInternal,
+      countFailedValues = false)
+    val json = JsonProtocol.accumulableInfoToJson(accum)
+    val newAccum = JsonProtocol.accumulableInfoFromJson(json)
+    assert(newAccum == accum.copy(update = expectedValue, value = expectedValue))
+  }
+
+  test("SPARK-31923: unexpected value type of internal accumulator") {
+    // Because a user may use `METRICS_PREFIX` in an accumulator name, we should test unexpected
+    // types to make sure we don't crash.
+    import InternalAccumulator.METRICS_PREFIX
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooString",
+      value = Some("foo"),
+      expectedValue = None)
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooList",
+      value = Some(java.util.Arrays.asList("string")),
+      expectedValue = Some(java.util.Collections.emptyList())
+    )
+    val blocks = Seq(
+      (TestBlockId("block1"), BlockStatus(StorageLevel.MEMORY_ONLY, 1L, 2L)),
+      (TestBlockId("block2"), BlockStatus(StorageLevel.DISK_ONLY, 3L, 4L)))
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooList",
+      value = Some(java.util.Arrays.asList(
+        "string",
+        blocks(0),
+        blocks(1))),
+      expectedValue = Some(blocks.asJava)
+    )
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooSet",
+      value = Some(Set("foo")),
+      expectedValue = None)
+  }
+
+  test("SPARK-30936: forwards compatibility - ignore unknown fields") {
+    val expected = TestListenerEvent("foo", 123)
+    val unknownFieldsJson =
+      """{
+        |  "Event" : "org.apache.spark.util.TestListenerEvent",
+        |  "foo" : "foo",
+        |  "bar" : 123,
+        |  "unknown" : "unknown"
+        |}""".stripMargin
+    assert(JsonProtocol.sparkEventFromJson(parse(unknownFieldsJson)) === expected)
+  }
+
+  test("SPARK-30936: backwards compatibility - set default values for missing fields") {
+    val expected = TestListenerEvent("foo", 0)
+    val unknownFieldsJson =
+      """{
+        |  "Event" : "org.apache.spark.util.TestListenerEvent",
+        |  "foo" : "foo"
+        |}""".stripMargin
+    assert(JsonProtocol.sparkEventFromJson(parse(unknownFieldsJson)) === expected)
+  }
 }
 
 
@@ -1041,6 +1122,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |          "Deserialized": true,
       |          "Replication": 1
       |        },
+      |        "Barrier" : false,
       |        "Number of Partitions": 201,
       |        "Number of Cached Partitions": 301,
       |        "Memory Size": 401,
@@ -1563,6 +1645,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 200,
       |          "Number of Cached Partitions": 300,
       |          "Memory Size": 400,
@@ -1607,6 +1690,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 400,
       |          "Number of Cached Partitions": 600,
       |          "Memory Size": 800,
@@ -1623,6 +1707,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 401,
       |          "Number of Cached Partitions": 601,
       |          "Memory Size": 801,
@@ -1667,6 +1752,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 600,
       |          "Number of Cached Partitions": 900,
       |          "Memory Size": 1200,
@@ -1683,6 +1769,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 601,
       |          "Number of Cached Partitions": 901,
       |          "Memory Size": 1201,
@@ -1699,6 +1786,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 602,
       |          "Number of Cached Partitions": 902,
       |          "Memory Size": 1202,
@@ -1743,6 +1831,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 800,
       |          "Number of Cached Partitions": 1200,
       |          "Memory Size": 1600,
@@ -1759,6 +1848,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 801,
       |          "Number of Cached Partitions": 1201,
       |          "Memory Size": 1601,
@@ -1775,6 +1865,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 802,
       |          "Number of Cached Partitions": 1202,
       |          "Memory Size": 1602,
@@ -1791,6 +1882,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 803,
       |          "Number of Cached Partitions": 1203,
       |          "Memory Size": 1603,
@@ -2310,3 +2402,5 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |}
     """.stripMargin
 }
+
+case class TestListenerEvent(foo: String, bar: Int) extends SparkListenerEvent
diff --git a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
index d4f2053e0b2f4..6183ba9faa6b4 100644
--- a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
@@ -73,15 +73,35 @@ class SizeEstimatorSuite
   with PrivateMethodTester
   with ResetSystemProperties {
 
+  // Save modified system properties so that we can restore them after tests.
+  val originalArch = System.getProperty("os.arch")
+  val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY)
+
+  def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = {
+    def set(k: String, v: String): Unit = {
+      if (v == null) {
+        System.clearProperty(k)
+      } else {
+        System.setProperty(k, v)
+      }
+    }
+    set("os.arch", arch)
+    set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops)
+    val initialize = PrivateMethod[Unit](Symbol("initialize"))
+    SizeEstimator invokePrivate initialize()
+  }
+
   override def beforeEach(): Unit = {
-    // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case
     super.beforeEach()
-    System.setProperty("os.arch", "amd64")
-    System.setProperty(TEST_USE_COMPRESSED_OOPS_KEY, "true")
+    // Set the arch to 64-bit and compressedOops to true so that SizeEstimator
+    // provides identical results accross all systems in these tests.
+    reinitializeSizeEstimator("amd64", "true")
   }
 
   override def afterEach(): Unit = {
     super.afterEach()
+    // Restore system properties and SizeEstimator to their original states.
+    reinitializeSizeEstimator(originalArch, originalCompressedOops)
   }
 
   test("simple classes") {
@@ -178,11 +198,7 @@ class SizeEstimatorSuite
   }
 
   test("32-bit arch") {
-    System.setProperty("os.arch", "x86")
-
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
-
+    reinitializeSizeEstimator("x86", "true")
     assertResult(40)(SizeEstimator.estimate(DummyString("")))
     assertResult(48)(SizeEstimator.estimate(DummyString("a")))
     assertResult(48)(SizeEstimator.estimate(DummyString("ab")))
@@ -192,11 +208,7 @@ class SizeEstimatorSuite
   // NOTE: The String class definition varies across JDK versions (1.6 vs. 1.7) and vendors
   // (Sun vs IBM). Use a DummyString class to make tests deterministic.
   test("64-bit arch with no compressed oops") {
-    System.setProperty("os.arch", "amd64")
-    System.setProperty(TEST_USE_COMPRESSED_OOPS_KEY, "false")
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
-
+    reinitializeSizeEstimator("amd64", "false")
     assertResult(56)(SizeEstimator.estimate(DummyString("")))
     assertResult(64)(SizeEstimator.estimate(DummyString("a")))
     assertResult(64)(SizeEstimator.estimate(DummyString("ab")))
@@ -214,14 +226,13 @@ class SizeEstimatorSuite
   }
 
   test("class field blocks rounding on 64-bit VM without useCompressedOops") {
+    reinitializeSizeEstimator("amd64", "false")
     assertResult(24)(SizeEstimator.estimate(new DummyClass5))
     assertResult(32)(SizeEstimator.estimate(new DummyClass6))
   }
 
   test("check 64-bit detection for s390x arch") {
-    System.setProperty("os.arch", "s390x")
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
+    reinitializeSizeEstimator("s390x", "true")
     // Class should be 32 bytes on s390x if recognised as 64 bit platform
     assertResult(32)(SizeEstimator.estimate(new DummyClass7))
   }
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index 8f8902e497d49..931eb6b5413f7 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -745,10 +745,14 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
     manager.add(3, () => output += 3)
     manager.add(2, () => output += 2)
     manager.add(4, () => output += 4)
+    manager.add(Int.MinValue, () => output += Int.MinValue)
+    manager.add(Int.MinValue, () => output += Int.MinValue)
+    manager.add(Int.MaxValue, () => output += Int.MaxValue)
+    manager.add(Int.MaxValue, () => output += Int.MaxValue)
     manager.remove(hook1)
 
     manager.runAll()
-    assert(output.toList === List(4, 3, 2))
+    assert(output.toList === List(Int.MaxValue, Int.MaxValue, 4, 3, 2, Int.MinValue, Int.MinValue))
   }
 
   test("isInDirectory") {
@@ -1243,6 +1247,10 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
     intercept[IllegalArgumentException] {
       Utils.checkAndGetK8sMasterUrl("k8s://foo://host:port")
     }
+
+    intercept[IllegalArgumentException] {
+      Utils.checkAndGetK8sMasterUrl("k8s:///https://host:port")
+    }
   }
 
   test("stringHalfWidth") {
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index 73f461255de43..20a4002def2fb 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -27,8 +27,8 @@ d3.min.js
 dagre-d3.min.js
 graphlib-dot.min.js
 sorttable.js
-vis.min.js
-vis.min.css
+vis-timeline-graph2d.min.js
+vis-timeline-graph2d.min.css
 dataTables.bootstrap.css
 dataTables.bootstrap.min.js
 dataTables.rowsGroup.js
@@ -39,11 +39,11 @@ jquery.dataTables.1.10.18.min.js
 jquery.mustache.js
 jsonFormatter.min.css
 jsonFormatter.min.js
-.*avsc
-.*txt
-.*json
-.*data
-.*log
+.*\.avsc
+.*\.txt
+.*\.json
+.*\.data
+.*\.log
 pyspark-coverage-site/
 cloudpickle.py
 heapq3.py
@@ -96,17 +96,17 @@ local-1430917381535_2
 DESCRIPTION
 NAMESPACE
 test_support/*
-.*Rd
+.*\.Rd
 help/*
 html/*
 INDEX
 .lintr
 gen-java.*
-.*avpr
-.*parquet
+.*\.avpr
+.*\.parquet
 spark-deps-.*
-.*csv
-.*tsv
+.*\.csv
+.*\.tsv
 .*\.sql
 .Rbuildignore
 META-INF/*
@@ -119,3 +119,11 @@ vote.tmpl
 SessionManager.java
 SessionHandler.java
 GangliaReporter.java
+_metadata
+_SUCCESS
+part-00000
+.*\.res
+flights_tiny.txt.1
+over1k
+over10k
+exported_table/*
diff --git a/dev/check-license b/dev/check-license
index 0cc17ffe55c67..bd255954d6db4 100755
--- a/dev/check-license
+++ b/dev/check-license
@@ -67,7 +67,7 @@ mkdir -p "$FWDIR"/lib
     exit 1
 }
 
-mkdir target
+mkdir -p target
 $java_cmd -jar "$rat_jar" -E "$FWDIR"/dev/.rat-excludes -d "$FWDIR" > target/rat-results.txt
 
 if [ $? -ne 0 ]; then
diff --git a/dev/create-release/do-release-docker.sh b/dev/create-release/do-release-docker.sh
index 694a87bf78084..8f53f4a4e13ad 100755
--- a/dev/create-release/do-release-docker.sh
+++ b/dev/create-release/do-release-docker.sh
@@ -54,7 +54,7 @@ WORKDIR=
 IMGTAG=latest
 JAVA=
 RELEASE_STEP=
-while getopts "d:hj:ns:t:" opt; do
+while getopts ":d:hj:ns:t:" opt; do
   case $opt in
     d) WORKDIR="$OPTARG" ;;
     n) DRY_RUN=1 ;;
@@ -62,7 +62,7 @@ while getopts "d:hj:ns:t:" opt; do
     j) JAVA="$OPTARG" ;;
     s) RELEASE_STEP="$OPTARG" ;;
     h) usage ;;
-    ?) error "Invalid option. Run with -h for help." ;;
+    \?) error "Invalid option. Run with -h for help." ;;
   esac
 done
 
@@ -93,7 +93,7 @@ done
 
 GPG_KEY_FILE="$WORKDIR/gpg.key"
 fcreate_secure "$GPG_KEY_FILE"
-$GPG --export-secret-key --armor "$GPG_KEY" > "$GPG_KEY_FILE"
+$GPG --export-secret-key --armor --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" "$GPG_KEY" > "$GPG_KEY_FILE"
 
 run_silent "Building spark-rm image with tag $IMGTAG..." "docker-build.log" \
   docker build -t "spark-rm:$IMGTAG" --build-arg UID=$UID "$SELF/spark-rm"
@@ -128,6 +128,7 @@ ASF_PASSWORD=$ASF_PASSWORD
 GPG_PASSPHRASE=$GPG_PASSPHRASE
 RELEASE_STEP=$RELEASE_STEP
 USER=$USER
+ZINC_OPTS=${RELEASE_ZINC_OPTS:-"-Xmx4g -XX:ReservedCodeCacheSize=2g"}
 EOF
 
 JAVA_VOL=
diff --git a/dev/create-release/do-release.sh b/dev/create-release/do-release.sh
index f1d4f3ab5ddec..4f18a55a3bceb 100755
--- a/dev/create-release/do-release.sh
+++ b/dev/create-release/do-release.sh
@@ -20,11 +20,11 @@
 SELF=$(cd $(dirname $0) && pwd)
 . "$SELF/release-util.sh"
 
-while getopts "bn" opt; do
+while getopts ":b:n" opt; do
   case $opt in
     b) GIT_BRANCH=$OPTARG ;;
     n) DRY_RUN=1 ;;
-    ?) error "Invalid option: $OPTARG" ;;
+    \?) error "Invalid option: $OPTARG" ;;
   esac
 done
 
diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index 022d3af95c05d..eb972589a995e 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -103,7 +103,7 @@ if [ -z "$SPARK_VERSION" ]; then
   # Run $MVN in a separate command so that 'set -e' does the right thing.
   TMP=$(mktemp)
   $MVN help:evaluate -Dexpression=project.version > $TMP
-  SPARK_VERSION=$(cat $TMP | grep -v INFO | grep -v WARNING | grep -v Download)
+  SPARK_VERSION=$(cat $TMP | grep -v INFO | grep -v WARNING | grep -vi Download)
   rm $TMP
 fi
 
@@ -164,7 +164,7 @@ fi
 DEST_DIR_NAME="$SPARK_PACKAGE_VERSION"
 
 git clean -d -f -x
-rm .gitignore
+rm -f .gitignore
 cd ..
 
 if [[ "$1" == "package" ]]; then
@@ -174,9 +174,9 @@ if [[ "$1" == "package" ]]; then
 
   # For source release in v2.4+, exclude copy of binary license/notice
   if [[ $SPARK_VERSION > "2.4" ]]; then
-    rm spark-$SPARK_VERSION/LICENSE-binary
-    rm spark-$SPARK_VERSION/NOTICE-binary
-    rm -r spark-$SPARK_VERSION/licenses-binary
+    rm -f spark-$SPARK_VERSION/LICENSE-binary
+    rm -f spark-$SPARK_VERSION/NOTICE-binary
+    rm -rf spark-$SPARK_VERSION/licenses-binary
   fi
 
   tar cvzf spark-$SPARK_VERSION.tgz --exclude spark-$SPARK_VERSION/.git spark-$SPARK_VERSION
@@ -380,7 +380,7 @@ if [[ "$1" == "publish-snapshot" ]]; then
   echo "</server></servers></settings>" >> $tmp_settings
 
   # Generate random point for Zinc
-  export ZINC_PORT=$(python -S -c "import random; print random.randrange(3030,4030)")
+  export ZINC_PORT=$(python -S -c "import random; print(random.randrange(3030,4030))")
 
   $MVN -DzincPort=$ZINC_PORT --settings $tmp_settings -DskipTests $SCALA_2_12_PROFILES $PUBLISH_PROFILES deploy
 
@@ -412,7 +412,7 @@ if [[ "$1" == "publish-release" ]]; then
   tmp_repo=$(mktemp -d spark-repo-XXXXX)
 
   # Generate random point for Zinc
-  export ZINC_PORT=$(python -S -c "import random; print random.randrange(3030,4030)")
+  export ZINC_PORT=$(python -S -c "import random; print(random.randrange(3030,4030))")
 
   # TODO: revisit for Scala 2.13 support
 
diff --git a/dev/create-release/release-tag.sh b/dev/create-release/release-tag.sh
index 39856a9955955..a9a518f9e10d7 100755
--- a/dev/create-release/release-tag.sh
+++ b/dev/create-release/release-tag.sh
@@ -64,8 +64,12 @@ init_maven_sbt
 
 ASF_SPARK_REPO="gitbox.apache.org/repos/asf/spark.git"
 
+function uriencode { jq -nSRr --arg v "$1" '$v|@uri'; }
+
+declare -r ENCODED_ASF_PASSWORD=$(uriencode "$ASF_PASSWORD")
+
 rm -rf spark
-git clone "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_SPARK_REPO" -b $GIT_BRANCH
+git clone "https://$ASF_USERNAME:$ENCODED_ASF_PASSWORD@$ASF_SPARK_REPO" -b $GIT_BRANCH
 cd spark
 
 git config user.name "$GIT_NAME"
diff --git a/dev/create-release/release-util.sh b/dev/create-release/release-util.sh
index 1282e15e79913..55e1d09da5898 100755
--- a/dev/create-release/release-util.sh
+++ b/dev/create-release/release-util.sh
@@ -159,10 +159,14 @@ function get_release_info {
   export SPARK_PACKAGE_VERSION="$RELEASE_TAG"
 
   # Gather some user information.
-  export ASF_USERNAME=$(read_config "ASF user" "$LOGNAME")
+  if [ -z "$ASF_USERNAME" ]; then
+    export ASF_USERNAME=$(read_config "ASF user" "$LOGNAME")
+  fi
 
-  GIT_NAME=$(git config user.name || echo "")
-  export GIT_NAME=$(read_config "Full name" "$GIT_NAME")
+  if [ -z "$GIT_NAME" ]; then
+    GIT_NAME=$(git config user.name || echo "")
+    export GIT_NAME=$(read_config "Full name" "$GIT_NAME")
+  fi
 
   export GIT_EMAIL="$ASF_USERNAME@apache.org"
   export GPG_KEY=$(read_config "GPG key" "$GIT_EMAIL")
@@ -225,7 +229,7 @@ function init_maven_sbt {
   if [[ $JAVA_VERSION < "1.8." ]]; then
     # Needed for maven central when using Java 7.
     SBT_OPTS="-Dhttps.protocols=TLSv1.1,TLSv1.2"
-    MVN_EXTRA_OPTS="-Dhttps.protocols=TLSv1.1,TLSv1.2"
+    MVN_EXTRA_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dhttps.protocols=TLSv1.1,TLSv1.2"
     MVN="$MVN $MVN_EXTRA_OPTS"
   fi
   export MVN MVN_EXTRA_OPTS SBT_OPTS
diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index 3ba8e97929613..ff6af6ff57d35 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -21,7 +21,7 @@
 # * Java 8
 # * Ivy
 # * Python (2.7.15/3.6.7)
-# * R-base/R-base-dev (3.6.1)
+# * R-base/R-base-dev (4.0.2)
 # * Ruby 2.3 build utilities
 
 FROM ubuntu:18.04
@@ -33,8 +33,8 @@ ENV DEBCONF_NONINTERACTIVE_SEEN true
 # These arguments are just for reuse and not really meant to be customized.
 ARG APT_INSTALL="apt-get install --no-install-recommends -y"
 
-ARG BASE_PIP_PKGS="setuptools wheel"
-ARG PIP_PKGS="pyopenssl numpy sphinx"
+ARG PIP_PKGS="sphinx==2.3.1 mkdocs==1.0.4 numpy==1.18.1"
+ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0"
 
 # Install extra needed repos and refresh.
 # - CRAN repo
@@ -43,7 +43,7 @@ ARG PIP_PKGS="pyopenssl numpy sphinx"
 # This is all in a single "RUN" command so that if anything changes, "apt update" is run to fetch
 # the most current package versions (instead of potentially using old versions cached by docker).
 RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
-  echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' >> /etc/apt/sources.list && \
+  echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/' >> /etc/apt/sources.list && \
   gpg --keyserver keyserver.ubuntu.com --recv-key E298A3A825C0D65DFD57CBB651716619E084DAB9 && \
   gpg -a --export E084DAB9 | apt-key add - && \
   apt-get clean && \
@@ -62,12 +62,13 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
   curl -sL https://deb.nodesource.com/setup_11.x | bash && \
   $APT_INSTALL nodejs && \
   # Install needed python packages. Use pip for installing packages (for consistency).
-  $APT_INSTALL libpython3-dev python3-pip && \
+  $APT_INSTALL libpython3-dev python3-pip python3-setuptools && \
+  # qpdf is required for CRAN checks to pass.
+  $APT_INSTALL qpdf jq && \
   # Change default python version to python3.
   update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && \
   update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2 && \
   update-alternatives --set python /usr/bin/python3.6 && \
-  pip3 install $BASE_PIP_PKGS && \
   pip3 install $PIP_PKGS && \
   # Install R packages and dependencies used when building.
   # R depends on pandoc*, libssl (which are installed above).
@@ -76,10 +77,8 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
   Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')" && \
   Rscript -e "devtools::install_github('jimhester/lintr')" && \
   # Install tools needed to build the documentation.
-  $APT_INSTALL ruby2.3 ruby2.3-dev mkdocs && \
-  gem install jekyll --no-rdoc --no-ri -v 3.8.6 && \
-  gem install jekyll-redirect-from && \
-  gem install rouge
+  $APT_INSTALL ruby2.5 ruby2.5-dev && \
+  gem install --no-document $GEM_PKGS
 
 WORKDIR /opt/spark-rm/output
 
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 62d5772ffc94a..8c6cc15c150b3 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -35,10 +35,10 @@ commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
 commons-cli/1.2//commons-cli-1.2.jar
 commons-codec/1.10//commons-codec-1.10.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
-commons-compiler/3.0.15//commons-compiler-3.0.15.jar
-commons-compress/1.8.1//commons-compress-1.8.1.jar
+commons-compiler/3.0.16//commons-compiler-3.0.16.jar
+commons-compress/1.20//commons-compress-1.20.jar
 commons-configuration/1.6//commons-configuration-1.6.jar
-commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-crypto/1.1.0//commons-crypto-1.1.0.jar
 commons-dbcp/1.4//commons-dbcp-1.4.jar
 commons-digester/1.8//commons-digester-1.8.jar
 commons-httpclient/3.1//commons-httpclient-3.1.jar
@@ -93,6 +93,7 @@ jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
 jackson-core/2.10.0//jackson-core-2.10.0.jar
 jackson-databind/2.10.0//jackson-databind-2.10.0.jar
 jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
+jackson-datatype-jsr310/2.10.3//jackson-datatype-jsr310-2.10.3.jar
 jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
 jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar
@@ -105,14 +106,14 @@ jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.15//janino-3.0.15.jar
+janino/3.0.16//janino-3.0.16.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
 javax.inject/1//javax.inject-1.jar
 javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
 javolution/5.5.1//javolution-5.5.1.jar
 jaxb-api/2.2.2//jaxb-api-2.2.2.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
-jcl-over-slf4j/1.7.16//jcl-over-slf4j-1.7.16.jar
+jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
 jersey-client/2.30//jersey-client-2.30.jar
 jersey-common/2.30//jersey-common-2.30.jar
@@ -135,11 +136,11 @@ json4s-scalap_2.12/3.6.6//json4s-scalap_2.12-3.6.6.jar
 jsp-api/2.1//jsp-api-2.1.jar
 jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
-jul-to-slf4j/1.7.16//jul-to-slf4j-1.7.16.jar
+jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.6.4//kubernetes-client-4.6.4.jar
-kubernetes-model-common/4.6.4//kubernetes-model-common-4.6.4.jar
-kubernetes-model/4.6.4//kubernetes-model-4.6.4.jar
+kubernetes-client/4.9.2//kubernetes-client-4.9.2.jar
+kubernetes-model-common/4.9.2//kubernetes-model-common-4.9.2.jar
+kubernetes-model/4.9.2//kubernetes-model-4.9.2.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
@@ -155,14 +156,14 @@ metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
 metrics-json/4.1.1//metrics-json-4.1.1.jar
 metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
 minlog/1.3.0//minlog-1.3.0.jar
-netty-all/4.1.42.Final//netty-all-4.1.42.Final.jar
+netty-all/4.1.47.Final//netty-all-4.1.47.Final.jar
 objenesis/2.5.1//objenesis-2.5.1.jar
 okhttp/3.12.6//okhttp-3.12.6.jar
 okio/1.15.0//okio-1.15.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.9/nohive/orc-core-1.5.9-nohive.jar
-orc-mapreduce/1.5.9/nohive/orc-mapreduce-1.5.9-nohive.jar
-orc-shims/1.5.9//orc-shims-1.5.9.jar
+orc-core/1.5.10/nohive/orc-core-1.5.10-nohive.jar
+orc-mapreduce/1.5.10/nohive/orc-mapreduce-1.5.10-nohive.jar
+orc-shims/1.5.10//orc-shims-1.5.10.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
@@ -174,7 +175,7 @@ parquet-hadoop-bundle/1.6.0//parquet-hadoop-bundle-1.6.0.jar
 parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
 parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
-py4j/0.10.8.1//py4j-0.10.8.1.jar
+py4j/0.10.9//py4j-0.10.9.jar
 pyrolite/4.30//pyrolite-4.30.jar
 scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
 scala-compiler/2.12.10//scala-compiler-2.12.10.jar
@@ -184,10 +185,10 @@ scala-reflect/2.12.10//scala-reflect-2.12.10.jar
 scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
 shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
 shims/0.7.45//shims-0.7.45.jar
-slf4j-api/1.7.16//slf4j-api-1.7.16.jar
-slf4j-log4j12/1.7.16//slf4j-log4j12-1.7.16.jar
+slf4j-api/1.7.30//slf4j-api-1.7.30.jar
+slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.3//snappy-java-1.1.7.3.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
 snappy/0.2//snappy-0.2.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
@@ -199,9 +200,10 @@ stream/2.9.6//stream-2.9.6.jar
 stringtemplate/3.2.1//stringtemplate-3.2.1.jar
 super-csv/2.2.0//super-csv-2.2.0.jar
 threeten-extra/1.5.0//threeten-extra-1.5.0.jar
-univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
+univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
-xercesImpl/2.9.1//xercesImpl-2.9.1.jar
+xercesImpl/2.12.0//xercesImpl-2.12.0.jar
+xml-apis/1.4.01//xml-apis-1.4.01.jar
 xmlenc/0.52//xmlenc-0.52.jar
 xz/1.5//xz-1.5.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 1b57250c1fb54..3e82df2bcdfa1 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -33,10 +33,10 @@ commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
 commons-cli/1.2//commons-cli-1.2.jar
 commons-codec/1.10//commons-codec-1.10.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
-commons-compiler/3.0.15//commons-compiler-3.0.15.jar
-commons-compress/1.8.1//commons-compress-1.8.1.jar
+commons-compiler/3.0.16//commons-compiler-3.0.16.jar
+commons-compress/1.20//commons-compress-1.20.jar
 commons-configuration/1.6//commons-configuration-1.6.jar
-commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-crypto/1.1.0//commons-crypto-1.1.0.jar
 commons-dbcp/1.4//commons-dbcp-1.4.jar
 commons-digester/1.8//commons-digester-1.8.jar
 commons-httpclient/3.1//commons-httpclient-3.1.jar
@@ -79,21 +79,20 @@ hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar
 hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar
 hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar
 hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar
-hive-beeline/2.3.6//hive-beeline-2.3.6.jar
-hive-cli/2.3.6//hive-cli-2.3.6.jar
-hive-common/2.3.6//hive-common-2.3.6.jar
-hive-exec/2.3.6/core/hive-exec-2.3.6-core.jar
-hive-jdbc/2.3.6//hive-jdbc-2.3.6.jar
-hive-llap-common/2.3.6//hive-llap-common-2.3.6.jar
-hive-metastore/2.3.6//hive-metastore-2.3.6.jar
-hive-serde/2.3.6//hive-serde-2.3.6.jar
-hive-service-rpc/2.3.6//hive-service-rpc-2.3.6.jar
-hive-shims-0.23/2.3.6//hive-shims-0.23-2.3.6.jar
-hive-shims-common/2.3.6//hive-shims-common-2.3.6.jar
-hive-shims-scheduler/2.3.6//hive-shims-scheduler-2.3.6.jar
-hive-shims/2.3.6//hive-shims-2.3.6.jar
+hive-beeline/2.3.7//hive-beeline-2.3.7.jar
+hive-cli/2.3.7//hive-cli-2.3.7.jar
+hive-common/2.3.7//hive-common-2.3.7.jar
+hive-exec/2.3.7/core/hive-exec-2.3.7-core.jar
+hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar
+hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar
+hive-metastore/2.3.7//hive-metastore-2.3.7.jar
+hive-serde/2.3.7//hive-serde-2.3.7.jar
+hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar
+hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar
+hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar
+hive-shims/2.3.7//hive-shims-2.3.7.jar
 hive-storage-api/2.7.1//hive-storage-api-2.7.1.jar
-hive-vector-code-gen/2.3.6//hive-vector-code-gen-2.3.6.jar
+hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
@@ -107,6 +106,7 @@ jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
 jackson-core/2.10.0//jackson-core-2.10.0.jar
 jackson-databind/2.10.0//jackson-databind-2.10.0.jar
 jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
+jackson-datatype-jsr310/2.10.3//jackson-datatype-jsr310-2.10.3.jar
 jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
 jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar
@@ -119,7 +119,7 @@ jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.15//janino-3.0.15.jar
+janino/3.0.16//janino-3.0.16.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
 javax.inject/1//javax.inject-1.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
@@ -127,7 +127,7 @@ javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
 javolution/5.5.1//javolution-5.5.1.jar
 jaxb-api/2.2.2//jaxb-api-2.2.2.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
-jcl-over-slf4j/1.7.16//jcl-over-slf4j-1.7.16.jar
+jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
 jersey-client/2.30//jersey-client-2.30.jar
 jersey-common/2.30//jersey-common-2.30.jar
@@ -151,11 +151,11 @@ json4s-scalap_2.12/3.6.6//json4s-scalap_2.12-3.6.6.jar
 jsp-api/2.1//jsp-api-2.1.jar
 jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
-jul-to-slf4j/1.7.16//jul-to-slf4j-1.7.16.jar
+jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.6.4//kubernetes-client-4.6.4.jar
-kubernetes-model-common/4.6.4//kubernetes-model-common-4.6.4.jar
-kubernetes-model/4.6.4//kubernetes-model-4.6.4.jar
+kubernetes-client/4.9.2//kubernetes-client-4.9.2.jar
+kubernetes-model-common/4.9.2//kubernetes-model-common-4.9.2.jar
+kubernetes-model/4.9.2//kubernetes-model-4.9.2.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
@@ -171,14 +171,14 @@ metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
 metrics-json/4.1.1//metrics-json-4.1.1.jar
 metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
 minlog/1.3.0//minlog-1.3.0.jar
-netty-all/4.1.42.Final//netty-all-4.1.42.Final.jar
+netty-all/4.1.47.Final//netty-all-4.1.47.Final.jar
 objenesis/2.5.1//objenesis-2.5.1.jar
 okhttp/3.12.6//okhttp-3.12.6.jar
 okio/1.15.0//okio-1.15.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.9//orc-core-1.5.9.jar
-orc-mapreduce/1.5.9//orc-mapreduce-1.5.9.jar
-orc-shims/1.5.9//orc-shims-1.5.9.jar
+orc-core/1.5.10//orc-core-1.5.10.jar
+orc-mapreduce/1.5.10//orc-mapreduce-1.5.10.jar
+orc-shims/1.5.10//orc-shims-1.5.10.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
@@ -189,7 +189,7 @@ parquet-format/2.4.0//parquet-format-2.4.0.jar
 parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
 parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
-py4j/0.10.8.1//py4j-0.10.8.1.jar
+py4j/0.10.9//py4j-0.10.9.jar
 pyrolite/4.30//pyrolite-4.30.jar
 scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
 scala-compiler/2.12.10//scala-compiler-2.12.10.jar
@@ -199,10 +199,10 @@ scala-reflect/2.12.10//scala-reflect-2.12.10.jar
 scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
 shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
 shims/0.7.45//shims-0.7.45.jar
-slf4j-api/1.7.16//slf4j-api-1.7.16.jar
-slf4j-log4j12/1.7.16//slf4j-log4j12-1.7.16.jar
+slf4j-api/1.7.30//slf4j-api-1.7.30.jar
+slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.3//snappy-java-1.1.7.3.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
@@ -213,10 +213,11 @@ stream/2.9.6//stream-2.9.6.jar
 super-csv/2.2.0//super-csv-2.2.0.jar
 threeten-extra/1.5.0//threeten-extra-1.5.0.jar
 transaction-api/1.1//transaction-api-1.1.jar
-univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
+univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar
 velocity/1.5//velocity-1.5.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
-xercesImpl/2.9.1//xercesImpl-2.9.1.jar
+xercesImpl/2.12.0//xercesImpl-2.12.0.jar
+xml-apis/1.4.01//xml-apis-1.4.01.jar
 xmlenc/0.52//xmlenc-0.52.jar
 xz/1.5//xz-1.5.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index ffd2364a51317..8de3433f2d9fb 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -30,10 +30,10 @@ commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
 commons-cli/1.2//commons-cli-1.2.jar
 commons-codec/1.10//commons-codec-1.10.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
-commons-compiler/3.0.15//commons-compiler-3.0.15.jar
-commons-compress/1.8.1//commons-compress-1.8.1.jar
+commons-compiler/3.0.16//commons-compiler-3.0.16.jar
+commons-compress/1.20//commons-compress-1.20.jar
 commons-configuration2/2.1.1//commons-configuration2-2.1.1.jar
-commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-crypto/1.1.0//commons-crypto-1.1.0.jar
 commons-daemon/1.0.13//commons-daemon-1.0.13.jar
 commons-dbcp/1.4//commons-dbcp-1.4.jar
 commons-httpclient/3.1//commons-httpclient-3.1.jar
@@ -78,21 +78,20 @@ hadoop-yarn-common/3.2.0//hadoop-yarn-common-3.2.0.jar
 hadoop-yarn-registry/3.2.0//hadoop-yarn-registry-3.2.0.jar
 hadoop-yarn-server-common/3.2.0//hadoop-yarn-server-common-3.2.0.jar
 hadoop-yarn-server-web-proxy/3.2.0//hadoop-yarn-server-web-proxy-3.2.0.jar
-hive-beeline/2.3.6//hive-beeline-2.3.6.jar
-hive-cli/2.3.6//hive-cli-2.3.6.jar
-hive-common/2.3.6//hive-common-2.3.6.jar
-hive-exec/2.3.6/core/hive-exec-2.3.6-core.jar
-hive-jdbc/2.3.6//hive-jdbc-2.3.6.jar
-hive-llap-common/2.3.6//hive-llap-common-2.3.6.jar
-hive-metastore/2.3.6//hive-metastore-2.3.6.jar
-hive-serde/2.3.6//hive-serde-2.3.6.jar
-hive-service-rpc/2.3.6//hive-service-rpc-2.3.6.jar
-hive-shims-0.23/2.3.6//hive-shims-0.23-2.3.6.jar
-hive-shims-common/2.3.6//hive-shims-common-2.3.6.jar
-hive-shims-scheduler/2.3.6//hive-shims-scheduler-2.3.6.jar
-hive-shims/2.3.6//hive-shims-2.3.6.jar
+hive-beeline/2.3.7//hive-beeline-2.3.7.jar
+hive-cli/2.3.7//hive-cli-2.3.7.jar
+hive-common/2.3.7//hive-common-2.3.7.jar
+hive-exec/2.3.7/core/hive-exec-2.3.7-core.jar
+hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar
+hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar
+hive-metastore/2.3.7//hive-metastore-2.3.7.jar
+hive-serde/2.3.7//hive-serde-2.3.7.jar
+hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar
+hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar
+hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar
+hive-shims/2.3.7//hive-shims-2.3.7.jar
 hive-storage-api/2.7.1//hive-storage-api-2.7.1.jar
-hive-vector-code-gen/2.3.6//hive-vector-code-gen-2.3.6.jar
+hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
@@ -106,6 +105,7 @@ jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
 jackson-core/2.10.0//jackson-core-2.10.0.jar
 jackson-databind/2.10.0//jackson-databind-2.10.0.jar
 jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
+jackson-datatype-jsr310/2.10.3//jackson-datatype-jsr310-2.10.3.jar
 jackson-jaxrs-base/2.9.5//jackson-jaxrs-base-2.9.5.jar
 jackson-jaxrs-json-provider/2.9.5//jackson-jaxrs-json-provider-2.9.5.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
@@ -118,7 +118,7 @@ jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.15//janino-3.0.15.jar
+janino/3.0.16//janino-3.0.16.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
 javax.inject/1//javax.inject-1.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
@@ -127,7 +127,7 @@ javolution/5.5.1//javolution-5.5.1.jar
 jaxb-api/2.2.11//jaxb-api-2.2.11.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
 jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar
-jcl-over-slf4j/1.7.16//jcl-over-slf4j-1.7.16.jar
+jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
 jersey-client/2.30//jersey-client-2.30.jar
 jersey-common/2.30//jersey-common-2.30.jar
@@ -149,7 +149,7 @@ json4s-scalap_2.12/3.6.6//json4s-scalap_2.12-3.6.6.jar
 jsp-api/2.1//jsp-api-2.1.jar
 jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
-jul-to-slf4j/1.7.16//jul-to-slf4j-1.7.16.jar
+jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kerb-admin/1.0.1//kerb-admin-1.0.1.jar
 kerb-client/1.0.1//kerb-client-1.0.1.jar
 kerb-common/1.0.1//kerb-common-1.0.1.jar
@@ -165,9 +165,9 @@ kerby-pkix/1.0.1//kerby-pkix-1.0.1.jar
 kerby-util/1.0.1//kerby-util-1.0.1.jar
 kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.6.4//kubernetes-client-4.6.4.jar
-kubernetes-model-common/4.6.4//kubernetes-model-common-4.6.4.jar
-kubernetes-model/4.6.4//kubernetes-model-4.6.4.jar
+kubernetes-client/4.9.2//kubernetes-client-4.9.2.jar
+kubernetes-model-common/4.9.2//kubernetes-model-common-4.9.2.jar
+kubernetes-model/4.9.2//kubernetes-model-4.9.2.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
@@ -183,17 +183,16 @@ metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
 metrics-json/4.1.1//metrics-json-4.1.1.jar
 metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
 minlog/1.3.0//minlog-1.3.0.jar
-mssql-jdbc/6.2.1.jre7//mssql-jdbc-6.2.1.jre7.jar
-netty-all/4.1.42.Final//netty-all-4.1.42.Final.jar
+netty-all/4.1.47.Final//netty-all-4.1.47.Final.jar
 nimbus-jose-jwt/4.41.1//nimbus-jose-jwt-4.41.1.jar
 objenesis/2.5.1//objenesis-2.5.1.jar
 okhttp/2.7.5//okhttp-2.7.5.jar
 okhttp/3.12.6//okhttp-3.12.6.jar
 okio/1.15.0//okio-1.15.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.9//orc-core-1.5.9.jar
-orc-mapreduce/1.5.9//orc-mapreduce-1.5.9.jar
-orc-shims/1.5.9//orc-shims-1.5.9.jar
+orc-core/1.5.10//orc-core-1.5.10.jar
+orc-mapreduce/1.5.10//orc-mapreduce-1.5.10.jar
+orc-shims/1.5.10//orc-shims-1.5.10.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
@@ -204,7 +203,7 @@ parquet-format/2.4.0//parquet-format-2.4.0.jar
 parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
 parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
-py4j/0.10.8.1//py4j-0.10.8.1.jar
+py4j/0.10.9//py4j-0.10.9.jar
 pyrolite/4.30//pyrolite-4.30.jar
 re2j/1.1//re2j-1.1.jar
 scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
@@ -215,10 +214,10 @@ scala-reflect/2.12.10//scala-reflect-2.12.10.jar
 scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
 shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
 shims/0.7.45//shims-0.7.45.jar
-slf4j-api/1.7.16//slf4j-api-1.7.16.jar
-slf4j-log4j12/1.7.16//slf4j-log4j12-1.7.16.jar
+slf4j-api/1.7.30//slf4j-api-1.7.30.jar
+slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.3//snappy-java-1.1.7.3.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
@@ -230,7 +229,7 @@ super-csv/2.2.0//super-csv-2.2.0.jar
 threeten-extra/1.5.0//threeten-extra-1.5.0.jar
 token-provider/1.0.1//token-provider-1.0.1.jar
 transaction-api/1.1//transaction-api-1.1.jar
-univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
+univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar
 velocity/1.5//velocity-1.5.jar
 woodstox-core/5.0.3//woodstox-core-5.0.3.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
diff --git a/dev/lint-python b/dev/lint-python
index e9ed83dec5fce..674896e3b5dd0 100755
--- a/dev/lint-python
+++ b/dev/lint-python
@@ -16,9 +16,6 @@
 # limitations under the License.
 #
 # define test binaries + versions
-PYDOCSTYLE_BUILD="pydocstyle"
-MINIMUM_PYDOCSTYLE="3.0.0"
-
 FLAKE8_BUILD="flake8"
 MINIMUM_FLAKE8="3.5.0"
 
@@ -27,6 +24,19 @@ MINIMUM_PYCODESTYLE="2.4.0"
 
 SPHINX_BUILD="sphinx-build"
 
+PYTHON_EXECUTABLE="python3"
+
+function satisfies_min_version {
+    local provided_version="$1"
+    local expected_version="$2"
+    echo "$(
+        "$PYTHON_EXECUTABLE" << EOM
+from setuptools.extern.packaging import version
+print(version.parse('$provided_version') >= version.parse('$expected_version'))
+EOM
+    )"
+}
+
 function compile_python_test {
     local COMPILE_STATUS=
     local COMPILE_REPORT=
@@ -56,7 +66,7 @@ function pycodestyle_test {
     local PYCODESTYLE_STATUS=
     local PYCODESTYLE_REPORT=
     local RUN_LOCAL_PYCODESTYLE=
-    local VERSION=
+    local PYCODESTYLE_VERSION=
     local EXPECTED_PYCODESTYLE=
     local PYCODESTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-$MINIMUM_PYCODESTYLE.py"
     local PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$MINIMUM_PYCODESTYLE/pycodestyle.py"
@@ -69,11 +79,8 @@ function pycodestyle_test {
     # check for locally installed pycodestyle & version
     RUN_LOCAL_PYCODESTYLE="False"
     if hash "$PYCODESTYLE_BUILD" 2> /dev/null; then
-        VERSION=$( $PYCODESTYLE_BUILD --version 2> /dev/null)
-        EXPECTED_PYCODESTYLE=$( (python3 -c 'from distutils.version import LooseVersion;
-                                print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_PYCODESTYLE'"""))')\
-                                2> /dev/null)
-
+        PYCODESTYLE_VERSION="$($PYCODESTYLE_BUILD --version)"
+        EXPECTED_PYCODESTYLE="$(satisfies_min_version $PYCODESTYLE_VERSION $MINIMUM_PYCODESTYLE)"
         if [ "$EXPECTED_PYCODESTYLE" == "True" ]; then
             RUN_LOCAL_PYCODESTYLE="True"
         fi
@@ -117,7 +124,6 @@ function pycodestyle_test {
 
 function flake8_test {
     local FLAKE8_VERSION=
-    local VERSION=
     local EXPECTED_FLAKE8=
     local FLAKE8_REPORT=
     local FLAKE8_STATUS=
@@ -128,11 +134,9 @@ function flake8_test {
         exit 1
     fi
 
-    FLAKE8_VERSION="$($FLAKE8_BUILD --version  2> /dev/null)"
-    VERSION=($FLAKE8_VERSION)
-    EXPECTED_FLAKE8=$( (python3 -c 'from distutils.version import LooseVersion;
-                       print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_FLAKE8'"""))') \
-                       2> /dev/null)
+    _FLAKE8_VERSION=($($FLAKE8_BUILD --version))
+    FLAKE8_VERSION="${_FLAKE8_VERSION[0]}"
+    EXPECTED_FLAKE8="$(satisfies_min_version $FLAKE8_VERSION $MINIMUM_FLAKE8)"
 
     if [[ "$EXPECTED_FLAKE8" == "False" ]]; then
         echo "\
@@ -158,50 +162,6 @@ flake8 checks failed."
     fi
 }
 
-function pydocstyle_test {
-    local PYDOCSTYLE_REPORT=
-    local PYDOCSTYLE_STATUS=
-    local PYDOCSTYLE_VERSION=
-    local EXPECTED_PYDOCSTYLE=
-
-    # Exclude auto-generated configuration file.
-    local DOC_PATHS_TO_CHECK="$( cd "${SPARK_ROOT_DIR}" && find . -name "*.py" | grep -vF 'functions.py' )"
-
-    # Check python document style, skip check if pydocstyle is not installed.
-    if ! hash "$PYDOCSTYLE_BUILD" 2> /dev/null; then
-        echo "The pydocstyle command was not found. Skipping pydocstyle checks for now."
-        echo
-        return
-    fi
-
-    PYDOCSTYLE_VERSION="$($PYDOCSTYLEBUILD --version 2> /dev/null)"
-    EXPECTED_PYDOCSTYLE=$(python3 -c 'from distutils.version import LooseVersion; \
-                             print(LooseVersion("""'$PYDOCSTYLE_VERSION'""") >= LooseVersion("""'$MINIMUM_PYDOCSTYLE'"""))' \
-                             2> /dev/null)
-
-    if [[ "$EXPECTED_PYDOCSTYLE" == "False" ]]; then
-        echo "\
-The minimum version of pydocstyle needs to be $MINIMUM_PYDOCSTYLE.
-Your current version is $PYDOCSTYLE_VERSION.
-Skipping pydocstyle checks for now."
-        echo
-        return
-    fi
-
-    echo "starting $PYDOCSTYLE_BUILD test..."
-    PYDOCSTYLE_REPORT=$( ($PYDOCSTYLE_BUILD --config=dev/tox.ini $DOC_PATHS_TO_CHECK) 2>&1)
-    PYDOCSTYLE_STATUS=$?
-
-    if [ "$PYDOCSTYLE_STATUS" -ne 0 ]; then
-        echo "pydocstyle checks failed:"
-        echo "$PYDOCSTYLE_REPORT"
-        exit "$PYDOCSTYLE_STATUS"
-    else
-        echo "pydocstyle checks passed."
-        echo
-    fi
-}
-
 function sphinx_test {
     local SPHINX_REPORT=
     local SPHINX_STATUS=
@@ -247,7 +207,6 @@ PYTHON_SOURCE="$(find . -name "*.py")"
 compile_python_test "$PYTHON_SOURCE"
 pycodestyle_test "$PYTHON_SOURCE"
 flake8_test
-pydocstyle_test
 sphinx_test
 
 echo
diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh
index 0b30eec76bb53..9e767ce5a3daa 100755
--- a/dev/make-distribution.sh
+++ b/dev/make-distribution.sh
@@ -39,6 +39,7 @@ NAME=none
 MVN="$SPARK_HOME/build/mvn"
 
 function exit_with_usage {
+  set +x
   echo "make-distribution.sh - tool for making binary distributions of Spark"
   echo ""
   echo "usage:"
diff --git a/dev/run-pip-tests b/dev/run-pip-tests
index 470f21e69d46a..b322d3f61b444 100755
--- a/dev/run-pip-tests
+++ b/dev/run-pip-tests
@@ -63,11 +63,15 @@ fi
 PYSPARK_VERSION=$(python3 -c "exec(open('python/pyspark/version.py').read());print(__version__)")
 PYSPARK_DIST="$FWDIR/python/dist/pyspark-$PYSPARK_VERSION.tar.gz"
 # The pip install options we use for all the pip commands
-PIP_OPTIONS="--upgrade --no-cache-dir --force-reinstall "
+PIP_OPTIONS="--upgrade --no-cache-dir --force-reinstall"
 # Test both regular user and edit/dev install modes.
 PIP_COMMANDS=("pip install $PIP_OPTIONS $PYSPARK_DIST"
 	      "pip install $PIP_OPTIONS -e python/")
 
+# Jenkins has PySpark installed under user sitepackages shared for some reasons.
+# In this test, explicitly exclude user sitepackages to prevent side effects
+export PYTHONNOUSERSITE=1
+
 for python in "${PYTHON_EXECS[@]}"; do
   for install_command in "${PIP_COMMANDS[@]}"; do
     echo "Testing pip installation with python $python"
@@ -76,8 +80,12 @@ for python in "${PYTHON_EXECS[@]}"; do
     VIRTUALENV_PATH="$VIRTUALENV_BASE"/$python
     rm -rf "$VIRTUALENV_PATH"
     if [ -n "$USE_CONDA" ]; then
+      if [ -f "$CONDA_PREFIX/etc/profile.d/conda.sh" ]; then
+        # See also https://github.com/conda/conda/issues/7980
+        source "$CONDA_PREFIX/etc/profile.d/conda.sh"
+      fi
       conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools
-      source activate "$VIRTUALENV_PATH"
+      source activate "$VIRTUALENV_PATH" || conda activate "$VIRTUALENV_PATH"
     else
       mkdir -p "$VIRTUALENV_PATH"
       virtualenv --python=$python "$VIRTUALENV_PATH"
@@ -120,7 +128,7 @@ for python in "${PYTHON_EXECS[@]}"; do
 
     # conda / virtualenv environments need to be deactivated differently
     if [ -n "$USE_CONDA" ]; then
-      source deactivate
+      source deactivate || conda deactivate
     else
       deactivate
     fi
diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins
index c3adc696a5122..c155d4ea3f076 100755
--- a/dev/run-tests-jenkins
+++ b/dev/run-tests-jenkins
@@ -26,6 +26,7 @@ FWDIR="$( cd "$( dirname "$0" )/.." && pwd )"
 cd "$FWDIR"
 
 export PATH=/home/anaconda/envs/py36/bin:$PATH
+export LANG="en_US.UTF-8"
 
 PYTHON_VERSION_CHECK=$(python3 -c 'import sys; print(sys.version_info < (3, 6, 0))')
 if [[ "$PYTHON_VERSION_CHECK" == "True" ]]; then
diff --git a/dev/run-tests-jenkins.py b/dev/run-tests-jenkins.py
index 72e32d4e16e14..13be9592d771f 100755
--- a/dev/run-tests-jenkins.py
+++ b/dev/run-tests-jenkins.py
@@ -198,7 +198,7 @@ def main():
     # format: http://linux.die.net/man/1/timeout
     # must be less than the timeout configured on Jenkins. Usually Jenkins's timeout is higher
     # then this. Please consult with the build manager or a committer when it should be increased.
-    tests_timeout = "400m"
+    tests_timeout = "500m"
 
     # Array to capture all test names to run on the pull request. These tests are represented
     # by their file equivalents in the dev/tests/ directory.
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 5255a77ec2081..d58884fc7783c 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -43,19 +43,21 @@ def determine_modules_for_files(filenames):
     """
     Given a list of filenames, return the set of modules that contain those files.
     If a file is not associated with a more specific submodule, then this method will consider that
-    file to belong to the 'root' module. GitHub Action and Appveyor files are ignored.
+    file to belong to the 'root' module. `.github` directory is counted only in GitHub Actions,
+    and `appveyor.yml` is always ignored because this file is dedicated only to AppVeyor builds.
 
     >>> sorted(x.name for x in determine_modules_for_files(["python/pyspark/a.py", "sql/core/foo"]))
     ['pyspark-core', 'sql']
     >>> [x.name for x in determine_modules_for_files(["file_not_matched_by_any_subproject"])]
     ['root']
-    >>> [x.name for x in determine_modules_for_files( \
-            [".github/workflows/master.yml", "appveyor.yml"])]
+    >>> [x.name for x in determine_modules_for_files(["appveyor.yml"])]
     []
     """
     changed_modules = set()
     for filename in filenames:
-        if filename in (".github/workflows/master.yml", "appveyor.yml"):
+        if filename in ("appveyor.yml",):
+            continue
+        if ("GITHUB_ACTIONS" not in os.environ) and filename.startswith(".github"):
             continue
         matched_at_least_one_module = False
         for module in modules.all_modules:
@@ -101,28 +103,52 @@ def setup_test_environ(environ):
         os.environ[k] = v
 
 
-def determine_modules_to_test(changed_modules):
+def determine_modules_to_test(changed_modules, deduplicated=True):
     """
     Given a set of modules that have changed, compute the transitive closure of those modules'
     dependent modules in order to determine the set of modules that should be tested.
 
     Returns a topologically-sorted list of modules (ties are broken by sorting on module names).
+    If ``deduplicated`` is disabled, the modules are returned without tacking the deduplication
+    by dependencies into account.
 
     >>> [x.name for x in determine_modules_to_test([modules.root])]
     ['root']
     >>> [x.name for x in determine_modules_to_test([modules.build])]
     ['root']
+    >>> [x.name for x in determine_modules_to_test([modules.core])]
+    ['root']
+    >>> [x.name for x in determine_modules_to_test([modules.launcher])]
+    ['root']
     >>> [x.name for x in determine_modules_to_test([modules.graphx])]
     ['graphx', 'examples']
-    >>> x = [x.name for x in determine_modules_to_test([modules.sql])]
-    >>> x # doctest: +NORMALIZE_WHITESPACE
+    >>> [x.name for x in determine_modules_to_test([modules.sql])]
+    ... # doctest: +NORMALIZE_WHITESPACE
     ['sql', 'avro', 'hive', 'mllib', 'sql-kafka-0-10', 'examples', 'hive-thriftserver',
      'pyspark-sql', 'repl', 'sparkr', 'pyspark-mllib', 'pyspark-ml']
+    >>> sorted([x.name for x in determine_modules_to_test(
+    ...     [modules.sparkr, modules.sql], deduplicated=False)])
+    ... # doctest: +NORMALIZE_WHITESPACE
+    ['avro', 'examples', 'hive', 'hive-thriftserver', 'mllib', 'pyspark-ml',
+     'pyspark-mllib', 'pyspark-sql', 'repl', 'sparkr', 'sql', 'sql-kafka-0-10']
+    >>> sorted([x.name for x in determine_modules_to_test(
+    ...     [modules.sql, modules.core], deduplicated=False)])
+    ... # doctest: +NORMALIZE_WHITESPACE
+    ['avro', 'catalyst', 'core', 'examples', 'graphx', 'hive', 'hive-thriftserver',
+     'mllib', 'mllib-local', 'pyspark-core', 'pyspark-ml', 'pyspark-mllib',
+     'pyspark-sql', 'pyspark-streaming', 'repl', 'root',
+     'sparkr', 'sql', 'sql-kafka-0-10', 'streaming', 'streaming-kafka-0-10',
+     'streaming-kinesis-asl']
     """
     modules_to_test = set()
     for module in changed_modules:
-        modules_to_test = modules_to_test.union(determine_modules_to_test(module.dependent_modules))
+        modules_to_test = modules_to_test.union(
+            determine_modules_to_test(module.dependent_modules, deduplicated))
     modules_to_test = modules_to_test.union(set(changed_modules))
+
+    if not deduplicated:
+        return modules_to_test
+
     # If we need to run all of the tests, then we should short-circuit and return 'root'
     if modules.root in modules_to_test:
         return [modules.root]
@@ -415,7 +441,7 @@ def run_scala_tests_sbt(test_modules, test_profiles):
     exec_sbt(profiles_and_goals)
 
 
-def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags):
+def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags):
     """Function to properly execute all tests passed in as a set from the
     `determine_test_suites` function"""
     set_title_and_block("Running Spark unit tests", "BLOCK_SPARK_UNIT_TESTS")
@@ -425,6 +451,8 @@ def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags):
     test_profiles = extra_profiles + \
         list(set(itertools.chain.from_iterable(m.build_profile_flags for m in test_modules)))
 
+    if included_tags:
+        test_profiles += ['-Dtest.include.tags=' + ",".join(included_tags)]
     if excluded_tags:
         test_profiles += ['-Dtest.exclude.tags=' + ",".join(excluded_tags)]
 
@@ -456,6 +484,12 @@ def run_python_tests(test_modules, parallelism, with_coverage=False):
     if test_modules != [modules.root]:
         command.append("--modules=%s" % ','.join(m.name for m in test_modules))
     command.append("--parallelism=%i" % parallelism)
+    if "GITHUB_ACTIONS" in os.environ:
+        # See SPARK-33565. Python 3.8 was temporarily removed as its default Python executables
+        # to test because of Jenkins environment issue. Once Jenkins has Python 3.8 to test,
+        # we should remove this change back and add python3.8 into python/run-tests.py script.
+        command.append("--python-executable=%s" % ','.join(
+            x for x in ["python3.8", "python2.7", "pypy3", "pypy"] if which(x)))
     run_cmd(command)
 
     if with_coverage:
@@ -532,6 +566,24 @@ def parse_opts():
         "-p", "--parallelism", type=int, default=8,
         help="The number of suites to test in parallel (default %(default)d)"
     )
+    parser.add_argument(
+        "-m", "--modules", type=str,
+        default=None,
+        help="A comma-separated list of modules to test "
+             "(default: %s)" % ",".join(sorted([m.name for m in modules.all_modules]))
+    )
+    parser.add_argument(
+        "-e", "--excluded-tags", type=str,
+        default=None,
+        help="A comma-separated list of tags to exclude in the tests, "
+             "e.g., org.apache.spark.tags.ExtendedHiveTest "
+    )
+    parser.add_argument(
+        "-i", "--included-tags", type=str,
+        default=None,
+        help="A comma-separated list of tags to include in the tests, "
+             "e.g., org.apache.spark.tags.ExtendedHiveTest "
+    )
 
     args, unknown = parser.parse_known_args()
     if unknown:
@@ -564,11 +616,20 @@ def main():
               " install one and retry.")
         sys.exit(2)
 
-    # install SparkR
-    if which("R"):
-        run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
-    else:
-        print("Cannot install SparkR as R was not found in PATH")
+    # Install SparkR
+    should_only_test_modules = opts.modules is not None
+    test_modules = []
+    if should_only_test_modules:
+        str_test_modules = [m.strip() for m in opts.modules.split(",")]
+        test_modules = [m for m in modules.all_modules if m.name in str_test_modules]
+
+    if not should_only_test_modules or modules.sparkr in test_modules:
+        # If tests modules are specified, we will not run R linter.
+        # SparkR needs the manual SparkR installation.
+        if which("R"):
+            run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
+        else:
+            print("Cannot install SparkR as R was not found in PATH")
 
     if os.environ.get("AMPLAB_JENKINS"):
         # if we're on the Amplab Jenkins build servers setup variables
@@ -582,27 +643,69 @@ def main():
         # /home/jenkins/anaconda2/envs/py36/bin
         os.environ["PATH"] = "/home/anaconda/envs/py36/bin:" + os.environ.get("PATH")
     else:
-        # else we're running locally and can use local settings
+        # else we're running locally or Github Actions.
         build_tool = "sbt"
         hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7")
         hive_version = os.environ.get("HIVE_PROFILE", "hive2.3")
-        test_env = "local"
+        if "GITHUB_ACTIONS" in os.environ:
+            test_env = "github_actions"
+        else:
+            test_env = "local"
 
     print("[info] Using build tool", build_tool, "with Hadoop profile", hadoop_version,
           "and Hive profile", hive_version, "under environment", test_env)
     extra_profiles = get_hadoop_profiles(hadoop_version) + get_hive_profiles(hive_version)
 
-    changed_modules = None
-    changed_files = None
-    if test_env == "amplab_jenkins" and os.environ.get("AMP_JENKINS_PRB"):
+    changed_modules = []
+    changed_files = []
+    included_tags = []
+    excluded_tags = []
+    if should_only_test_modules:
+        # If we're running the tests in Github Actions, attempt to detect and test
+        # only the affected modules.
+        if test_env == "github_actions":
+            if os.environ["GITHUB_BASE_REF"] != "":
+                # Pull requests
+                changed_files = identify_changed_files_from_git_commits(
+                    os.environ["GITHUB_SHA"], target_branch=os.environ["GITHUB_BASE_REF"])
+            else:
+                # Build for each commit.
+                changed_files = identify_changed_files_from_git_commits(
+                    os.environ["GITHUB_SHA"], target_ref=os.environ["GITHUB_PREV_SHA"])
+
+            modules_to_test = determine_modules_to_test(
+                determine_modules_for_files(changed_files), deduplicated=False)
+
+            if modules.root not in modules_to_test:
+                # If root module is not found, only test the intersected modules.
+                # If root module is found, just run the modules as specified initially.
+                test_modules = list(set(modules_to_test).intersection(test_modules))
+
+        changed_modules = test_modules
+        if len(changed_modules) == 0:
+            print("[info] There are no modules to test, exiting without testing.")
+            return
+
+    # If we're running the tests in AMPLab Jenkins, calculate the diff from the targeted branch, and
+    # detect modules to test.
+    elif test_env == "amplab_jenkins" and os.environ.get("AMP_JENKINS_PRB"):
         target_branch = os.environ["ghprbTargetBranch"]
         changed_files = identify_changed_files_from_git_commits("HEAD", target_branch=target_branch)
         changed_modules = determine_modules_for_files(changed_files)
+        test_modules = determine_modules_to_test(changed_modules)
         excluded_tags = determine_tags_to_exclude(changed_modules)
 
+    # If there is no changed module found, tests all.
     if not changed_modules:
         changed_modules = [modules.root]
-        excluded_tags = []
+    if not test_modules:
+        test_modules = determine_modules_to_test(changed_modules)
+
+    if opts.excluded_tags:
+        excluded_tags.extend([t.strip() for t in opts.excluded_tags.split(",")])
+    if opts.included_tags:
+        included_tags.extend([t.strip() for t in opts.included_tags.split(",")])
+
     print("[info] Found the following changed modules:",
           ", ".join(x.name for x in changed_modules))
 
@@ -615,33 +718,32 @@ def main():
         test_environ.update(m.environ)
     setup_test_environ(test_environ)
 
-    test_modules = determine_modules_to_test(changed_modules)
-
-    # license checks
-    run_apache_rat_checks()
-
-    # style checks
-    if not changed_files or any(f.endswith(".scala")
-                                or f.endswith("scalastyle-config.xml")
-                                for f in changed_files):
-        run_scala_style_checks(extra_profiles)
     should_run_java_style_checks = False
-    if not changed_files or any(f.endswith(".java")
-                                or f.endswith("checkstyle.xml")
-                                or f.endswith("checkstyle-suppressions.xml")
-                                for f in changed_files):
-        # Run SBT Checkstyle after the build to prevent a side-effect to the build.
-        should_run_java_style_checks = True
-    if not changed_files or any(f.endswith("lint-python")
-                                or f.endswith("tox.ini")
-                                or f.endswith(".py")
-                                for f in changed_files):
-        run_python_style_checks()
-    if not changed_files or any(f.endswith(".R")
-                                or f.endswith("lint-r")
-                                or f.endswith(".lintr")
-                                for f in changed_files):
-        run_sparkr_style_checks()
+    if not should_only_test_modules:
+        # license checks
+        run_apache_rat_checks()
+
+        # style checks
+        if not changed_files or any(f.endswith(".scala")
+                                    or f.endswith("scalastyle-config.xml")
+                                    for f in changed_files):
+            run_scala_style_checks(extra_profiles)
+        if not changed_files or any(f.endswith(".java")
+                                    or f.endswith("checkstyle.xml")
+                                    or f.endswith("checkstyle-suppressions.xml")
+                                    for f in changed_files):
+            # Run SBT Checkstyle after the build to prevent a side-effect to the build.
+            should_run_java_style_checks = True
+        if not changed_files or any(f.endswith("lint-python")
+                                    or f.endswith("tox.ini")
+                                    or f.endswith(".py")
+                                    for f in changed_files):
+            run_python_style_checks()
+        if not changed_files or any(f.endswith(".R")
+                                    or f.endswith("lint-r")
+                                    or f.endswith(".lintr")
+                                    for f in changed_files):
+            run_sparkr_style_checks()
 
     # determine if docs were changed and if we're inside the amplab environment
     # note - the below commented out until *all* Jenkins workers can get `jekyll` installed
@@ -663,7 +765,7 @@ def main():
         build_spark_assembly_sbt(extra_profiles, should_run_java_style_checks)
 
     # run the test suites
-    run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags)
+    run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags)
 
     modules_with_python_tests = [m for m in test_modules if m.python_test_goals]
     if modules_with_python_tests:
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 40f2ca288d694..8705d523208fe 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -100,9 +100,75 @@ def __hash__(self):
     ]
 )
 
+kvstore = Module(
+    name="kvstore",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/kvstore/",
+    ],
+    sbt_test_goals=[
+        "kvstore/test",
+    ],
+)
+
+network_common = Module(
+    name="network-common",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/network-common/",
+    ],
+    sbt_test_goals=[
+        "network-common/test",
+    ],
+)
+
+network_shuffle = Module(
+    name="network-shuffle",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/network-shuffle/",
+    ],
+    sbt_test_goals=[
+        "network-shuffle/test",
+    ],
+)
+
+unsafe = Module(
+    name="unsafe",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/unsafe",
+    ],
+    sbt_test_goals=[
+        "unsafe/test",
+    ],
+)
+
+launcher = Module(
+    name="launcher",
+    dependencies=[tags],
+    source_file_regexes=[
+        "launcher/",
+    ],
+    sbt_test_goals=[
+        "launcher/test",
+    ],
+)
+
+core = Module(
+    name="core",
+    dependencies=[kvstore, network_common, network_shuffle, unsafe, launcher],
+    source_file_regexes=[
+        "core/",
+    ],
+    sbt_test_goals=[
+        "core/test",
+    ],
+)
+
 catalyst = Module(
     name="catalyst",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "sql/catalyst/",
     ],
@@ -111,7 +177,6 @@ def __hash__(self):
     ],
 )
 
-
 sql = Module(
     name="sql",
     dependencies=[catalyst],
@@ -123,7 +188,6 @@ def __hash__(self):
     ],
 )
 
-
 hive = Module(
     name="hive",
     dependencies=[sql],
@@ -142,7 +206,6 @@ def __hash__(self):
     ]
 )
 
-
 repl = Module(
     name="repl",
     dependencies=[hive],
@@ -154,7 +217,6 @@ def __hash__(self):
     ],
 )
 
-
 hive_thriftserver = Module(
     name="hive-thriftserver",
     dependencies=[hive],
@@ -192,7 +254,6 @@ def __hash__(self):
     ]
 )
 
-
 sketch = Module(
     name="sketch",
     dependencies=[tags],
@@ -204,10 +265,9 @@ def __hash__(self):
     ]
 )
 
-
 graphx = Module(
     name="graphx",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "graphx/",
     ],
@@ -216,10 +276,9 @@ def __hash__(self):
     ]
 )
 
-
 streaming = Module(
     name="streaming",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "streaming",
     ],
@@ -235,7 +294,7 @@ def __hash__(self):
 # fail other PRs.
 streaming_kinesis_asl = Module(
     name="streaming-kinesis-asl",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "external/kinesis-asl/",
         "external/kinesis-asl-assembly/",
@@ -254,21 +313,23 @@ def __hash__(self):
 
 streaming_kafka_0_10 = Module(
     name="streaming-kafka-0-10",
-    dependencies=[streaming],
+    dependencies=[streaming, core],
     source_file_regexes=[
         # The ending "/" is necessary otherwise it will include "sql-kafka" codes
         "external/kafka-0-10/",
         "external/kafka-0-10-assembly",
+        "external/kafka-0-10-token-provider",
     ],
     sbt_test_goals=[
         "streaming-kafka-0-10/test",
+        "token-provider-kafka-0-10/test"
     ]
 )
 
 
 mllib_local = Module(
     name="mllib-local",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "mllib-local",
     ],
@@ -302,10 +363,9 @@ def __hash__(self):
     ]
 )
 
-
 pyspark_core = Module(
     name="pyspark-core",
-    dependencies=[],
+    dependencies=[core],
     source_file_regexes=[
         "python/(?!pyspark/(ml|mllib|sql|streaming))"
     ],
@@ -339,7 +399,6 @@ def __hash__(self):
     ]
 )
 
-
 pyspark_sql = Module(
     name="pyspark-sql",
     dependencies=[pyspark_core, hive, avro],
@@ -364,7 +423,6 @@ def __hash__(self):
         "pyspark.sql.avro.functions",
         "pyspark.sql.pandas.conversion",
         "pyspark.sql.pandas.map_ops",
-        "pyspark.sql.pandas.functions",
         "pyspark.sql.pandas.group_ops",
         "pyspark.sql.pandas.types",
         "pyspark.sql.pandas.serializers",
@@ -490,6 +548,7 @@ def __hash__(self):
         "pyspark.ml.tests.test_stat",
         "pyspark.ml.tests.test_training_summary",
         "pyspark.ml.tests.test_tuning",
+        "pyspark.ml.tests.test_util",
         "pyspark.ml.tests.test_wrapper",
     ],
     blacklisted_python_implementations=[
@@ -579,7 +638,7 @@ def __hash__(self):
 # No other modules should directly depend on this module.
 root = Module(
     name="root",
-    dependencies=[build],  # Changes to build should trigger all tests.
+    dependencies=[build, core],  # Changes to build should trigger all tests.
     source_file_regexes=[],
     # In order to run all of the tests, enable every test profile:
     build_profile_flags=list(set(
diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh
index 936ac00f6b9e7..46f1f55938120 100755
--- a/dev/test-dependencies.sh
+++ b/dev/test-dependencies.sh
@@ -47,7 +47,7 @@ OLD_VERSION=$($MVN -q \
     -Dexec.executable="echo" \
     -Dexec.args='${project.version}' \
     --non-recursive \
-    org.codehaus.mojo:exec-maven-plugin:1.6.0:exec)
+    org.codehaus.mojo:exec-maven-plugin:1.6.0:exec | grep -E '[0-9]+\.[0-9]+\.[0-9]+')
 if [ $? != 0 ]; then
     echo -e "Error while getting version string from Maven:\n$OLD_VERSION"
     exit 1
diff --git a/dev/tox.ini b/dev/tox.ini
index 54f65692c8303..49b673bd63db6 100644
--- a/dev/tox.ini
+++ b/dev/tox.ini
@@ -16,6 +16,4 @@
 [pycodestyle]
 ignore=E226,E241,E305,E402,E722,E731,E741,W503,W504
 max-line-length=100
-exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*
-[pydocstyle]
-ignore=D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D203,D204,D205,D206,D207,D208,D209,D210,D211,D212,D213,D214,D215,D300,D301,D302,D400,D401,D402,D403,D404,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414
+exclude=*/target/*,cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*
diff --git a/docs/.gitignore b/docs/.gitignore
new file mode 100644
index 0000000000000..9df83f37815b7
--- /dev/null
+++ b/docs/.gitignore
@@ -0,0 +1 @@
+generated-*.html
diff --git a/docs/README.md b/docs/README.md
index 22039871cf63d..a1c569ed8712c 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -39,6 +39,12 @@ installed. Also install the following libraries:
 $ sudo gem install jekyll jekyll-redirect-from rouge
 ```
 
+If your ruby version is 3.0 or higher, you should also install `webrick`.
+
+```sh
+$ sudo gem install jekyll jekyll-redirect-from webrick
+```
+
 Note: If you are on a system with both Ruby 1.9 and Ruby 2.0 you may need to replace gem with gem2.0.
 
 ### R Documentation
diff --git a/docs/_config.yml b/docs/_config.yml
index a888620139207..abb3b49fff54d 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -8,14 +8,17 @@ gems:
 kramdown:
   entity_output: numeric
 
+plugins:
+  - jekyll-redirect-from
+
 include:
   - _static
   - _modules
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.0-SNAPSHOT
-SPARK_VERSION_SHORT: 3.0.0
+SPARK_VERSION: 3.0.2-SNAPSHOT
+SPARK_VERSION_SHORT: 3.0.2
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
 MESOS_VERSION: 1.0.0
diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 3e4db7107ec34..581fd1de6ad2d 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -24,6 +24,8 @@
   subitems:
     - text: "Generic Load/Save Functions"
       url: sql-data-sources-load-save-functions.html
+    - text: "Generic File Source Options"
+      url: sql-data-sources-generic-options.html
     - text: Parquet Files
       url: sql-data-sources-parquet.html
     - text: ORC Files
@@ -65,6 +67,8 @@
       url: sql-pyspark-pandas-with-arrow.html#enabling-for-conversion-tofrom-pandas
     - text: "Pandas UDFs (a.k.a. Vectorized UDFs)"
       url: sql-pyspark-pandas-with-arrow.html#pandas-udfs-aka-vectorized-udfs
+    - text: "Pandas Function APIs"
+      url: sql-pyspark-pandas-with-arrow.html#pandas-function-apis
     - text: Usage Notes
       url: sql-pyspark-pandas-with-arrow.html#usage-notes
 - text: Migration Guide
@@ -72,12 +76,36 @@
 - text: SQL Reference
   url: sql-ref.html
   subitems:
+    - text: ANSI Compliance
+      url: sql-ref-ansi-compliance.html
+      subitems:
+        - text: Arithmetic Operations
+          url: sql-ref-ansi-compliance.html#arithmetic-operations
+        - text: Type Conversion
+          url: sql-ref-ansi-compliance.html#type-conversion
+        - text: SQL Keywords
+          url: sql-ref-ansi-compliance.html#sql-keywords
     - text: Data Types
       url: sql-ref-datatypes.html
+    - text: Datetime Pattern
+      url: sql-ref-datetime-pattern.html
+    - text: Functions
+      url: sql-ref-functions.html
+      subitems:
+      - text: Built-in Functions
+        url: sql-ref-functions-builtin.html
+      - text: Scalar UDFs (User-Defined Functions)
+        url: sql-ref-functions-udf-scalar.html
+      - text: UDAFs (User-Defined Aggregate Functions)
+        url: sql-ref-functions-udf-aggregate.html
+      - text: Integration with Hive UDFs/UDAFs/UDTFs
+        url: sql-ref-functions-udf-hive.html
+    - text: Identifiers
+      url: sql-ref-identifier.html
+    - text: Literals
+      url: sql-ref-literals.html
     - text: Null Semantics
       url: sql-ref-null-semantics.html
-    - text: NaN Semantics
-      url: sql-ref-nan-semantics.html
     - text: SQL Syntax
       url: sql-ref-syntax.html
       subitems:
@@ -110,6 +138,8 @@
               url: sql-ref-syntax-ddl-truncate-table.html
             - text: REPAIR TABLE
               url: sql-ref-syntax-ddl-repair-table.html
+            - text: USE DATABASE
+              url: sql-ref-syntax-ddl-usedb.html
         - text: Data Manipulation Statements
           url: sql-ref-syntax-dml.html
           subitems:
@@ -139,19 +169,43 @@
                   url: sql-ref-syntax-qry-select-distribute-by.html
                 - text: LIMIT Clause 
                   url: sql-ref-syntax-qry-select-limit.html
-                - text: USE database
-                  url: sql-ref-syntax-qry-select-usedb.html
+                - text: Common Table Expression
+                  url: sql-ref-syntax-qry-select-cte.html
+                - text: Hints
+                  url: sql-ref-syntax-qry-select-hints.html
+                - text: Inline Table
+                  url: sql-ref-syntax-qry-select-inline-table.html
+                - text: File
+                  url: sql-ref-syntax-qry-select-file.html
+                - text: JOIN
+                  url: sql-ref-syntax-qry-select-join.html
+                - text: LIKE Predicate
+                  url: sql-ref-syntax-qry-select-like.html
+                - text: Set Operators
+                  url: sql-ref-syntax-qry-select-setops.html
+                - text: TABLESAMPLE
+                  url: sql-ref-syntax-qry-select-sampling.html
+                - text: Table-valued Function
+                  url: sql-ref-syntax-qry-select-tvf.html
+                - text: Window Function
+                  url: sql-ref-syntax-qry-select-window.html
+                - text: CASE Clause
+                  url: sql-ref-syntax-qry-select-case.html
+                - text: LATERAL VIEW Clause
+                  url: sql-ref-syntax-qry-select-lateral-view.html
+                - text: PIVOT Clause
+                  url: sql-ref-syntax-qry-select-pivot.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
           url: sql-ref-syntax-aux.html
           subitems:
-            - text: Analyze statement
+            - text: ANALYZE
               url: sql-ref-syntax-aux-analyze.html
               subitems: 
                 - text: ANALYZE TABLE
                   url: sql-ref-syntax-aux-analyze-table.html
-            - text: Caching statements 
+            - text: CACHE
               url: sql-ref-syntax-aux-cache.html
               subitems:
                 - text: CACHE TABLE
@@ -161,10 +215,10 @@
                 - text: CLEAR CACHE
                   url: sql-ref-syntax-aux-cache-clear-cache.html
                 - text: REFRESH TABLE
-                  url: sql-ref-syntax-aux-refresh-table.html
+                  url: sql-ref-syntax-aux-cache-refresh-table.html
                 - text: REFRESH
-                  url: sql-ref-syntax-aux-cache-refresh.md
-            - text: Describe Commands
+                  url: sql-ref-syntax-aux-cache-refresh.html
+            - text: DESCRIBE
               url: sql-ref-syntax-aux-describe.html
               subitems:
                 - text: DESCRIBE DATABASE
@@ -175,33 +229,35 @@
                   url: sql-ref-syntax-aux-describe-function.html
                 - text: DESCRIBE QUERY
                   url: sql-ref-syntax-aux-describe-query.html
-            - text: Show commands
+            - text: SHOW
               url: sql-ref-syntax-aux-show.html
               subitems:
                 - text: SHOW COLUMNS 
                   url: sql-ref-syntax-aux-show-columns.html
+                - text: SHOW CREATE TABLE
+                  url: sql-ref-syntax-aux-show-create-table.html
                 - text: SHOW DATABASES
                   url: sql-ref-syntax-aux-show-databases.html
                 - text: SHOW FUNCTIONS
                   url: sql-ref-syntax-aux-show-functions.html
+                - text: SHOW PARTITIONS
+                  url: sql-ref-syntax-aux-show-partitions.html
                 - text: SHOW TABLE
                   url: sql-ref-syntax-aux-show-table.html
                 - text: SHOW TABLES
                   url: sql-ref-syntax-aux-show-tables.html
                 - text: SHOW TBLPROPERTIES
                   url: sql-ref-syntax-aux-show-tblproperties.html
-                - text: SHOW PARTITIONS 
-                  url: sql-ref-syntax-aux-show-partitions.html
-                - text: SHOW CREATE TABLE
-                  url: sql-ref-syntax-aux-show-create-table.html
-            - text: Configuration Management Commands
+                - text: SHOW VIEWS
+                  url: sql-ref-syntax-aux-show-views.html
+            - text: CONFIGURATION MANAGEMENT
               url: sql-ref-syntax-aux-conf-mgmt.html
               subitems:
                 - text: SET 
                   url: sql-ref-syntax-aux-conf-mgmt-set.html
                 - text: RESET
                   url: sql-ref-syntax-aux-conf-mgmt-reset.html
-            - text: Resource Management Commands
+            - text: RESOURCE MANAGEMENT
               url: sql-ref-syntax-aux-resource-mgmt.html
               subitems:
                 - text: ADD FILE
@@ -212,5 +268,3 @@
                   url: sql-ref-syntax-aux-resource-mgmt-list-file.html
                 - text: LIST JAR
                   url: sql-ref-syntax-aux-resource-mgmt-list-jar.html
-    - text: Arithmetic operations
-      url: sql-ref-arithmetic-ops.html
diff --git a/docs/_layouts/global.html b/docs/_layouts/global.html
index d5fb18bfb06c0..d05ac6bbe129d 100755
--- a/docs/_layouts/global.html
+++ b/docs/_layouts/global.html
@@ -82,7 +82,7 @@
                         <li class="dropdown">
                             <a href="#" class="dropdown-toggle" data-toggle="dropdown">API Docs<b class="caret"></b></a>
                             <ul class="dropdown-menu">
-                                <li><a href="api/scala/index.html#org.apache.spark.package">Scala</a></li>
+                                <li><a href="api/scala/org/apache/spark/index.html">Scala</a></li>
                                 <li><a href="api/java/index.html">Java</a></li>
                                 <li><a href="api/python/index.html">Python</a></li>
                                 <li><a href="api/R/index.html">R</a></li>
diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
index f95e4e2f97792..8e2a06e4bc9a8 100644
--- a/docs/_plugins/copy_api_dirs.rb
+++ b/docs/_plugins/copy_api_dirs.rb
@@ -157,8 +157,8 @@
     curr_dir = pwd
     cd("..")
 
-    puts "Running 'build/sbt clean package' from " + pwd + "; this may take a few minutes..."
-    system("build/sbt clean package") || raise("SQL doc generation failed")
+    puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..."
+    system("build/sbt clean package -Phive") || raise("SQL doc generation failed")
 
     puts "Moving back into docs dir."
     cd("docs")
diff --git a/docs/building-spark.md b/docs/building-spark.md
index 77ab7900dc4a2..3d12a60e2b974 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -83,9 +83,9 @@ Example:
 
 To enable Hive integration for Spark SQL along with its JDBC server and CLI,
 add the `-Phive` and `-Phive-thriftserver` profiles to your existing build options.
-By default Spark will build with Hive 2.3.6.
+By default Spark will build with Hive 2.3.7.
 
-    # With Hive 2.3.6 support
+    # With Hive 2.3.7 support
     ./build/mvn -Pyarn -Phive -Phive-thriftserver -DskipTests clean package
 
 ## Packaging without Hadoop Dependencies for YARN
diff --git a/docs/cloud-integration.md b/docs/cloud-integration.md
index b2a3e77f1ee9d..1eaa8ab4eeee0 100644
--- a/docs/cloud-integration.md
+++ b/docs/cloud-integration.md
@@ -103,7 +103,7 @@ for talking to cloud infrastructures, in which case this module may not be neede
 Spark jobs must authenticate with the object stores to access data within them.
 
 1. When Spark is running in a cloud infrastructure, the credentials are usually automatically set up.
-1. `spark-submit` reads the `AWS_ACCESS_KEY`, `AWS_SECRET_KEY`
+1. `spark-submit` reads the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`
 and `AWS_SESSION_TOKEN` environment variables and sets the associated authentication options
 for the `s3n` and `s3a` connectors to Amazon S3.
 1. In a Hadoop cluster, settings may be set in the `core-site.xml` file.
@@ -163,7 +163,7 @@ different stores and connectors when renaming directories:
 | Amazon S3     | s3a       | Unsafe                  | O(data) |
 | Azure Storage | wasb      | Safe                    | O(files) |
 | Azure Datalake Gen 2 | abfs | Safe                  | O(1) |
-| Google GCS    | gs        | Safe                    | O(1) |
+| Google Cloud Storage | gs        | Safe                    | O(1) |
 
 As storing temporary files can run up charges; delete
 directories called `"_temporary"` on a regular basis.
@@ -253,9 +253,9 @@ Here is the documentation on the standard connectors both from Apache and the cl
 * [Azure Blob Storage and Azure Datalake Gen 2](https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html).
 * [Azure Data Lake Gen 1](https://hadoop.apache.org/docs/current/hadoop-azure-datalake/index.html).
 * [Hadoop-AWS module (Hadoop 3.x)](https://hadoop.apache.org/docs/current3/hadoop-aws/tools/hadoop-aws/index.html).
-* [Amazon S3 via S3A and S3N (Hadoop 2.x)](https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html).
-* [Amazon EMR File System (EMRFS)](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-fs.html). From Amazon
-* [Google Cloud Storage Connector for Spark and Hadoop](https://cloud.google.com/hadoop/google-cloud-storage-connector). From Google
+* [Amazon S3 via S3A and S3N (Hadoop 2.x)](https://hadoop.apache.org/docs/current2/hadoop-aws/tools/hadoop-aws/index.html).
+* [Amazon EMR File System (EMRFS)](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-fs.html). From Amazon.
+* [Google Cloud Storage Connector for Spark and Hadoop](https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage). From Google.
 * [The Azure Blob Filesystem driver (ABFS)](https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-abfs-driver)
-* IBM Cloud Object Storage connector for Apache Spark: [Stocator](https://github.com/CODAIT/stocator), [IBM Object Storage](https://www.ibm.com/cloud/object-storage), [how-to-use-connector](https://developer.ibm.com/code/2018/08/16/installing-running-stocator-apache-spark-ibm-cloud-object-storage). From IBM
+* IBM Cloud Object Storage connector for Apache Spark: [Stocator](https://github.com/CODAIT/stocator), [IBM Object Storage](https://www.ibm.com/cloud/object-storage). From IBM.
 
diff --git a/docs/configuration.md b/docs/configuration.md
index 2febfe9744d5c..a9daabe366fd8 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -24,7 +24,7 @@ license: |
 Spark provides three locations to configure the system:
 
 * [Spark properties](#spark-properties) control most application parameters and can be set by using
-  a [SparkConf](api/scala/index.html#org.apache.spark.SparkConf) object, or through Java
+  a [SparkConf](api/scala/org/apache/spark/SparkConf.html) object, or through Java
   system properties.
 * [Environment variables](#environment-variables) can be used to set per-machine settings, such as
   the IP address, through the `conf/spark-env.sh` script on each node.
@@ -34,7 +34,7 @@ Spark provides three locations to configure the system:
 
 Spark properties control most application settings and are configured separately for each
 application. These properties can be set directly on a
-[SparkConf](api/scala/index.html#org.apache.spark.SparkConf) passed to your
+[SparkConf](api/scala/org/apache/spark/SparkConf.html) passed to your
 `SparkContext`. `SparkConf` allows you to configure some of the common properties
 (e.g. master URL and application name), as well as arbitrary key-value pairs through the
 `set()` method. For example, we could initialize an application with two threads as follows:
@@ -95,7 +95,7 @@ Then, you can supply configuration values at runtime:
 
 The Spark shell and [`spark-submit`](submitting-applications.html)
 tool support two ways to load configurations dynamically. The first is command line options,
-such as `--master`, as shown above. `spark-submit` can accept any Spark property using the `--conf`
+such as `--master`, as shown above. `spark-submit` can accept any Spark property using the `--conf/-c`
 flag, but uses special flags for properties that play a part in launching the Spark application.
 Running `./bin/spark-submit --help` will show the entire list of these options.
 
@@ -136,13 +136,14 @@ of the most common options to set are:
 ### Application Properties
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.app.name</code></td>
   <td>(none)</td>
   <td>
     The name of your application. This will appear in the UI and in log data.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.cores</code></td>
@@ -150,6 +151,7 @@ of the most common options to set are:
   <td>
     Number of cores to use for the driver process, only in cluster mode.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.maxResultSize</code></td>
@@ -162,6 +164,7 @@ of the most common options to set are:
     and memory overhead of objects in JVM). Setting a proper limit can protect the driver from
     out-of-memory errors.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.memory</code></td>
@@ -176,6 +179,7 @@ of the most common options to set are:
     Instead, please set this through the <code>--driver-memory</code> command line option
     or in your default properties file.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.driver.memoryOverhead</code></td>
@@ -192,25 +196,7 @@ of the most common options to set are:
     driver is determined by the sum of <code>spark.driver.memoryOverhead</code> 
     and <code>spark.driver.memory</code>.
   </td>
-</tr>
-<tr>
- <td><code>spark.resources.coordinate.enable</code></td>
-  <td>true</td>
-  <td>
-    Whether to coordinate resources automatically among workers/drivers(client only) 
-    in Standalone. If false, the user is responsible for configuring different resources
-    for workers/drivers that run on the same host.
-  </td>
-</tr>
-<tr>
- <td><code>spark.resources.dir</code></td>
-  <td>SPARK_HOME</td>
-  <td>
-    Directory used to coordinate resources among workers/drivers(client only) in Standalone.
-    Default is SPARK_HOME. Make sure to use the same directory for worker and drivers in
-    client mode that run on the same host. Don't clean up this directory while workers/drivers
-    are still alive to avoid the most likely resources conflict. 
-  </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
  <td><code>spark.driver.resource.{resourceName}.amount</code></td>
@@ -221,6 +207,7 @@ of the most common options to set are:
     <code>spark.driver.resource.{resourceName}.discoveryScript</code>
     for the driver to find the resource on startup.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
  <td><code>spark.driver.resource.{resourceName}.discoveryScript</code></td>
@@ -228,10 +215,10 @@ of the most common options to set are:
   <td>
     A script for the driver to run to discover a particular resource type. This should
     write to STDOUT a JSON string in the format of the ResourceInformation class. This has a
-    name and an array of addresses. For a client-submitted driver in Standalone, discovery
-    script must assign different resource addresses to this driver comparing to workers' and
-    other drivers' when <code>spark.resources.coordinate.enable</code> is off.
+    name and an array of addresses. For a client-submitted driver, discovery script must assign
+    different resource addresses to this driver comparing to other drivers on the same host.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
  <td><code>spark.driver.resource.{resourceName}.vendor</code></td>
@@ -242,9 +229,10 @@ of the most common options to set are:
     the Kubernetes device plugin naming convention. (e.g. For GPUs on Kubernetes
     this config would be set to nvidia.com or amd.com)
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
- <td><code>spark.resources.discovery.plugin</code></td>
+ <td><code>spark.resources.discoveryPlugin</code></td>
   <td>org.apache.spark.resource.ResourceDiscoveryScriptPlugin</td>
   <td>
     Comma-separated list of class names implementing
@@ -254,6 +242,7 @@ of the most common options to set are:
     returns the resource information for that resource. It tries the discovery
     script last if none of the plugins return information for that resource.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.memory</code></td>
@@ -262,6 +251,7 @@ of the most common options to set are:
     Amount of memory to use per executor process, in the same format as JVM memory strings with
     a size unit suffix ("k", "m", "g" or "t") (e.g. <code>512m</code>, <code>2g</code>).
   </td>
+  <td>0.7.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.pyspark.memory</code></td>
@@ -278,15 +268,15 @@ of the most common options to set are:
     limitations are inherited. For instance, Windows does not support resource limiting and actual 
     resource is not limited on MacOS.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.memoryOverhead</code></td>
   <td>executorMemory * 0.10, with minimum of 384 </td>
   <td>
-    Amount of additional memory to be allocated per executor process in cluster mode, in MiB unless
-    otherwise specified. This is memory that accounts for things like VM overheads, interned strings,
-    other native overheads, etc. This tends to grow with the executor size (typically 6-10%).
-    This option is currently supported on YARN and Kubernetes.
+    Amount of additional memory to be allocated per executor process, in MiB unless otherwise specified.
+    This is memory that accounts for things like VM overheads, interned strings, other native overheads, etc.
+    This tends to grow with the executor size (typically 6-10%). This option is currently supported on YARN and Kubernetes.
     <br/>
     <em>Note:</em> Additional memory includes PySpark executor memory 
     (when <code>spark.executor.pyspark.memory</code> is not configured) and memory used by other
@@ -295,6 +285,7 @@ of the most common options to set are:
     <code>spark.executor.memory</code>, <code>spark.memory.offHeap.size</code> and 
     <code>spark.executor.pyspark.memory</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.resource.{resourceName}.amount</code></td>
@@ -305,6 +296,7 @@ of the most common options to set are:
     <code>spark.executor.resource.{resourceName}.discoveryScript</code>
     for the executor to find the resource on startup.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.resource.{resourceName}.discoveryScript</code></td>
@@ -314,6 +306,7 @@ of the most common options to set are:
     write to STDOUT a JSON string in the format of the ResourceInformation class. This has a
     name and an array of addresses.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.resource.{resourceName}.vendor</code></td>
@@ -324,6 +317,7 @@ of the most common options to set are:
     the Kubernetes device plugin naming convention. (e.g. For GPUs on Kubernetes
     this config would be set to nvidia.com or amd.com)
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.extraListeners</code></td>
@@ -335,6 +329,7 @@ of the most common options to set are:
     will be called; otherwise, a zero-argument constructor will be called. If no valid constructor
     can be found, the SparkContext creation will fail with an exception.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.local.dir</code></td>
@@ -348,6 +343,7 @@ of the most common options to set are:
     <em>Note:</em> This will be overridden by SPARK_LOCAL_DIRS (Standalone), MESOS_SANDBOX (Mesos) or
     LOCAL_DIRS (YARN) environment variables set by the cluster manager.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.logConf</code></td>
@@ -355,6 +351,7 @@ of the most common options to set are:
   <td>
     Logs the effective SparkConf as INFO when a SparkContext is started.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.master</code></td>
@@ -363,6 +360,7 @@ of the most common options to set are:
     The cluster manager to connect to. See the list of
     <a href="submitting-applications.html#master-urls"> allowed master URL's</a>.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.submit.deployMode</code></td>
@@ -372,6 +370,7 @@ of the most common options to set are:
     Which means to launch driver program locally ("client")
     or remotely ("cluster") on one of the nodes inside the cluster.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.log.callerContext</code></td>
@@ -381,6 +380,7 @@ of the most common options to set are:
     Its length depends on the Hadoop configuration <code>hadoop.caller.context.max.size</code>. It should be concise,
     and typically can have up to 50 characters.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.supervise</code></td>
@@ -389,6 +389,7 @@ of the most common options to set are:
     If true, restarts the driver automatically if it fails with a non-zero exit status.
     Only has effect in Spark standalone mode or Mesos cluster deploy mode.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.log.dfsDir</code></td>
@@ -403,6 +404,7 @@ of the most common options to set are:
     <code>spark.history.fs.driverlog.cleaner.enabled</code> is true and, if they are older than max age configured
     by setting <code>spark.history.fs.driverlog.cleaner.maxAge</code>.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.log.persistToDfs.enabled</code></td>
@@ -413,6 +415,7 @@ of the most common options to set are:
     will not be persisted. Additionally, enable the cleaner by setting <code>spark.history.fs.driverlog.cleaner.enabled</code>
     to true in <a href="monitoring.html#spark-history-server-configuration-options">Spark History Server</a>.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.log.layout</code></td>
@@ -422,6 +425,7 @@ of the most common options to set are:
     it uses the layout for the first appender defined in log4j.properties. If that is also not configured, driver logs
     use the default layout.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.log.allowErasureCoding</code></td>
@@ -432,6 +436,7 @@ of the most common options to set are:
     written by the application. Note that even if this is true, Spark will still not force the
     file to use erasure coding, it will simply use file system defaults.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -440,7 +445,7 @@ Apart from these, the following properties are also available, and may be useful
 ### Runtime Environment
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.driver.extraClassPath</code></td>
   <td>(none)</td>
@@ -452,6 +457,7 @@ Apart from these, the following properties are also available, and may be useful
     Instead, please set this through the <code>--driver-class-path</code> command line option or in
     your default properties file.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.defaultJavaOptions</code></td>
@@ -470,6 +476,7 @@ Apart from these, the following properties are also available, and may be useful
     Instead, please set this through the <code>--driver-java-options</code> command line option or in
     your default properties file.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.extraJavaOptions</code></td>
@@ -489,6 +496,7 @@ Apart from these, the following properties are also available, and may be useful
 
     <code>spark.driver.defaultJavaOptions</code> will be prepended to this configuration.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.extraLibraryPath</code></td>
@@ -501,6 +509,7 @@ Apart from these, the following properties are also available, and may be useful
     Instead, please set this through the <code>--driver-library-path</code> command line option or in
     your default properties file.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.userClassPathFirst</code></td>
@@ -512,6 +521,7 @@ Apart from these, the following properties are also available, and may be useful
 
     This is used in cluster mode only.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.extraClassPath</code></td>
@@ -521,6 +531,7 @@ Apart from these, the following properties are also available, and may be useful
     backwards-compatibility with older versions of Spark. Users typically should not need to set
     this option.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.defaultJavaOptions</code></td>
@@ -539,6 +550,7 @@ Apart from these, the following properties are also available, and may be useful
     verbose gc logging to a file named for the executor ID of the app in /tmp, pass a 'value' of:
     <code>-verbose:gc -Xloggc:/tmp/{{APP_ID}}-{{EXECUTOR_ID}}.gc</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.extraJavaOptions</code></td>
@@ -558,6 +570,7 @@ Apart from these, the following properties are also available, and may be useful
 
     <code>spark.executor.defaultJavaOptions</code> will be prepended to this configuration.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.extraLibraryPath</code></td>
@@ -565,6 +578,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Set a special library path to use when launching executor JVM's.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.logs.rolling.maxRetainedFiles</code></td>
@@ -573,6 +587,7 @@ Apart from these, the following properties are also available, and may be useful
     Sets the number of latest rolling log files that are going to be retained by the system.
     Older log files will be deleted. Disabled by default.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.logs.rolling.enableCompression</code></td>
@@ -581,6 +596,7 @@ Apart from these, the following properties are also available, and may be useful
     Enable executor log compression. If it is enabled, the rolled executor logs will be compressed.
     Disabled by default.
   </td>
+  <td>2.0.2</td>
 </tr>
 <tr>
   <td><code>spark.executor.logs.rolling.maxSize</code></td>
@@ -590,6 +606,7 @@ Apart from these, the following properties are also available, and may be useful
     Rolling is disabled by default. See <code>spark.executor.logs.rolling.maxRetainedFiles</code>
     for automatic cleaning of old logs.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.logs.rolling.strategy</code></td>
@@ -601,6 +618,7 @@ Apart from these, the following properties are also available, and may be useful
     For "size", use <code>spark.executor.logs.rolling.maxSize</code> to set
     the maximum file size for rolling.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.logs.rolling.time.interval</code></td>
@@ -611,6 +629,7 @@ Apart from these, the following properties are also available, and may be useful
     any interval in seconds. See <code>spark.executor.logs.rolling.maxRetainedFiles</code>
     for automatic cleaning of old logs.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.userClassPathFirst</code></td>
@@ -619,6 +638,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) Same functionality as <code>spark.driver.userClassPathFirst</code>, but
     applied to executor instances.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.executorEnv.[EnvironmentVariableName]</code></td>
@@ -627,6 +647,7 @@ Apart from these, the following properties are also available, and may be useful
     Add the environment variable specified by <code>EnvironmentVariableName</code> to the Executor
     process. The user can specify multiple of these to set multiple environment variables.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.redaction.regex</code></td>
@@ -636,6 +657,7 @@ Apart from these, the following properties are also available, and may be useful
     executor environments contain sensitive information. When this regex matches a property key or
     value, the value is redacted from the environment UI and various logs like YARN and event logs.
   </td>
+  <td>2.1.2</td>
 </tr>
 <tr>
   <td><code>spark.python.profile</code></td>
@@ -649,6 +671,7 @@ Apart from these, the following properties are also available, and may be useful
     By default the <code>pyspark.profiler.BasicProfiler</code> will be used, but this can be overridden by
     passing a profiler class in as a parameter to the <code>SparkContext</code> constructor.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.python.profile.dump</code></td>
@@ -659,6 +682,7 @@ Apart from these, the following properties are also available, and may be useful
     by <code>pstats.Stats()</code>. If this is specified, the profile result will not be displayed
     automatically.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.python.worker.memory</code></td>
@@ -669,6 +693,7 @@ Apart from these, the following properties are also available, and may be useful
     (e.g. <code>512m</code>, <code>2g</code>).
     If the memory used during aggregation goes above this amount, it will spill the data into disks.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.python.worker.reuse</code></td>
@@ -679,6 +704,7 @@ Apart from these, the following properties are also available, and may be useful
     if there is a large broadcast, then the broadcast will not need to be transferred
     from JVM to Python worker for every task.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.files</code></td>
@@ -686,6 +712,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Comma-separated list of files to be placed in the working directory of each executor. Globs are allowed.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.submit.pyFiles</code></td>
@@ -693,6 +720,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Comma-separated list of .zip, .egg, or .py files to place on the PYTHONPATH for Python apps. Globs are allowed.
   </td>
+  <td>1.0.1</td>
 </tr>
 <tr>
   <td><code>spark.jars</code></td>
@@ -700,6 +728,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Comma-separated list of jars to include on the driver and executor classpaths. Globs are allowed.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.jars.packages</code></td>
@@ -712,6 +741,7 @@ Apart from these, the following properties are also available, and may be useful
     repositories given by the command-line option <code>--repositories</code>. For more details, see
     <a href="submitting-applications.html#advanced-dependency-management">Advanced Dependency Management</a>.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.jars.excludes</code></td>
@@ -720,6 +750,7 @@ Apart from these, the following properties are also available, and may be useful
     Comma-separated list of groupId:artifactId, to exclude while resolving the dependencies
     provided in <code>spark.jars.packages</code> to avoid dependency conflicts.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.jars.ivy</code></td>
@@ -729,6 +760,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.jars.packages</code>. This will override the Ivy property <code>ivy.default.ivy.user.dir</code>
     which defaults to ~/.ivy2.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.jars.ivySettings</code></td>
@@ -741,6 +773,7 @@ Apart from these, the following properties are also available, and may be useful
     artifact server like Artifactory. Details on the settings file format can be
     found at <a href="http://ant.apache.org/ivy/history/latest-milestone/settings.html">Settings Files</a>
   </td>
+  <td>2.2.0</td>
 </tr>
  <tr>
   <td><code>spark.jars.repositories</code></td>
@@ -749,6 +782,7 @@ Apart from these, the following properties are also available, and may be useful
     Comma-separated list of additional remote repositories to search for the maven coordinates
     given with <code>--packages</code> or <code>spark.jars.packages</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.pyspark.driver.python</code></td>
@@ -757,6 +791,7 @@ Apart from these, the following properties are also available, and may be useful
     Python binary executable to use for PySpark in driver.
     (default is <code>spark.pyspark.python</code>)
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.pyspark.python</code></td>
@@ -764,13 +799,14 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Python binary executable to use for PySpark in both driver and executors.
   </td>
+  <td>2.1.0</td>
 </tr>
 </table>
 
 ### Shuffle Behavior
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.reducer.maxSizeInFlight</code></td>
   <td>48m</td>
@@ -780,6 +816,7 @@ Apart from these, the following properties are also available, and may be useful
     represents a fixed memory overhead per reduce task, so keep it small unless you have a
     large amount of memory.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.reducer.maxReqsInFlight</code></td>
@@ -790,6 +827,7 @@ Apart from these, the following properties are also available, and may be useful
     of inbound connections to one or more nodes, causing the workers to fail under load.
     By allowing it to limit the number of fetch requests, this scenario can be mitigated.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.reducer.maxBlocksInFlightPerAddress</code></td>
@@ -801,6 +839,7 @@ Apart from these, the following properties are also available, and may be useful
     is especially useful to reduce the load on the Node Manager when external shuffle is enabled.
     You can mitigate this issue by setting it to a lower value.
   </td>
+  <td>2.2.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.compress</code></td>
@@ -809,6 +848,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to compress map output files. Generally a good idea. Compression will use
     <code>spark.io.compression.codec</code>.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.file.buffer</code></td>
@@ -818,6 +858,7 @@ Apart from these, the following properties are also available, and may be useful
     specified. These buffers reduce the number of disk seeks and system calls made in creating 
     intermediate shuffle files.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.io.maxRetries</code></td>
@@ -827,6 +868,7 @@ Apart from these, the following properties are also available, and may be useful
     set to a non-zero value. This retry logic helps stabilize large shuffles in the face of long GC
     pauses or transient network connectivity issues.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.io.numConnectionsPerPeer</code></td>
@@ -836,6 +878,7 @@ Apart from these, the following properties are also available, and may be useful
     large clusters. For clusters with many hard disks and few hosts, this may result in insufficient
     concurrency to saturate all disks, and so users may consider increasing this value.
   </td>
+  <td>1.2.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.io.preferDirectBufs</code></td>
@@ -845,6 +888,7 @@ Apart from these, the following properties are also available, and may be useful
     block transfer. For environments where off-heap memory is tightly limited, users may wish to
     turn this off to force all allocations from Netty to be on-heap.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.io.retryWait</code></td>
@@ -853,6 +897,7 @@ Apart from these, the following properties are also available, and may be useful
     (Netty only) How long to wait between retries of fetches. The maximum delay caused by retrying
     is 15 seconds by default, calculated as <code>maxRetries * retryWait</code>.
   </td>
+  <td>1.2.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.io.backLog</code></td>
@@ -865,18 +910,19 @@ Apart from these, the following properties are also available, and may be useful
     application (see <code>spark.shuffle.service.enabled</code> option below). If set below 1,
     will fallback to OS default defined by Netty's <code>io.netty.util.NetUtil#SOMAXCONN</code>.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.service.enabled</code></td>
   <td>false</td>
   <td>
     Enables the external shuffle service. This service preserves the shuffle files written by
-    executors so the executors can be safely removed. This must be enabled if
-    <code>spark.dynamicAllocation.enabled</code> is "true". The external shuffle service
+    executors so the executors can be safely removed. The external shuffle service
     must be set up in order to enable it. See
     <a href="job-scheduling.html#configuration-and-setup">dynamic allocation
     configuration and setup documentation</a> for more information.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.service.port</code></td>
@@ -884,6 +930,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Port on which the external shuffle service will run.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.service.index.cache.size</code></td>
@@ -891,6 +938,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Cache entries limited to the specified memory footprint, in bytes unless otherwise specified.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.maxChunksBeingTransferred</code></td>
@@ -902,6 +950,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.shuffle.io.retryWait</code>), if those limits are reached the task will fail with
     fetch failure.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.sort.bypassMergeThreshold</code></td>
@@ -910,6 +959,7 @@ Apart from these, the following properties are also available, and may be useful
     (Advanced) In the sort-based shuffle manager, avoid merge-sorting data if there is no
     map-side aggregation and there are at most this many reduce partitions.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.spill.compress</code></td>
@@ -918,6 +968,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to compress data spilled during shuffles. Compression will use
     <code>spark.io.compression.codec</code>.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.accurateBlockThreshold</code></td>
@@ -927,6 +978,7 @@ Apart from these, the following properties are also available, and may be useful
     accurately recorded. This helps to prevent OOM by avoiding underestimating shuffle 
     block size when fetch shuffle blocks.
   </td>
+  <td>2.2.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.registration.timeout</code></td>
@@ -934,6 +986,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Timeout in milliseconds for registration to the external shuffle service.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.registration.maxAttempts</code></td>
@@ -941,13 +994,14 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     When we fail to register to the external shuffle service, we will retry for maxAttempts times.
   </td>
+  <td>2.3.0</td>
 </tr>
 </table>
 
 ### Spark UI
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.eventLog.logBlockUpdates.enabled</code></td>
   <td>false</td>
@@ -955,6 +1009,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to log events for every block update, if <code>spark.eventLog.enabled</code> is true.
     *Warning*: This will increase the size of the event log considerably.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.longForm.enabled</code></td>
@@ -962,6 +1017,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     If true, use the long form of call sites in the event log. Otherwise use the short form.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.compress</code></td>
@@ -969,6 +1025,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Whether to compress logged events, if <code>spark.eventLog.enabled</code> is true.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.compression.codec</code></td>
@@ -977,6 +1034,7 @@ Apart from these, the following properties are also available, and may be useful
     The codec to compress logged events. If this is not given,
     <code>spark.io.compression.codec</code> will be used.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.erasureCoding.enabled</code></td>
@@ -988,6 +1046,7 @@ Apart from these, the following properties are also available, and may be useful
     Note that even if this is true, Spark will still not force the file to use erasure coding, it
     will simply use filesystem defaults.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.dir</code></td>
@@ -998,6 +1057,7 @@ Apart from these, the following properties are also available, and may be useful
     events specific to the application in this directory. Users may want to set this to
     a unified location like an HDFS directory so history files can be read by the history server.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.enabled</code></td>
@@ -1006,6 +1066,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to log Spark events, useful for reconstructing the Web UI after the application has
     finished.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.overwrite</code></td>
@@ -1013,6 +1074,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Whether to overwrite any existing files.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.buffer.kb</code></td>
@@ -1020,6 +1082,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Buffer size to use when writing to output streams, in KiB unless otherwise specified.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.rolling.enabled</code></td>
@@ -1028,13 +1091,15 @@ Apart from these, the following properties are also available, and may be useful
     Whether rolling over event log files is enabled. If set to true, it cuts down each event
     log file to the configured size.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.rolling.maxFileSize</code></td>
   <td>128m</td>
   <td>
-    The max size of event log file before it's rolled over.
+    When <code>spark.eventLog.rolling.enabled=true</code>, specifies the max size of event log file before it's rolled over.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.dagGraph.retainedRootRDDs</code></td>
@@ -1042,6 +1107,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many DAG graph nodes the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.enabled</code></td>
@@ -1049,6 +1115,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Whether to run the web UI for the Spark application.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.ui.killEnabled</code></td>
@@ -1056,6 +1123,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Allows jobs and stages to be killed from the web UI.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.liveUpdate.period</code></td>
@@ -1065,6 +1133,7 @@ Apart from these, the following properties are also available, and may be useful
     meaning only the last write will happen. For live applications, this avoids a few
     operations that we can live without when rapidly processing incoming task events.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.liveUpdate.minFlushPeriod</code></td>
@@ -1073,6 +1142,7 @@ Apart from these, the following properties are also available, and may be useful
     Minimum time elapsed before stale UI data is flushed. This avoids UI staleness when incoming
     task events are not fired frequently.
   </td>
+  <td>2.4.2</td>
 </tr>
 <tr>
   <td><code>spark.ui.port</code></td>
@@ -1080,6 +1150,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Port for your application's dashboard, which shows memory and workload data.
   </td>
+  <td>0.7.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.retainedJobs</code></td>
@@ -1088,6 +1159,7 @@ Apart from these, the following properties are also available, and may be useful
     How many jobs the Spark UI and status APIs remember before garbage collecting.
     This is a target maximum, and fewer elements may be retained in some circumstances.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.retainedStages</code></td>
@@ -1096,14 +1168,16 @@ Apart from these, the following properties are also available, and may be useful
     How many stages the Spark UI and status APIs remember before garbage collecting.
     This is a target maximum, and fewer elements may be retained in some circumstances.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.retainedTasks</code></td>
   <td>100000</td>
   <td>
-    How many tasks the Spark UI and status APIs remember before garbage collecting.
+    How many tasks in one stage the Spark UI and status APIs remember before garbage collecting.
     This is a target maximum, and fewer elements may be retained in some circumstances.
   </td>
+  <td>2.0.1</td>
 </tr>
 <tr>
   <td><code>spark.ui.reverseProxy</code></td>
@@ -1111,6 +1185,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Enable running Spark Master as reverse proxy for worker and application UIs. In this mode, Spark master will reverse proxy the worker and application UIs to enable access without requiring direct access to their hosts. Use it with caution, as worker and application UI will not be accessible directly, you will only be able to access them through spark master/proxy public URL. This setting affects all the workers and application UIs running in the cluster and must be set on all the workers, drivers and masters.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.reverseProxyUrl</code></td>
@@ -1118,6 +1193,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     This is the URL where your proxy is running. This URL is for proxy which is running in front of Spark Master. This is useful when running proxy for authentication e.g. OAuth proxy. Make sure this is a complete URL including scheme (http/https) and port to reach your proxy.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.proxyRedirectUri</code></td>
@@ -1130,6 +1206,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>X-Forwarded-Context</code> request header), or by setting the proxy base in the Spark
     app's configuration.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.showConsoleProgress</code></td>
@@ -1141,6 +1218,7 @@ Apart from these, the following properties are also available, and may be useful
     <br/>
     <em>Note:</em> In shell environment, the default value of spark.ui.showConsoleProgress is true.
   </td>
+  <td>1.2.1</td>
 </tr>
 <tr>
   <td><code>spark.ui.custom.executor.log.url</code></td>
@@ -1156,6 +1234,7 @@ Apart from these, the following properties are also available, and may be useful
     <p/>
     For now, only YARN mode supports this configuration
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.ui.retainedExecutors</code></td>
@@ -1163,6 +1242,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished executors the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.ui.retainedDrivers</code></td>
@@ -1170,6 +1250,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished drivers the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.ui.retainedExecutions</code></td>
@@ -1177,6 +1258,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished executions the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.ui.retainedBatches</code></td>
@@ -1184,6 +1266,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished batches the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.retainedDeadExecutors</code></td>
@@ -1191,6 +1274,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many dead executors the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.filters</code></td>
@@ -1208,6 +1292,7 @@ Apart from these, the following properties are also available, and may be useful
     <br /><code>spark.com.test.filter1.param.name1=foo</code>
     <br /><code>spark.com.test.filter1.param.name2=bar</code>
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.requestHeaderSize</code></td>
@@ -1216,13 +1301,14 @@ Apart from these, the following properties are also available, and may be useful
     The maximum allowed size for a HTTP request header, in bytes unless otherwise specified.
     This setting applies for the Spark History Server too.
   </td>
+  <td>2.2.3</td>
 </tr>
 </table>
 
 ### Compression and Serialization
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.broadcast.compress</code></td>
   <td>true</td>
@@ -1230,6 +1316,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to compress broadcast variables before sending them. Generally a good idea.
     Compression will use <code>spark.io.compression.codec</code>.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.checkpoint.compress</code></td>
@@ -1237,7 +1324,8 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Whether to compress RDD checkpoints. Generally a good idea.
     Compression will use <code>spark.io.compression.codec</code>.
-   </td>
+  </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.io.compression.codec</code></td>
@@ -1252,6 +1340,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>org.apache.spark.io.SnappyCompressionCodec</code>,
     and <code>org.apache.spark.io.ZStdCompressionCodec</code>.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.io.compression.lz4.blockSize</code></td>
@@ -1261,6 +1350,7 @@ Apart from these, the following properties are also available, and may be useful
     is used. Lowering this block size will also lower shuffle memory usage when LZ4 is used.
     Default unit is bytes, unless otherwise specified.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.io.compression.snappy.blockSize</code></td>
@@ -1270,6 +1360,7 @@ Apart from these, the following properties are also available, and may be useful
     Lowering this block size will also lower shuffle memory usage when Snappy is used.
     Default unit is bytes, unless otherwise specified.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.io.compression.zstd.level</code></td>
@@ -1278,6 +1369,7 @@ Apart from these, the following properties are also available, and may be useful
     Compression level for Zstd compression codec. Increasing the compression level will result in better
     compression at the expense of more CPU and memory.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.io.compression.zstd.bufferSize</code></td>
@@ -1287,6 +1379,7 @@ Apart from these, the following properties are also available, and may be useful
     is used. Lowering this size will lower the shuffle memory usage when Zstd is used, but it
     might increase the compression cost because of excessive JNI call overhead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kryo.classesToRegister</code></td>
@@ -1296,6 +1389,7 @@ Apart from these, the following properties are also available, and may be useful
     with Kryo.
     See the <a href="tuning.html#data-serialization">tuning guide</a> for more details.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.kryo.referenceTracking</code></td>
@@ -1306,6 +1400,7 @@ Apart from these, the following properties are also available, and may be useful
     copies of the same object. Can be disabled to improve performance if you know this is not the
     case.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.kryo.registrationRequired</code></td>
@@ -1317,6 +1412,7 @@ Apart from these, the following properties are also available, and may be useful
     significant performance overhead, so enabling this option can enforce strictly that a
     user has not omitted classes from registration.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.kryo.registrator</code></td>
@@ -1326,10 +1422,11 @@ Apart from these, the following properties are also available, and may be useful
     property is useful if you need to register your classes in a custom way, e.g. to specify a custom
     field serializer. Otherwise <code>spark.kryo.classesToRegister</code> is simpler. It should be
     set to classes that extend
-    <a href="api/scala/index.html#org.apache.spark.serializer.KryoRegistrator">
+    <a href="api/scala/org/apache/spark/serializer/KryoRegistrator.html">
     <code>KryoRegistrator</code></a>.
     See the <a href="tuning.html#data-serialization">tuning guide</a> for more details.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.kryo.unsafe</code></td>
@@ -1338,6 +1435,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to use unsafe based Kryo serializer. Can be
     substantially faster by using Unsafe Based IO.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.kryoserializer.buffer.max</code></td>
@@ -1347,6 +1445,7 @@ Apart from these, the following properties are also available, and may be useful
     This must be larger than any object you attempt to serialize and must be less than 2048m.
     Increase this if you get a "buffer limit exceeded" exception inside Kryo.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kryoserializer.buffer</code></td>
@@ -1356,6 +1455,7 @@ Apart from these, the following properties are also available, and may be useful
     Note that there will be one buffer <i>per core</i> on each worker. This buffer will grow up to
     <code>spark.kryoserializer.buffer.max</code> if needed.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.rdd.compress</code></td>
@@ -1367,6 +1467,7 @@ Apart from these, the following properties are also available, and may be useful
     Can save substantial space at the cost of some extra CPU time.
     Compression will use <code>spark.io.compression.codec</code>.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.serializer</code></td>
@@ -1379,9 +1480,10 @@ Apart from these, the following properties are also available, and may be useful
     but is quite slow, so we recommend <a href="tuning.html">using
     <code>org.apache.spark.serializer.KryoSerializer</code> and configuring Kryo serialization</a>
     when speed is necessary. Can be any subclass of
-    <a href="api/scala/index.html#org.apache.spark.serializer.Serializer">
+    <a href="api/scala/org/apache/spark/serializer/Serializer.html">
     <code>org.apache.spark.Serializer</code></a>.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.serializer.objectStreamReset</code></td>
@@ -1393,13 +1495,14 @@ Apart from these, the following properties are also available, and may be useful
     objects to be collected. To turn off this periodic reset set it to -1.
     By default it will reset the serializer every 100 objects.
   </td>
+  <td>1.0.0</td>
 </tr>
 </table>
 
 ### Memory Management
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.memory.fraction</code></td>
   <td>0.6</td>
@@ -1412,6 +1515,7 @@ Apart from these, the following properties are also available, and may be useful
     garbage collection when increasing this value, see
     <a href="tuning.html#memory-management-overview">this description</a>.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.memory.storageFraction</code></td>
@@ -1423,6 +1527,7 @@ Apart from these, the following properties are also available, and may be useful
     Leaving this at the default value is recommended. For more detail, see
     <a href="tuning.html#memory-management-overview">this description</a>.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.memory.offHeap.enabled</code></td>
@@ -1431,6 +1536,7 @@ Apart from these, the following properties are also available, and may be useful
     If true, Spark will attempt to use off-heap memory for certain operations. If off-heap memory 
     use is enabled, then <code>spark.memory.offHeap.size</code> must be positive.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.memory.offHeap.size</code></td>
@@ -1441,6 +1547,7 @@ Apart from these, the following properties are also available, and may be useful
     must fit within some hard limit then be sure to shrink your JVM heap size accordingly.
     This must be set to a positive value when <code>spark.memory.offHeap.enabled=true</code>.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.storage.replication.proactive</code></td>
@@ -1450,6 +1557,7 @@ Apart from these, the following properties are also available, and may be useful
     executor failures are replenished if there are any existing available replicas. This tries
     to get the replication level of the block to the initial number.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.cleaner.periodicGC.interval</code></td>
@@ -1461,6 +1569,7 @@ Apart from these, the following properties are also available, and may be useful
     on the driver, this may happen very occasionally or not at all. Not cleaning at all may
     lead to executors running out of disk space after a while.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.cleaner.referenceTracking</code></td>
@@ -1468,6 +1577,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Enables or disables context cleaning.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.cleaner.referenceTracking.blocking</code></td>
@@ -1476,6 +1586,7 @@ Apart from these, the following properties are also available, and may be useful
     Controls whether the cleaning thread should block on cleanup tasks (other than shuffle, which is controlled by
     <code>spark.cleaner.referenceTracking.blocking.shuffle</code> Spark property).
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.cleaner.referenceTracking.blocking.shuffle</code></td>
@@ -1483,6 +1594,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Controls whether the cleaning thread should block on shuffle cleanup tasks.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.cleaner.referenceTracking.cleanCheckpoints</code></td>
@@ -1490,13 +1602,14 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Controls whether to clean checkpoint files if the reference is out of scope.
   </td>
+  <td>1.4.0</td>
 </tr>
 </table>
 
 ### Execution Behavior
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.broadcast.blockSize</code></td>
   <td>4m</td>
@@ -1505,6 +1618,7 @@ Apart from these, the following properties are also available, and may be useful
     specified. Too large a value decreases parallelism during broadcast (makes it slower); however, 
     if it is too small, <code>BlockManager</code> might take a performance hit.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.broadcast.checksum</code></td>
@@ -1514,6 +1628,7 @@ Apart from these, the following properties are also available, and may be useful
     help detect corrupted blocks, at the cost of computing and sending a little more data. It's possible
     to disable it if the network has other mechanisms to guarantee data won't be corrupted during broadcast.
   </td>
+  <td>2.1.1</td>
 </tr>
 <tr>
   <td><code>spark.executor.cores</code></td>
@@ -1527,6 +1642,7 @@ Apart from these, the following properties are also available, and may be useful
     In standalone and Mesos coarse-grained modes, for more detail, see
     <a href="spark-standalone.html#Executors Scheduling">this description</a>.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.default.parallelism</code></td>
@@ -1544,14 +1660,18 @@ Apart from these, the following properties are also available, and may be useful
     Default number of partitions in RDDs returned by transformations like <code>join</code>,
     <code>reduceByKey</code>, and <code>parallelize</code> when not set by user.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
-    <td><code>spark.executor.heartbeatInterval</code></td>
-    <td>10s</td>
-    <td>Interval between each executor's heartbeats to the driver.  Heartbeats let
+  <td><code>spark.executor.heartbeatInterval</code></td>
+  <td>10s</td>
+  <td>
+    Interval between each executor's heartbeats to the driver.  Heartbeats let
     the driver know that the executor is still alive and update it with metrics for in-progress
     tasks. spark.executor.heartbeatInterval should be significantly less than
-    spark.network.timeout</td>
+    spark.network.timeout
+  </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.files.fetchTimeout</code></td>
@@ -1560,6 +1680,7 @@ Apart from these, the following properties are also available, and may be useful
     Communication timeout to use when fetching files added through SparkContext.addFile() from
     the driver.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.files.useFetchCache</code></td>
@@ -1572,6 +1693,7 @@ Apart from these, the following properties are also available, and may be useful
     disabled in order to use Spark local directories that reside on NFS filesystems (see
     <a href="https://issues.apache.org/jira/browse/SPARK-6313">SPARK-6313</a> for more details).
   </td>
+  <td>1.2.2</td>
 </tr>
 <tr>
   <td><code>spark.files.overwrite</code></td>
@@ -1580,6 +1702,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to overwrite files added through SparkContext.addFile() when the target file exists and
     its contents do not match those of the source.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.files.maxPartitionBytes</code></td>
@@ -1587,6 +1710,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     The maximum number of bytes to pack into a single partition when reading files.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.files.openCostInBytes</code></td>
@@ -1596,25 +1720,32 @@ Apart from these, the following properties are also available, and may be useful
     time. This is used when putting multiple files into a partition. It is better to overestimate,
     then the partitions with small files will be faster than partitions with bigger files.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
-    <td><code>spark.hadoop.cloneConf</code></td>
-    <td>false</td>
-    <td>If set to true, clones a new Hadoop <code>Configuration</code> object for each task.  This
+  <td><code>spark.hadoop.cloneConf</code></td>
+  <td>false</td>
+  <td>
+    If set to true, clones a new Hadoop <code>Configuration</code> object for each task.  This
     option should be enabled to work around <code>Configuration</code> thread-safety issues (see
     <a href="https://issues.apache.org/jira/browse/SPARK-2546">SPARK-2546</a> for more details).
     This is disabled by default in order to avoid unexpected performance regressions for jobs that
-    are not affected by these issues.</td>
+    are not affected by these issues.
+  </td>
+  <td>1.0.3</td>
 </tr>
 <tr>
-    <td><code>spark.hadoop.validateOutputSpecs</code></td>
-    <td>true</td>
-    <td>If set to true, validates the output specification (e.g. checking if the output directory already exists)
+  <td><code>spark.hadoop.validateOutputSpecs</code></td>
+  <td>true</td>
+  <td>
+    If set to true, validates the output specification (e.g. checking if the output directory already exists)
     used in saveAsHadoopFile and other variants. This can be disabled to silence exceptions due to pre-existing
-    output directories. We recommend that users do not disable this except if trying to achieve compatibility with
-    previous versions of Spark. Simply use Hadoop's FileSystem API to delete output directories by hand.
-    This setting is ignored for jobs generated through Spark Streaming's StreamingContext, since
-    data may need to be rewritten to pre-existing output directories during checkpoint recovery.</td>
+    output directories. We recommend that users do not disable this except if trying to achieve compatibility
+    with previous versions of Spark. Simply use Hadoop's FileSystem API to delete output directories by hand.
+    This setting is ignored for jobs generated through Spark Streaming's StreamingContext, since data may
+    need to be rewritten to pre-existing output directories during checkpoint recovery.
+  </td>
+  <td>1.0.1</td>
 </tr>
 <tr>
   <td><code>spark.storage.memoryMapThreshold</code></td>
@@ -1624,24 +1755,25 @@ Apart from these, the following properties are also available, and may be useful
     unless specified otherwise. This prevents Spark from memory mapping very small blocks. In general, 
     memory mapping has high overhead for blocks close to or below the page size of the operating system.
   </td>
+  <td>0.9.2</td>
 </tr>
 <tr>
   <td><code>spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version</code></td>
   <td>1</td>
   <td>
     The file output committer algorithm version, valid algorithm version number: 1 or 2.
-    Version 2 may have better performance, but version 1 may handle failures better in certain situations,
-    as per <a href="https://issues.apache.org/jira/browse/MAPREDUCE-4815">MAPREDUCE-4815</a>.
+    Note that 2 may cause a correctness issue like MAPREDUCE-7282.
   </td>
+  <td>2.2.0</td>
 </tr>
 </table>
 
 ### Executor Metrics
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
-  <td><code>spark.eventLog.logStageExecutorMetrics.enabled</code></td>
+  <td><code>spark.eventLog.logStageExecutorMetrics</code></td>
   <td>false</td>
   <td>
     Whether to write per-stage peaks of executor metrics (for each executor) to the event log.
@@ -1650,6 +1782,7 @@ Apart from these, the following properties are also available, and may be useful
     and this is always done; this configuration is only to determine if aggregated metric peaks
     are written to the event log.
   </td>
+  <td>3.0.0</td>
 </tr>
   <td><code>spark.executor.processTreeMetrics.enabled</code></td>
   <td>false</td>
@@ -1660,6 +1793,7 @@ Apart from these, the following properties are also available, and may be useful
     <em>Note:</em> The process tree metrics are collected only if the /proc filesystem
     exists.
   </td>
+  <td>3.0.0</td>
 <tr>
   <td><code>spark.executor.metrics.pollingInterval</code></td>
   <td>0</td>
@@ -1670,13 +1804,14 @@ Apart from these, the following properties are also available, and may be useful
     specified by <code>spark.executor.heartbeatInterval</code>).
     If positive, the polling is done at this interval.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
 ### Networking
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.rpc.message.maxSize</code></td>
   <td>128</td>
@@ -1685,6 +1820,7 @@ Apart from these, the following properties are also available, and may be useful
     output size information sent between executors and the driver. Increase this if you are running
     jobs with many thousands of map and reduce tasks and see messages about the RPC message size.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.blockManager.port</code></td>
@@ -1692,6 +1828,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Port for all block managers to listen on. These exist on both the driver and the executors.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.blockManager.port</code></td>
@@ -1700,6 +1837,7 @@ Apart from these, the following properties are also available, and may be useful
     Driver-specific port for the block manager to listen on, for cases where it cannot use the same
     configuration as executors.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.bindAddress</code></td>
@@ -1713,6 +1851,7 @@ Apart from these, the following properties are also available, and may be useful
     the different ports used by the driver (RPC, block manager and UI) need to be forwarded from the
     container's host.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.host</code></td>
@@ -1721,6 +1860,7 @@ Apart from these, the following properties are also available, and may be useful
     Hostname or IP address for the driver.
     This is used for communicating with the executors and the standalone Master.
   </td>
+  <td>0.7.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.port</code></td>
@@ -1729,6 +1869,7 @@ Apart from these, the following properties are also available, and may be useful
     Port for the driver to listen on.
     This is used for communicating with the executors and the standalone Master.
   </td>
+  <td>0.7.0</td>
 </tr>
 <tr>
   <td><code>spark.rpc.io.backLog</code></td>
@@ -1738,17 +1879,18 @@ Apart from these, the following properties are also available, and may be useful
     need to be increased, so that incoming connections are not dropped when a large number of
     connections arrives in a short period of time.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.network.timeout</code></td>
   <td>120s</td>
   <td>
     Default timeout for all network interactions. This config will be used in place of
-    <code>spark.core.connection.ack.wait.timeout</code>,
     <code>spark.storage.blockManagerSlaveTimeoutMs</code>,
     <code>spark.shuffle.io.connectionTimeout</code>, <code>spark.rpc.askTimeout</code> or
     <code>spark.rpc.lookupTimeout</code> if they are not configured.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.network.io.preferDirectBufs</code></td>
@@ -1758,7 +1900,8 @@ Apart from these, the following properties are also available, and may be useful
     Off-heap buffers are used to reduce garbage collection during shuffle and cache
     block transfer. For environments where off-heap memory is tightly limited, users may wish to
     turn this off to force all allocations to be on-heap.
-    </td>
+  </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.port.maxRetries</code></td>
@@ -1770,6 +1913,7 @@ Apart from these, the following properties are also available, and may be useful
     essentially allows it to try a range of ports from the start port specified
     to port + maxRetries.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.rpc.numRetries</code></td>
@@ -1778,6 +1922,7 @@ Apart from these, the following properties are also available, and may be useful
     Number of times to retry before an RPC task gives up.
     An RPC task will run at most times of this number.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.rpc.retry.wait</code></td>
@@ -1785,6 +1930,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Duration for an RPC ask operation to wait before retrying.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.rpc.askTimeout</code></td>
@@ -1792,6 +1938,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Duration for an RPC ask operation to wait before timing out.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.rpc.lookupTimeout</code></td>
@@ -1799,18 +1946,10 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Duration for an RPC remote endpoint lookup operation to wait before timing out.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
-  <td><code>spark.core.connection.ack.wait.timeout</code></td>
-  <td><code>spark.network.timeout</code></td>
-  <td>
-    How long for the connection to wait for ack to occur before timing
-    out and giving up. To avoid unwilling timeout caused by long pause like GC,
-    you can set larger value.
-  </td>
-</tr>
-<tr>
-  <td><code>spark.maxRemoteBlockSizeFetchToMem</code></td>
+  <td><code>spark.network.maxRemoteBlockSizeFetchToMem</code></td>
   <td>200m</td>
   <td>
     Remote block will be fetched to disk when size of the block is above this threshold
@@ -1819,13 +1958,14 @@ Apart from these, the following properties are also available, and may be useful
     For users who enabled external shuffle service, this feature can only work when
     external shuffle service is at least 2.3.0.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
 ### Scheduling
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.cores.max</code></td>
   <td>(not set)</td>
@@ -1837,6 +1977,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.deploy.defaultCores</code> on Spark's standalone cluster manager, or
     infinite (all available cores) on Mesos.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.locality.wait</code></td>
@@ -1849,6 +1990,7 @@ Apart from these, the following properties are also available, and may be useful
     You should increase this setting if your tasks are long and see poor locality, but the
     default usually works well.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.locality.wait.node</code></td>
@@ -1857,6 +1999,7 @@ Apart from these, the following properties are also available, and may be useful
     Customize the locality wait for node locality. For example, you can set this to 0 to skip
     node locality and search immediately for rack locality (if your cluster has rack information).
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.locality.wait.process</code></td>
@@ -1865,6 +2008,7 @@ Apart from these, the following properties are also available, and may be useful
     Customize the locality wait for process locality. This affects tasks that attempt to access
     cached data in a particular executor process.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.locality.wait.rack</code></td>
@@ -1872,6 +2016,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Customize the locality wait for rack locality.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.maxRegisteredResourcesWaitingTime</code></td>
@@ -1879,6 +2024,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Maximum amount of time to wait for resources to register before scheduling begins.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.minRegisteredResourcesRatio</code></td>
@@ -1892,6 +2038,7 @@ Apart from these, the following properties are also available, and may be useful
     the maximum amount of time it will wait before scheduling begins is controlled by config
     <code>spark.scheduler.maxRegisteredResourcesWaitingTime</code>.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.mode</code></td>
@@ -1902,6 +2049,7 @@ Apart from these, the following properties are also available, and may be useful
     to use fair sharing instead of queueing jobs one after another. Useful for
     multi-user services.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.revive.interval</code></td>
@@ -1909,6 +2057,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     The interval length for the scheduler to revive the worker resource offers to run tasks.
   </td>
+  <td>0.8.1</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.capacity</code></td>
@@ -1920,6 +2069,7 @@ Apart from these, the following properties are also available, and may be useful
     config. Note that capacity must be greater than 0. Consider increasing value (e.g. 20000) 
     if listener events are dropped. Increasing this value may result in the driver using more memory.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.shared.capacity</code></td>
@@ -1929,6 +2079,7 @@ Apart from these, the following properties are also available, and may be useful
     that register to the listener bus. Consider increasing value, if the listener events corresponding
     to shared queue are dropped. Increasing this value may result in the driver using more memory.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.appStatus.capacity</code></td>
@@ -1938,6 +2089,7 @@ Apart from these, the following properties are also available, and may be useful
     Consider increasing value, if the listener events corresponding to appStatus queue are dropped.
     Increasing this value may result in the driver using more memory.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.executorManagement.capacity</code></td>
@@ -1947,6 +2099,7 @@ Apart from these, the following properties are also available, and may be useful
     executor management listeners. Consider increasing value if the listener events corresponding to
     executorManagement queue are dropped. Increasing this value may result in the driver using more memory.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.eventLog.capacity</code></td>
@@ -1956,6 +2109,7 @@ Apart from these, the following properties are also available, and may be useful
     that write events to eventLogs. Consider increasing value if the listener events corresponding to eventLog queue
     are dropped. Increasing this value may result in the driver using more memory.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.streams.capacity</code></td>
@@ -1965,6 +2119,7 @@ Apart from these, the following properties are also available, and may be useful
     Consider increasing value if the listener events corresponding to streams queue are dropped. Increasing
     this value may result in the driver using more memory.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.blacklist.unschedulableTaskSetTimeout</code></td>
@@ -1973,6 +2128,7 @@ Apart from these, the following properties are also available, and may be useful
     The timeout in seconds to wait to acquire a new executor and schedule a task before aborting a
     TaskSet which is unschedulable because of being completely blacklisted.
   </td>
+  <td>2.4.1</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.enabled</code></td>
@@ -1984,6 +2140,7 @@ Apart from these, the following properties are also available, and may be useful
     due to too many task failures. The blacklisting algorithm can be further controlled by the
     other "spark.blacklist" configuration options.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.timeout</code></td>
@@ -1992,6 +2149,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) How long a node or executor is blacklisted for the entire application, before it
     is unconditionally removed from the blacklist to attempt running new tasks.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.task.maxTaskAttemptsPerExecutor</code></td>
@@ -2000,6 +2158,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) For a given task, how many times it can be retried on one executor before the
     executor is blacklisted for that task.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.task.maxTaskAttemptsPerNode</code></td>
@@ -2008,6 +2167,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) For a given task, how many times it can be retried on one node, before the entire
     node is blacklisted for that task.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.stage.maxFailedTasksPerExecutor</code></td>
@@ -2016,6 +2176,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) How many different tasks must fail on one executor, within one stage, before the
     executor is blacklisted for that stage.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.stage.maxFailedExecutorsPerNode</code></td>
@@ -2024,6 +2185,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) How many different executors are marked as blacklisted for a given stage, before
     the entire node is marked as failed for the stage.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.application.maxFailedTasksPerExecutor</code></td>
@@ -2035,6 +2197,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.blacklist.timeout</code>.  Note that with dynamic allocation, though, the executors
     may get marked as idle and be reclaimed by the cluster manager.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.application.maxFailedExecutorsPerNode</code></td>
@@ -2046,6 +2209,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.blacklist.timeout</code>.  Note that with dynamic allocation, though, the executors
     on the node may get marked as idle and be reclaimed by the cluster manager.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.killBlacklistedExecutors</code></td>
@@ -2056,6 +2220,7 @@ Apart from these, the following properties are also available, and may be useful
     as controlled by spark.blacklist.application.*. Note that, when an entire node is added 
     to the blacklist, all of the executors on that node will be killed.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.application.fetchFailure.enabled</code></td>
@@ -2065,6 +2230,7 @@ Apart from these, the following properties are also available, and may be useful
     failure happens. If external shuffle service is enabled, then the whole node will be
     blacklisted.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.speculation</code></td>
@@ -2073,6 +2239,7 @@ Apart from these, the following properties are also available, and may be useful
     If set to "true", performs speculative execution of tasks. This means if one or more tasks are
     running slowly in a stage, they will be re-launched.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.speculation.interval</code></td>
@@ -2080,6 +2247,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How often Spark will check for tasks to speculate.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.speculation.multiplier</code></td>
@@ -2087,6 +2255,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many times slower a task is than the median to be considered for speculation.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.speculation.quantile</code></td>
@@ -2094,6 +2263,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Fraction of tasks which must be complete before speculation is enabled for a particular stage.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.speculation.task.duration.threshold</code></td>
@@ -2108,6 +2278,7 @@ Apart from these, the following properties are also available, and may be useful
     the conf values of spark.executor.cores and spark.task.cpus minimum 1.
     Default unit is bytes, unless otherwise specified.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.task.cpus</code></td>
@@ -2115,6 +2286,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Number of cores to allocate for each task.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.task.resource.{resourceName}.amount</code></td>
@@ -2130,6 +2302,7 @@ Apart from these, the following properties are also available, and may be useful
     in order to assign resource slots (e.g. a 0.2222 configuration, or 1/0.2222 slots will become 
     4 tasks/resource, not 5).
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.task.maxFailures</code></td>
@@ -2140,6 +2313,7 @@ Apart from these, the following properties are also available, and may be useful
     to fail; a particular task has to fail this number of attempts.
     Should be greater than or equal to 1. Number of allowed retries = this value - 1.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.task.reaper.enabled</code></td>
@@ -2151,6 +2325,7 @@ Apart from these, the following properties are also available, and may be useful
     of this monitoring. When set to false (the default), task killing will use an older code
     path which lacks such monitoring.
   </td>
+  <td>2.0.3</td>
 </tr>
 <tr>
   <td><code>spark.task.reaper.pollingInterval</code></td>
@@ -2162,6 +2337,7 @@ Apart from these, the following properties are also available, and may be useful
     (this thread dump can be disabled via the <code>spark.task.reaper.threadDump</code> setting,
     which is documented below).
   </td>
+  <td>2.0.3</td>
 </tr>
 <tr>
   <td><code>spark.task.reaper.threadDump</code></td>
@@ -2171,6 +2347,7 @@ Apart from these, the following properties are also available, and may be useful
     dumps are logged during periodic polling of killed tasks. Set this to false to disable
     collection of thread dumps.
   </td>
+  <td>2.0.3</td>
 </tr>
 <tr>
   <td><code>spark.task.reaper.killTimeout</code></td>
@@ -2182,6 +2359,7 @@ Apart from these, the following properties are also available, and may be useful
     of this setting is to act as a safety-net to prevent runaway noncancellable tasks from rendering
     an executor unusable.
   </td>
+  <td>2.0.3</td>
 </tr>
 <tr>
   <td><code>spark.stage.maxConsecutiveAttempts</code></td>
@@ -2189,13 +2367,14 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Number of consecutive stage attempts allowed before a stage is aborted.
   </td>
+  <td>2.2.0</td>
 </tr>
 </table>
 
 ### Barrier Execution Mode
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.barrier.sync.timeout</code></td>
   <td>365d</td>
@@ -2205,6 +2384,7 @@ Apart from these, the following properties are also available, and may be useful
     configured time, throw a SparkException to fail all the tasks. The default value is set
     to 31536000(3600 * 24 * 365) so the <code>barrier()</code> call shall wait for one year.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.barrier.maxConcurrentTasksCheck.interval</code></td>
@@ -2219,6 +2399,7 @@ Apart from these, the following properties are also available, and may be useful
     config only applies to jobs that contain one or more barrier stages, we won't perform
     the check on non-barrier jobs.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.barrier.maxConcurrentTasksCheck.maxFailures</code></td>
@@ -2233,13 +2414,14 @@ Apart from these, the following properties are also available, and may be useful
     applies to jobs that contain one or more barrier stages, we won't perform the check on
     non-barrier jobs.
   </td>
+  <td>2.4.0</td>
 </tr>
 </table>
 
 ### Dynamic Allocation
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.dynamicAllocation.enabled</code></td>
   <td>false</td>
@@ -2249,13 +2431,15 @@ Apart from these, the following properties are also available, and may be useful
     For more detail, see the description
     <a href="job-scheduling.html#dynamic-resource-allocation">here</a>.
     <br><br>
-    This requires <code>spark.shuffle.service.enabled</code> to be set.
+    This requires <code>spark.shuffle.service.enabled</code> or
+    <code>spark.dynamicAllocation.shuffleTracking.enabled</code> to be set.
     The following configurations are also relevant:
     <code>spark.dynamicAllocation.minExecutors</code>,
     <code>spark.dynamicAllocation.maxExecutors</code>, and
     <code>spark.dynamicAllocation.initialExecutors</code>
     <code>spark.dynamicAllocation.executorAllocationRatio</code>
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.executorIdleTimeout</code></td>
@@ -2265,6 +2449,7 @@ Apart from these, the following properties are also available, and may be useful
     the executor will be removed. For more detail, see this
     <a href="job-scheduling.html#resource-allocation-policy">description</a>.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code></td>
@@ -2274,6 +2459,7 @@ Apart from these, the following properties are also available, and may be useful
     the executor will be removed. For more details, see this
     <a href="job-scheduling.html#resource-allocation-policy">description</a>.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.initialExecutors</code></td>
@@ -2284,6 +2470,7 @@ Apart from these, the following properties are also available, and may be useful
     If `--num-executors` (or `spark.executor.instances`) is set and larger than this value, it will
     be used as the initial number of executors.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.maxExecutors</code></td>
@@ -2291,6 +2478,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Upper bound for the number of executors if dynamic allocation is enabled.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.minExecutors</code></td>
@@ -2298,6 +2486,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Lower bound for the number of executors if dynamic allocation is enabled.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.executorAllocationRatio</code></td>
@@ -2315,6 +2504,7 @@ Apart from these, the following properties are also available, and may be useful
     by the <code>spark.dynamicAllocation.minExecutors</code> and
     <code>spark.dynamicAllocation.maxExecutors</code> settings
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.schedulerBacklogTimeout</code></td>
@@ -2324,6 +2514,7 @@ Apart from these, the following properties are also available, and may be useful
     this duration, new executors will be requested. For more detail, see this
     <a href="job-scheduling.html#resource-allocation-policy">description</a>.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</code></td>
@@ -2333,6 +2524,7 @@ Apart from these, the following properties are also available, and may be useful
     subsequent executor requests. For more detail, see this
     <a href="job-scheduling.html#resource-allocation-policy">description</a>.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.shuffleTracking.enabled</code></td>
@@ -2342,9 +2534,10 @@ Apart from these, the following properties are also available, and may be useful
     without the need for an external shuffle service. This option will try to keep alive executors
     that are storing shuffle data for active jobs.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
-  <td><code>spark.dynamicAllocation.shuffleTimeout</code></td>
+  <td><code>spark.dynamicAllocation.shuffleTracking.timeout</code></td>
   <td><code>infinity</code></td>
   <td>
     When shuffle tracking is enabled, controls the timeout for executors that are holding shuffle
@@ -2353,6 +2546,7 @@ Apart from these, the following properties are also available, and may be useful
     quickly enough, this option can be used to control when to time out executors even when they are
     storing shuffle data.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -2366,13 +2560,14 @@ like shuffle, just replace "rpc" with "shuffle" in the property names except
 <code>spark.{driver|executor}.rpc.netty.dispatcher.numThreads</code>, which is only for RPC module.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.{driver|executor}.rpc.io.serverThreads</code></td>
   <td>
     Fall back on <code>spark.rpc.io.serverThreads</code>
   </td>
   <td>Number of threads used in the server thread pool</td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.{driver|executor}.rpc.io.clientThreads</code></td>
@@ -2380,6 +2575,7 @@ like shuffle, just replace "rpc" with "shuffle" in the property names except
     Fall back on <code>spark.rpc.io.clientThreads</code>
   </td>
   <td>Number of threads used in the client thread pool</td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.{driver|executor}.rpc.netty.dispatcher.numThreads</code></td>
@@ -2387,6 +2583,7 @@ like shuffle, just replace "rpc" with "shuffle" in the property names except
     Fall back on <code>spark.rpc.netty.dispatcher.numThreads</code>
   </td>
   <td>Number of threads used in RPC message dispatcher thread pool</td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -2399,53 +2596,43 @@ the driver or executor, or, in the absence of that value, the number of cores av
 Please refer to the [Security](security.html) page for available options on how to secure different
 Spark subsystems.
 
-### Spark SQL
-
-Running the <code>SET -v</code> command will show the entire list of the SQL configuration.
-
-<div class="codetabs">
-<div data-lang="scala"  markdown="1">
-
-{% highlight scala %}
-// spark is an existing SparkSession
-spark.sql("SET -v").show(numRows = 200, truncate = false)
-{% endhighlight %}
 
-</div>
+### Spark SQL
 
-<div data-lang="java"  markdown="1">
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-runtime-sql-config-table.html' %}
 
-{% highlight java %}
-// spark is an existing SparkSession
-spark.sql("SET -v").show(200, false);
-{% endhighlight %}
-</div>
+#### Runtime SQL Configuration
 
-<div data-lang="python"  markdown="1">
+Runtime SQL configurations are per-session, mutable Spark SQL configurations. They can be set with initial values by the config file
+and command-line options with `--conf/-c` prefixed, or by setting `SparkConf` that are used to create `SparkSession`.
+Also, they can be set and queried by SET commands and rest to their initial values by RESET command,
+or by `SparkSession.conf`'s setter and getter methods in runtime.
 
-{% highlight python %}
-# spark is an existing SparkSession
-spark.sql("SET -v").show(n=200, truncate=False)
-{% endhighlight %}
+{% include_relative generated-runtime-sql-config-table.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
 
-</div>
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-static-sql-config-table.html' %}
 
-<div data-lang="r"  markdown="1">
+#### Static SQL Configuration
 
-{% highlight r %}
-sparkR.session()
-properties <- sql("SET -v")
-showDF(properties, numRows = 200, truncate = FALSE)
-{% endhighlight %}
+Static SQL configurations are cross-session, immutable Spark SQL configurations. They can be set with final values by the config file
+and command-line options with `--conf/-c` prefixed, or by setting `SparkConf` that are used to create `SparkSession`.
+External users can query the static sql config values via `SparkSession.conf` or via set command, e.g. `SET spark.sql.extensions;`, but cannot set/unset them.
 
-</div>
-</div>
+{% include_relative generated-static-sql-config-table.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
 
 
 ### Spark Streaming
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.streaming.backpressure.enabled</code></td>
   <td>false</td>
@@ -2458,6 +2645,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
     <code>spark.streaming.receiver.maxRate</code> and <code>spark.streaming.kafka.maxRatePerPartition</code>
     if they are set (see below).
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.backpressure.initialRate</code></td>
@@ -2466,6 +2654,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
     This is the initial maximum receiving rate at which each receiver will receive data for the
     first batch when the backpressure mechanism is enabled.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.blockInterval</code></td>
@@ -2476,6 +2665,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
     <a href="streaming-programming-guide.html#level-of-parallelism-in-data-receiving">performance
      tuning</a> section in the Spark Streaming programming guide for more details.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.receiver.maxRate</code></td>
@@ -2487,6 +2677,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
     See the <a href="streaming-programming-guide.html#deploying-applications">deployment guide</a>
     in the Spark Streaming programming guide for mode details.
   </td>
+  <td>1.0.2</td>
 </tr>
 <tr>
   <td><code>spark.streaming.receiver.writeAheadLog.enable</code></td>
@@ -2497,6 +2688,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
     See the <a href="streaming-programming-guide.html#deploying-applications">deployment guide</a>
     in the Spark Streaming programming guide for more details.
   </td>
+  <td>1.2.1</td>
 </tr>
 <tr>
   <td><code>spark.streaming.unpersist</code></td>
@@ -2508,6 +2700,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
     streaming application as they will not be cleared automatically. But it comes at the cost of
     higher memory usage in Spark.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.stopGracefullyOnShutdown</code></td>
@@ -2516,6 +2709,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
     If <code>true</code>, Spark shuts down the <code>StreamingContext</code> gracefully on JVM
     shutdown rather than immediately.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.kafka.maxRatePerPartition</code></td>
@@ -2526,14 +2720,16 @@ showDF(properties, numRows = 200, truncate = FALSE)
     <a href="streaming-kafka-0-10-integration.html">Kafka Integration guide</a>
     for more details.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
-    <td><code>spark.streaming.kafka.minRatePerPartition</code></td>
-    <td>1</td>
-    <td>
-      Minimum rate (number of records per second) at which data will be read from each Kafka
-      partition when using the new Kafka direct stream API.
-    </td>
+  <td><code>spark.streaming.kafka.minRatePerPartition</code></td>
+  <td>1</td>
+  <td>
+    Minimum rate (number of records per second) at which data will be read from each Kafka
+    partition when using the new Kafka direct stream API.
+  </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.ui.retainedBatches</code></td>
@@ -2541,6 +2737,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
   <td>
     How many batches the Spark Streaming UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.driver.writeAheadLog.closeFileAfterWrite</code></td>
@@ -2550,6 +2747,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
     when you want to use S3 (or any file system that does not support flushing) for the metadata WAL
     on the driver.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.receiver.writeAheadLog.closeFileAfterWrite</code></td>
@@ -2559,19 +2757,21 @@ showDF(properties, numRows = 200, truncate = FALSE)
     when you want to use S3 (or any file system that does not support flushing) for the data WAL
     on the receivers.
   </td>
+  <td>1.6.0</td>
 </tr>
 </table>
 
 ### SparkR
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.r.numRBackendThreads</code></td>
   <td>2</td>
   <td>
     Number of threads used by RBackend to handle RPC calls from SparkR package.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.r.command</code></td>
@@ -2579,6 +2779,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
   <td>
     Executable for executing R scripts in cluster modes for both driver and workers.
   </td>
+  <td>1.5.3</td>
 </tr>
 <tr>
   <td><code>spark.r.driver.command</code></td>
@@ -2586,6 +2787,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
   <td>
     Executable for executing R scripts in client modes for driver. Ignored in cluster modes.
   </td>
+  <td>1.5.3</td>
 </tr>
 <tr>
   <td><code>spark.r.shell.command</code></td>
@@ -2594,6 +2796,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
     Executable for executing sparkR shell in client modes for driver. Ignored in cluster modes. It is the same as environment variable <code>SPARKR_DRIVER_R</code>, but take precedence over it.
     <code>spark.r.shell.command</code> is used for sparkR shell while <code>spark.r.driver.command</code> is used for running R script.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.r.backendConnectionTimeout</code></td>
@@ -2601,6 +2804,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
   <td>
     Connection timeout set by R process on its connection to RBackend in seconds.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.r.heartBeatInterval</code></td>
@@ -2608,6 +2812,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
   <td>
     Interval for heartbeats sent from SparkR backend to R process to prevent connection timeout.
   </td>
+  <td>2.1.0</td>
 </tr>
 
 </table>
@@ -2615,7 +2820,7 @@ showDF(properties, numRows = 200, truncate = FALSE)
 ### GraphX
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.graphx.pregel.checkpointInterval</code></td>
   <td>-1</td>
@@ -2623,28 +2828,32 @@ showDF(properties, numRows = 200, truncate = FALSE)
     Checkpoint interval for graph and message in Pregel. It used to avoid stackOverflowError due to long lineage chains
   after lots of iterations. The checkpoint is disabled by default.
   </td>
+  <td>2.2.0</td>
 </tr>
 </table>
 
 ### Deploy
 
 <table class="table">
-  <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+  <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.deploy.recoveryMode</code></td>
     <td>NONE</td>
     <td>The recovery mode setting to recover submitted Spark jobs with cluster mode when it failed and relaunches.
     This is only applicable for cluster mode when running with Standalone or Mesos.</td>
+    <td>0.8.1</td>
   </tr>
   <tr>
     <td><code>spark.deploy.zookeeper.url</code></td>
     <td>None</td>
     <td>When `spark.deploy.recoveryMode` is set to ZOOKEEPER, this configuration is used to set the zookeeper URL to connect to.</td>
+    <td>0.8.1</td>
   </tr>
   <tr>
     <td><code>spark.deploy.zookeeper.dir</code></td>
     <td>None</td>
     <td>When `spark.deploy.recoveryMode` is set to ZOOKEEPER, this configuration is used to set the zookeeper directory to store recovery state.</td>
+    <td>0.8.1</td>
   </tr>
 </table>
 
@@ -2787,5 +2996,4 @@ There are configurations available to request resources for the driver: <code>sp
 
 Spark will use the configurations specified to first request containers with the corresponding resources from the cluster manager. Once it gets the container, Spark launches an Executor in that container which will discover what resources the container has and the addresses associated with each resource. The Executor will register with the Driver and report back the resources available to that Executor. The Spark scheduler can then schedule tasks to each Executor and assign specific resource addresses based on the resource requirements the user specified. The user can see the resources assigned to a task using the <code>TaskContext.get().resources</code> api. On the driver, the user can see the resources assigned with the SparkContext <code>resources</code> call. It's then up to the user to use the assignedaddresses to do the processing they want or pass those into the ML/AI framework they are using.
 
-See your cluster manager specific page for requirements and details on each of - [YARN](running-on-yarn.html#resource-allocation-and-configuration-overview), [Kubernetes](running-on-kubernetes.html#resource-allocation-and-configuration-overview) and [Standalone Mode](spark-standalone.html#resource-allocation-and-configuration-overview). It is currently not available with Mesos or local mode. If using local-cluster mode see the Spark Standalone documentation but be aware only a single worker resources file or discovery script can be specified the is shared by all the Workers so you should enable resource coordination (see <code>spark.resources.coordinate.enable</code>).
-
+See your cluster manager specific page for requirements and details on each of - [YARN](running-on-yarn.html#resource-allocation-and-configuration-overview), [Kubernetes](running-on-kubernetes.html#resource-allocation-and-configuration-overview) and [Standalone Mode](spark-standalone.html#resource-allocation-and-configuration-overview). It is currently not available with Mesos or local mode. And please also note that local-cluster mode with multiple workers is not supported(see Standalone documentation).
diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md
index fdb0afad6af9b..63baef145f01e 100644
--- a/docs/core-migration-guide.md
+++ b/docs/core-migration-guide.md
@@ -25,8 +25,8 @@ license: |
 ## Upgrading from Core 2.4 to 3.0
 
 - The `org.apache.spark.ExecutorPlugin` interface and related configuration has been replaced with
-  `org.apache.spark.plugin.SparkPlugin`, which adds new functionality. Plugins using the old
-  interface need to be modified to extend the new interfaces. Check the
+  `org.apache.spark.api.plugin.SparkPlugin`, which adds new functionality. Plugins using the old
+  interface must be modified to extend the new interfaces. Check the
   [Monitoring](monitoring.html) guide for more details.
 
 - Deprecated method `TaskContext.isRunningLocally` has been removed. Local execution was removed and it always has returned `false`.
@@ -35,6 +35,10 @@ license: |
 
 - Deprecated method `AccumulableInfo.apply` have been removed because creating `AccumulableInfo` is disallowed.
 
-- Event log file will be written as UTF-8 encoding, and Spark History Server will replay event log files as UTF-8 encoding. Previously Spark writes event log file as default charset of driver JVM process, so Spark History Server of Spark 2.x is needed to read the old event log files in case of incompatible encoding.
+- Deprecated accumulator v1 APIs have been removed and please use v2 APIs instead.
 
-- A new protocol for fetching shuffle blocks is used. It's recommended that external shuffle services be upgraded when running Spark 3.0 apps. Old external shuffle services can still be used by setting the configuration `spark.shuffle.useOldFetchProtocol` to `true`. Otherwise, Spark may run into errors with messages like `IllegalArgumentException: Unexpected message type: <number>`.
+- Event log file will be written as UTF-8 encoding, and Spark History Server will replay event log files as UTF-8 encoding. Previously Spark wrote the event log file as default charset of driver JVM process, so Spark History Server of Spark 2.x is needed to read the old event log files in case of incompatible encoding.
+
+- A new protocol for fetching shuffle blocks is used. It's recommended that external shuffle services be upgraded when running Spark 3.0 apps. You can still use old external shuffle services by setting the configuration `spark.shuffle.useOldFetchProtocol` to `true`. Otherwise, Spark may run into errors with messages like `IllegalArgumentException: Unexpected message type: <number>`.
+
+- `SPARK_WORKER_INSTANCES` is deprecated in Standalone mode. It's recommended to launch multiple executors in one worker and launch one worker per node instead of launching multiple workers per node and launching one executor per worker.
diff --git a/docs/css/main.css b/docs/css/main.css
index e24dff8531f24..b873992dee8d0 100755
--- a/docs/css/main.css
+++ b/docs/css/main.css
@@ -2,6 +2,37 @@
    Author's custom styles
    ========================================================================== */
 
+table {
+  margin: 15px 0;
+  padding: 0;
+}
+
+table tr {
+  border-top: 1px solid #cccccc;
+  background-color: white;
+  margin: 0;
+  padding: 0;
+}
+
+table tr:nth-child(2n) {
+  background-color: #F1F4F5;
+}
+
+table tr th {
+  font-weight: bold;
+  border: 1px solid #cccccc;
+  text-align: left;
+  margin: 0;
+  padding: 6px 13px;
+}
+
+table tr td {
+  border: 1px solid #cccccc;
+  text-align: left;
+  margin: 0;
+  padding: 6px 13px;
+}
+
 .navbar .brand {
   height: 50px;
   width: 110px;
@@ -48,6 +79,7 @@ body .container-wrapper {
   margin-right: auto;
   border-radius: 15px;
   position: relative;
+  min-height: 100vh;
 }
 
 .title {
@@ -128,6 +160,7 @@ a:hover code {
   max-width: 914px;
   line-height: 1.6; /* Inspired by Github's wiki style */
   padding-left: 30px;
+  min-height: 100vh;
 }
 
 .dropdown-menu {
@@ -208,11 +241,10 @@ a.anchorjs-link:hover { text-decoration: none; }
   border-bottom-width: 0px;
   margin-top: 0px;
   width: 210px;
+  height: 80%;
   float: left;
   position: fixed;
   overflow-y: scroll;
-  top: 0;
-  bottom: 0;
 }
 
 .left-menu {
diff --git a/docs/graphx-programming-guide.md b/docs/graphx-programming-guide.md
index 167c44aa1b2e9..50c9366a0999f 100644
--- a/docs/graphx-programming-guide.md
+++ b/docs/graphx-programming-guide.md
@@ -25,38 +25,38 @@ license: |
 
 <!-- All the documentation links  -->
 
-[EdgeRDD]: api/scala/index.html#org.apache.spark.graphx.EdgeRDD
-[VertexRDD]: api/scala/index.html#org.apache.spark.graphx.VertexRDD
-[Edge]: api/scala/index.html#org.apache.spark.graphx.Edge
-[EdgeTriplet]: api/scala/index.html#org.apache.spark.graphx.EdgeTriplet
-[Graph]: api/scala/index.html#org.apache.spark.graphx.Graph
-[GraphOps]: api/scala/index.html#org.apache.spark.graphx.GraphOps
-[Graph.mapVertices]: api/scala/index.html#org.apache.spark.graphx.Graph@mapVertices[VD2]((VertexId,VD)⇒VD2)(ClassTag[VD2]):Graph[VD2,ED]
-[Graph.reverse]: api/scala/index.html#org.apache.spark.graphx.Graph@reverse:Graph[VD,ED]
-[Graph.subgraph]: api/scala/index.html#org.apache.spark.graphx.Graph@subgraph((EdgeTriplet[VD,ED])⇒Boolean,(VertexId,VD)⇒Boolean):Graph[VD,ED]
-[Graph.mask]: api/scala/index.html#org.apache.spark.graphx.Graph@mask[VD2,ED2](Graph[VD2,ED2])(ClassTag[VD2],ClassTag[ED2]):Graph[VD,ED]
-[Graph.groupEdges]: api/scala/index.html#org.apache.spark.graphx.Graph@groupEdges((ED,ED)⇒ED):Graph[VD,ED]
-[GraphOps.joinVertices]: api/scala/index.html#org.apache.spark.graphx.GraphOps@joinVertices[U](RDD[(VertexId,U)])((VertexId,VD,U)⇒VD)(ClassTag[U]):Graph[VD,ED]
-[Graph.outerJoinVertices]: api/scala/index.html#org.apache.spark.graphx.Graph@outerJoinVertices[U,VD2](RDD[(VertexId,U)])((VertexId,VD,Option[U])⇒VD2)(ClassTag[U],ClassTag[VD2]):Graph[VD2,ED]
-[Graph.aggregateMessages]: api/scala/index.html#org.apache.spark.graphx.Graph@aggregateMessages[A]((EdgeContext[VD,ED,A])⇒Unit,(A,A)⇒A,TripletFields)(ClassTag[A]):VertexRDD[A]
-[EdgeContext]: api/scala/index.html#org.apache.spark.graphx.EdgeContext
-[GraphOps.collectNeighborIds]: api/scala/index.html#org.apache.spark.graphx.GraphOps@collectNeighborIds(EdgeDirection):VertexRDD[Array[VertexId]]
-[GraphOps.collectNeighbors]: api/scala/index.html#org.apache.spark.graphx.GraphOps@collectNeighbors(EdgeDirection):VertexRDD[Array[(VertexId,VD)]]
+[EdgeRDD]: api/scala/org/apache/spark/graphx/EdgeRDD.html
+[VertexRDD]: api/scala/org/apache/spark/graphx/VertexRDD.html
+[Edge]: api/scala/org/apache/spark/graphx/Edge.html
+[EdgeTriplet]: api/scala/org/apache/spark/graphx/EdgeTriplet.html
+[Graph]: api/scala/org/apache/spark/graphx/Graph$.html
+[GraphOps]: api/scala/org/apache/spark/graphx/GraphOps.html
+[Graph.mapVertices]: api/scala/org/apache/spark/graphx/Graph.html#mapVertices[VD2]((VertexId,VD)⇒VD2)(ClassTag[VD2]):Graph[VD2,ED]
+[Graph.reverse]: api/scala/org/apache/spark/graphx/Graph.html#reverse:Graph[VD,ED]
+[Graph.subgraph]: api/scala/org/apache/spark/graphx/Graph.html#subgraph((EdgeTriplet[VD,ED])⇒Boolean,(VertexId,VD)⇒Boolean):Graph[VD,ED]
+[Graph.mask]: api/scala/org/apache/spark/graphx/Graph.html#mask[VD2,ED2](Graph[VD2,ED2])(ClassTag[VD2],ClassTag[ED2]):Graph[VD,ED]
+[Graph.groupEdges]: api/scala/org/apache/spark/graphx/Graph.html#groupEdges((ED,ED)⇒ED):Graph[VD,ED]
+[GraphOps.joinVertices]: api/scala/org/apache/spark/graphx/GraphOps.html#joinVertices[U](RDD[(VertexId,U)])((VertexId,VD,U)⇒VD)(ClassTag[U]):Graph[VD,ED]
+[Graph.outerJoinVertices]: api/scala/org/apache/spark/graphx/Graph.html#outerJoinVertices[U,VD2](RDD[(VertexId,U)])((VertexId,VD,Option[U])⇒VD2)(ClassTag[U],ClassTag[VD2]):Graph[VD2,ED]
+[Graph.aggregateMessages]: api/scala/org/apache/spark/graphx/Graph.html#aggregateMessages[A]((EdgeContext[VD,ED,A])⇒Unit,(A,A)⇒A,TripletFields)(ClassTag[A]):VertexRDD[A]
+[EdgeContext]: api/scala/org/apache/spark/graphx/EdgeContext.html
+[GraphOps.collectNeighborIds]: api/scala/org/apache/spark/graphx/GraphOps.html#collectNeighborIds(EdgeDirection):VertexRDD[Array[VertexId]]
+[GraphOps.collectNeighbors]: api/scala/org/apache/spark/graphx/GraphOps.html#collectNeighbors(EdgeDirection):VertexRDD[Array[(VertexId,VD)]]
 [RDD Persistence]: rdd-programming-guide.html#rdd-persistence
-[Graph.cache]: api/scala/index.html#org.apache.spark.graphx.Graph@cache():Graph[VD,ED]
-[GraphOps.pregel]: api/scala/index.html#org.apache.spark.graphx.GraphOps@pregel[A](A,Int,EdgeDirection)((VertexId,VD,A)⇒VD,(EdgeTriplet[VD,ED])⇒Iterator[(VertexId,A)],(A,A)⇒A)(ClassTag[A]):Graph[VD,ED]
-[PartitionStrategy]: api/scala/index.html#org.apache.spark.graphx.PartitionStrategy$
-[GraphLoader.edgeListFile]: api/scala/index.html#org.apache.spark.graphx.GraphLoader$@edgeListFile(SparkContext,String,Boolean,Int):Graph[Int,Int]
-[Graph.apply]: api/scala/index.html#org.apache.spark.graphx.Graph$@apply[VD,ED](RDD[(VertexId,VD)],RDD[Edge[ED]],VD)(ClassTag[VD],ClassTag[ED]):Graph[VD,ED]
-[Graph.fromEdgeTuples]: api/scala/index.html#org.apache.spark.graphx.Graph$@fromEdgeTuples[VD](RDD[(VertexId,VertexId)],VD,Option[PartitionStrategy])(ClassTag[VD]):Graph[VD,Int]
-[Graph.fromEdges]: api/scala/index.html#org.apache.spark.graphx.Graph$@fromEdges[VD,ED](RDD[Edge[ED]],VD)(ClassTag[VD],ClassTag[ED]):Graph[VD,ED]
-[PartitionStrategy]: api/scala/index.html#org.apache.spark.graphx.PartitionStrategy
-[PageRank]: api/scala/index.html#org.apache.spark.graphx.lib.PageRank$
-[ConnectedComponents]: api/scala/index.html#org.apache.spark.graphx.lib.ConnectedComponents$
-[TriangleCount]: api/scala/index.html#org.apache.spark.graphx.lib.TriangleCount$
-[Graph.partitionBy]: api/scala/index.html#org.apache.spark.graphx.Graph@partitionBy(PartitionStrategy):Graph[VD,ED]
-[EdgeContext.sendToSrc]: api/scala/index.html#org.apache.spark.graphx.EdgeContext@sendToSrc(msg:A):Unit
-[EdgeContext.sendToDst]: api/scala/index.html#org.apache.spark.graphx.EdgeContext@sendToDst(msg:A):Unit
+[Graph.cache]: api/scala/org/apache/spark/graphx/Graph.html#cache():Graph[VD,ED]
+[GraphOps.pregel]: api/scala/org/apache/spark/graphx/GraphOps.html#pregel[A](A,Int,EdgeDirection)((VertexId,VD,A)⇒VD,(EdgeTriplet[VD,ED])⇒Iterator[(VertexId,A)],(A,A)⇒A)(ClassTag[A]):Graph[VD,ED]
+[PartitionStrategy]: api/scala/org/apache/spark/graphx/PartitionStrategy$.html
+[GraphLoader.edgeListFile]: api/scala/org/apache/spark/graphx/GraphLoader$.html#edgeListFile(SparkContext,String,Boolean,Int):Graph[Int,Int]
+[Graph.apply]: api/scala/org/apache/spark/graphx/Graph$.html#apply[VD,ED](RDD[(VertexId,VD)],RDD[Edge[ED]],VD)(ClassTag[VD],ClassTag[ED]):Graph[VD,ED]
+[Graph.fromEdgeTuples]: api/scala/org/apache/spark/graphx/Graph$.html#fromEdgeTuples[VD](RDD[(VertexId,VertexId)],VD,Option[PartitionStrategy])(ClassTag[VD]):Graph[VD,Int]
+[Graph.fromEdges]: api/scala/org/apache/spark/graphx/Graph$.html#fromEdges[VD,ED](RDD[Edge[ED]],VD)(ClassTag[VD],ClassTag[ED]):Graph[VD,ED]
+[PartitionStrategy]: api/scala/org/apache/spark/graphx/PartitionStrategy$.html
+[PageRank]: api/scala/org/apache/spark/graphx/lib/PageRank$.html
+[ConnectedComponents]: api/scala/org/apache/spark/graphx/lib/ConnectedComponents$.html
+[TriangleCount]: api/scala/org/apache/spark/graphx/lib/TriangleCount$.html
+[Graph.partitionBy]: api/scala/org/apache/spark/graphx/Graph.html#partitionBy(PartitionStrategy):Graph[VD,ED]
+[EdgeContext.sendToSrc]: api/scala/org/apache/spark/graphx/EdgeContext.html#sendToSrc(msg:A):Unit
+[EdgeContext.sendToDst]: api/scala/org/apache/spark/graphx/EdgeContext.html#sendToDst(msg:A):Unit
 [TripletFields]: api/java/org/apache/spark/graphx/TripletFields.html
 [TripletFields.All]: api/java/org/apache/spark/graphx/TripletFields.html#All
 [TripletFields.None]: api/java/org/apache/spark/graphx/TripletFields.html#None
@@ -74,7 +74,7 @@ license: |
 # Overview
 
 GraphX is a new component in Spark for graphs and graph-parallel computation. At a high level,
-GraphX extends the Spark [RDD](api/scala/index.html#org.apache.spark.rdd.RDD) by introducing a
+GraphX extends the Spark [RDD](api/scala/org/apache/spark/rdd/RDD.html) by introducing a
 new [Graph](#property_graph) abstraction: a directed multigraph with properties
 attached to each vertex and edge.  To support graph computation, GraphX exposes a set of fundamental
 operators (e.g., [subgraph](#structural_operators), [joinVertices](#join_operators), and
@@ -99,7 +99,7 @@ getting started with Spark refer to the [Spark Quick Start Guide](quick-start.ht
 
 # The Property Graph
 
-The [property graph](api/scala/index.html#org.apache.spark.graphx.Graph) is a directed multigraph
+The [property graph](api/scala/org/apache/spark/graphx/Graph.html) is a directed multigraph
 with user defined objects attached to each vertex and edge.  A directed multigraph is a directed
 graph with potentially multiple parallel edges sharing the same source and destination vertex.  The
 ability to support parallel edges simplifies modeling scenarios where there can be multiple
@@ -175,7 +175,7 @@ val userGraph: Graph[(String, String), String]
 There are numerous ways to construct a property graph from raw files, RDDs, and even synthetic
 generators and these are discussed in more detail in the section on
 [graph builders](#graph_builders).  Probably the most general method is to use the
-[Graph object](api/scala/index.html#org.apache.spark.graphx.Graph$).  For example the following
+[Graph object](api/scala/org/apache/spark/graphx/Graph$.html).  For example the following
 code constructs a graph from a collection of RDDs:
 
 {% highlight scala %}
diff --git a/docs/hardware-provisioning.md b/docs/hardware-provisioning.md
index 4e5d681962c8d..fc87995f9817f 100644
--- a/docs/hardware-provisioning.md
+++ b/docs/hardware-provisioning.md
@@ -63,10 +63,10 @@ Note that memory usage is greatly affected by storage level and serialization fo
 the [tuning guide](tuning.html) for tips on how to reduce it.
 
 Finally, note that the Java VM does not always behave well with more than 200 GiB of RAM. If you
-purchase machines with more RAM than this, you can run _multiple worker JVMs per node_. In
-Spark's [standalone mode](spark-standalone.html), you can set the number of workers per node
-with the `SPARK_WORKER_INSTANCES` variable in `conf/spark-env.sh`, and the number of cores
-per worker with `SPARK_WORKER_CORES`.
+purchase machines with more RAM than this, you can launch multiple executors in a single node. In
+Spark's [standalone mode](spark-standalone.html), a worker is responsible for launching multiple
+executors according to its available memory and cores, and each executor will be launched in a
+separate Java VM.
 
 # Network
 
diff --git a/docs/img/webui-structured-streaming-detail.png b/docs/img/webui-structured-streaming-detail.png
new file mode 100644
index 0000000000000..f4850523c5c2f
Binary files /dev/null and b/docs/img/webui-structured-streaming-detail.png differ
diff --git a/docs/index.md b/docs/index.md
index f6ec595231f9e..c0771ca170af5 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -44,10 +44,9 @@ source, visit [Building Spark](building-spark.html).
 
 Spark runs on both Windows and UNIX-like systems (e.g. Linux, Mac OS), and it should run on any platform that runs a supported version of Java. This should include JVMs on x86_64 and ARM64. It's easy to run locally on one machine --- all you need is to have `java` installed on your system `PATH`, or the `JAVA_HOME` environment variable pointing to a Java installation.
 
-Spark runs on Java 8/11, Scala 2.12, Python 2.7+/3.4+ and R 3.1+.
+Spark runs on Java 8/11, Scala 2.12, Python 2.7+/3.4+ and R 3.5+.
 Java 8 prior to version 8u92 support is deprecated as of Spark 3.0.0.
 Python 2 and Python 3 prior to version 3.6 support is deprecated as of Spark 3.0.0.
-R prior to version 3.4 support is deprecated as of Spark 3.0.0.
 For the Scala API, Spark {{site.SPARK_VERSION}}
 uses Scala {{site.SCALA_BINARY_VERSION}}. You will need to use a compatible Scala version
 ({{site.SCALA_BINARY_VERSION}}.x).
@@ -118,7 +117,7 @@ options for deployment:
 
 **API Docs:**
 
-* [Spark Scala API (Scaladoc)](api/scala/index.html#org.apache.spark.package)
+* [Spark Scala API (Scaladoc)](api/scala/org/apache/spark/index.html)
 * [Spark Java API (Javadoc)](api/java/index.html)
 * [Spark Python API (Sphinx)](api/python/index.html)
 * [Spark R API (Roxygen2)](api/R/index.html)
diff --git a/docs/job-scheduling.md b/docs/job-scheduling.md
index eaacfa49c657c..2164b2eb2555c 100644
--- a/docs/job-scheduling.md
+++ b/docs/job-scheduling.md
@@ -79,23 +79,24 @@ are no longer used and request them again later when there is demand. This featu
 useful if multiple applications share resources in your Spark cluster.
 
 This feature is disabled by default and available on all coarse-grained cluster managers, i.e.
-[standalone mode](spark-standalone.html), [YARN mode](running-on-yarn.html), and
-[Mesos coarse-grained mode](running-on-mesos.html#mesos-run-modes).
+[standalone mode](spark-standalone.html), [YARN mode](running-on-yarn.html),
+[Mesos coarse-grained mode](running-on-mesos.html#mesos-run-modes) and [K8s mode](running-on-kubernetes.html).
+
 
 ### Configuration and Setup
 
-There are two requirements for using this feature. First, your application must set
-`spark.dynamicAllocation.enabled` to `true`. Second, you must set up an *external shuffle service*
-on each worker node in the same cluster and set `spark.shuffle.service.enabled` to true in your
-application. The purpose of the external shuffle service is to allow executors to be removed
+There are two ways for using this feature.
+First, your application must set both `spark.dynamicAllocation.enabled` and `spark.dynamicAllocation.shuffleTracking.enabled` to `true`.
+Second, your application must set both `spark.dynamicAllocation.enabled` and `spark.shuffle.service.enabled` to `true`
+after you set up an *external shuffle service* on each worker node in the same cluster.
+The purpose of the shuffle tracking or the external shuffle service is to allow executors to be removed
 without deleting shuffle files written by them (more detail described
-[below](job-scheduling.html#graceful-decommission-of-executors)). The way to set up this service
-varies across cluster managers:
+[below](job-scheduling.html#graceful-decommission-of-executors)). While it is simple to enable shuffle tracking, the way to set up the external shuffle service varies across cluster managers:
 
 In standalone mode, simply start your workers with `spark.shuffle.service.enabled` set to `true`.
 
 In Mesos coarse-grained mode, run `$SPARK_HOME/sbin/start-mesos-shuffle-service.sh` on all
-slave nodes with `spark.shuffle.service.enabled` set to `true`. For instance, you may do so
+worker nodes with `spark.shuffle.service.enabled` set to `true`. For instance, you may do so
 through Marathon.
 
 In YARN mode, follow the instructions [here](running-on-yarn.html#configuring-the-external-shuffle-service).
diff --git a/docs/ml-advanced.md b/docs/ml-advanced.md
index 5787fe914ce7a..0e19bca92d19d 100644
--- a/docs/ml-advanced.md
+++ b/docs/ml-advanced.md
@@ -55,10 +55,10 @@ other first-order optimizations.
 Quasi-Newton](https://www.microsoft.com/en-us/research/wp-content/uploads/2007/01/andrew07scalable.pdf)
 (OWL-QN) is an extension of L-BFGS that can effectively handle L1 and elastic net regularization.
 
-L-BFGS is used as a solver for [LinearRegression](api/scala/index.html#org.apache.spark.ml.regression.LinearRegression),
-[LogisticRegression](api/scala/index.html#org.apache.spark.ml.classification.LogisticRegression),
-[AFTSurvivalRegression](api/scala/index.html#org.apache.spark.ml.regression.AFTSurvivalRegression)
-and [MultilayerPerceptronClassifier](api/scala/index.html#org.apache.spark.ml.classification.MultilayerPerceptronClassifier).
+L-BFGS is used as a solver for [LinearRegression](api/scala/org/apache/spark/ml/regression/LinearRegression.html),
+[LogisticRegression](api/scala/org/apache/spark/ml/classification/LogisticRegression.html),
+[AFTSurvivalRegression](api/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.html)
+and [MultilayerPerceptronClassifier](api/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.html).
 
 MLlib L-BFGS solver calls the corresponding implementation in [breeze](https://github.com/scalanlp/breeze/blob/master/math/src/main/scala/breeze/optimize/LBFGS.scala).
 
@@ -108,4 +108,4 @@ It solves certain optimization problems iteratively through the following proced
 
 Since it involves solving a weighted least squares (WLS) problem by `WeightedLeastSquares` in each iteration,
 it also requires the number of features to be no more than 4096.
-Currently IRLS is used as the default solver of [GeneralizedLinearRegression](api/scala/index.html#org.apache.spark.ml.regression.GeneralizedLinearRegression).
+Currently IRLS is used as the default solver of [GeneralizedLinearRegression](api/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.html).
diff --git a/docs/ml-classification-regression.md b/docs/ml-classification-regression.md
index 630a15d8535f4..9d5388005e587 100644
--- a/docs/ml-classification-regression.md
+++ b/docs/ml-classification-regression.md
@@ -71,7 +71,7 @@ $\alpha$ and `regParam` corresponds to $\lambda$.
 
 <div data-lang="scala" markdown="1">
 
-More details on parameters can be found in the [Scala API documentation](api/scala/index.html#org.apache.spark.ml.classification.LogisticRegression).
+More details on parameters can be found in the [Scala API documentation](api/scala/org/apache/spark/ml/classification/LogisticRegression.html).
 
 {% include_example scala/org/apache/spark/examples/ml/LogisticRegressionWithElasticNetExample.scala %}
 </div>
@@ -109,12 +109,12 @@ only available on the driver.
 
 <div data-lang="scala" markdown="1">
 
-[`LogisticRegressionTrainingSummary`](api/scala/index.html#org.apache.spark.ml.classification.LogisticRegressionTrainingSummary)
+[`LogisticRegressionTrainingSummary`](api/scala/org/apache/spark/ml/classification/LogisticRegressionTrainingSummary.html)
 provides a summary for a
-[`LogisticRegressionModel`](api/scala/index.html#org.apache.spark.ml.classification.LogisticRegressionModel).
+[`LogisticRegressionModel`](api/scala/org/apache/spark/ml/classification/LogisticRegressionModel.html).
 In the case of binary classification, certain additional metrics are
 available, e.g. ROC curve. The binary summary can be accessed via the
-`binarySummary` method. See [`BinaryLogisticRegressionTrainingSummary`](api/scala/index.html#org.apache.spark.ml.classification.BinaryLogisticRegressionTrainingSummary).
+`binarySummary` method. See [`BinaryLogisticRegressionTrainingSummary`](api/scala/org/apache/spark/ml/classification/BinaryLogisticRegressionTrainingSummary.html).
 
 Continuing the earlier example:
 
@@ -216,7 +216,7 @@ We use two feature transformers to prepare the data; these help index categories
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-More details on parameters can be found in the [Scala API documentation](api/scala/index.html#org.apache.spark.ml.classification.DecisionTreeClassifier).
+More details on parameters can be found in the [Scala API documentation](api/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.html).
 
 {% include_example scala/org/apache/spark/examples/ml/DecisionTreeClassificationExample.scala %}
 
@@ -261,7 +261,7 @@ We use two feature transformers to prepare the data; these help index categories
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.RandomForestClassifier) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/RandomForestClassifier.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/RandomForestClassifierExample.scala %}
 </div>
@@ -302,7 +302,7 @@ We use two feature transformers to prepare the data; these help index categories
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.GBTClassifier) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/GBTClassifier.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/GradientBoostedTreeClassifierExample.scala %}
 </div>
@@ -358,7 +358,7 @@ MLPC employs backpropagation for learning the model. We use the logistic loss fu
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.MultilayerPerceptronClassifier) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/MultilayerPerceptronClassifierExample.scala %}
 </div>
@@ -403,7 +403,7 @@ in Spark ML supports binary classification with linear SVM. Internally, it optim
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.LinearSVC) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/LinearSVC.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/LinearSVCExample.scala %}
 </div>
@@ -447,7 +447,7 @@ The example below demonstrates how to load the
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.OneVsRest) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/OneVsRest.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/OneVsRestExample.scala %}
 </div>
@@ -501,7 +501,7 @@ setting the parameter $\lambda$ (default to $1.0$).
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.NaiveBayes) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/NaiveBayes.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/NaiveBayesExample.scala %}
 </div>
@@ -544,7 +544,7 @@ We scale features to be between 0 and 1 to prevent the exploding gradient proble
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.FMClassifier) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/FMClassifier.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/FMClassifierExample.scala %}
 </div>
@@ -585,7 +585,7 @@ regression model and extracting model summary statistics.
 
 <div data-lang="scala" markdown="1">
 
-More details on parameters can be found in the [Scala API documentation](api/scala/index.html#org.apache.spark.ml.regression.LinearRegression).
+More details on parameters can be found in the [Scala API documentation](api/scala/org/apache/spark/ml/regression/LinearRegression.html).
 
 {% include_example scala/org/apache/spark/examples/ml/LinearRegressionWithElasticNetExample.scala %}
 </div>
@@ -726,7 +726,7 @@ function and extracting model summary statistics.
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.regression.GeneralizedLinearRegression) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/GeneralizedLinearRegressionExample.scala %}
 </div>
@@ -768,7 +768,7 @@ We use a feature transformer to index categorical features, adding metadata to t
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-More details on parameters can be found in the [Scala API documentation](api/scala/index.html#org.apache.spark.ml.regression.DecisionTreeRegressor).
+More details on parameters can be found in the [Scala API documentation](api/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.html).
 
 {% include_example scala/org/apache/spark/examples/ml/DecisionTreeRegressionExample.scala %}
 </div>
@@ -810,7 +810,7 @@ We use a feature transformer to index categorical features, adding metadata to t
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.regression.RandomForestRegressor) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/regression/RandomForestRegressor.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/RandomForestRegressorExample.scala %}
 </div>
@@ -851,7 +851,7 @@ be true in general.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.regression.GBTRegressor) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/regression/GBTRegressor.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/GradientBoostedTreeRegressorExample.scala %}
 </div>
@@ -945,7 +945,7 @@ The implementation matches the result from R's survival function
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.regression.AFTSurvivalRegression) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/AFTSurvivalRegressionExample.scala %}
 </div>
@@ -1025,7 +1025,7 @@ is treated as piecewise linear function. The rules for prediction therefore are:
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [`IsotonicRegression` Scala docs](api/scala/index.html#org.apache.spark.ml.regression.IsotonicRegression) for details on the API.
+Refer to the [`IsotonicRegression` Scala docs](api/scala/org/apache/spark/ml/regression/IsotonicRegression.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/IsotonicRegressionExample.scala %}
 </div>
@@ -1066,7 +1066,7 @@ We scale features to be between 0 and 1 to prevent the exploding gradient proble
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.regression.FMRegressor) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/regression/FMRegressor.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/FMRegressorExample.scala %}
 </div>
diff --git a/docs/ml-clustering.md b/docs/ml-clustering.md
index 2775d0421ccca..4574567fa9d50 100644
--- a/docs/ml-clustering.md
+++ b/docs/ml-clustering.md
@@ -85,7 +85,7 @@ called [kmeans||](http://theory.stanford.edu/~sergei/papers/vldb12-kmpar.pdf).
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.clustering.KMeans) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/clustering/KMeans.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/KMeansExample.scala %}
 </div>
@@ -123,7 +123,7 @@ and generates a `LDAModel` as the base model. Expert users may cast a `LDAModel`
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.clustering.LDA) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/clustering/LDA.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/LDAExample.scala %}
 </div>
@@ -166,7 +166,7 @@ Bisecting K-means can often be much faster than regular K-means, but it will gen
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.clustering.BisectingKMeans) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/clustering/BisectingKMeans.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/BisectingKMeansExample.scala %}
 </div>
@@ -255,7 +255,7 @@ model.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.clustering.GaussianMixture) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/clustering/GaussianMixture.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/GaussianMixtureExample.scala %}
 </div>
@@ -302,7 +302,7 @@ using truncated power iteration on a normalized pair-wise similarity matrix of t
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.clustering.PowerIterationClustering) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/clustering/PowerIterationClustering.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/PowerIterationClusteringExample.scala %}
 </div>
diff --git a/docs/ml-collaborative-filtering.md b/docs/ml-collaborative-filtering.md
index e6e596bed110b..6c41efd5cc306 100644
--- a/docs/ml-collaborative-filtering.md
+++ b/docs/ml-collaborative-filtering.md
@@ -115,7 +115,7 @@ explicit (`implicitPrefs` is `false`).
 We evaluate the recommendation model by measuring the root-mean-square error of
 rating prediction.
 
-Refer to the [`ALS` Scala docs](api/scala/index.html#org.apache.spark.ml.recommendation.ALS)
+Refer to the [`ALS` Scala docs](api/scala/org/apache/spark/ml/recommendation/ALS.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ALSExample.scala %}
diff --git a/docs/ml-datasource.md b/docs/ml-datasource.md
index 5dc2d057a9163..8e9c947b75f38 100644
--- a/docs/ml-datasource.md
+++ b/docs/ml-datasource.md
@@ -42,7 +42,7 @@ The schema of the `image` column is:
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`ImageDataSource`](api/scala/index.html#org.apache.spark.ml.source.image.ImageDataSource)
+[`ImageDataSource`](api/scala/org/apache/spark/ml/source/image/ImageDataSource.html)
 implements a Spark SQL data source API for loading image data as a DataFrame.
 
 {% highlight scala %}
@@ -86,7 +86,7 @@ Will output:
 In PySpark we provide Spark SQL data source API for loading image data as a DataFrame.
 
 {% highlight python %}
->>> df = spark.read.format("image").option("dropInvalid", true).load("data/mllib/images/origin/kittens")
+>>> df = spark.read.format("image").option("dropInvalid", True).load("data/mllib/images/origin/kittens")
 >>> df.select("image.origin", "image.width", "image.height").show(truncate=False)
 +-----------------------------------------------------------------------+-----+------+
 |origin                                                                 |width|height|
@@ -133,7 +133,7 @@ The schemas of the columns are:
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`LibSVMDataSource`](api/scala/index.html#org.apache.spark.ml.source.libsvm.LibSVMDataSource)
+[`LibSVMDataSource`](api/scala/org/apache/spark/ml/source/libsvm/LibSVMDataSource.html)
 implements a Spark SQL data source API for loading `LIBSVM` data as a DataFrame.
 
 {% highlight scala %}
diff --git a/docs/ml-features.md b/docs/ml-features.md
index 92d2f3d0b418d..05ef848aefef8 100644
--- a/docs/ml-features.md
+++ b/docs/ml-features.md
@@ -96,8 +96,8 @@ when using text as features.  Our feature vectors could then be passed to a lear
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [HashingTF Scala docs](api/scala/index.html#org.apache.spark.ml.feature.HashingTF) and
-the [IDF Scala docs](api/scala/index.html#org.apache.spark.ml.feature.IDF) for more details on the API.
+Refer to the [HashingTF Scala docs](api/scala/org/apache/spark/ml/feature/HashingTF.html) and
+the [IDF Scala docs](api/scala/org/apache/spark/ml/feature/IDF.html) for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/TfIdfExample.scala %}
 </div>
@@ -135,7 +135,7 @@ In the following code segment, we start with a set of documents, each of which i
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Word2Vec Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Word2Vec)
+Refer to the [Word2Vec Scala docs](api/scala/org/apache/spark/ml/feature/Word2Vec.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/Word2VecExample.scala %}
@@ -200,8 +200,8 @@ Each vector represents the token counts of the document over the vocabulary.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [CountVectorizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.CountVectorizer)
-and the [CountVectorizerModel Scala docs](api/scala/index.html#org.apache.spark.ml.feature.CountVectorizerModel)
+Refer to the [CountVectorizer Scala docs](api/scala/org/apache/spark/ml/feature/CountVectorizer.html)
+and the [CountVectorizerModel Scala docs](api/scala/org/apache/spark/ml/feature/CountVectorizerModel.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/CountVectorizerExample.scala %}
@@ -286,7 +286,7 @@ The resulting feature vectors could then be passed to a learning algorithm.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [FeatureHasher Scala docs](api/scala/index.html#org.apache.spark.ml.feature.FeatureHasher)
+Refer to the [FeatureHasher Scala docs](api/scala/org/apache/spark/ml/feature/FeatureHasher.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/FeatureHasherExample.scala %}
@@ -313,9 +313,9 @@ for more details on the API.
 
 ## Tokenizer
 
-[Tokenization](http://en.wikipedia.org/wiki/Lexical_analysis#Tokenization) is the process of taking text (such as a sentence) and breaking it into individual terms (usually words).  A simple [Tokenizer](api/scala/index.html#org.apache.spark.ml.feature.Tokenizer) class provides this functionality.  The example below shows how to split sentences into sequences of words.
+[Tokenization](http://en.wikipedia.org/wiki/Lexical_analysis#Tokenization) is the process of taking text (such as a sentence) and breaking it into individual terms (usually words).  A simple [Tokenizer](api/scala/org/apache/spark/ml/feature/Tokenizer.html) class provides this functionality.  The example below shows how to split sentences into sequences of words.
 
-[RegexTokenizer](api/scala/index.html#org.apache.spark.ml.feature.RegexTokenizer) allows more
+[RegexTokenizer](api/scala/org/apache/spark/ml/feature/RegexTokenizer.html) allows more
  advanced tokenization based on regular expression (regex) matching.
  By default, the parameter "pattern" (regex, default: `"\\s+"`) is used as delimiters to split the input text.
  Alternatively, users can set parameter "gaps" to false indicating the regex "pattern" denotes
@@ -326,8 +326,8 @@ for more details on the API.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Tokenizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Tokenizer)
-and the [RegexTokenizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.RegexTokenizer)
+Refer to the [Tokenizer Scala docs](api/scala/org/apache/spark/ml/feature/Tokenizer.html)
+and the [RegexTokenizer Scala docs](api/scala/org/apache/spark/ml/feature/RegexTokenizer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/TokenizerExample.scala %}
@@ -395,7 +395,7 @@ filtered out.
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [StopWordsRemover Scala docs](api/scala/index.html#org.apache.spark.ml.feature.StopWordsRemover)
+Refer to the [StopWordsRemover Scala docs](api/scala/org/apache/spark/ml/feature/StopWordsRemover.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala %}
@@ -430,7 +430,7 @@ An [n-gram](https://en.wikipedia.org/wiki/N-gram) is a sequence of $n$ tokens (t
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [NGram Scala docs](api/scala/index.html#org.apache.spark.ml.feature.NGram)
+Refer to the [NGram Scala docs](api/scala/org/apache/spark/ml/feature/NGram.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/NGramExample.scala %}
@@ -468,7 +468,7 @@ for `inputCol`.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Binarizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Binarizer)
+Refer to the [Binarizer Scala docs](api/scala/org/apache/spark/ml/feature/Binarizer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/BinarizerExample.scala %}
@@ -493,14 +493,14 @@ for more details on the API.
 
 ## PCA
 
-[PCA](http://en.wikipedia.org/wiki/Principal_component_analysis) is a statistical procedure that uses an orthogonal transformation to convert a set of observations of possibly correlated variables into a set of values of linearly uncorrelated variables called principal components. A [PCA](api/scala/index.html#org.apache.spark.ml.feature.PCA) class trains a model to project vectors to a low-dimensional space using PCA. The example below shows how to project 5-dimensional feature vectors into 3-dimensional principal components.
+[PCA](http://en.wikipedia.org/wiki/Principal_component_analysis) is a statistical procedure that uses an orthogonal transformation to convert a set of observations of possibly correlated variables into a set of values of linearly uncorrelated variables called principal components. A [PCA](api/scala/org/apache/spark/ml/feature/PCA.html) class trains a model to project vectors to a low-dimensional space using PCA. The example below shows how to project 5-dimensional feature vectors into 3-dimensional principal components.
 
 **Examples**
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [PCA Scala docs](api/scala/index.html#org.apache.spark.ml.feature.PCA)
+Refer to the [PCA Scala docs](api/scala/org/apache/spark/ml/feature/PCA.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/PCAExample.scala %}
@@ -525,14 +525,14 @@ for more details on the API.
 
 ## PolynomialExpansion
 
-[Polynomial expansion](http://en.wikipedia.org/wiki/Polynomial_expansion) is the process of expanding your features into a polynomial space, which is formulated by an n-degree combination of original dimensions. A [PolynomialExpansion](api/scala/index.html#org.apache.spark.ml.feature.PolynomialExpansion) class provides this functionality.  The example below shows how to expand your features into a 3-degree polynomial space.
+[Polynomial expansion](http://en.wikipedia.org/wiki/Polynomial_expansion) is the process of expanding your features into a polynomial space, which is formulated by an n-degree combination of original dimensions. A [PolynomialExpansion](api/scala/org/apache/spark/ml/feature/PolynomialExpansion.html) class provides this functionality.  The example below shows how to expand your features into a 3-degree polynomial space.
 
 **Examples**
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [PolynomialExpansion Scala docs](api/scala/index.html#org.apache.spark.ml.feature.PolynomialExpansion)
+Refer to the [PolynomialExpansion Scala docs](api/scala/org/apache/spark/ml/feature/PolynomialExpansion.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala %}
@@ -561,7 +561,7 @@ The [Discrete Cosine
 Transform](https://en.wikipedia.org/wiki/Discrete_cosine_transform)
 transforms a length $N$ real-valued sequence in the time domain into
 another length $N$ real-valued sequence in the frequency domain. A
-[DCT](api/scala/index.html#org.apache.spark.ml.feature.DCT) class
+[DCT](api/scala/org/apache/spark/ml/feature/DCT.html) class
 provides this functionality, implementing the
 [DCT-II](https://en.wikipedia.org/wiki/Discrete_cosine_transform#DCT-II)
 and scaling the result by $1/\sqrt{2}$ such that the representing matrix
@@ -574,7 +574,7 @@ $0$th DCT coefficient and _not_ the $N/2$th).
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [DCT Scala docs](api/scala/index.html#org.apache.spark.ml.feature.DCT)
+Refer to the [DCT Scala docs](api/scala/org/apache/spark/ml/feature/DCT.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/DCTExample.scala %}
@@ -704,7 +704,7 @@ Notice that the rows containing "d" or "e" are mapped to index "3.0"
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [StringIndexer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.StringIndexer)
+Refer to the [StringIndexer Scala docs](api/scala/org/apache/spark/ml/feature/StringIndexer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/StringIndexerExample.scala %}
@@ -770,7 +770,7 @@ labels (they will be inferred from the columns' metadata):
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [IndexToString Scala docs](api/scala/index.html#org.apache.spark.ml.feature.IndexToString)
+Refer to the [IndexToString Scala docs](api/scala/org/apache/spark/ml/feature/IndexToString.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/IndexToStringExample.scala %}
@@ -809,7 +809,7 @@ for more details on the API.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [OneHotEncoder Scala docs](api/scala/index.html#org.apache.spark.ml.feature.OneHotEncoder) for more details on the API.
+Refer to the [OneHotEncoder Scala docs](api/scala/org/apache/spark/ml/feature/OneHotEncoder.html) for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala %}
 </div>
@@ -835,7 +835,7 @@ Refer to the [OneHotEncoder Python docs](api/python/pyspark.ml.html#pyspark.ml.f
 `VectorIndexer` helps index categorical features in datasets of `Vector`s.
 It can both automatically decide which features are categorical and convert original values to category indices.  Specifically, it does the following:
 
-1. Take an input column of type [Vector](api/scala/index.html#org.apache.spark.ml.linalg.Vector) and a parameter `maxCategories`.
+1. Take an input column of type [Vector](api/scala/org/apache/spark/ml/linalg/Vector.html) and a parameter `maxCategories`.
 2. Decide which features should be categorical based on the number of distinct values, where features with at most `maxCategories` are declared categorical.
 3. Compute 0-based category indices for each categorical feature.
 4. Index categorical features and transform original feature values to indices.
@@ -849,7 +849,7 @@ In the example below, we read in a dataset of labeled points and then use `Vecto
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [VectorIndexer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.VectorIndexer)
+Refer to the [VectorIndexer Scala docs](api/scala/org/apache/spark/ml/feature/VectorIndexer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/VectorIndexerExample.scala %}
@@ -910,7 +910,7 @@ then `interactedCol` as the output column contains:
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Interaction Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Interaction)
+Refer to the [Interaction Scala docs](api/scala/org/apache/spark/ml/feature/Interaction.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/InteractionExample.scala %}
@@ -944,7 +944,7 @@ The following example demonstrates how to load a dataset in libsvm format and th
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Normalizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Normalizer)
+Refer to the [Normalizer Scala docs](api/scala/org/apache/spark/ml/feature/Normalizer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/NormalizerExample.scala %}
@@ -986,7 +986,7 @@ The following example demonstrates how to load a dataset in libsvm format and th
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [StandardScaler Scala docs](api/scala/index.html#org.apache.spark.ml.feature.StandardScaler)
+Refer to the [StandardScaler Scala docs](api/scala/org/apache/spark/ml/feature/StandardScaler.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/StandardScalerExample.scala %}
@@ -1030,7 +1030,7 @@ The following example demonstrates how to load a dataset in libsvm format and th
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [RobustScaler Scala docs](api/scala/index.html#org.apache.spark.ml.feature.RobustScaler)
+Refer to the [RobustScaler Scala docs](api/scala/org/apache/spark/ml/feature/RobustScaler.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/RobustScalerExample.scala %}
@@ -1078,8 +1078,8 @@ The following example demonstrates how to load a dataset in libsvm format and th
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [MinMaxScaler Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MinMaxScaler)
-and the [MinMaxScalerModel Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MinMaxScalerModel)
+Refer to the [MinMaxScaler Scala docs](api/scala/org/apache/spark/ml/feature/MinMaxScaler.html)
+and the [MinMaxScalerModel Scala docs](api/scala/org/apache/spark/ml/feature/MinMaxScalerModel.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala %}
@@ -1121,8 +1121,8 @@ The following example demonstrates how to load a dataset in libsvm format and th
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [MaxAbsScaler Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MaxAbsScaler)
-and the [MaxAbsScalerModel Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MaxAbsScalerModel)
+Refer to the [MaxAbsScaler Scala docs](api/scala/org/apache/spark/ml/feature/MaxAbsScaler.html)
+and the [MaxAbsScalerModel Scala docs](api/scala/org/apache/spark/ml/feature/MaxAbsScalerModel.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/MaxAbsScalerExample.scala %}
@@ -1157,7 +1157,7 @@ Note that if you have no idea of the upper and lower bounds of the targeted colu
 
 Note also that the splits that you provided have to be in strictly increasing order, i.e. `s0 < s1 < s2 < ... < sn`.
 
-More details can be found in the API docs for [Bucketizer](api/scala/index.html#org.apache.spark.ml.feature.Bucketizer).
+More details can be found in the API docs for [Bucketizer](api/scala/org/apache/spark/ml/feature/Bucketizer.html).
 
 **Examples**
 
@@ -1166,7 +1166,7 @@ The following example demonstrates how to bucketize a column of `Double`s into a
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Bucketizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Bucketizer)
+Refer to the [Bucketizer Scala docs](api/scala/org/apache/spark/ml/feature/Bucketizer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/BucketizerExample.scala %}
@@ -1216,7 +1216,7 @@ This example below demonstrates how to transform vectors using a transforming ve
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [ElementwiseProduct Scala docs](api/scala/index.html#org.apache.spark.ml.feature.ElementwiseProduct)
+Refer to the [ElementwiseProduct Scala docs](api/scala/org/apache/spark/ml/feature/ElementwiseProduct.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ElementwiseProductExample.scala %}
@@ -1276,7 +1276,7 @@ This is the output of the `SQLTransformer` with statement `"SELECT *, (v1 + v2)
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [SQLTransformer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.SQLTransformer)
+Refer to the [SQLTransformer Scala docs](api/scala/org/apache/spark/ml/feature/SQLTransformer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/SQLTransformerExample.scala %}
@@ -1336,7 +1336,7 @@ output column to `features`, after transformation we should get the following Da
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [VectorAssembler Scala docs](api/scala/index.html#org.apache.spark.ml.feature.VectorAssembler)
+Refer to the [VectorAssembler Scala docs](api/scala/org/apache/spark/ml/feature/VectorAssembler.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala %}
@@ -1387,7 +1387,7 @@ to avoid this kind of inconsistent state.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [VectorSizeHint Scala docs](api/scala/index.html#org.apache.spark.ml.feature.VectorSizeHint)
+Refer to the [VectorSizeHint Scala docs](api/scala/org/apache/spark/ml/feature/VectorSizeHint.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/VectorSizeHintExample.scala %}
@@ -1426,7 +1426,7 @@ NaN values, they will be handled specially and placed into their own bucket, for
 are used, then non-NaN data will be put into buckets[0-3], but NaNs will be counted in a special bucket[4].
 
 Algorithm: The bin ranges are chosen using an approximate algorithm (see the documentation for
-[approxQuantile](api/scala/index.html#org.apache.spark.sql.DataFrameStatFunctions) for a
+[approxQuantile](api/scala/org/apache/spark/sql/DataFrameStatFunctions.html) for a
 detailed description). The precision of the approximation can be controlled with the
 `relativeError` parameter. When set to zero, exact quantiles are calculated
 (**Note:** Computing exact quantiles is an expensive operation). The lower and upper bin bounds
@@ -1470,7 +1470,7 @@ a categorical one. Given `numBuckets = 3`, we should get the following DataFrame
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [QuantileDiscretizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.QuantileDiscretizer)
+Refer to the [QuantileDiscretizer Scala docs](api/scala/org/apache/spark/ml/feature/QuantileDiscretizer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/QuantileDiscretizerExample.scala %}
@@ -1499,7 +1499,7 @@ for more details on the API.
 
 The `Imputer` estimator completes missing values in a dataset, either using the mean or the 
 median of the columns in which the missing values are located. The input columns should be of
-`DoubleType` or `FloatType`. Currently `Imputer` does not support categorical features and possibly
+numeric type. Currently `Imputer` does not support categorical features and possibly
 creates incorrect values for columns containing categorical features. Imputer can impute custom values 
 other than 'NaN' by `.setMissingValue(custom_value)`. For example, `.setMissingValue(0)` will impute 
 all occurrences of (0).
@@ -1539,7 +1539,7 @@ the relevant column.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Imputer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Imputer)
+Refer to the [Imputer Scala docs](api/scala/org/apache/spark/ml/feature/Imputer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ImputerExample.scala %}
@@ -1620,7 +1620,7 @@ Suppose also that we have potential input attributes for the `userFeatures`, i.e
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [VectorSlicer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.VectorSlicer)
+Refer to the [VectorSlicer Scala docs](api/scala/org/apache/spark/ml/feature/VectorSlicer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/VectorSlicerExample.scala %}
@@ -1706,7 +1706,7 @@ id | country | hour | clicked | features         | label
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [RFormula Scala docs](api/scala/index.html#org.apache.spark.ml.feature.RFormula)
+Refer to the [RFormula Scala docs](api/scala/org/apache/spark/ml/feature/RFormula.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/RFormulaExample.scala %}
@@ -1770,7 +1770,7 @@ id | features              | clicked | selectedFeatures
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [ChiSqSelector Scala docs](api/scala/index.html#org.apache.spark.ml.feature.ChiSqSelector)
+Refer to the [ChiSqSelector Scala docs](api/scala/org/apache/spark/ml/feature/ChiSqSelector.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ChiSqSelectorExample.scala %}
@@ -1856,7 +1856,7 @@ Bucketed Random Projection accepts arbitrary vectors as input features, and supp
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [BucketedRandomProjectionLSH Scala docs](api/scala/index.html#org.apache.spark.ml.feature.BucketedRandomProjectionLSH)
+Refer to the [BucketedRandomProjectionLSH Scala docs](api/scala/org/apache/spark/ml/feature/BucketedRandomProjectionLSH.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/BucketedRandomProjectionLSHExample.scala %}
@@ -1897,7 +1897,7 @@ The input sets for MinHash are represented as binary vectors, where the vector i
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [MinHashLSH Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MinHashLSH)
+Refer to the [MinHashLSH Scala docs](api/scala/org/apache/spark/ml/feature/MinHashLSH.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/MinHashLSHExample.scala %}
diff --git a/docs/ml-frequent-pattern-mining.md b/docs/ml-frequent-pattern-mining.md
index a243188603997..42d7e50357391 100644
--- a/docs/ml-frequent-pattern-mining.md
+++ b/docs/ml-frequent-pattern-mining.md
@@ -75,7 +75,7 @@ The `FPGrowthModel` provides:
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.fpm.FPGrowth) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/fpm/FPGrowth.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/FPGrowthExample.scala %}
 </div>
@@ -128,7 +128,7 @@ pattern mining problem.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.fpm.PrefixSpan) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/fpm/PrefixSpan.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/PrefixSpanExample.scala %}
 </div>
diff --git a/docs/ml-guide.md b/docs/ml-guide.md
index 7b4fa4f651e64..1b4a3e4eee557 100644
--- a/docs/ml-guide.md
+++ b/docs/ml-guide.md
@@ -41,8 +41,6 @@ The primary Machine Learning API for Spark is now the [DataFrame](sql-programmin
 * MLlib will still support the RDD-based API in `spark.mllib` with bug fixes.
 * MLlib will not add new features to the RDD-based API.
 * In the Spark 2.x releases, MLlib will add features to the DataFrames-based API to reach feature parity with the RDD-based API.
-* After reaching feature parity (roughly estimated for Spark 2.3), the RDD-based API will be deprecated.
-* The RDD-based API is expected to be removed in Spark 3.0.
 
 *Why is MLlib switching to the DataFrame-based API?*
 
@@ -64,54 +62,44 @@ The primary Machine Learning API for Spark is now the [DataFrame](sql-programmin
 
 # Dependencies
 
-MLlib uses the linear algebra package [Breeze](http://www.scalanlp.org/), which depends on
-[netlib-java](https://github.com/fommil/netlib-java) for optimised numerical processing.
-If native libraries[^1] are not available at runtime, you will see a warning message and a pure JVM
-implementation will be used instead.
+MLlib uses linear algebra packages [Breeze](http://www.scalanlp.org/) and [netlib-java](https://github.com/fommil/netlib-java) for optimised numerical processing[^1]. Those packages may call native acceleration libraries such as [Intel MKL](https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html) or [OpenBLAS](http://www.openblas.net) if they are available as system libraries or in runtime library paths. 
 
-Due to licensing issues with runtime proprietary binaries, we do not include `netlib-java`'s native
-proxies by default.
-To configure `netlib-java` / Breeze to use system optimised binaries, include
-`com.github.fommil.netlib:all:1.1.2` (or build Spark with `-Pnetlib-lgpl`) as a dependency of your
-project and read the [netlib-java](https://github.com/fommil/netlib-java) documentation for your
-platform's additional installation instructions.
-
-The most popular native BLAS such as [Intel MKL](https://software.intel.com/en-us/mkl), [OpenBLAS](http://www.openblas.net), can use multiple threads in a single operation, which can conflict with Spark's execution model.
-
-Configuring these BLAS implementations to use a single thread for operations may actually improve performance (see [SPARK-21305](https://issues.apache.org/jira/browse/SPARK-21305)). It is usually optimal to match this to the number of cores each Spark task is configured to use, which is 1 by default and typically left at 1.
-
-Please refer to resources like the following to understand how to configure the number of threads these BLAS implementations use: [Intel MKL](https://software.intel.com/en-us/articles/recommended-settings-for-calling-intel-mkl-routines-from-multi-threaded-applications) and [OpenBLAS](https://github.com/xianyi/OpenBLAS/wiki/faq#multi-threaded).
+Due to differing OSS licenses, `netlib-java`'s native proxies can't be distributed with Spark. See [MLlib Linear Algebra Acceleration Guide](ml-linalg-guide.html) for how to enable accelerated linear algebra processing. If accelerated native libraries are not enabled, you will see a warning message like below and a pure JVM implementation will be used instead:
+```
+WARN BLAS: Failed to load implementation from:com.github.fommil.netlib.NativeSystemBLAS
+WARN BLAS: Failed to load implementation from:com.github.fommil.netlib.NativeRefBLAS
+```
 
 To use MLlib in Python, you will need [NumPy](http://www.numpy.org) version 1.4 or newer.
 
 [^1]: To learn more about the benefits and background of system optimised natives, you may wish to
     watch Sam Halliday's ScalaX talk on [High Performance Linear Algebra in Scala](http://fommil.github.io/scalax14/#/).
 
-# Highlights in 2.3
+# Highlights in 3.0
 
-The list below highlights some of the new features and enhancements added to MLlib in the `2.3`
+The list below highlights some of the new features and enhancements added to MLlib in the `3.0`
 release of Spark:
 
-* Built-in support for reading images into a `DataFrame` was added
-([SPARK-21866](https://issues.apache.org/jira/browse/SPARK-21866)).
-* [`OneHotEncoderEstimator`](ml-features.html#onehotencoderestimator) was added, and should be
-used instead of the existing `OneHotEncoder` transformer. The new estimator supports
-transforming multiple columns.
-* Multiple column support was also added to `QuantileDiscretizer` and `Bucketizer`
-([SPARK-22397](https://issues.apache.org/jira/browse/SPARK-22397) and
-[SPARK-20542](https://issues.apache.org/jira/browse/SPARK-20542))
-* A new [`FeatureHasher`](ml-features.html#featurehasher) transformer was added
- ([SPARK-13969](https://issues.apache.org/jira/browse/SPARK-13969)).
-* Added support for evaluating multiple models in parallel when performing cross-validation using
-[`TrainValidationSplit` or `CrossValidator`](ml-tuning.html)
-([SPARK-19357](https://issues.apache.org/jira/browse/SPARK-19357)).
-* Improved support for custom pipeline components in Python (see
-[SPARK-21633](https://issues.apache.org/jira/browse/SPARK-21633) and 
-[SPARK-21542](https://issues.apache.org/jira/browse/SPARK-21542)).
-* `DataFrame` functions for descriptive summary statistics over vector columns
-([SPARK-19634](https://issues.apache.org/jira/browse/SPARK-19634)).
-* Robust linear regression with Huber loss
-([SPARK-3181](https://issues.apache.org/jira/browse/SPARK-3181)).
+* Multiple columns support was added to `Binarizer` ([SPARK-23578](https://issues.apache.org/jira/browse/SPARK-23578)), `StringIndexer` ([SPARK-11215](https://issues.apache.org/jira/browse/SPARK-11215)), `StopWordsRemover` ([SPARK-29808](https://issues.apache.org/jira/browse/SPARK-29808)) and PySpark `QuantileDiscretizer` ([SPARK-22796](https://issues.apache.org/jira/browse/SPARK-22796)).
+* Tree-Based Feature Transformation was added
+([SPARK-13677](https://issues.apache.org/jira/browse/SPARK-13677)).
+* Two new evaluators `MultilabelClassificationEvaluator` ([SPARK-16692](https://issues.apache.org/jira/browse/SPARK-16692)) and `RankingEvaluator` ([SPARK-28045](https://issues.apache.org/jira/browse/SPARK-28045)) were added.
+* Sample weights support was added in `DecisionTreeClassifier/Regressor` ([SPARK-19591](https://issues.apache.org/jira/browse/SPARK-19591)), `RandomForestClassifier/Regressor` ([SPARK-9478](https://issues.apache.org/jira/browse/SPARK-9478)), `GBTClassifier/Regressor` ([SPARK-9612](https://issues.apache.org/jira/browse/SPARK-9612)),  `MulticlassClassificationEvaluator` ([SPARK-24101](https://issues.apache.org/jira/browse/SPARK-24101)), `RegressionEvaluator` ([SPARK-24102](https://issues.apache.org/jira/browse/SPARK-24102)), `BinaryClassificationEvaluator` ([SPARK-24103](https://issues.apache.org/jira/browse/SPARK-24103)), `BisectingKMeans` ([SPARK-30351](https://issues.apache.org/jira/browse/SPARK-30351)), `KMeans` ([SPARK-29967](https://issues.apache.org/jira/browse/SPARK-29967)) and `GaussianMixture` ([SPARK-30102](https://issues.apache.org/jira/browse/SPARK-30102)).
+* R API for `PowerIterationClustering` was added
+([SPARK-19827](https://issues.apache.org/jira/browse/SPARK-19827)).
+* Added Spark ML listener for tracking ML pipeline status
+([SPARK-23674](https://issues.apache.org/jira/browse/SPARK-23674)).
+* Fit with validation set was added to Gradient Boosted Trees in Python
+([SPARK-24333](https://issues.apache.org/jira/browse/SPARK-24333)).
+* [`RobustScaler`](ml-features.html#robustscaler) transformer was added
+([SPARK-28399](https://issues.apache.org/jira/browse/SPARK-28399)).
+* [`Factorization Machines`](ml-classification-regression.html#factorization-machines) classifier and regressor were added
+([SPARK-29224](https://issues.apache.org/jira/browse/SPARK-29224)).
+* Gaussian Naive Bayes Classifier ([SPARK-16872](https://issues.apache.org/jira/browse/SPARK-16872)) and Complement Naive Bayes Classifier ([SPARK-29942](https://issues.apache.org/jira/browse/SPARK-29942)) were added.
+* ML function parity between Scala and Python
+([SPARK-28958](https://issues.apache.org/jira/browse/SPARK-28958)).
+* `predictRaw` is made public in all the Classification models. `predictProbability` is made public in all the Classification models except `LinearSVCModel`
+([SPARK-30358](https://issues.apache.org/jira/browse/SPARK-30358)).
 
 # Migration Guide
 
diff --git a/docs/ml-linalg-guide.md b/docs/ml-linalg-guide.md
new file mode 100644
index 0000000000000..739091363473f
--- /dev/null
+++ b/docs/ml-linalg-guide.md
@@ -0,0 +1,103 @@
+---
+layout: global
+title: MLlib Linear Algebra Acceleration Guide
+displayTitle: MLlib Linear Algebra Acceleration Guide
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+## Introduction
+
+This guide provides necessary information to enable accelerated linear algebra processing for Spark MLlib.
+
+Spark MLlib defines Vector and Matrix as basic data types for machine learning algorithms. On top of them, [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) and [LAPACK](https://en.wikipedia.org/wiki/LAPACK) operations are implemented and supported by [netlib-java](https://github.com/fommil/netlib-Java) (the algorithms may call [Breeze](https://github.com/scalanlp/breeze) and it will in turn call `netlib-java`). `netlib-java` can use optimized native linear algebra libraries (refered to as "native libraries" or "BLAS libraries" hereafter) for faster numerical processing. [Intel MKL](https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html) and [OpenBLAS](http://www.openblas.net) are two popular ones.
+
+However due to license differences, the official released Spark binaries by default don't contain native libraries support for `netlib-java`.
+
+The following sections describe how to enable `netlib-java` with native libraries support for Spark MLlib and how to install native libraries and configure them properly.
+
+## Enable `netlib-java` with native library proxies 
+
+`netlib-java` depends on `libgfortran`. It requires GFORTRAN 1.4 or above. This can be obtained by installing `libgfortran` package. After installation, the following command can be used to verify if it is installed properly.
+```
+strings /path/to/libgfortran.so.3.0.0 | grep GFORTRAN_1.4
+```
+
+To build Spark with `netlib-java` native library proxies, you need to add `-Pnetlib-lgpl` to Maven build command line. For example:
+```
+$SPARK_SOURCE_HOME/build/mvn -Pnetlib-lgpl -DskipTests -Pyarn -Phadoop-2.7 clean package
+```
+
+If you only want to enable it in your project, include `com.github.fommil.netlib:all:1.1.2` as a dependency of your project.
+
+## Install native linear algebra libraries
+
+Intel MKL and OpenBLAS are two popular native linear algebra libraries. You can choose one of them based on your preference. We provide basic instructions as below. You can refer to [netlib-java documentation](https://github.com/fommil/netlib-java) for more advanced installation instructions.
+
+### Intel MKL
+
+- Download and install Intel MKL. The installation should be done on all nodes of the cluster. We assume the installation location is $MKLROOT (e.g. /opt/intel/mkl).
+- Create soft links to `libmkl_rt.so` with specific names in system library search paths. For instance, make sure `/usr/local/lib` is in system library search paths and run the following commands:
+```
+$ ln -sf $MKLROOT/lib/intel64/libmkl_rt.so /usr/local/lib/libblas.so.3
+$ ln -sf $MKLROOT/lib/intel64/libmkl_rt.so /usr/local/lib/liblapack.so.3
+```
+
+### OpenBLAS
+
+The installation should be done on all nodes of the cluster. Generic version of OpenBLAS are available with most distributions. You can install it with a distribution package manager like `apt` or `yum`.
+
+For Debian / Ubuntu:
+```
+sudo apt-get install libopenblas-base
+sudo update-alternatives --config libblas.so.3
+```
+For CentOS / RHEL:
+```
+sudo yum install openblas
+```
+
+## Check if native libraries are enabled for MLlib
+
+To verify native libraries are properly loaded, start `spark-shell` and run the following code:
+```
+scala> import com.github.fommil.netlib.BLAS;
+scala> System.out.println(BLAS.getInstance().getClass().getName());
+```
+
+If they are correctly loaded, it should print `com.github.fommil.netlib.NativeSystemBLAS`. Otherwise the warnings should be printed:
+```
+WARN BLAS: Failed to load implementation from:com.github.fommil.netlib.NativeSystemBLAS
+WARN BLAS: Failed to load implementation from:com.github.fommil.netlib.NativeRefBLAS
+```
+
+If native libraries are not properly configured in the system, the Java implementation (f2jBLAS) will be used as fallback option.
+
+## Spark Configuration
+
+The default behavior of multi-threading in either Intel MKL or OpenBLAS may not be optimal with Spark's execution model [^1].
+
+Therefore configuring these native libraries to use a single thread for operations may actually improve performance (see [SPARK-21305](https://issues.apache.org/jira/browse/SPARK-21305)). It is usually optimal to match this to the number of `spark.task.cpus`, which is `1` by default and typically left at `1`.
+
+You can use the options in `config/spark-env.sh` to set thread number for Intel MKL or OpenBLAS:
+* For Intel MKL:
+```
+MKL_NUM_THREADS=1
+```
+* For OpenBLAS:
+```
+OPENBLAS_NUM_THREADS=1
+```
+
+[^1]: Please refer to the following resources to understand how to configure the number of threads for these BLAS implementations: [Intel MKL](https://software.intel.com/en-us/articles/recommended-settings-for-calling-intel-mkl-routines-from-multi-threaded-applications) or [Intel oneMKL](https://software.intel.com/en-us/onemkl-linux-developer-guide-improving-performance-with-threading) and [OpenBLAS](https://github.com/xianyi/OpenBLAS/wiki/faq#multi-threaded).
diff --git a/docs/ml-migration-guide.md b/docs/ml-migration-guide.md
index 49f701b2156b3..4e6d68f5a8cf4 100644
--- a/docs/ml-migration-guide.md
+++ b/docs/ml-migration-guide.md
@@ -32,16 +32,66 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
 {:.no_toc}
 
 * `OneHotEncoder` which is deprecated in 2.3, is removed in 3.0 and `OneHotEncoderEstimator` is now renamed to `OneHotEncoder`.
+* `org.apache.spark.ml.image.ImageSchema.readImages` which is deprecated in 2.3, is removed in 3.0, use `spark.read.format('image')` instead.
+* `org.apache.spark.mllib.clustering.KMeans.train` with param Int `runs` which is deprecated in 2.1, is removed in 3.0. Use `train` method without `runs` instead.
+* `org.apache.spark.mllib.classification.LogisticRegressionWithSGD` which is deprecated in 2.0, is removed in 3.0, use `org.apache.spark.ml.classification.LogisticRegression` or `spark.mllib.classification.LogisticRegressionWithLBFGS` instead.
+* `org.apache.spark.mllib.feature.ChiSqSelectorModel.isSorted ` which is deprecated in 2.1, is removed in 3.0, is not intended for subclasses to use.
+* `org.apache.spark.mllib.regression.RidgeRegressionWithSGD` which is deprecated in 2.0, is removed in 3.0, use `org.apache.spark.ml.regression.LinearRegression` with `elasticNetParam` = 0.0. Note the default `regParam` is 0.01 for `RidgeRegressionWithSGD`, but is 0.0 for `LinearRegression`.
+* `org.apache.spark.mllib.regression.LassoWithSGD` which is deprecated in 2.0, is removed in 3.0, use `org.apache.spark.ml.regression.LinearRegression` with `elasticNetParam` = 1.0. Note the default `regParam` is 0.01 for `LassoWithSGD`, but is 0.0 for `LinearRegression`.
+* `org.apache.spark.mllib.regression.LinearRegressionWithSGD` which is deprecated in 2.0, is removed in 3.0, use `org.apache.spark.ml.regression.LinearRegression` or `LBFGS` instead.
+* `org.apache.spark.mllib.clustering.KMeans.getRuns` and `setRuns` which are deprecated in 2.1, are removed in 3.0, have no effect since Spark 2.0.0.
+* `org.apache.spark.ml.LinearSVCModel.setWeightCol` which is deprecated in 2.4, is removed in 3.0, is not intended for users.
+* From 3.0, `org.apache.spark.ml.classification.MultilayerPerceptronClassificationModel` extends `MultilayerPerceptronParams` to expose the training params. As a result, `layers` in `MultilayerPerceptronClassificationModel` has been changed from `Array[Int]` to `IntArrayParam`. Users should use `MultilayerPerceptronClassificationModel.getLayers` instead of `MultilayerPerceptronClassificationModel.layers` to retrieve the size of layers.
+* `org.apache.spark.ml.classification.GBTClassifier.numTrees`  which is deprecated in 2.4.5, is removed in 3.0, use `getNumTrees` instead.
+* `org.apache.spark.ml.clustering.KMeansModel.computeCost` which is deprecated in 2.4, is removed in 3.0, use `ClusteringEvaluator` instead.
+* The member variable `precision` in `org.apache.spark.mllib.evaluation.MulticlassMetrics` which is deprecated in 2.0, is removed in 3.0. Use `accuracy` instead.
+* The member variable `recall` in `org.apache.spark.mllib.evaluation.MulticlassMetrics` which is deprecated in 2.0, is removed in 3.0. Use `accuracy` instead.
+* The member variable `fMeasure` in `org.apache.spark.mllib.evaluation.MulticlassMetrics` which is deprecated in 2.0, is removed in 3.0. Use `accuracy` instead.
+* `org.apache.spark.ml.util.GeneralMLWriter.context` which is deprecated in 2.0, is removed in 3.0, use `session` instead.
+* `org.apache.spark.ml.util.MLWriter.context` which is deprecated in 2.0, is removed in 3.0, use `session` instead.
+* `org.apache.spark.ml.util.MLReader.context` which is deprecated in 2.0, is removed in 3.0, use `session` instead.
+* `abstract class UnaryTransformer[IN, OUT, T <: UnaryTransformer[IN, OUT, T]]` is changed to `abstract class UnaryTransformer[IN: TypeTag, OUT: TypeTag, T <: UnaryTransformer[IN, OUT, T]]` in 3.0.
 
-### Changes of behavior
+### Deprecations and changes of behavior
 {:.no_toc}
 
+**Deprecations**
+
+* [SPARK-11215](https://issues.apache.org/jira/browse/SPARK-11215):
+`labels` in `StringIndexerModel` is deprecated and will be removed in 3.1.0. Use `labelsArray` instead.
+* [SPARK-25758](https://issues.apache.org/jira/browse/SPARK-25758):
+`computeCost` in `BisectingKMeansModel` is deprecated and will be removed in future versions. Use `ClusteringEvaluator` instead.
+
+**Changes of behavior**
+
 * [SPARK-11215](https://issues.apache.org/jira/browse/SPARK-11215):
  In Spark 2.4 and previous versions, when specifying `frequencyDesc` or `frequencyAsc` as
  `stringOrderType` param in `StringIndexer`, in case of equal frequency, the order of
  strings is undefined. Since Spark 3.0, the strings with equal frequency are further
  sorted by alphabet. And since Spark 3.0, `StringIndexer` supports encoding multiple
  columns.
+ * [SPARK-20604](https://issues.apache.org/jira/browse/SPARK-20604):
+ In prior to 3.0 releases, `Imputer` requires input column to be Double or Float. In 3.0, this
+ restriction is lifted so `Imputer` can handle all numeric types.
+* [SPARK-23469](https://issues.apache.org/jira/browse/SPARK-23469):
+In Spark 3.0, the `HashingTF` Transformer uses a corrected implementation of the murmur3 hash
+function to hash elements to vectors. `HashingTF` in Spark 3.0 will map elements to
+different positions in vectors than in Spark 2. However, `HashingTF` created with Spark 2.x
+and loaded with Spark 3.0 will still use the previous hash function and will not change behavior.
+* [SPARK-28969](https://issues.apache.org/jira/browse/SPARK-28969):
+The `setClassifier` method in PySpark's `OneVsRestModel` has been removed in 3.0 for parity with
+the Scala implementation. Callers should not need to set the classifier in the model after
+creation.
+* [SPARK-25790](https://issues.apache.org/jira/browse/SPARK-25790):
+ PCA adds the support for more than 65535 column matrix in Spark 3.0.
+* [SPARK-28927](https://issues.apache.org/jira/browse/SPARK-28927):
+ When fitting ALS model on nondeterministic input data, previously if rerun happens, users
+ would see ArrayIndexOutOfBoundsException caused by mismatch between In/Out user/item blocks.
+ From 3.0, a SparkException with more clear message will be thrown, and original
+ ArrayIndexOutOfBoundsException is wrapped.
+* [SPARK-29232](https://issues.apache.org/jira/browse/SPARK-29232):
+ In prior to 3.0 releases, `RandomForestRegressionModel` doesn't update the parameter maps
+ of the DecisionTreeRegressionModels underneath. This is fixed in 3.0.
 
 ## Upgrading from MLlib 2.2 to 2.3
 
@@ -186,7 +236,7 @@ val mlVec: org.apache.spark.ml.linalg.Vector = mllibVec.asML
 val mlMat: org.apache.spark.ml.linalg.Matrix = mllibMat.asML
 {% endhighlight %}
 
-Refer to the [`MLUtils` Scala docs](api/scala/index.html#org.apache.spark.mllib.util.MLUtils$) for further detail.
+Refer to the [`MLUtils` Scala docs](api/scala/org/apache/spark/mllib/util/MLUtils$.html) for further detail.
 </div>
 
 <div data-lang="java" markdown="1">
@@ -340,9 +390,9 @@ In the `spark.ml` package, there exists one breaking API change and one behavior
 In the `spark.mllib` package, there were several breaking changes, but all in `DeveloperApi` or `Experimental` APIs:
 
 * Gradient-Boosted Trees
-    * *(Breaking change)* The signature of the [`Loss.gradient`](api/scala/index.html#org.apache.spark.mllib.tree.loss.Loss) method was changed.  This is only an issues for users who wrote their own losses for GBTs.
-    * *(Breaking change)* The `apply` and `copy` methods for the case class [`BoostingStrategy`](api/scala/index.html#org.apache.spark.mllib.tree.configuration.BoostingStrategy) have been changed because of a modification to the case class fields.  This could be an issue for users who use `BoostingStrategy` to set GBT parameters.
-* *(Breaking change)* The return value of [`LDA.run`](api/scala/index.html#org.apache.spark.mllib.clustering.LDA) has changed.  It now returns an abstract class `LDAModel` instead of the concrete class `DistributedLDAModel`.  The object of type `LDAModel` can still be cast to the appropriate concrete type, which depends on the optimization algorithm.
+    * *(Breaking change)* The signature of the [`Loss.gradient`](api/scala/org/apache/spark/mllib/tree/loss/Loss.html) method was changed.  This is only an issues for users who wrote their own losses for GBTs.
+    * *(Breaking change)* The `apply` and `copy` methods for the case class [`BoostingStrategy`](api/scala/org/apache/spark/mllib/tree/configuration/BoostingStrategy.html) have been changed because of a modification to the case class fields.  This could be an issue for users who use `BoostingStrategy` to set GBT parameters.
+* *(Breaking change)* The return value of [`LDA.run`](api/scala/org/apache/spark/mllib/clustering/LDA.html) has changed.  It now returns an abstract class `LDAModel` instead of the concrete class `DistributedLDAModel`.  The object of type `LDAModel` can still be cast to the appropriate concrete type, which depends on the optimization algorithm.
 
 In the `spark.ml` package, several major API changes occurred, including:
 
@@ -358,12 +408,12 @@ changes for future releases.
 
 In the `spark.mllib` package, there were several breaking changes.  The first change (in `ALS`) is the only one in a component not marked as Alpha or Experimental.
 
-* *(Breaking change)* In [`ALS`](api/scala/index.html#org.apache.spark.mllib.recommendation.ALS), the extraneous method `solveLeastSquares` has been removed.  The `DeveloperApi` method `analyzeBlocks` was also removed.
-* *(Breaking change)* [`StandardScalerModel`](api/scala/index.html#org.apache.spark.mllib.feature.StandardScalerModel) remains an Alpha component. In it, the `variance` method has been replaced with the `std` method.  To compute the column variance values returned by the original `variance` method, simply square the standard deviation values returned by `std`.
-* *(Breaking change)* [`StreamingLinearRegressionWithSGD`](api/scala/index.html#org.apache.spark.mllib.regression.StreamingLinearRegressionWithSGD) remains an Experimental component.  In it, there were two changes:
+* *(Breaking change)* In [`ALS`](api/scala/org/apache/spark/mllib/recommendation/ALS.html), the extraneous method `solveLeastSquares` has been removed.  The `DeveloperApi` method `analyzeBlocks` was also removed.
+* *(Breaking change)* [`StandardScalerModel`](api/scala/org/apache/spark/mllib/feature/StandardScalerModel.html) remains an Alpha component. In it, the `variance` method has been replaced with the `std` method.  To compute the column variance values returned by the original `variance` method, simply square the standard deviation values returned by `std`.
+* *(Breaking change)* [`StreamingLinearRegressionWithSGD`](api/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionWithSGD.html) remains an Experimental component.  In it, there were two changes:
     * The constructor taking arguments was removed in favor of a builder pattern using the default constructor plus parameter setter methods.
     * Variable `model` is no longer public.
-* *(Breaking change)* [`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree) remains an Experimental component.  In it and its associated classes, there were several changes:
+* *(Breaking change)* [`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html) remains an Experimental component.  In it and its associated classes, there were several changes:
     * In `DecisionTree`, the deprecated class method `train` has been removed.  (The object/static `train` methods remain.)
     * In `Strategy`, the `checkpointDir` parameter has been removed.  Checkpointing is still supported, but the checkpoint directory must be set before calling tree and tree ensemble training.
 * `PythonMLlibAPI` (the interface between Scala/Java and Python for MLlib) was a public API but is now private, declared `private[python]`.  This was never meant for external use.
@@ -372,31 +422,31 @@ In the `spark.mllib` package, there were several breaking changes.  The first ch
 
 In the `spark.ml` package, the main API changes are from Spark SQL.  We list the most important changes here:
 
-* The old [SchemaRDD](https://spark.apache.org/docs/1.2.1/api/scala/index.html#org.apache.spark.sql.SchemaRDD) has been replaced with [DataFrame](api/scala/index.html#org.apache.spark.sql.DataFrame) with a somewhat modified API.  All algorithms in `spark.ml` which used to use SchemaRDD now use DataFrame.
+* The old [SchemaRDD](https://spark.apache.org/docs/1.2.1/api/scala/index.html#org.apache.spark.sql.SchemaRDD) has been replaced with [DataFrame](api/scala/org/apache/spark/sql/DataFrame.html) with a somewhat modified API.  All algorithms in `spark.ml` which used to use SchemaRDD now use DataFrame.
 * In Spark 1.2, we used implicit conversions from `RDD`s of `LabeledPoint` into `SchemaRDD`s by calling `import sqlContext._` where `sqlContext` was an instance of `SQLContext`.  These implicits have been moved, so we now call `import sqlContext.implicits._`.
 * Java APIs for SQL have also changed accordingly.  Please see the examples above and the [Spark SQL Programming Guide](sql-programming-guide.html) for details.
 
 Other changes were in `LogisticRegression`:
 
 * The `scoreCol` output column (with default value "score") was renamed to be `probabilityCol` (with default value "probability").  The type was originally `Double` (for the probability of class 1.0), but it is now `Vector` (for the probability of each class, to support multiclass classification in the future).
-* In Spark 1.2, `LogisticRegressionModel` did not include an intercept.  In Spark 1.3, it includes an intercept; however, it will always be 0.0 since it uses the default settings for [spark.mllib.LogisticRegressionWithLBFGS](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS).  The option to use an intercept will be added in the future.
+* In Spark 1.2, `LogisticRegressionModel` did not include an intercept.  In Spark 1.3, it includes an intercept; however, it will always be 0.0 since it uses the default settings for [spark.mllib.LogisticRegressionWithLBFGS](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html).  The option to use an intercept will be added in the future.
 
 ## Upgrading from MLlib 1.1 to 1.2
 
 The only API changes in MLlib v1.2 are in
-[`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree),
+[`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html),
 which continues to be an experimental API in MLlib 1.2:
 
 1. *(Breaking change)* The Scala API for classification takes a named argument specifying the number
 of classes.  In MLlib v1.1, this argument was called `numClasses` in Python and
 `numClassesForClassification` in Scala.  In MLlib v1.2, the names are both set to `numClasses`.
 This `numClasses` parameter is specified either via
-[`Strategy`](api/scala/index.html#org.apache.spark.mllib.tree.configuration.Strategy)
-or via [`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree)
+[`Strategy`](api/scala/org/apache/spark/mllib/tree/configuration/Strategy.html)
+or via [`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html)
 static `trainClassifier` and `trainRegressor` methods.
 
 2. *(Breaking change)* The API for
-[`Node`](api/scala/index.html#org.apache.spark.mllib.tree.model.Node) has changed.
+[`Node`](api/scala/org/apache/spark/mllib/tree/model/Node.html) has changed.
 This should generally not affect user code, unless the user manually constructs decision trees
 (instead of using the `trainClassifier` or `trainRegressor` methods).
 The tree `Node` now includes more information, including the probability of the predicted label
@@ -410,7 +460,7 @@ Examples in the Spark distribution and examples in the
 ## Upgrading from MLlib 1.0 to 1.1
 
 The only API changes in MLlib v1.1 are in
-[`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree),
+[`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html),
 which continues to be an experimental API in MLlib 1.1:
 
 1. *(Breaking change)* The meaning of tree depth has been changed by 1 in order to match
@@ -420,12 +470,12 @@ and in [rpart](http://cran.r-project.org/web/packages/rpart/index.html).
 In MLlib v1.0, a depth-1 tree had 1 leaf node, and a depth-2 tree had 1 root node and 2 leaf nodes.
 In MLlib v1.1, a depth-0 tree has 1 leaf node, and a depth-1 tree has 1 root node and 2 leaf nodes.
 This depth is specified by the `maxDepth` parameter in
-[`Strategy`](api/scala/index.html#org.apache.spark.mllib.tree.configuration.Strategy)
-or via [`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree)
+[`Strategy`](api/scala/org/apache/spark/mllib/tree/configuration/Strategy.html)
+or via [`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html)
 static `trainClassifier` and `trainRegressor` methods.
 
 2. *(Non-breaking change)* We recommend using the newly added `trainClassifier` and `trainRegressor`
-methods to build a [`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree),
+methods to build a [`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html),
 rather than using the old parameter class `Strategy`.  These new training methods explicitly
 separate classification and regression, and they replace specialized parameter types with
 simple `String` types.
diff --git a/docs/ml-pipeline.md b/docs/ml-pipeline.md
index 993a428ab5489..0b581e1a09c97 100644
--- a/docs/ml-pipeline.md
+++ b/docs/ml-pipeline.md
@@ -238,7 +238,7 @@ notes, then it should be treated as a bug to be fixed.
 
 This section gives code examples illustrating the functionality discussed above.
 For more info, please refer to the API documentation
-([Scala](api/scala/index.html#org.apache.spark.ml.package),
+([Scala](api/scala/org/apache/spark/ml/package.html),
 [Java](api/java/org/apache/spark/ml/package-summary.html),
 and [Python](api/python/pyspark.ml.html)).
 
@@ -250,9 +250,9 @@ This example covers the concepts of `Estimator`, `Transformer`, and `Param`.
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [`Estimator` Scala docs](api/scala/index.html#org.apache.spark.ml.Estimator),
-the [`Transformer` Scala docs](api/scala/index.html#org.apache.spark.ml.Transformer) and
-the [`Params` Scala docs](api/scala/index.html#org.apache.spark.ml.param.Params) for details on the API.
+Refer to the [`Estimator` Scala docs](api/scala/org/apache/spark/ml/Estimator.html),
+the [`Transformer` Scala docs](api/scala/org/apache/spark/ml/Transformer.html) and
+the [`Params` Scala docs](api/scala/org/apache/spark/ml/param/Params.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/EstimatorTransformerParamExample.scala %}
 </div>
@@ -285,7 +285,7 @@ This example follows the simple text document `Pipeline` illustrated in the figu
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [`Pipeline` Scala docs](api/scala/index.html#org.apache.spark.ml.Pipeline) for details on the API.
+Refer to the [`Pipeline` Scala docs](api/scala/org/apache/spark/ml/Pipeline.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/PipelineExample.scala %}
 </div>
diff --git a/docs/ml-statistics.md b/docs/ml-statistics.md
index 997c57244b2ef..a3d57ff7d266b 100644
--- a/docs/ml-statistics.md
+++ b/docs/ml-statistics.md
@@ -50,7 +50,7 @@ correlation methods are currently Pearson's and Spearman's correlation.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`Correlation`](api/scala/index.html#org.apache.spark.ml.stat.Correlation$)
+[`Correlation`](api/scala/org/apache/spark/ml/stat/Correlation$.html)
 computes the correlation matrix for the input Dataset of Vectors using the specified method.
 The output will be a DataFrame that contains the correlation matrix of the column of vectors.
 
@@ -87,7 +87,7 @@ the Chi-squared statistic is computed. All label and feature values must be cate
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`ChiSquareTest` Scala docs](api/scala/index.html#org.apache.spark.ml.stat.ChiSquareTest$) for details on the API.
+Refer to the [`ChiSquareTest` Scala docs](api/scala/org/apache/spark/ml/stat/ChiSquareTest$.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ChiSquareTestExample.scala %}
 </div>
@@ -114,7 +114,7 @@ as well as the total count.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-The following example demonstrates using [`Summarizer`](api/scala/index.html#org.apache.spark.ml.stat.Summarizer$)
+The following example demonstrates using [`Summarizer`](api/scala/org/apache/spark/ml/stat/Summarizer$.html)
 to compute the mean and variance for a vector column of the input dataframe, with and without a weight column.
 
 {% include_example scala/org/apache/spark/examples/ml/SummarizerExample.scala %}
@@ -133,4 +133,4 @@ Refer to the [`Summarizer` Python docs](api/python/index.html#pyspark.ml.stat.Su
 {% include_example python/ml/summarizer_example.py %}
 </div>
 
-</div>
\ No newline at end of file
+</div>
diff --git a/docs/ml-tuning.md b/docs/ml-tuning.md
index 0717cce538bf3..274f1951011bd 100644
--- a/docs/ml-tuning.md
+++ b/docs/ml-tuning.md
@@ -49,12 +49,12 @@ Built-in Cross-Validation and other tooling allow users to optimize hyperparamet
 An important task in ML is *model selection*, or using data to find the best model or parameters for a given task.  This is also called *tuning*.
 Tuning may be done for individual `Estimator`s such as `LogisticRegression`, or for entire `Pipeline`s which include multiple algorithms, featurization, and other steps.  Users can tune an entire `Pipeline` at once, rather than tuning each element in the `Pipeline` separately.
 
-MLlib supports model selection using tools such as [`CrossValidator`](api/scala/index.html#org.apache.spark.ml.tuning.CrossValidator) and [`TrainValidationSplit`](api/scala/index.html#org.apache.spark.ml.tuning.TrainValidationSplit).
+MLlib supports model selection using tools such as [`CrossValidator`](api/scala/org/apache/spark/ml/tuning/CrossValidator.html) and [`TrainValidationSplit`](api/scala/org/apache/spark/ml/tuning/TrainValidationSplit.html).
 These tools require the following items:
 
-* [`Estimator`](api/scala/index.html#org.apache.spark.ml.Estimator): algorithm or `Pipeline` to tune
+* [`Estimator`](api/scala/org/apache/spark/ml/Estimator.html): algorithm or `Pipeline` to tune
 * Set of `ParamMap`s: parameters to choose from, sometimes called a "parameter grid" to search over
-* [`Evaluator`](api/scala/index.html#org.apache.spark.ml.evaluation.Evaluator): metric to measure how well a fitted `Model` does on held-out test data
+* [`Evaluator`](api/scala/org/apache/spark/ml/evaluation/Evaluator.html): metric to measure how well a fitted `Model` does on held-out test data
 
 At a high level, these model selection tools work as follows:
 
@@ -63,13 +63,15 @@ At a high level, these model selection tools work as follows:
   * For each `ParamMap`, they fit the `Estimator` using those parameters, get the fitted `Model`, and evaluate the `Model`'s performance using the `Evaluator`.
 * They select the `Model` produced by the best-performing set of parameters.
 
-The `Evaluator` can be a [`RegressionEvaluator`](api/scala/index.html#org.apache.spark.ml.evaluation.RegressionEvaluator)
-for regression problems, a [`BinaryClassificationEvaluator`](api/scala/index.html#org.apache.spark.ml.evaluation.BinaryClassificationEvaluator)
-for binary data, or a [`MulticlassClassificationEvaluator`](api/scala/index.html#org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator)
-for multiclass problems. The default metric used to choose the best `ParamMap` can be overridden by the `setMetricName`
-method in each of these evaluators.
+The `Evaluator` can be a [`RegressionEvaluator`](api/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.html)
+for regression problems, a [`BinaryClassificationEvaluator`](api/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.html)
+for binary data, a [`MulticlassClassificationEvaluator`](api/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.html)
+for multiclass problems, a [`MultilabelClassificationEvaluator`](api/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.html)
+ for multi-label classifications, or a
+[`RankingEvaluator`](api/scala/org/apache/spark/ml/evaluation/RankingEvaluator.html) for ranking problems. The default metric used to
+choose the best `ParamMap` can be overridden by the `setMetricName` method in each of these evaluators.
 
-To help construct the parameter grid, users can use the [`ParamGridBuilder`](api/scala/index.html#org.apache.spark.ml.tuning.ParamGridBuilder) utility.
+To help construct the parameter grid, users can use the [`ParamGridBuilder`](api/scala/org/apache/spark/ml/tuning/ParamGridBuilder.html) utility.
 By default, sets of parameters from the parameter grid are evaluated in serial. Parameter evaluation can be done in parallel by setting `parallelism` with a value of 2 or more (a value of 1 will be serial) before running model selection with `CrossValidator` or `TrainValidationSplit`.
 The value of `parallelism` should be chosen carefully to maximize parallelism without exceeding cluster resources, and larger values may not always lead to improved performance.  Generally speaking, a value up to 10 should be sufficient for most clusters.
 
@@ -93,7 +95,7 @@ However, it is also a well-established method for choosing parameters which is m
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [`CrossValidator` Scala docs](api/scala/index.html#org.apache.spark.ml.tuning.CrossValidator) for details on the API.
+Refer to the [`CrossValidator` Scala docs](api/scala/org/apache/spark/ml/tuning/CrossValidator.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ModelSelectionViaCrossValidationExample.scala %}
 </div>
@@ -133,7 +135,7 @@ Like `CrossValidator`, `TrainValidationSplit` finally fits the `Estimator` using
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [`TrainValidationSplit` Scala docs](api/scala/index.html#org.apache.spark.ml.tuning.TrainValidationSplit) for details on the API.
+Refer to the [`TrainValidationSplit` Scala docs](api/scala/org/apache/spark/ml/tuning/TrainValidationSplit.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ModelSelectionViaTrainValidationSplitExample.scala %}
 </div>
diff --git a/docs/mllib-clustering.md b/docs/mllib-clustering.md
index 12c33a5e38049..4cb2e259ccfbc 100644
--- a/docs/mllib-clustering.md
+++ b/docs/mllib-clustering.md
@@ -55,12 +55,12 @@ initialization via k-means\|\|.
 The following code snippets can be executed in `spark-shell`.
 
 In the following example after loading and parsing data, we use the
-[`KMeans`](api/scala/index.html#org.apache.spark.mllib.clustering.KMeans) object to cluster the data
+[`KMeans`](api/scala/org/apache/spark/mllib/clustering/KMeans.html) object to cluster the data
 into two clusters. The number of desired clusters is passed to the algorithm. We then compute Within
 Set Sum of Squared Error (WSSSE). You can reduce this error measure by increasing *k*. In fact, the
 optimal *k* is usually one where there is an "elbow" in the WSSSE graph.
 
-Refer to the [`KMeans` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.KMeans) and [`KMeansModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.KMeansModel) for details on the API.
+Refer to the [`KMeans` Scala docs](api/scala/org/apache/spark/mllib/clustering/KMeans.html) and [`KMeansModel` Scala docs](api/scala/org/apache/spark/mllib/clustering/KMeansModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/KMeansExample.scala %}
 </div>
@@ -111,11 +111,11 @@ has the following parameters:
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 In the following example after loading and parsing data, we use a
-[GaussianMixture](api/scala/index.html#org.apache.spark.mllib.clustering.GaussianMixture)
+[GaussianMixture](api/scala/org/apache/spark/mllib/clustering/GaussianMixture.html)
 object to cluster the data into two clusters. The number of desired clusters is passed
 to the algorithm. We then output the parameters of the mixture model.
 
-Refer to the [`GaussianMixture` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.GaussianMixture) and [`GaussianMixtureModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.GaussianMixtureModel) for details on the API.
+Refer to the [`GaussianMixture` Scala docs](api/scala/org/apache/spark/mllib/clustering/GaussianMixture.html) and [`GaussianMixtureModel` Scala docs](api/scala/org/apache/spark/mllib/clustering/GaussianMixtureModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/GaussianMixtureExample.scala %}
 </div>
@@ -172,15 +172,15 @@ In the following, we show code snippets to demonstrate how to use PIC in `spark.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[`PowerIterationClustering`](api/scala/index.html#org.apache.spark.mllib.clustering.PowerIterationClustering) 
+[`PowerIterationClustering`](api/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.html) 
 implements the PIC algorithm.
 It takes an `RDD` of `(srcId: Long, dstId: Long, similarity: Double)` tuples representing the
 affinity matrix.
 Calling `PowerIterationClustering.run` returns a
-[`PowerIterationClusteringModel`](api/scala/index.html#org.apache.spark.mllib.clustering.PowerIterationClusteringModel),
+[`PowerIterationClusteringModel`](api/scala/org/apache/spark/mllib/clustering/PowerIterationClusteringModel.html),
 which contains the computed clustering assignments.
 
-Refer to the [`PowerIterationClustering` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.PowerIterationClustering) and [`PowerIterationClusteringModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.PowerIterationClusteringModel) for details on the API.
+Refer to the [`PowerIterationClustering` Scala docs](api/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.html) and [`PowerIterationClusteringModel` Scala docs](api/scala/org/apache/spark/mllib/clustering/PowerIterationClusteringModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/PowerIterationClusteringExample.scala %}
 </div>
@@ -278,9 +278,9 @@ separately.
 **Expectation Maximization**
 
 Implemented in
-[`EMLDAOptimizer`](api/scala/index.html#org.apache.spark.mllib.clustering.EMLDAOptimizer)
+[`EMLDAOptimizer`](api/scala/org/apache/spark/mllib/clustering/EMLDAOptimizer.html)
 and
-[`DistributedLDAModel`](api/scala/index.html#org.apache.spark.mllib.clustering.DistributedLDAModel).
+[`DistributedLDAModel`](api/scala/org/apache/spark/mllib/clustering/DistributedLDAModel.html).
 
 For the parameters provided to `LDA`:
 
@@ -350,13 +350,13 @@ perplexity of the provided `documents` given the inferred topics.
 **Examples**
 
 In the following example, we load word count vectors representing a corpus of documents.
-We then use [LDA](api/scala/index.html#org.apache.spark.mllib.clustering.LDA)
+We then use [LDA](api/scala/org/apache/spark/mllib/clustering/LDA.html)
 to infer three topics from the documents. The number of desired clusters is passed
 to the algorithm. We then output the topics, represented as probability distributions over words.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`LDA` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.LDA) and [`DistributedLDAModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.DistributedLDAModel) for details on the API.
+Refer to the [`LDA` Scala docs](api/scala/org/apache/spark/mllib/clustering/LDA.html) and [`DistributedLDAModel` Scala docs](api/scala/org/apache/spark/mllib/clustering/DistributedLDAModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/LatentDirichletAllocationExample.scala %}
 </div>
@@ -398,7 +398,7 @@ The implementation in MLlib has the following parameters:
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`BisectingKMeans` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.BisectingKMeans) and [`BisectingKMeansModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.BisectingKMeansModel) for details on the API.
+Refer to the [`BisectingKMeans` Scala docs](api/scala/org/apache/spark/mllib/clustering/BisectingKMeans.html) and [`BisectingKMeansModel` Scala docs](api/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/BisectingKMeansExample.scala %}
 </div>
@@ -451,7 +451,7 @@ This example shows how to estimate clusters on streaming data.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`StreamingKMeans` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.StreamingKMeans) for details on the API.
+Refer to the [`StreamingKMeans` Scala docs](api/scala/org/apache/spark/mllib/clustering/StreamingKMeans.html) for details on the API.
 And Refer to [Spark Streaming Programming Guide](streaming-programming-guide.html#initializing) for details on StreamingContext.
 
 {% include_example scala/org/apache/spark/examples/mllib/StreamingKMeansExample.scala %}
diff --git a/docs/mllib-collaborative-filtering.md b/docs/mllib-collaborative-filtering.md
index 21546a63263f9..aaefa59c4a9c3 100644
--- a/docs/mllib-collaborative-filtering.md
+++ b/docs/mllib-collaborative-filtering.md
@@ -76,11 +76,11 @@ best parameter learned from a sampled subset to the full dataset and expect simi
 
 <div data-lang="scala" markdown="1">
 In the following example, we load rating data. Each row consists of a user, a product and a rating.
-We use the default [ALS.train()](api/scala/index.html#org.apache.spark.mllib.recommendation.ALS$) 
+We use the default [ALS.train()](api/scala/org/apache/spark/mllib/recommendation/ALS$.html)
 method which assumes ratings are explicit. We evaluate the
 recommendation model by measuring the Mean Squared Error of rating prediction.
 
-Refer to the [`ALS` Scala docs](api/scala/index.html#org.apache.spark.mllib.recommendation.ALS) for more details on the API.
+Refer to the [`ALS` Scala docs](api/scala/org/apache/spark/mllib/recommendation/ALS.html) for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/RecommendationExample.scala %}
 
diff --git a/docs/mllib-data-types.md b/docs/mllib-data-types.md
index cdac46284b6be..6d3b1a599d48b 100644
--- a/docs/mllib-data-types.md
+++ b/docs/mllib-data-types.md
@@ -42,13 +42,13 @@ of the vector.
 <div data-lang="scala" markdown="1">
 
 The base class of local vectors is
-[`Vector`](api/scala/index.html#org.apache.spark.mllib.linalg.Vector), and we provide two
-implementations: [`DenseVector`](api/scala/index.html#org.apache.spark.mllib.linalg.DenseVector) and
-[`SparseVector`](api/scala/index.html#org.apache.spark.mllib.linalg.SparseVector).  We recommend
+[`Vector`](api/scala/org/apache/spark/mllib/linalg/Vector.html), and we provide two
+implementations: [`DenseVector`](api/scala/org/apache/spark/mllib/linalg/DenseVector.html) and
+[`SparseVector`](api/scala/org/apache/spark/mllib/linalg/SparseVector.html).  We recommend
 using the factory methods implemented in
-[`Vectors`](api/scala/index.html#org.apache.spark.mllib.linalg.Vectors$) to create local vectors.
+[`Vectors`](api/scala/org/apache/spark/mllib/linalg/Vectors$.html) to create local vectors.
 
-Refer to the [`Vector` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.Vector) and [`Vectors` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.Vectors$) for details on the API.
+Refer to the [`Vector` Scala docs](api/scala/org/apache/spark/mllib/linalg/Vector.html) and [`Vectors` Scala docs](api/scala/org/apache/spark/mllib/linalg/Vectors$.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
@@ -138,9 +138,9 @@ For multiclass classification, labels should be class indices starting from zero
 <div data-lang="scala" markdown="1">
 
 A labeled point is represented by the case class
-[`LabeledPoint`](api/scala/index.html#org.apache.spark.mllib.regression.LabeledPoint).
+[`LabeledPoint`](api/scala/org/apache/spark/mllib/regression/LabeledPoint.html).
 
-Refer to the [`LabeledPoint` Scala docs](api/scala/index.html#org.apache.spark.mllib.regression.LabeledPoint) for details on the API.
+Refer to the [`LabeledPoint` Scala docs](api/scala/org/apache/spark/mllib/regression/LabeledPoint.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.Vectors
@@ -211,10 +211,10 @@ After loading, the feature indices are converted to zero-based.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[`MLUtils.loadLibSVMFile`](api/scala/index.html#org.apache.spark.mllib.util.MLUtils$) reads training
+[`MLUtils.loadLibSVMFile`](api/scala/org/apache/spark/mllib/util/MLUtils$.html) reads training
 examples stored in LIBSVM format.
 
-Refer to the [`MLUtils` Scala docs](api/scala/index.html#org.apache.spark.mllib.util.MLUtils$) for details on the API.
+Refer to the [`MLUtils` Scala docs](api/scala/org/apache/spark/mllib/util/MLUtils$.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.regression.LabeledPoint
@@ -272,14 +272,14 @@ is stored in a one-dimensional array `[1.0, 3.0, 5.0, 2.0, 4.0, 6.0]` with the m
 <div data-lang="scala" markdown="1">
 
 The base class of local matrices is
-[`Matrix`](api/scala/index.html#org.apache.spark.mllib.linalg.Matrix), and we provide two
-implementations: [`DenseMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.DenseMatrix),
-and [`SparseMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.SparseMatrix).
+[`Matrix`](api/scala/org/apache/spark/mllib/linalg/Matrix.html), and we provide two
+implementations: [`DenseMatrix`](api/scala/org/apache/spark/mllib/linalg/DenseMatrix.html),
+and [`SparseMatrix`](api/scala/org/apache/spark/mllib/linalg/SparseMatrix.html).
 We recommend using the factory methods implemented
-in [`Matrices`](api/scala/index.html#org.apache.spark.mllib.linalg.Matrices$) to create local
+in [`Matrices`](api/scala/org/apache/spark/mllib/linalg/Matrices$.html) to create local
 matrices. Remember, local matrices in MLlib are stored in column-major order.
 
-Refer to the [`Matrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.Matrix) and [`Matrices` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.Matrices$) for details on the API.
+Refer to the [`Matrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/Matrix.html) and [`Matrices` Scala docs](api/scala/org/apache/spark/mllib/linalg/Matrices$.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.{Matrix, Matrices}
@@ -377,12 +377,12 @@ limited by the integer range but it should be much smaller in practice.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-A [`RowMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.RowMatrix) can be
+A [`RowMatrix`](api/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.html) can be
 created from an `RDD[Vector]` instance.  Then we can compute its column summary statistics and decompositions.
 [QR decomposition](https://en.wikipedia.org/wiki/QR_decomposition) is of the form A = QR where Q is an orthogonal matrix and R is an upper triangular matrix.
 For [singular value decomposition (SVD)](https://en.wikipedia.org/wiki/Singular_value_decomposition) and [principal component analysis (PCA)](https://en.wikipedia.org/wiki/Principal_component_analysis), please refer to [Dimensionality reduction](mllib-dimensionality-reduction.html).
 
-Refer to the [`RowMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.RowMatrix) for details on the API.
+Refer to the [`RowMatrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.Vector
@@ -463,13 +463,13 @@ vector.
 <div data-lang="scala" markdown="1">
 
 An
-[`IndexedRowMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.IndexedRowMatrix)
+[`IndexedRowMatrix`](api/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.html)
 can be created from an `RDD[IndexedRow]` instance, where
-[`IndexedRow`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.IndexedRow) is a
+[`IndexedRow`](api/scala/org/apache/spark/mllib/linalg/distributed/IndexedRow.html) is a
 wrapper over `(Long, Vector)`.  An `IndexedRowMatrix` can be converted to a `RowMatrix` by dropping
 its row indices.
 
-Refer to the [`IndexedRowMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.IndexedRowMatrix) for details on the API.
+Refer to the [`IndexedRowMatrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.distributed.{IndexedRow, IndexedRowMatrix, RowMatrix}
@@ -568,14 +568,14 @@ dimensions of the matrix are huge and the matrix is very sparse.
 <div data-lang="scala" markdown="1">
 
 A
-[`CoordinateMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.CoordinateMatrix)
+[`CoordinateMatrix`](api/scala/org/apache/spark/mllib/linalg/distributed/CoordinateMatrix.html)
 can be created from an `RDD[MatrixEntry]` instance, where
-[`MatrixEntry`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.MatrixEntry) is a
+[`MatrixEntry`](api/scala/org/apache/spark/mllib/linalg/distributed/MatrixEntry.html) is a
 wrapper over `(Long, Long, Double)`.  A `CoordinateMatrix` can be converted to an `IndexedRowMatrix`
 with sparse rows by calling `toIndexedRowMatrix`.  Other computations for 
 `CoordinateMatrix` are not currently supported.
 
-Refer to the [`CoordinateMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.CoordinateMatrix) for details on the API.
+Refer to the [`CoordinateMatrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/distributed/CoordinateMatrix.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.distributed.{CoordinateMatrix, MatrixEntry}
@@ -678,12 +678,12 @@ the sub-matrix at the given index with size `rowsPerBlock` x `colsPerBlock`.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-A [`BlockMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.BlockMatrix) can be
+A [`BlockMatrix`](api/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.html) can be
 most easily created from an `IndexedRowMatrix` or `CoordinateMatrix` by calling `toBlockMatrix`.
 `toBlockMatrix` creates blocks of size 1024 x 1024 by default.
 Users may change the block size by supplying the values through `toBlockMatrix(rowsPerBlock, colsPerBlock)`.
 
-Refer to the [`BlockMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.BlockMatrix) for details on the API.
+Refer to the [`BlockMatrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.distributed.{BlockMatrix, CoordinateMatrix, MatrixEntry}
diff --git a/docs/mllib-decision-tree.md b/docs/mllib-decision-tree.md
index 045da744239b9..455649c8e686e 100644
--- a/docs/mllib-decision-tree.md
+++ b/docs/mllib-decision-tree.md
@@ -151,7 +151,7 @@ When tuning these parameters, be careful to validate on held-out test data to av
 
 * **`maxDepth`**: Maximum depth of a tree.  Deeper trees are more expressive (potentially allowing higher accuracy), but they are also more costly to train and are more likely to overfit.
 
-* **`minInstancesPerNode`**: For a node to be split further, each of its children must receive at least this number of training instances.  This is commonly used with [RandomForest](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$) since those are often trained deeper than individual trees.
+* **`minInstancesPerNode`**: For a node to be split further, each of its children must receive at least this number of training instances.  This is commonly used with [RandomForest](api/scala/org/apache/spark/mllib/tree/RandomForest$.html) since those are often trained deeper than individual trees.
 
 * **`minInfoGain`**: For a node to be split further, the split must improve at least this much (in terms of information gain).
 
@@ -167,13 +167,13 @@ These parameters may be tuned.  Be careful to validate on held-out test data whe
   * The default value is conservatively chosen to be 256 MiB to allow the decision algorithm to work in most scenarios.  Increasing `maxMemoryInMB` can lead to faster training (if the memory is available) by allowing fewer passes over the data.  However, there may be decreasing returns as `maxMemoryInMB` grows since the amount of communication on each iteration can be proportional to `maxMemoryInMB`.
   * *Implementation details*: For faster processing, the decision tree algorithm collects statistics about groups of nodes to split (rather than 1 node at a time).  The number of nodes which can be handled in one group is determined by the memory requirements (which vary per features).  The `maxMemoryInMB` parameter specifies the memory limit in terms of megabytes which each worker can use for these statistics.
 
-* **`subsamplingRate`**: Fraction of the training data used for learning the decision tree.  This parameter is most relevant for training ensembles of trees (using [`RandomForest`](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$) and [`GradientBoostedTrees`](api/scala/index.html#org.apache.spark.mllib.tree.GradientBoostedTrees)), where it can be useful to subsample the original data.  For training a single decision tree, this parameter is less useful since the number of training instances is generally not the main constraint.
+* **`subsamplingRate`**: Fraction of the training data used for learning the decision tree.  This parameter is most relevant for training ensembles of trees (using [`RandomForest`](api/scala/org/apache/spark/mllib/tree/RandomForest$.html) and [`GradientBoostedTrees`](api/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.html)), where it can be useful to subsample the original data.  For training a single decision tree, this parameter is less useful since the number of training instances is generally not the main constraint.
 
 * **`impurity`**: Impurity measure (discussed above) used to choose between candidate splits.  This measure must match the `algo` parameter.
 
 ### Caching and checkpointing
 
-MLlib 1.2 adds several features for scaling up to larger (deeper) trees and tree ensembles.  When `maxDepth` is set to be large, it can be useful to turn on node ID caching and checkpointing.  These parameters are also useful for [RandomForest](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$) when `numTrees` is set to be large.
+MLlib 1.2 adds several features for scaling up to larger (deeper) trees and tree ensembles.  When `maxDepth` is set to be large, it can be useful to turn on node ID caching and checkpointing.  These parameters are also useful for [RandomForest](api/scala/org/apache/spark/mllib/tree/RandomForest$.html) when `numTrees` is set to be large.
 
 * **`useNodeIdCache`**: If this is set to true, the algorithm will avoid passing the current model (tree or trees) to executors on each iteration.
   * This can be useful with deep trees (speeding up computation on workers) and for large Random Forests (reducing communication on each iteration).
@@ -207,7 +207,7 @@ maximum tree depth of 5. The test error is calculated to measure the algorithm a
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`DecisionTree` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree) and [`DecisionTreeModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.DecisionTreeModel) for details on the API.
+Refer to the [`DecisionTree` Scala docs](api/scala/org/apache/spark/mllib/tree/DecisionTree.html) and [`DecisionTreeModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/DecisionTreeClassificationExample.scala %}
 </div>
@@ -238,7 +238,7 @@ depth of 5. The Mean Squared Error (MSE) is computed at the end to evaluate
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`DecisionTree` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree) and [`DecisionTreeModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.DecisionTreeModel) for details on the API.
+Refer to the [`DecisionTree` Scala docs](api/scala/org/apache/spark/mllib/tree/DecisionTree.html) and [`DecisionTreeModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/DecisionTreeRegressionExample.scala %}
 </div>
diff --git a/docs/mllib-dimensionality-reduction.md b/docs/mllib-dimensionality-reduction.md
index 5eb36b4228ca3..8818e403aabbc 100644
--- a/docs/mllib-dimensionality-reduction.md
+++ b/docs/mllib-dimensionality-reduction.md
@@ -77,7 +77,7 @@ passes, $O(n)$ storage on each executor, and $O(n k)$ storage on the driver.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`SingularValueDecomposition` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.SingularValueDecomposition) for details on the API.
+Refer to the [`SingularValueDecomposition` Scala docs](api/scala/org/apache/spark/mllib/linalg/SingularValueDecomposition.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/SVDExample.scala %}
 
@@ -117,14 +117,14 @@ the rotation matrix are called principal components. PCA is used widely in dimen
 The following code demonstrates how to compute principal components on a `RowMatrix`
 and use them to project the vectors into a low-dimensional space.
 
-Refer to the [`RowMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.RowMatrix) for details on the API.
+Refer to the [`RowMatrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala %}
 
 The following code demonstrates how to compute principal components on source vectors
 and use them to project the vectors into a low-dimensional space while keeping associated labels:
 
-Refer to the [`PCA` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.PCA) for details on the API.
+Refer to the [`PCA` Scala docs](api/scala/org/apache/spark/mllib/feature/PCA.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala %}
 
diff --git a/docs/mllib-ensembles.md b/docs/mllib-ensembles.md
index 6149f458214e6..27a9fe67f3f6d 100644
--- a/docs/mllib-ensembles.md
+++ b/docs/mllib-ensembles.md
@@ -24,7 +24,7 @@ license: |
 
 An [ensemble method](http://en.wikipedia.org/wiki/Ensemble_learning)
 is a learning algorithm which creates a model composed of a set of other base models.
-`spark.mllib` supports two major ensemble algorithms: [`GradientBoostedTrees`](api/scala/index.html#org.apache.spark.mllib.tree.GradientBoostedTrees) and [`RandomForest`](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$).
+`spark.mllib` supports two major ensemble algorithms: [`GradientBoostedTrees`](api/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.html) and [`RandomForest`](api/scala/org/apache/spark/mllib/tree/RandomForest$.html).
 Both use [decision trees](mllib-decision-tree.html) as their base models.
 
 ## Gradient-Boosted Trees vs. Random Forests
@@ -111,7 +111,7 @@ The test error is calculated to measure the algorithm accuracy.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`RandomForest` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$) and [`RandomForestModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.RandomForestModel) for details on the API.
+Refer to the [`RandomForest` Scala docs](api/scala/org/apache/spark/mllib/tree/RandomForest$.html) and [`RandomForestModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/RandomForestModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/RandomForestClassificationExample.scala %}
 </div>
@@ -142,7 +142,7 @@ The Mean Squared Error (MSE) is computed at the end to evaluate
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`RandomForest` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$) and [`RandomForestModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.RandomForestModel) for details on the API.
+Refer to the [`RandomForest` Scala docs](api/scala/org/apache/spark/mllib/tree/RandomForest$.html) and [`RandomForestModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/RandomForestModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/RandomForestRegressionExample.scala %}
 </div>
@@ -252,7 +252,7 @@ The test error is calculated to measure the algorithm accuracy.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`GradientBoostedTrees` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.GradientBoostedTrees) and [`GradientBoostedTreesModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.GradientBoostedTreesModel) for details on the API.
+Refer to the [`GradientBoostedTrees` Scala docs](api/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.html) and [`GradientBoostedTreesModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/GradientBoostedTreesModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/GradientBoostingClassificationExample.scala %}
 </div>
@@ -283,7 +283,7 @@ The Mean Squared Error (MSE) is computed at the end to evaluate
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`GradientBoostedTrees` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.GradientBoostedTrees) and [`GradientBoostedTreesModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.GradientBoostedTreesModel) for details on the API.
+Refer to the [`GradientBoostedTrees` Scala docs](api/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.html) and [`GradientBoostedTreesModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/GradientBoostedTreesModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/GradientBoostingRegressionExample.scala %}
 </div>
diff --git a/docs/mllib-evaluation-metrics.md b/docs/mllib-evaluation-metrics.md
index fb2883de6810a..f9efa769fc140 100644
--- a/docs/mllib-evaluation-metrics.md
+++ b/docs/mllib-evaluation-metrics.md
@@ -117,7 +117,7 @@ The following code snippets illustrate how to load a sample dataset, train a bin
 data, and evaluate the performance of the algorithm by several binary evaluation metrics.
 
 <div data-lang="scala" markdown="1">
-Refer to the [`LogisticRegressionWithLBFGS` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS) and [`BinaryClassificationMetrics` Scala docs](api/scala/index.html#org.apache.spark.mllib.evaluation.BinaryClassificationMetrics) for details on the API.
+Refer to the [`LogisticRegressionWithLBFGS` Scala docs](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html) and [`BinaryClassificationMetrics` Scala docs](api/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala %}
 
@@ -243,7 +243,7 @@ The following code snippets illustrate how to load a sample dataset, train a mul
 the data, and evaluate the performance of the algorithm by several multiclass classification evaluation metrics.
 
 <div data-lang="scala" markdown="1">
-Refer to the [`MulticlassMetrics` Scala docs](api/scala/index.html#org.apache.spark.mllib.evaluation.MulticlassMetrics) for details on the API.
+Refer to the [`MulticlassMetrics` Scala docs](api/scala/org/apache/spark/mllib/evaluation/MulticlassMetrics.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/MulticlassMetricsExample.scala %}
 
@@ -393,7 +393,7 @@ True classes:
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`MultilabelMetrics` Scala docs](api/scala/index.html#org.apache.spark.mllib.evaluation.MultilabelMetrics) for details on the API.
+Refer to the [`MultilabelMetrics` Scala docs](api/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/MultiLabelMetricsExample.scala %}
 
@@ -521,7 +521,7 @@ expanded world of non-positive weights are "the same as never having interacted
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`RegressionMetrics` Scala docs](api/scala/index.html#org.apache.spark.mllib.evaluation.RegressionMetrics) and [`RankingMetrics` Scala docs](api/scala/index.html#org.apache.spark.mllib.evaluation.RankingMetrics) for details on the API.
+Refer to the [`RegressionMetrics` Scala docs](api/scala/org/apache/spark/mllib/evaluation/RegressionMetrics.html) and [`RankingMetrics` Scala docs](api/scala/org/apache/spark/mllib/evaluation/RankingMetrics.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala %}
 
diff --git a/docs/mllib-feature-extraction.md b/docs/mllib-feature-extraction.md
index 33a223ad486af..8df9699150f6a 100644
--- a/docs/mllib-feature-extraction.md
+++ b/docs/mllib-feature-extraction.md
@@ -69,12 +69,12 @@ We refer users to the [Stanford NLP Group](http://nlp.stanford.edu/) and
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-TF and IDF are implemented in [HashingTF](api/scala/index.html#org.apache.spark.mllib.feature.HashingTF)
-and [IDF](api/scala/index.html#org.apache.spark.mllib.feature.IDF).
+TF and IDF are implemented in [HashingTF](api/scala/org/apache/spark/mllib/feature/HashingTF.html)
+and [IDF](api/scala/org/apache/spark/mllib/feature/IDF.html).
 `HashingTF` takes an `RDD[Iterable[_]]` as the input.
 Each record could be an iterable of strings or other types.
 
-Refer to the [`HashingTF` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.HashingTF) for details on the API.
+Refer to the [`HashingTF` Scala docs](api/scala/org/apache/spark/mllib/feature/HashingTF.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/TFIDFExample.scala %}
 </div>
@@ -135,7 +135,7 @@ Here we assume the extracted file is `text8` and in same directory as you run th
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`Word2Vec` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.Word2Vec) for details on the API.
+Refer to the [`Word2Vec` Scala docs](api/scala/org/apache/spark/mllib/feature/Word2Vec.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/Word2VecExample.scala %}
 </div>
@@ -159,19 +159,19 @@ against features with very large variances exerting an overly large influence du
 
 ### Model Fitting
 
-[`StandardScaler`](api/scala/index.html#org.apache.spark.mllib.feature.StandardScaler) has the
+[`StandardScaler`](api/scala/org/apache/spark/mllib/feature/StandardScaler.html) has the
 following parameters in the constructor:
 
 * `withMean` False by default. Centers the data with mean before scaling. It will build a dense
 output, so take care when applying to sparse input.
 * `withStd` True by default. Scales the data to unit standard deviation.
 
-We provide a [`fit`](api/scala/index.html#org.apache.spark.mllib.feature.StandardScaler) method in
+We provide a [`fit`](api/scala/org/apache/spark/mllib/feature/StandardScaler.html) method in
 `StandardScaler` which can take an input of `RDD[Vector]`, learn the summary statistics, and then
 return a model which can transform the input dataset into unit standard deviation and/or zero mean features
 depending how we configure the `StandardScaler`.
 
-This model implements [`VectorTransformer`](api/scala/index.html#org.apache.spark.mllib.feature.VectorTransformer)
+This model implements [`VectorTransformer`](api/scala/org/apache/spark/mllib/feature/VectorTransformer.html)
 which can apply the standardization on a `Vector` to produce a transformed `Vector` or on
 an `RDD[Vector]` to produce a transformed `RDD[Vector]`.
 
@@ -185,7 +185,7 @@ so that the new features have unit standard deviation and/or zero mean.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`StandardScaler` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.StandardScaler) for details on the API.
+Refer to the [`StandardScaler` Scala docs](api/scala/org/apache/spark/mllib/feature/StandardScaler.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/StandardScalerExample.scala %}
 </div>
@@ -203,12 +203,12 @@ Normalizer scales individual samples to have unit $L^p$ norm. This is a common o
 classification or clustering. For example, the dot product of two $L^2$ normalized TF-IDF vectors
 is the cosine similarity of the vectors.
 
-[`Normalizer`](api/scala/index.html#org.apache.spark.mllib.feature.Normalizer) has the following
+[`Normalizer`](api/scala/org/apache/spark/mllib/feature/Normalizer.html) has the following
 parameter in the constructor:
 
 * `p` Normalization in $L^p$ space, $p = 2$ by default.
 
-`Normalizer` implements [`VectorTransformer`](api/scala/index.html#org.apache.spark.mllib.feature.VectorTransformer)
+`Normalizer` implements [`VectorTransformer`](api/scala/org/apache/spark/mllib/feature/VectorTransformer.html)
 which can apply the normalization on a `Vector` to produce a transformed `Vector` or on
 an `RDD[Vector]` to produce a transformed `RDD[Vector]`.
 
@@ -221,7 +221,7 @@ with $L^2$ norm, and $L^\infty$ norm.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`Normalizer` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.Normalizer) for details on the API.
+Refer to the [`Normalizer` Scala docs](api/scala/org/apache/spark/mllib/feature/Normalizer.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/NormalizerExample.scala %}
 </div>
@@ -239,7 +239,7 @@ Refer to the [`Normalizer` Python docs](api/python/pyspark.mllib.html#pyspark.ml
 features for use in model construction. It reduces the size of the feature space, which can improve
 both speed and statistical learning behavior.
 
-[`ChiSqSelector`](api/scala/index.html#org.apache.spark.mllib.feature.ChiSqSelector) implements
+[`ChiSqSelector`](api/scala/org/apache/spark/mllib/feature/ChiSqSelector.html) implements
 Chi-Squared feature selection. It operates on labeled data with categorical features. ChiSqSelector uses the
 [Chi-Squared test of independence](https://en.wikipedia.org/wiki/Chi-squared_test) to decide which
 features to choose. It supports five selection methods: `numTopFeatures`, `percentile`, `fpr`, `fdr`, `fwe`:
@@ -257,7 +257,7 @@ The number of features to select can be tuned using a held-out validation set.
 
 ### Model Fitting
 
-The [`fit`](api/scala/index.html#org.apache.spark.mllib.feature.ChiSqSelector) method takes
+The [`fit`](api/scala/org/apache/spark/mllib/feature/ChiSqSelector.html) method takes
 an input of `RDD[LabeledPoint]` with categorical features, learns the summary statistics, and then
 returns a `ChiSqSelectorModel` which can transform an input dataset into the reduced feature space.
 The `ChiSqSelectorModel` can be applied either to a `Vector` to produce a reduced `Vector`, or to
@@ -272,7 +272,7 @@ The following example shows the basic use of ChiSqSelector. The data set used ha
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [`ChiSqSelector` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.ChiSqSelector)
+Refer to the [`ChiSqSelector` Scala docs](api/scala/org/apache/spark/mllib/feature/ChiSqSelector.html)
 for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala %}
@@ -312,11 +312,11 @@ v_N
   \end{pmatrix}
 \]`
 
-[`ElementwiseProduct`](api/scala/index.html#org.apache.spark.mllib.feature.ElementwiseProduct) has the following parameter in the constructor:
+[`ElementwiseProduct`](api/scala/org/apache/spark/mllib/feature/ElementwiseProduct.html) has the following parameter in the constructor:
 
 * `scalingVec`: the transforming vector.
 
-`ElementwiseProduct` implements [`VectorTransformer`](api/scala/index.html#org.apache.spark.mllib.feature.VectorTransformer) which can apply the weighting on a `Vector` to produce a transformed `Vector` or on an `RDD[Vector]` to produce a transformed `RDD[Vector]`.
+`ElementwiseProduct` implements [`VectorTransformer`](api/scala/org/apache/spark/mllib/feature/VectorTransformer.html) which can apply the weighting on a `Vector` to produce a transformed `Vector` or on an `RDD[Vector]` to produce a transformed `RDD[Vector]`.
 
 ### Example
 
@@ -325,7 +325,7 @@ This example below demonstrates how to transform vectors using a transforming ve
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [`ElementwiseProduct` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.ElementwiseProduct) for details on the API.
+Refer to the [`ElementwiseProduct` Scala docs](api/scala/org/apache/spark/mllib/feature/ElementwiseProduct.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala %}
 </div>
diff --git a/docs/mllib-frequent-pattern-mining.md b/docs/mllib-frequent-pattern-mining.md
index 8bc93ac2e8adf..709acde062d7e 100644
--- a/docs/mllib-frequent-pattern-mining.md
+++ b/docs/mllib-frequent-pattern-mining.md
@@ -54,18 +54,18 @@ We refer users to the papers for more details.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[`FPGrowth`](api/scala/index.html#org.apache.spark.mllib.fpm.FPGrowth) implements the
+[`FPGrowth`](api/scala/org/apache/spark/mllib/fpm/FPGrowth.html) implements the
 FP-growth algorithm.
 It takes an `RDD` of transactions, where each transaction is an `Array` of items of a generic type.
 Calling `FPGrowth.run` with transactions returns an
-[`FPGrowthModel`](api/scala/index.html#org.apache.spark.mllib.fpm.FPGrowthModel)
+[`FPGrowthModel`](api/scala/org/apache/spark/mllib/fpm/FPGrowthModel.html)
 that stores the frequent itemsets with their frequencies.  The following
 example illustrates how to mine frequent itemsets and association rules
 (see [Association
 Rules](mllib-frequent-pattern-mining.html#association-rules) for
 details) from `transactions`.
 
-Refer to the [`FPGrowth` Scala docs](api/scala/index.html#org.apache.spark.mllib.fpm.FPGrowth) for details on the API.
+Refer to the [`FPGrowth` Scala docs](api/scala/org/apache/spark/mllib/fpm/FPGrowth.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/SimpleFPGrowth.scala %}
 
@@ -111,7 +111,7 @@ Refer to the [`FPGrowth` Python docs](api/python/pyspark.mllib.html#pyspark.mlli
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[AssociationRules](api/scala/index.html#org.apache.spark.mllib.fpm.AssociationRules)
+[AssociationRules](api/scala/org/apache/spark/mllib/fpm/AssociationRules.html)
 implements a parallel rule generation algorithm for constructing rules
 that have a single item as the consequent.
 
@@ -168,13 +168,13 @@ The following example illustrates PrefixSpan running on the sequences
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[`PrefixSpan`](api/scala/index.html#org.apache.spark.mllib.fpm.PrefixSpan) implements the
+[`PrefixSpan`](api/scala/org/apache/spark/mllib/fpm/PrefixSpan.html) implements the
 PrefixSpan algorithm.
 Calling `PrefixSpan.run` returns a
-[`PrefixSpanModel`](api/scala/index.html#org.apache.spark.mllib.fpm.PrefixSpanModel)
+[`PrefixSpanModel`](api/scala/org/apache/spark/mllib/fpm/PrefixSpanModel.html)
 that stores the frequent sequences with their frequencies.
 
-Refer to the [`PrefixSpan` Scala docs](api/scala/index.html#org.apache.spark.mllib.fpm.PrefixSpan) and [`PrefixSpanModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.fpm.PrefixSpanModel) for details on the API.
+Refer to the [`PrefixSpan` Scala docs](api/scala/org/apache/spark/mllib/fpm/PrefixSpan.html) and [`PrefixSpanModel` Scala docs](api/scala/org/apache/spark/mllib/fpm/PrefixSpanModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/PrefixSpanExample.scala %}
 
diff --git a/docs/mllib-isotonic-regression.md b/docs/mllib-isotonic-regression.md
index d9cc775547bb1..94ffadaf65c55 100644
--- a/docs/mllib-isotonic-regression.md
+++ b/docs/mllib-isotonic-regression.md
@@ -74,7 +74,7 @@ i.e. 4710.28,500.00. The data are split to training and testing set.
 Model is created using the training set and a mean squared error is calculated from the predicted
 labels and real labels in the test set.
 
-Refer to the [`IsotonicRegression` Scala docs](api/scala/index.html#org.apache.spark.mllib.regression.IsotonicRegression) and [`IsotonicRegressionModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.regression.IsotonicRegressionModel) for details on the API.
+Refer to the [`IsotonicRegression` Scala docs](api/scala/org/apache/spark/mllib/regression/IsotonicRegression.html) and [`IsotonicRegressionModel` Scala docs](api/scala/org/apache/spark/mllib/regression/IsotonicRegressionModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/IsotonicRegressionExample.scala %}
 </div>
diff --git a/docs/mllib-linear-methods.md b/docs/mllib-linear-methods.md
index 801876dbffa79..e7726271ccb72 100644
--- a/docs/mllib-linear-methods.md
+++ b/docs/mllib-linear-methods.md
@@ -184,7 +184,7 @@ training algorithm on this training data using a static method in the algorithm
 object, and make predictions with the resulting model to compute the training
 error.
 
-Refer to the [`SVMWithSGD` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.SVMWithSGD) and [`SVMModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.SVMModel) for details on the API.
+Refer to the [`SVMWithSGD` Scala docs](api/scala/org/apache/spark/mllib/classification/SVMWithSGD.html) and [`SVMModel` Scala docs](api/scala/org/apache/spark/mllib/classification/SVMModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/SVMWithSGDExample.scala %}
 
@@ -305,11 +305,11 @@ We recommend L-BFGS over mini-batch gradient descent for faster convergence.
 <div data-lang="scala" markdown="1">
 The following code illustrates how to load a sample multiclass dataset, split it into train and
 test, and use
-[LogisticRegressionWithLBFGS](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS)
+[LogisticRegressionWithLBFGS](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html)
 to fit a logistic regression model.
 Then the model is evaluated against the test dataset and saved to disk.
 
-Refer to the [`LogisticRegressionWithLBFGS` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS) and [`LogisticRegressionModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionModel) for details on the API.
+Refer to the [`LogisticRegressionWithLBFGS` Scala docs](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html) and [`LogisticRegressionModel` Scala docs](api/scala/org/apache/spark/mllib/classification/LogisticRegressionModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/LogisticRegressionWithLBFGSExample.scala %}
 
@@ -438,8 +438,8 @@ regularization parameter (`regParam`) along with various parameters associated w
 gradient descent (`stepSize`, `numIterations`, `miniBatchFraction`).  For each of them, we support
 all three possible regularizations (none, L1 or L2).
 
-For Logistic Regression, [L-BFGS](api/scala/index.html#org.apache.spark.mllib.optimization.LBFGS)
-version is implemented under [LogisticRegressionWithLBFGS](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS), and this
+For Logistic Regression, [L-BFGS](api/scala/org/apache/spark/mllib/optimization/LBFGS.html)
+version is implemented under [LogisticRegressionWithLBFGS](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html), and this
 version supports both binary and multinomial Logistic Regression while SGD version only supports
 binary Logistic Regression. However, L-BFGS version doesn't support L1 regularization but SGD one
 supports L1 regularization. When L1 regularization is not required, L-BFGS version is strongly
@@ -448,10 +448,10 @@ inverse Hessian matrix using quasi-Newton method.
 
 Algorithms are all implemented in Scala:
 
-* [SVMWithSGD](api/scala/index.html#org.apache.spark.mllib.classification.SVMWithSGD)
-* [LogisticRegressionWithLBFGS](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS)
-* [LogisticRegressionWithSGD](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithSGD)
-* [LinearRegressionWithSGD](api/scala/index.html#org.apache.spark.mllib.regression.LinearRegressionWithSGD)
-* [RidgeRegressionWithSGD](api/scala/index.html#org.apache.spark.mllib.regression.RidgeRegressionWithSGD)
-* [LassoWithSGD](api/scala/index.html#org.apache.spark.mllib.regression.LassoWithSGD)
+* [SVMWithSGD](api/scala/org/apache/spark/mllib/classification/SVMWithSGD.html)
+* [LogisticRegressionWithLBFGS](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html)
+* [LogisticRegressionWithSGD](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithSGD.html)
+* [LinearRegressionWithSGD](api/scala/org/apache/spark/mllib/regression/LinearRegressionWithSGD.html)
+* [RidgeRegressionWithSGD](api/scala/org/apache/spark/mllib/regression/RidgeRegressionWithSGD.html)
+* [LassoWithSGD](api/scala/org/apache/spark/mllib/regression/LassoWithSGD.html)
 
diff --git a/docs/mllib-naive-bayes.md b/docs/mllib-naive-bayes.md
index 09b15876a3914..a3602667b5835 100644
--- a/docs/mllib-naive-bayes.md
+++ b/docs/mllib-naive-bayes.md
@@ -46,14 +46,14 @@ sparsity. Since the training data is only used once, it is not necessary to cach
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[NaiveBayes](api/scala/index.html#org.apache.spark.mllib.classification.NaiveBayes$) implements
+[NaiveBayes](api/scala/org/apache/spark/mllib/classification/NaiveBayes$.html) implements
 multinomial naive Bayes. It takes an RDD of
-[LabeledPoint](api/scala/index.html#org.apache.spark.mllib.regression.LabeledPoint) and an optional
+[LabeledPoint](api/scala/org/apache/spark/mllib/regression/LabeledPoint.html) and an optional
 smoothing parameter `lambda` as input, an optional model type parameter (default is "multinomial"), and outputs a
-[NaiveBayesModel](api/scala/index.html#org.apache.spark.mllib.classification.NaiveBayesModel), which
+[NaiveBayesModel](api/scala/org/apache/spark/mllib/classification/NaiveBayesModel.html), which
 can be used for evaluation and prediction.
 
-Refer to the [`NaiveBayes` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.NaiveBayes) and [`NaiveBayesModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.NaiveBayesModel) for details on the API.
+Refer to the [`NaiveBayes` Scala docs](api/scala/org/apache/spark/mllib/classification/NaiveBayes$.html) and [`NaiveBayesModel` Scala docs](api/scala/org/apache/spark/mllib/classification/NaiveBayesModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/NaiveBayesExample.scala %}
 </div>
diff --git a/docs/mllib-optimization.md b/docs/mllib-optimization.md
index f2e128ec215a1..f0b0c817ed3f6 100644
--- a/docs/mllib-optimization.md
+++ b/docs/mllib-optimization.md
@@ -111,12 +111,12 @@ As an alternative to just use the subgradient `$R'(\wv)$` of the regularizer in
 direction, an improved update for some cases can be obtained by using the proximal operator
 instead.
 For the L1-regularizer, the proximal operator is given by soft thresholding, as implemented in
-[L1Updater](api/scala/index.html#org.apache.spark.mllib.optimization.L1Updater).
+[L1Updater](api/scala/org/apache/spark/mllib/optimization/L1Updater.html).
 
 
 ### Update schemes for distributed SGD
 The SGD implementation in
-[GradientDescent](api/scala/index.html#org.apache.spark.mllib.optimization.GradientDescent) uses
+[GradientDescent](api/scala/org/apache/spark/mllib/optimization/GradientDescent.html) uses
 a simple (distributed) sampling of the data examples.
 We recall that the loss part of the optimization problem `$\eqref{eq:regPrimal}$` is
 `$\frac1n \sum_{i=1}^n L(\wv;\x_i,y_i)$`, and therefore `$\frac1n \sum_{i=1}^n L'_{\wv,i}$` would
@@ -169,7 +169,7 @@ are developed, see the
 section for example.
 
 The SGD class
-[GradientDescent](api/scala/index.html#org.apache.spark.mllib.optimization.GradientDescent)
+[GradientDescent](api/scala/org/apache/spark/mllib/optimization/GradientDescent.html)
 sets the following parameters:
 
 * `Gradient` is a class that computes the stochastic gradient of the function
@@ -195,15 +195,15 @@ each iteration, to compute the gradient direction.
 L-BFGS is currently only a low-level optimization primitive in `MLlib`. If you want to use L-BFGS in various 
 ML algorithms such as Linear Regression, and Logistic Regression, you have to pass the gradient of objective
 function, and updater into optimizer yourself instead of using the training APIs like 
-[LogisticRegressionWithSGD](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithSGD).
+[LogisticRegressionWithSGD](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithSGD.html).
 See the example below. It will be addressed in the next release. 
 
 The L1 regularization by using 
-[L1Updater](api/scala/index.html#org.apache.spark.mllib.optimization.L1Updater) will not work since the 
+[L1Updater](api/scala/org/apache/spark/mllib/optimization/L1Updater.html) will not work since the 
 soft-thresholding logic in L1Updater is designed for gradient descent. See the developer's note.
 
 The L-BFGS method
-[LBFGS.runLBFGS](api/scala/index.html#org.apache.spark.mllib.optimization.LBFGS)
+[LBFGS.runLBFGS](api/scala/org/apache/spark/mllib/optimization/LBFGS.html)
 has the following parameters:
 
 * `Gradient` is a class that computes the gradient of the objective function
@@ -233,7 +233,7 @@ L-BFGS optimizer.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`LBFGS` Scala docs](api/scala/index.html#org.apache.spark.mllib.optimization.LBFGS) and [`SquaredL2Updater` Scala docs](api/scala/index.html#org.apache.spark.mllib.optimization.SquaredL2Updater) for details on the API.
+Refer to the [`LBFGS` Scala docs](api/scala/org/apache/spark/mllib/optimization/LBFGS.html) and [`SquaredL2Updater` Scala docs](api/scala/org/apache/spark/mllib/optimization/SquaredL2Updater.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/LBFGSExample.scala %}
 </div>
diff --git a/docs/mllib-pmml-model-export.md b/docs/mllib-pmml-model-export.md
index fbc14cefd14ec..d7eb51d69a1da 100644
--- a/docs/mllib-pmml-model-export.md
+++ b/docs/mllib-pmml-model-export.md
@@ -62,7 +62,7 @@ To export a supported `model` (see table above) to PMML, simply call `model.toPM
 
 As well as exporting the PMML model to a String (`model.toPMML` as in the example above), you can export the PMML model to other formats.
 
-Refer to the [`KMeans` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.KMeans) and [`Vectors` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.Vectors$) for details on the API.
+Refer to the [`KMeans` Scala docs](api/scala/org/apache/spark/mllib/clustering/KMeans.html) and [`Vectors` Scala docs](api/scala/org/apache/spark/mllib/linalg/Vectors$.html) for details on the API.
 
 Here a complete example of building a KMeansModel and print it out in PMML format:
 {% include_example scala/org/apache/spark/examples/mllib/PMMLModelExportExample.scala %}
diff --git a/docs/mllib-statistics.md b/docs/mllib-statistics.md
index 4698d3e6347e7..7de214bb6b6a6 100644
--- a/docs/mllib-statistics.md
+++ b/docs/mllib-statistics.md
@@ -48,12 +48,12 @@ available in `Statistics`.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[`colStats()`](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) returns an instance of
-[`MultivariateStatisticalSummary`](api/scala/index.html#org.apache.spark.mllib.stat.MultivariateStatisticalSummary),
+[`colStats()`](api/scala/org/apache/spark/mllib/stat/Statistics$.html) returns an instance of
+[`MultivariateStatisticalSummary`](api/scala/org/apache/spark/mllib/stat/MultivariateStatisticalSummary.html),
 which contains the column-wise max, min, mean, variance, and number of nonzeros, as well as the
 total count.
 
-Refer to the [`MultivariateStatisticalSummary` Scala docs](api/scala/index.html#org.apache.spark.mllib.stat.MultivariateStatisticalSummary) for details on the API.
+Refer to the [`MultivariateStatisticalSummary` Scala docs](api/scala/org/apache/spark/mllib/stat/MultivariateStatisticalSummary.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/SummaryStatisticsExample.scala %}
 </div>
@@ -91,11 +91,11 @@ correlation methods are currently Pearson's and Spearman's correlation.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`Statistics`](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) provides methods to
+[`Statistics`](api/scala/org/apache/spark/mllib/stat/Statistics$.html) provides methods to
 calculate correlations between series. Depending on the type of input, two `RDD[Double]`s or
 an `RDD[Vector]`, the output will be a `Double` or the correlation `Matrix` respectively.
 
-Refer to the [`Statistics` Scala docs](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) for details on the API.
+Refer to the [`Statistics` Scala docs](api/scala/org/apache/spark/mllib/stat/Statistics$.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/CorrelationsExample.scala %}
 </div>
@@ -137,7 +137,7 @@ python.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`sampleByKeyExact()`](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions) allows users to
+[`sampleByKeyExact()`](api/scala/org/apache/spark/rdd/PairRDDFunctions.html) allows users to
 sample exactly $\lceil f_k \cdot n_k \rceil \, \forall k \in K$ items, where $f_k$ is the desired
 fraction for key $k$, $n_k$ is the number of key-value pairs for key $k$, and $K$ is the set of
 keys. Sampling without replacement requires one additional pass over the RDD to guarantee sample
@@ -181,7 +181,7 @@ independence tests.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`Statistics`](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) provides methods to
+[`Statistics`](api/scala/org/apache/spark/mllib/stat/Statistics$.html) provides methods to
 run Pearson's chi-squared tests. The following example demonstrates how to run and interpret
 hypothesis tests.
 
@@ -221,11 +221,11 @@ message.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`Statistics`](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) provides methods to
+[`Statistics`](api/scala/org/apache/spark/mllib/stat/Statistics$.html) provides methods to
 run a 1-sample, 2-sided Kolmogorov-Smirnov test. The following example demonstrates how to run
 and interpret the hypothesis tests.
 
-Refer to the [`Statistics` Scala docs](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) for details on the API.
+Refer to the [`Statistics` Scala docs](api/scala/org/apache/spark/mllib/stat/Statistics$.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/HypothesisTestingKolmogorovSmirnovTestExample.scala %}
 </div>
@@ -269,7 +269,7 @@ all prior batches.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`StreamingTest`](api/scala/index.html#org.apache.spark.mllib.stat.test.StreamingTest)
+[`StreamingTest`](api/scala/org/apache/spark/mllib/stat/test/StreamingTest.html)
 provides streaming hypothesis testing.
 
 {% include_example scala/org/apache/spark/examples/mllib/StreamingTestExample.scala %}
@@ -292,12 +292,12 @@ uniform, standard normal, or Poisson.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`RandomRDDs`](api/scala/index.html#org.apache.spark.mllib.random.RandomRDDs$) provides factory
+[`RandomRDDs`](api/scala/org/apache/spark/mllib/random/RandomRDDs$.html) provides factory
 methods to generate random double RDDs or vector RDDs.
 The following example generates a random double RDD, whose values follows the standard normal
 distribution `N(0, 1)`, and then map it to `N(1, 4)`.
 
-Refer to the [`RandomRDDs` Scala docs](api/scala/index.html#org.apache.spark.mllib.random.RandomRDDs$) for details on the API.
+Refer to the [`RandomRDDs` Scala docs](api/scala/org/apache/spark/mllib/random/RandomRDDs$.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.SparkContext
@@ -370,11 +370,11 @@ mean of PDFs of normal distributions centered around each of the samples.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-[`KernelDensity`](api/scala/index.html#org.apache.spark.mllib.stat.KernelDensity) provides methods
+[`KernelDensity`](api/scala/org/apache/spark/mllib/stat/KernelDensity.html) provides methods
 to compute kernel density estimates from an RDD of samples. The following example demonstrates how
 to do so.
 
-Refer to the [`KernelDensity` Scala docs](api/scala/index.html#org.apache.spark.mllib.stat.KernelDensity) for details on the API.
+Refer to the [`KernelDensity` Scala docs](api/scala/org/apache/spark/mllib/stat/KernelDensity.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/KernelDensityEstimationExample.scala %}
 </div>
diff --git a/docs/monitoring.md b/docs/monitoring.md
index fc95f6a8d3191..ac6b693c0fc82 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -95,19 +95,62 @@ The history server can be configured as follows:
   </tr>
 </table>
 
+### Applying compaction on rolling event log files
+
+A long-running application (e.g. streaming) can bring a huge single event log file which may cost a lot to maintain and
+also requires a bunch of resource to replay per each update in Spark History Server.
+
+Enabling <code>spark.eventLog.rolling.enabled</code> and <code>spark.eventLog.rolling.maxFileSize</code> would
+let you have rolling event log files instead of single huge event log file which may help some scenarios on its own,
+but it still doesn't help you reducing the overall size of logs.
+
+Spark History Server can apply compaction on the rolling event log files to reduce the overall size of
+logs, via setting the configuration <code>spark.history.fs.eventLog.rolling.maxFilesToRetain</code> on the
+Spark History Server.
+
+Details will be described below, but please note in prior that compaction is LOSSY operation.
+Compaction will discard some events which will be no longer seen on UI - you may want to check which events will be discarded
+before enabling the option.
+
+When the compaction happens, the History Server lists all the available event log files for the application, and considers
+the event log files having less index than the file with smallest index which will be retained as target of compaction.
+For example, if the application A has 5 event log files and <code>spark.history.fs.eventLog.rolling.maxFilesToRetain</code> is set to 2, then first 3 log files will be selected to be compacted.
+
+Once it selects the target, it analyzes them to figure out which events can be excluded, and rewrites them
+into one compact file with discarding events which are decided to exclude.
+
+The compaction tries to exclude the events which point to the outdated data. As of now, below describes the candidates of events to be excluded:
+
+* Events for the job which is finished, and related stage/tasks events
+* Events for the executor which is terminated
+* Events for the SQL execution which is finished, and related job/stage/tasks events
+
+Once rewriting is done, original log files will be deleted, via best-effort manner. The History Server may not be able to delete
+the original log files, but it will not affect the operation of the History Server.
+
+Please note that Spark History Server may not compact the old event log files if figures out not a lot of space
+would be reduced during compaction. For streaming query we normally expect compaction
+will run as each micro-batch will trigger one or more jobs which will be finished shortly, but compaction won't run
+in many cases for batch query.
+
+Please also note that this is a new feature introduced in Spark 3.0, and may not be completely stable. Under some circumstances,
+the compaction may exclude more events than you expect, leading some UI issues on History Server for the application.
+Use it with caution.
+
 ### Spark History Server Configuration Options
 
 Security options for the Spark History Server are covered more detail in the
 [Security](security.html#web-ui) page.
 
 <table class="table">
-  <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+  <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td>spark.history.provider</td>
     <td><code>org.apache.spark.deploy.history.FsHistoryProvider</code></td>
     <td>Name of the class implementing the application history backend. Currently there is only
     one implementation, provided by Spark, which looks for application logs stored in the
     file system.</td>
+    <td>1.1.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.logDirectory</td>
@@ -118,6 +161,7 @@ Security options for the Spark History Server are covered more detail in the
     an HDFS path <code>hdfs://namenode/shared/spark-logs</code>
     or that of an alternative filesystem supported by the Hadoop APIs.
     </td>
+    <td>1.1.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.update.interval</td>
@@ -129,6 +173,7 @@ Security options for the Spark History Server are covered more detail in the
       As soon as an update has completed, listings of the completed and incomplete applications
       will reflect the changes.
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td>spark.history.retainedApplications</td>
@@ -138,6 +183,7 @@ Security options for the Spark History Server are covered more detail in the
       the oldest applications will be removed from the cache. If an application is not in the cache,
       it will have to be loaded from disk if it is accessed from the UI.
     </td>
+    <td>1.0.0</td>
   </tr>
   <tr>
     <td>spark.history.ui.maxApplications</td>
@@ -146,6 +192,7 @@ Security options for the Spark History Server are covered more detail in the
       The number of applications to display on the history summary page. Application UIs are still
       available by accessing their URLs directly even if they are not displayed on the history summary page.
     </td>
+    <td>2.0.1</td>
   </tr>
   <tr>
     <td>spark.history.ui.port</td>
@@ -153,30 +200,32 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       The port to which the web interface of the history server binds.
     </td>
+    <td>1.0.0</td>
   </tr>
   <tr>
     <td>spark.history.kerberos.enabled</td>
     <td>false</td>
     <td>
       Indicates whether the history server should use kerberos to login. This is required
-      if the history server is accessing HDFS files on a secure Hadoop cluster. If this is
-      true, it uses the configs <code>spark.history.kerberos.principal</code> and
-      <code>spark.history.kerberos.keytab</code>.
+      if the history server is accessing HDFS files on a secure Hadoop cluster.
     </td>
+    <td>1.0.1</td>
   </tr>
   <tr>
     <td>spark.history.kerberos.principal</td>
     <td>(none)</td>
     <td>
-      Kerberos principal name for the History Server.
+      When <code>spark.history.kerberos.enabled=true</code>, specifies kerberos principal name for the History Server.
     </td>
+    <td>1.0.1</td>
   </tr>
   <tr>
     <td>spark.history.kerberos.keytab</td>
     <td>(none)</td>
     <td>
-      Location of the kerberos keytab file for the History Server.
+      When <code>spark.history.kerberos.enabled=true</code>, specifies location of the kerberos keytab file for the History Server.
     </td>
+    <td>1.0.1</td>
   </tr>
   <tr>
     <td>spark.history.fs.cleaner.enabled</td>
@@ -184,35 +233,39 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       Specifies whether the History Server should periodically clean up event logs from storage.
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.cleaner.interval</td>
     <td>1d</td>
     <td>
-      How often the filesystem job history cleaner checks for files to delete.
+      When <code>spark.history.fs.cleaner.enabled=true</code>, specifies how often the filesystem job history cleaner checks for files to delete.
       Files are deleted if at least one of two conditions holds.
       First, they're deleted if they're older than <code>spark.history.fs.cleaner.maxAge</code>.
       They are also deleted if the number of files is more than
       <code>spark.history.fs.cleaner.maxNum</code>, Spark tries to clean up the completed attempts
       from the applications based on the order of their oldest attempt time.
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.cleaner.maxAge</td>
     <td>7d</td>
     <td>
-      Job history files older than this will be deleted when the filesystem history cleaner runs.
+      When <code>spark.history.fs.cleaner.enabled=true</code>, job history files older than this will be deleted when the filesystem history cleaner runs.
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.cleaner.maxNum</td>
     <td>Int.MaxValue</td>
     <td>
-      The maximum number of files in the event log directory.
+      When <code>spark.history.fs.cleaner.enabled=true</code>, specifies the maximum number of files in the event log directory.
       Spark tries to clean up the completed attempt logs to maintain the log directory under this limit.
       This should be smaller than the underlying file system limit like
       `dfs.namenode.fs-limits.max-directory-items` in HDFS.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.endEventReparseChunkSize</td>
@@ -222,6 +275,7 @@ Security options for the Spark History Server are covered more detail in the
       This is used to speed up generation of application listings by skipping unnecessary
       parts of event log files. It can be disabled by setting this config to 0.
     </td>
+    <td>2.4.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.inProgressOptimization.enabled</td>
@@ -230,6 +284,7 @@ Security options for the Spark History Server are covered more detail in the
       Enable optimized handling of in-progress logs. This option may leave finished
       applications that fail to rename their event logs listed as in-progress.
     </td>
+    <td>2.4.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.driverlog.cleaner.enabled</td>
@@ -237,21 +292,24 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       Specifies whether the History Server should periodically clean up driver logs from storage.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.driverlog.cleaner.interval</td>
     <td><code>spark.history.fs.cleaner.interval</code></td>
     <td>
-      How often the filesystem driver log cleaner checks for files to delete.
+      When <code>spark.history.fs.driverlog.cleaner.enabled=true</code>, specifies how often the filesystem driver log cleaner checks for files to delete.
       Files are only deleted if they are older than <code>spark.history.fs.driverlog.cleaner.maxAge</code>
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.driverlog.cleaner.maxAge</td>
     <td><code>spark.history.fs.cleaner.maxAge</code></td>
     <td>
-      Driver log files older than this will be deleted when the driver log cleaner runs.
+      When <code>spark.history.fs.driverlog.cleaner.enabled=true</code>, driver log files older than this will be deleted when the driver log cleaner runs.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.numReplayThreads</td>
@@ -259,6 +317,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       Number of threads that will be used by history server to process event logs.
     </td>
+    <td>2.0.0</td>
   </tr>
   <tr>
     <td>spark.history.store.maxDiskUsage</td>
@@ -267,6 +326,7 @@ Security options for the Spark History Server are covered more detail in the
       Maximum disk usage for the local directory where the cache application history information
       are stored.
     </td>
+    <td>2.3.0</td>
   </tr>
   <tr>
     <td>spark.history.store.path</td>
@@ -276,6 +336,7 @@ Security options for the Spark History Server are covered more detail in the
         server will store application data on disk instead of keeping it in memory. The data
         written to disk will be re-used in the event of a history server restart.
     </td>
+    <td>2.3.0</td>
   </tr>
   <tr>
     <td>spark.history.custom.executor.log.url</td>
@@ -289,6 +350,7 @@ Security options for the Spark History Server are covered more detail in the
         <p/>
         For now, only YARN mode supports this configuration
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.custom.executor.log.url.applyIncompleteApplication</td>
@@ -299,26 +361,17 @@ Security options for the Spark History Server are covered more detail in the
         Please note that incomplete applications may include applications which didn't shutdown gracefully.
         Even this is set to `true`, this configuration has no effect on a live application, it only affects the history server.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.eventLog.rolling.maxFilesToRetain</td>
     <td>Int.MaxValue</td>
     <td>
       The maximum number of event log files which will be retained as non-compacted. By default,
-      all event log files will be retained.<br/>
-      Please note that compaction will happen in Spark History Server, which means this configuration
-      should be set to the configuration of Spark History server, and the same value will be applied
-      across applications which are being loaded in Spark History Server. This also means compaction
-      and cleanup would require running Spark History Server.<br/>
-      Please set the configuration in Spark History Server, and <code>spark.eventLog.rolling.maxFileSize</code>
-      in each application accordingly if you want to control the overall size of event log files.
-      The event log files older than these retained files will be compacted into single file and
-      deleted afterwards.<br/>
-      NOTE: Spark History Server may not compact the old event log files if it figures
-      out not a lot of space would be reduced during compaction. For streaming query
-      (including Structured Streaming) we normally expect compaction will run, but for
-      batch query compaction won't run in many cases.
+      all event log files will be retained. The lowest value is 1 for technical reason.<br/>
+      Please read the section of "Applying compaction of old event log files" for more details.
     </td>
+    <td>3.0.0</td>
   </tr>
 </table>
 
@@ -566,7 +619,7 @@ A list of the available metrics, with a short description:
   </tr>
   <tr>
     <td>inputMetrics.*</td>
-    <td>Metrics related to reading data from [[org.apache.spark.rdd.HadoopRDD]] 
+    <td>Metrics related to reading data from <code>org.apache.spark.rdd.HadoopRDD</code>
     or from persisted data.</td>
   </tr>
   <tr>
@@ -660,31 +713,75 @@ A list of the available metrics, with a short description:
 ### Executor Metrics
 
 Executor-level metrics are sent from each executor to the driver as part of the Heartbeat to describe the performance metrics of Executor itself like JVM heap memory, GC information.
-Executor metric values and their measured peak values per executor are exposed via the REST API at the end point `/applications/[app-id]/executors`.
-In addition, aggregated per-stage peak values of the executor metrics are written to the event log if `spark.eventLog.logStageExecutorMetrics.enabled` is true.
-Executor metrics are also exposed via the Spark metrics system based on the Dropwizard metrics library.
+Executor metric values and their measured memory peak values per executor are exposed via the REST API in JSON format and in Prometheus format.
+The JSON end point is exposed at: `/applications/[app-id]/executors`, and the Prometheus endpoint at: `/metrics/executors/prometheus`.
+The Prometheus endpoint is experimental and conditional to a configuration parameter: `spark.ui.prometheus.enabled=true` (the default is `false`).
+In addition, aggregated per-stage peak values of the executor memory metrics are written to the event log if
+`spark.eventLog.logStageExecutorMetrics` is true.  
+Executor memory metrics are also exposed via the Spark metrics system based on the [Dropwizard metrics library](http://metrics.dropwizard.io/4.1.1).
 A list of the available metrics, with a short description:
 
 <table class="table">
   <tr><th>Executor Level Metric name</th>
       <th>Short description</th>
   </tr>
+  <tr>
+    <td>rddBlocks</td>
+    <td>RDD blocks in the block manager of this executor.</td>
+  </tr>
+  <tr>
+    <td>memoryUsed</td>
+    <td>Storage memory used by this executor.</td>
+  </tr>
+  <tr>
+    <td>diskUsed</td>
+    <td>Disk space used for RDD storage by this executor.</td>
+  </tr>
+  <tr>
+    <td>totalCores</td>
+    <td>Number of cores available in this executor.</td>
+  </tr>
+  <tr>
+    <td>maxTasks</td>
+    <td>Maximum number of tasks that can run concurrently in this executor.</td>
+  </tr>
+  <tr>
+    <td>activeTasks</td>
+    <td>Number of tasks currently executing.</td>
+  </tr>
+  <tr>
+    <td>failedTasks</td>
+    <td>Number of tasks that have failed in this executor.</td>
+  </tr>
+  <tr>
+    <td>completedTasks</td>
+    <td>Number of tasks that have completed in this executor.</td>
+  </tr>
+  <tr>
+    <td>totalTasks</td>
+    <td>Total number of tasks (running, failed and completed) in this executor.</td>
+  </tr>
+  <tr>
+    <td>totalDuration</td>
+    <td>Elapsed time the JVM spent executing tasks in this executor.
+    The value is expressed in milliseconds.</td>
+  </tr>
   <tr>
     <td>totalGCTime</td>
-    <td>Elapsed time the JVM spent in garbage collection summed in this Executor.
+    <td>Elapsed time the JVM spent in garbage collection summed in this executor.
     The value is expressed in milliseconds.</td>
   </tr>
   <tr>
     <td>totalInputBytes</td>
-    <td>Total input bytes summed in this Executor.</td>
+    <td>Total input bytes summed in this executor.</td>
   </tr>
   <tr>
     <td>totalShuffleRead</td>
-    <td>Total shuffer read bytes summed in this Executor.</td>
+    <td>Total shuffle read bytes summed in this executor.</td>
   </tr>
   <tr>
     <td>totalShuffleWrite</td>
-    <td>Total shuffer write bytes summed in this Executor.</td>
+    <td>Total shuffle write bytes summed in this executor.</td>
   </tr>
   <tr>
     <td>maxMemory</td>
@@ -750,11 +847,11 @@ A list of the available metrics, with a short description:
   </tr>
   <tr>
     <td>&nbsp;&nbsp;&nbsp;&nbsp;.DirectPoolMemory</td>
-    <td>Peak memory that the JVM is using for direct buffer pool ([[java.lang.management.BufferPoolMXBean]])</td>
+    <td>Peak memory that the JVM is using for direct buffer pool (<code>java.lang.management.BufferPoolMXBean</code>)</td>
   </tr>
   <tr>
     <td>&nbsp;&nbsp;&nbsp;&nbsp;.MappedPoolMemory</td>
-    <td>Peak memory that the JVM is using for mapped buffer pool ([[java.lang.management.BufferPoolMXBean]])</td>
+    <td>Peak memory that the JVM is using for mapped buffer pool (<code>java.lang.management.BufferPoolMXBean</code>)</td>
   </tr>
   <tr>
     <td>&nbsp;&nbsp;&nbsp;&nbsp;.ProcessTreeJVMVMemory</td>
@@ -825,7 +922,7 @@ keep the paths consistent in both modes.
 # Metrics
 
 Spark has a configurable metrics system based on the
-[Dropwizard Metrics Library](http://metrics.dropwizard.io/).
+[Dropwizard Metrics Library](http://metrics.dropwizard.io/4.1.1).
 This allows users to report Spark metrics to a variety of sinks including HTTP, JMX, and CSV
 files. The metrics are generated by sources embedded in the Spark code base. They
 provide instrumentation for specific activities and Spark components.
@@ -866,7 +963,7 @@ Each instance can report to zero or more _sinks_. Sinks are contained in the
 * `CSVSink`: Exports metrics data to CSV files at regular intervals.
 * `JmxSink`: Registers metrics for viewing in a JMX console.
 * `MetricsServlet`: Adds a servlet within the existing Spark UI to serve metrics data as JSON data.
-* `PrometheusServlet`: Adds a servlet within the existing Spark UI to serve metrics data in Prometheus format.
+* `PrometheusServlet`: (Experimental) Adds a servlet within the existing Spark UI to serve metrics data in Prometheus format.
 * `GraphiteSink`: Sends metrics to a Graphite node.
 * `Slf4jSink`: Sends metrics to slf4j as log entries.
 * `StatsdSink`: Sends metrics to a StatsD node.
@@ -919,7 +1016,7 @@ activates the JVM source:
 ## List of available metrics providers 
 
 Metrics used by Spark are of multiple types: gauge, counter, histogram, meter and timer, 
-see [Dropwizard library documentation for details](https://metrics.dropwizard.io/3.1.0/getting-started/).
+see [Dropwizard library documentation for details](https://metrics.dropwizard.io/4.1.1/getting-started.html).
 The following list of components and metrics reports the name and some details about the available metrics,
 grouped per component instance and source namespace.
 The most common time of metrics used in Spark instrumentation are gauges and counters. 
@@ -945,7 +1042,7 @@ This is the component with the largest amount of instrumented metrics
   - memory.remainingOnHeapMem_MB
 
 - namespace=HiveExternalCatalog
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - fileCacheHits.count
   - filesDiscovered.count
@@ -954,12 +1051,11 @@ This is the component with the largest amount of instrumented metrics
   - partitionsFetched.count
 
 - namespace=CodeGenerator
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - compilationTime (histogram)
   - generatedClassSize (histogram)
   - generatedMethodSize (histogram)
-  - hiveClientCalls.count
   - sourceCodeSize (histogram)
 
 - namespace=DAGScheduler
@@ -1118,7 +1214,7 @@ when running in local mode.
   - shuffle-server.usedHeapMemory
 
 - namespace=HiveExternalCatalog
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - fileCacheHits.count
   - filesDiscovered.count
@@ -1127,12 +1223,11 @@ when running in local mode.
   - partitionsFetched.count
 
 - namespace=CodeGenerator
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - compilationTime (histogram)
   - generatedClassSize (histogram)
   - generatedMethodSize (histogram)
-  - hiveClientCalls.count
   - sourceCodeSize (histogram)
 
 - namespace=plugin.\<Plugin Class Name>
@@ -1144,9 +1239,11 @@ when running in local mode.
 Notes: 
   - Activate this source by setting the relevant `metrics.properties` file entry or the 
   configuration parameter:`spark.metrics.conf.*.source.jvm.class=org.apache.spark.metrics.source.JvmSource`  
+  - These metrics are conditional to a configuration parameter:
+    `spark.metrics.staticSources.enabled` (default is true)
   - This source is available for driver and executor instances and is also available for other instances.  
   - This source provides information on JVM metrics using the 
-  [Dropwizard/Codahale Metric Sets for JVM instrumentation](https://metrics.dropwizard.io/3.1.0/manual/jvm/)
+  [Dropwizard/Codahale Metric Sets for JVM instrumentation](https://metrics.dropwizard.io/4.1.1/manual/jvm.html)
    and in particular the metric sets BufferPoolMetricSet, GarbageCollectorMetricSet and MemoryUsageGaugeSet. 
 
 ### Component instance = applicationMaster
@@ -1228,9 +1325,3 @@ Both take a comma-separated list of class names that implement the
 possible for one list to be placed in the Spark default config file, allowing users to
 easily add other plugins from the command line without overwriting the config file's list. Duplicate
 plugins are ignored.
-
-Distribution of the jar files containing the plugin code is currently not done by Spark. The user
-or admin should make sure that the jar files are available to Spark applications, for example, by
-including the plugin jar with the Spark distribution. The exception to this rule is the YARN
-backend, where the <code>--jars</code> command line option (or equivalent config entry) can be
-used to make the plugin code available to both executors and cluster-mode drivers.
diff --git a/docs/pyspark-migration-guide.md b/docs/pyspark-migration-guide.md
index 8ea4fec75edf8..2c9ea410f217e 100644
--- a/docs/pyspark-migration-guide.md
+++ b/docs/pyspark-migration-guide.md
@@ -27,67 +27,25 @@ Many items of SQL migration can be applied when migrating PySpark to higher vers
 Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.html).
 
 ## Upgrading from PySpark 2.4 to 3.0
+- In Spark 3.0, PySpark requires a pandas version of 0.23.2 or higher to use pandas related functionality, such as `toPandas`, `createDataFrame` from pandas DataFrame, and so on.
 
-  - Since Spark 3.0, PySpark requires a Pandas version of 0.23.2 or higher to use Pandas related functionality, such as `toPandas`, `createDataFrame` from Pandas DataFrame, etc.
-
-  - Since Spark 3.0, PySpark requires a PyArrow version of 0.12.1 or higher to use PyArrow related functionality, such as `pandas_udf`, `toPandas` and `createDataFrame` with "spark.sql.execution.arrow.enabled=true", etc.
-
-  - In PySpark, when creating a `SparkSession` with `SparkSession.builder.getOrCreate()`, if there is an existing `SparkContext`, the builder was trying to update the `SparkConf` of the existing `SparkContext` with configurations specified to the builder, but the `SparkContext` is shared by all `SparkSession`s, so we should not update them. Since 3.0, the builder comes to not update the configurations. This is the same behavior as Java/Scala API in 2.3 and above. If you want to update them, you need to update them prior to creating a `SparkSession`.
-
-  - In PySpark, when Arrow optimization is enabled, if Arrow version is higher than 0.11.0, Arrow can perform safe type conversion when converting Pandas.Series to Arrow array during serialization. Arrow will raise errors when detecting unsafe type conversion like overflow. Setting `spark.sql.execution.pandas.arrowSafeTypeConversion` to true can enable it. The default setting is false. PySpark's behavior for Arrow versions is illustrated in the table below:
-    <table class="table">
-        <tr>
-          <th>
-            <b>PyArrow version</b>
-          </th>
-          <th>
-            <b>Integer Overflow</b>
-          </th>
-          <th>
-            <b>Floating Point Truncation</b>
-          </th>
-        </tr>
-        <tr>
-          <td>
-            version < 0.11.0
-          </td>
-          <td>
-            Raise error
-          </td>
-          <td>
-            Silently allows
-          </td>
-        </tr>
-        <tr>
-          <td>
-            version > 0.11.0, arrowSafeTypeConversion=false
-          </td>
-          <td>
-            Silent overflow
-          </td>
-          <td>
-            Silently allows
-          </td>
-        </tr>
-        <tr>
-          <td>
-            version > 0.11.0, arrowSafeTypeConversion=true
-          </td>
-          <td>
-            Raise error
-          </td>
-          <td>
-            Raise error
-          </td>
-        </tr>
-    </table>
-
-  - Since Spark 3.0, `createDataFrame(..., verifySchema=True)` validates `LongType` as well in PySpark. Previously, `LongType` was not verified and resulted in `None` in case the value overflows. To restore this behavior, `verifySchema` can be set to `False` to disable the validation.
-
-  - Since Spark 3.0, `Column.getItem` is fixed such that it does not call `Column.apply`. Consequently, if `Column` is used as an argument to `getItem`, the indexing operator should be used.
-    For example, `map_col.getItem(col('id'))` should be replaced with `map_col[col('id')]`.
-
-  - As of Spark 3.0 `Row` field names are no longer sorted alphabetically when constructing with named arguments for Python versions 3.6 and above, and the order of fields will match that as entered. To enable sorted fields by default, as in Spark 2.4, set the environment variable `PYSPARK_ROW_FIELD_SORTING_ENABLED` to "true". For Python versions less than 3.6, the field names will be sorted alphabetically as the only option.
+- In Spark 3.0, PySpark requires a PyArrow version of 0.12.1 or higher to use PyArrow related functionality, such as `pandas_udf`, `toPandas` and `createDataFrame` with "spark.sql.execution.arrow.enabled=true", etc.
+
+- In PySpark, when creating a `SparkSession` with `SparkSession.builder.getOrCreate()`, if there is an existing `SparkContext`, the builder was trying to update the `SparkConf` of the existing `SparkContext` with configurations specified to the builder, but the `SparkContext` is shared by all `SparkSession`s, so we should not update them. In 3.0, the builder comes to not update the configurations. This is the same behavior as Java/Scala API in 2.3 and above. If you want to update them, you need to update them prior to creating a `SparkSession`.
+
+- In PySpark, when Arrow optimization is enabled, if Arrow version is higher than 0.11.0, Arrow can perform safe type conversion when converting `pandas.Series` to an Arrow array during serialization. Arrow raises errors when detecting unsafe type conversions like overflow. You enable it by setting `spark.sql.execution.pandas.convertToArrowArraySafely` to `true`. The default setting is `false`. PySpark behavior for Arrow versions is illustrated in the following table:
+
+  | PyArrow version  | Integer overflow | Floating point truncation |
+  | ---------------- | ---------------- | ------------------------- |
+  | 0.11.0 and below | Raise error      | Silently allows           |
+  | \> 0.11.0, arrowSafeTypeConversion=false | Silent overflow  | Silently allows |
+  | \> 0.11.0, arrowSafeTypeConversion=true  | Raise error      | Raise error |
+   
+- In Spark 3.0, `createDataFrame(..., verifySchema=True)` validates `LongType` as well in PySpark. Previously, `LongType` was not verified and resulted in `None` in case the value overflows. To restore this behavior, `verifySchema` can be set to `False` to disable the validation.
+
+- As of Spark 3.0, `Row` field names are no longer sorted alphabetically when constructing with named arguments for Python versions 3.6 and above, and the order of fields will match that as entered. To enable sorted fields by default, as in Spark 2.4, set the environment variable `PYSPARK_ROW_FIELD_SORTING_ENABLED` to `true` for both executors and driver - this environment variable must be consistent on all executors and driver; otherwise, it may cause failures or incorrect answers. For Python versions less than 3.6, the field names will be sorted alphabetically as the only option.
+
+- In Spark 3.0, `pyspark.ml.param.shared.Has*` mixins do not provide any `set*(self, value)` setter methods anymore, use the respective `self.set(self.*, value)` instead. See [SPARK-29093](https://issues.apache.org/jira/browse/SPARK-29093) for details.
 
 ## Upgrading from PySpark 2.3 to 2.4
 
diff --git a/docs/quick-start.md b/docs/quick-start.md
index 93abb25f20f47..557fc187fb81d 100644
--- a/docs/quick-start.md
+++ b/docs/quick-start.md
@@ -57,7 +57,7 @@ scala> val textFile = spark.read.textFile("README.md")
 textFile: org.apache.spark.sql.Dataset[String] = [value: string]
 {% endhighlight %}
 
-You can get values from Dataset directly, by calling some actions, or transform the Dataset to get a new one. For more details, please read the _[API doc](api/scala/index.html#org.apache.spark.sql.Dataset)_.
+You can get values from Dataset directly, by calling some actions, or transform the Dataset to get a new one. For more details, please read the _[API doc](api/scala/org/apache/spark/sql/Dataset.html)_.
 
 {% highlight scala %}
 scala> textFile.count() // Number of items in this Dataset
@@ -264,7 +264,7 @@ Spark README. Note that you'll need to replace YOUR_SPARK_HOME with the location
 installed. Unlike the earlier examples with the Spark shell, which initializes its own SparkSession,
 we initialize a SparkSession as part of the program.
 
-We call `SparkSession.builder` to construct a [[SparkSession]], then set the application name, and finally call `getOrCreate` to get the [[SparkSession]] instance.
+We call `SparkSession.builder` to construct a `SparkSession`, then set the application name, and finally call `getOrCreate` to get the `SparkSession` instance.
 
 Our application depends on the Spark API, so we'll also include an sbt configuration file,
 `build.sbt`, which explains that Spark is a dependency. This file also adds a repository that
@@ -399,7 +399,7 @@ If you are building a packaged PySpark application or library you can add it to
 
 {% highlight python %}
     install_requires=[
-        'pyspark=={site.SPARK_VERSION}'
+        'pyspark=={{site.SPARK_VERSION}}'
     ]
 {% endhighlight %}
 
diff --git a/docs/rdd-programming-guide.md b/docs/rdd-programming-guide.md
index 5e55c93c4148e..ba99007aaf639 100644
--- a/docs/rdd-programming-guide.md
+++ b/docs/rdd-programming-guide.md
@@ -149,8 +149,8 @@ $ PYSPARK_PYTHON=/opt/pypy-2.5/bin/pypy bin/spark-submit examples/src/main/pytho
 
 <div data-lang="scala"  markdown="1">
 
-The first thing a Spark program must do is to create a [SparkContext](api/scala/index.html#org.apache.spark.SparkContext) object, which tells Spark
-how to access a cluster. To create a `SparkContext` you first need to build a [SparkConf](api/scala/index.html#org.apache.spark.SparkConf) object
+The first thing a Spark program must do is to create a [SparkContext](api/scala/org/apache/spark/SparkContext.html) object, which tells Spark
+how to access a cluster. To create a `SparkContext` you first need to build a [SparkConf](api/scala/org/apache/spark/SparkConf.html) object
 that contains information about your application.
 
 Only one SparkContext should be active per JVM. You must `stop()` the active SparkContext before creating a new one.
@@ -500,7 +500,7 @@ then this approach should work well for such cases.
 
 If you have custom serialized binary data (such as loading data from Cassandra / HBase), then you will first need to
 transform that data on the Scala/Java side to something which can be handled by Pyrolite's pickler.
-A [Converter](api/scala/index.html#org.apache.spark.api.python.Converter) trait is provided
+A [Converter](api/scala/org/apache/spark/api/python/Converter.html) trait is provided
 for this. Simply extend this trait and implement your transformation code in the ```convert```
 method. Remember to ensure that this class, along with any dependencies required to access your ```InputFormat```, are packaged into your Spark job jar and included on the PySpark
 classpath.
@@ -856,7 +856,7 @@ by a key.
 In Scala, these operations are automatically available on RDDs containing
 [Tuple2](http://www.scala-lang.org/api/{{site.SCALA_VERSION}}/index.html#scala.Tuple2) objects
 (the built-in tuples in the language, created by simply writing `(a, b)`). The key-value pair operations are available in the
-[PairRDDFunctions](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions) class,
+[PairRDDFunctions](api/scala/org/apache/spark/rdd/PairRDDFunctions.html) class,
 which automatically wraps around an RDD of tuples.
 
 For example, the following code uses the `reduceByKey` operation on key-value pairs to count how
@@ -946,12 +946,12 @@ We could also use `counts.sortByKey()`, for example, to sort the pairs alphabeti
 
 The following table lists some of the common transformations supported by Spark. Refer to the
 RDD API doc
-([Scala](api/scala/index.html#org.apache.spark.rdd.RDD),
+([Scala](api/scala/org/apache/spark/rdd/RDD.html),
  [Java](api/java/index.html?org/apache/spark/api/java/JavaRDD.html),
  [Python](api/python/pyspark.html#pyspark.RDD),
  [R](api/R/index.html))
 and pair RDD functions doc
-([Scala](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions),
+([Scala](api/scala/org/apache/spark/rdd/PairRDDFunctions.html),
  [Java](api/java/index.html?org/apache/spark/api/java/JavaPairRDD.html))
 for details.
 
@@ -1060,13 +1060,13 @@ for details.
 
 The following table lists some of the common actions supported by Spark. Refer to the
 RDD API doc
-([Scala](api/scala/index.html#org.apache.spark.rdd.RDD),
+([Scala](api/scala/org/apache/spark/rdd/RDD.html),
  [Java](api/java/index.html?org/apache/spark/api/java/JavaRDD.html),
  [Python](api/python/pyspark.html#pyspark.RDD),
  [R](api/R/index.html))
 
 and pair RDD functions doc
-([Scala](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions),
+([Scala](api/scala/org/apache/spark/rdd/PairRDDFunctions.html),
  [Java](api/java/index.html?org/apache/spark/api/java/JavaPairRDD.html))
 for details.
 
@@ -1208,7 +1208,7 @@ In addition, each persisted RDD can be stored using a different *storage level*,
 to persist the dataset on disk, persist it in memory but as serialized Java objects (to save space),
 replicate it across nodes.
 These levels are set by passing a
-`StorageLevel` object ([Scala](api/scala/index.html#org.apache.spark.storage.StorageLevel),
+`StorageLevel` object ([Scala](api/scala/org/apache/spark/storage/StorageLevel.html),
 [Java](api/java/index.html?org/apache/spark/storage/StorageLevel.html),
 [Python](api/python/pyspark.html#pyspark.StorageLevel))
 to `persist()`. The `cache()` method is a shorthand for using the default storage level,
@@ -1404,11 +1404,11 @@ res2: Long = 10
 {% endhighlight %}
 
 While this code used the built-in support for accumulators of type Long, programmers can also
-create their own types by subclassing [AccumulatorV2](api/scala/index.html#org.apache.spark.util.AccumulatorV2).
+create their own types by subclassing [AccumulatorV2](api/scala/org/apache/spark/util/AccumulatorV2.html).
 The AccumulatorV2 abstract class has several methods which one has to override: `reset` for resetting
 the accumulator to zero, `add` for adding another value into the accumulator,
 `merge` for merging another same-type accumulator into this one. Other methods that must be overridden
-are contained in the [API documentation](api/scala/index.html#org.apache.spark.util.AccumulatorV2). For example, supposing we had a `MyVector` class
+are contained in the [API documentation](api/scala/org/apache/spark/util/AccumulatorV2.html). For example, supposing we had a `MyVector` class
 representing mathematical vectors, we could write:
 
 {% highlight scala %}
@@ -1457,11 +1457,11 @@ accum.value();
 {% endhighlight %}
 
 While this code used the built-in support for accumulators of type Long, programmers can also
-create their own types by subclassing [AccumulatorV2](api/scala/index.html#org.apache.spark.util.AccumulatorV2).
+create their own types by subclassing [AccumulatorV2](api/scala/org/apache/spark/util/AccumulatorV2.html).
 The AccumulatorV2 abstract class has several methods which one has to override: `reset` for resetting
 the accumulator to zero, `add` for adding another value into the accumulator,
 `merge` for merging another same-type accumulator into this one. Other methods that must be overridden
-are contained in the [API documentation](api/scala/index.html#org.apache.spark.util.AccumulatorV2). For example, supposing we had a `MyVector` class
+are contained in the [API documentation](api/scala/org/apache/spark/util/AccumulatorV2.html). For example, supposing we had a `MyVector` class
 representing mathematical vectors, we could write:
 
 {% highlight java %}
@@ -1620,4 +1620,4 @@ For help on deploying, the [cluster mode overview](cluster-overview.html) descri
 in distributed operation and supported cluster managers.
 
 Finally, full API documentation is available in
-[Scala](api/scala/#org.apache.spark.package), [Java](api/java/), [Python](api/python/) and [R](api/R/).
+[Scala](api/scala/org/apache/spark/), [Java](api/java/), [Python](api/python/) and [R](api/R/).
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 61d6154ccb084..f49f953938391 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -77,8 +77,7 @@ logs and remains in "completed" state in the Kubernetes API until it's eventuall
 
 Note that in the completed state, the driver pod does *not* use any computational or memory resources.
 
-The driver and executor pod scheduling is handled by Kubernetes. Communication to the Kubernetes API is done via fabric8, and we are
-currently running <code>kubernetes-client</code> version <code>4.1.0</code>. Make sure that when you are making infrastructure additions that you are aware of said version. It is possible to schedule the
+The driver and executor pod scheduling is handled by Kubernetes. Communication to the Kubernetes API is done via fabric8. It is possible to schedule the
 driver and executor pods on a subset of available nodes through a [node selector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector)
 using the configuration property for it. It will be possible to use more advanced
 scheduling hints like [node/pod affinities](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) in a future release.
@@ -495,7 +494,7 @@ See the [configuration page](configuration.html) for information on Spark config
 #### Spark Properties
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.kubernetes.context</code></td>
   <td><code>(none)</code></td>
@@ -506,6 +505,7 @@ See the [configuration page](configuration.html) for information on Spark config
     auto-configured settings can be overridden by the use of other Spark
     configuration properties e.g. <code>spark.kubernetes.namespace</code>.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.master</code></td>
@@ -513,6 +513,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     The internal Kubernetes master (API server) address to be used for driver to request executors.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.namespace</code></td>
@@ -520,6 +521,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     The namespace that will be used for running the driver and executor pods.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.container.image</code></td>
@@ -530,6 +532,7 @@ See the [configuration page](configuration.html) for information on Spark config
     This configuration is required and must be provided by the user, unless explicit
     images are provided for each different container type.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.container.image</code></td>
@@ -537,6 +540,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Custom container image to use for the driver.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.container.image</code></td>
@@ -544,6 +548,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Custom container image to use for executors.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.container.image.pullPolicy</code></td>
@@ -551,6 +556,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Container image pull policy used when pulling images within Kubernetes.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.container.image.pullSecrets</code></td>
@@ -558,6 +564,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Comma separated list of Kubernetes secrets used to pull images from private image registries.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.allocation.batch.size</code></td>
@@ -565,6 +572,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Number of pods to launch at once in each round of executor pod allocation.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.allocation.batch.delay</code></td>
@@ -573,6 +581,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Time to wait between each round of executor pod allocation. Specifying values less than 1 second may lead to
     excessive CPU usage on the spark driver.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.submission.caCertFile</code></td>
@@ -582,6 +591,7 @@ See the [configuration page](configuration.html) for information on Spark config
     must be located on the submitting machine's disk. Specify this as a path as opposed to a URI (i.e. do not provide
     a scheme). In client mode, use <code>spark.kubernetes.authenticate.caCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.submission.clientKeyFile</code></td>
@@ -591,6 +601,7 @@ See the [configuration page](configuration.html) for information on Spark config
     must be located on the submitting machine's disk. Specify this as a path as opposed to a URI (i.e. do not provide
     a scheme). In client mode, use <code>spark.kubernetes.authenticate.clientKeyFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.submission.clientCertFile</code></td>
@@ -600,6 +611,7 @@ See the [configuration page](configuration.html) for information on Spark config
     file must be located on the submitting machine's disk. Specify this as a path as opposed to a URI (i.e. do not
     provide a scheme). In client mode, use <code>spark.kubernetes.authenticate.clientCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.submission.oauthToken</code></td>
@@ -609,6 +621,7 @@ See the [configuration page](configuration.html) for information on Spark config
     that unlike the other authentication options, this is expected to be the exact string value of the token to use for
     the authentication. In client mode, use <code>spark.kubernetes.authenticate.oauthToken</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.submission.oauthTokenFile</code></td>
@@ -618,6 +631,7 @@ See the [configuration page](configuration.html) for information on Spark config
     This file must be located on the submitting machine's disk. Specify this as a path as opposed to a URI (i.e. do not
     provide a scheme). In client mode, use <code>spark.kubernetes.authenticate.oauthTokenFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.caCertFile</code></td>
@@ -628,6 +642,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify this as a path as opposed to a URI (i.e. do not provide a scheme). In client mode, use
     <code>spark.kubernetes.authenticate.caCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.clientKeyFile</code></td>
@@ -638,6 +653,7 @@ See the [configuration page](configuration.html) for information on Spark config
     a Kubernetes secret. Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
     In client mode, use <code>spark.kubernetes.authenticate.clientKeyFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.clientCertFile</code></td>
@@ -648,6 +664,7 @@ See the [configuration page](configuration.html) for information on Spark config
     driver pod as a Kubernetes secret. Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
     In client mode, use <code>spark.kubernetes.authenticate.clientCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.oauthToken</code></td>
@@ -658,6 +675,7 @@ See the [configuration page](configuration.html) for information on Spark config
     the token to use for the authentication. This token value is uploaded to the driver pod as a Kubernetes secret.
     In client mode, use <code>spark.kubernetes.authenticate.oauthToken</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.oauthTokenFile</code></td>
@@ -668,6 +686,7 @@ See the [configuration page](configuration.html) for information on Spark config
     the token to use for the authentication. This token value is uploaded to the driver pod as a secret. In client mode, use
     <code>spark.kubernetes.authenticate.oauthTokenFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.mounted.caCertFile</code></td>
@@ -678,6 +697,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify this as a path as opposed to a URI (i.e. do not provide a scheme). In client mode, use
     <code>spark.kubernetes.authenticate.caCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.mounted.clientKeyFile</code></td>
@@ -688,6 +708,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify this as a path as opposed to a URI (i.e. do not provide a scheme). In client mode, use
     <code>spark.kubernetes.authenticate.clientKeyFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.mounted.clientCertFile</code></td>
@@ -698,6 +719,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify this as a path as opposed to a URI (i.e. do not provide a scheme). In client mode, use
     <code>spark.kubernetes.authenticate.clientCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.mounted.oauthTokenFile</code></td>
@@ -708,6 +730,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Note that unlike the other authentication options, this file must contain the exact string value of the token to use
     for the authentication. In client mode, use <code>spark.kubernetes.authenticate.oauthTokenFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.serviceAccountName</code></td>
@@ -717,6 +740,7 @@ See the [configuration page](configuration.html) for information on Spark config
     executor pods from the API server. Note that this cannot be specified alongside a CA cert file, client key file,
     client cert file, and/or OAuth token. In client mode, use <code>spark.kubernetes.authenticate.serviceAccountName</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.caCertFile</code></td>
@@ -725,6 +749,7 @@ See the [configuration page](configuration.html) for information on Spark config
     In client mode, path to the CA cert file for connecting to the Kubernetes API server over TLS when
     requesting executors. Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.clientKeyFile</code></td>
@@ -733,6 +758,7 @@ See the [configuration page](configuration.html) for information on Spark config
     In client mode, path to the client key file for authenticating against the Kubernetes API server
     when requesting executors. Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.clientCertFile</code></td>
@@ -741,6 +767,7 @@ See the [configuration page](configuration.html) for information on Spark config
     In client mode, path to the client cert file for authenticating against the Kubernetes API server
     when requesting executors. Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.oauthToken</code></td>
@@ -750,6 +777,7 @@ See the [configuration page](configuration.html) for information on Spark config
     requesting executors. Note that unlike the other authentication options, this must be the exact string value of
     the token to use for the authentication.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.oauthTokenFile</code></td>
@@ -758,6 +786,7 @@ See the [configuration page](configuration.html) for information on Spark config
     In client mode, path to the file containing the OAuth token to use when authenticating against the Kubernetes API
     server when requesting executors.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.label.[LabelName]</code></td>
@@ -768,14 +797,25 @@ See the [configuration page](configuration.html) for information on Spark config
     Note that Spark also adds its own labels to the driver pod
     for bookkeeping purposes.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.annotation.[AnnotationName]</code></td>
   <td>(none)</td>
   <td>
-    Add the annotation specified by <code>AnnotationName</code> to the driver pod.
+    Add the Kubernetes <a href="https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/">annotation</a> specified by <code>AnnotationName</code> to the driver pod.
     For example, <code>spark.kubernetes.driver.annotation.something=true</code>.
   </td>
+  <td>2.3.0</td>
+</tr>
+<tr>
+  <td><code>spark.kubernetes.driver.service.annotation.[AnnotationName]</code></td>
+  <td>(none)</td>
+  <td>
+    Add the Kubernetes <a href="https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/">annotation</a> specified by <code>AnnotationName</code> to the driver service.
+    For example, <code>spark.kubernetes.driver.service.annotation.something=true</code>.
+  </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.label.[LabelName]</code></td>
@@ -786,14 +826,16 @@ See the [configuration page](configuration.html) for information on Spark config
     Note that Spark also adds its own labels to the executor pod
     for bookkeeping purposes.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.annotation.[AnnotationName]</code></td>
   <td>(none)</td>
   <td>
-    Add the annotation specified by <code>AnnotationName</code> to the executor pods.
+    Add the Kubernetes <a href="https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/">annotation</a> specified by <code>AnnotationName</code> to the executor pods.
     For example, <code>spark.kubernetes.executor.annotation.something=true</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.pod.name</code></td>
@@ -805,6 +847,7 @@ See the [configuration page](configuration.html) for information on Spark config
     value in client mode allows the driver to become the owner of its executor pods, which in turn allows the executor
     pods to be garbage collected by the cluster.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.lostCheck.maxAttempts</code></td>
@@ -814,6 +857,7 @@ See the [configuration page](configuration.html) for information on Spark config
     The loss reason is used to ascertain whether the executor failure is due to a framework or an application error
     which in turn decides whether the executor is removed and replaced, or placed into a failed state for debugging.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.submission.waitAppCompletion</code></td>
@@ -822,6 +866,7 @@ See the [configuration page](configuration.html) for information on Spark config
     In cluster mode, whether to wait for the application to finish before exiting the launcher process.  When changed to
     false, the launcher has a "fire-and-forget" behavior when launching the Spark job.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.report.interval</code></td>
@@ -829,6 +874,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Interval between reports of the current Spark job status in cluster mode.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.request.cores</code></td>
@@ -838,6 +884,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Example values include 0.1, 500m, 1.5, 5, etc., with the definition of cpu units documented in <a href="https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units">CPU units</a>.
     This takes precedence over <code>spark.driver.cores</code> for specifying the driver pod cpu request if set.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.limit.cores</code></td>
@@ -845,6 +892,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Specify a hard cpu <a href="https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container">limit</a> for the driver pod.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.request.cores</code></td>
@@ -855,6 +903,7 @@ See the [configuration page](configuration.html) for information on Spark config
     This is distinct from <code>spark.executor.cores</code>: it is only used and takes precedence over <code>spark.executor.cores</code> for specifying the executor pod cpu request if set. Task
     parallelism, e.g., number of tasks an executor can run concurrently is not affected by this.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.limit.cores</code></td>
@@ -862,6 +911,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Specify a hard cpu <a href="https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container">limit</a> for each executor pod launched for the Spark Application.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.node.selector.[labelKey]</code></td>
@@ -872,6 +922,7 @@ See the [configuration page](configuration.html) for information on Spark config
     will result in the driver pod and executors having a node selector with key <code>identifier</code> and value
      <code>myIdentifier</code>. Multiple node selector keys can be added by setting multiple configurations with this prefix.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driverEnv.[EnvironmentVariableName]</code></td>
@@ -880,6 +931,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Add the environment variable specified by <code>EnvironmentVariableName</code> to
     the Driver process. The user can specify multiple of these to set multiple environment variables.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.secrets.[SecretName]</code></td>
@@ -888,6 +940,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add the <a href="https://kubernetes.io/docs/concepts/configuration/secret/">Kubernetes Secret</a> named <code>SecretName</code> to the driver pod on the path specified in the value. For example,
    <code>spark.kubernetes.driver.secrets.spark-secret=/etc/secrets</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.secrets.[SecretName]</code></td>
@@ -896,6 +949,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add the <a href="https://kubernetes.io/docs/concepts/configuration/secret/">Kubernetes Secret</a> named <code>SecretName</code> to the executor pod on the path specified in the value. For example,
    <code>spark.kubernetes.executor.secrets.spark-secret=/etc/secrets</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.secretKeyRef.[EnvName]</code></td>
@@ -904,6 +958,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add as an environment variable to the driver container with name EnvName (case sensitive), the value referenced by key <code> key </code> in the data of the referenced <a href="https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables">Kubernetes Secret</a>. For example,
    <code>spark.kubernetes.driver.secretKeyRef.ENV_VAR=spark-secret:key</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.secretKeyRef.[EnvName]</code></td>
@@ -912,6 +967,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add as an environment variable to the executor container with name EnvName (case sensitive), the value referenced by key <code> key </code> in the data of the referenced <a href="https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables">Kubernetes Secret</a>. For example,
    <code>spark.kubernetes.executor.secrets.ENV_VAR=spark-secret:key</code>.
   </td>
+  <td>2.4.0</td>
 </tr>   
 <tr>
   <td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.path</code></td>
@@ -920,6 +976,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add the <a href="https://kubernetes.io/docs/concepts/storage/volumes/">Kubernetes Volume</a> named <code>VolumeName</code> of the <code>VolumeType</code> type to the driver pod on the path specified in the value. For example,
    <code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.mount.path=/checkpoint</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.subPath</code></td>
@@ -928,6 +985,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specifies a <a href="https://kubernetes.io/docs/concepts/storage/volumes/#using-subpath">subpath</a> to be mounted from the volume into the driver pod.
    <code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.mount.subPath=checkpoint</code>.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.readOnly</code></td>
@@ -936,6 +994,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify if the mounted volume is read only or not. For example,
    <code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.mount.readOnly=false</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].options.[OptionName]</code></td>
@@ -944,6 +1003,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Configure <a href="https://kubernetes.io/docs/concepts/storage/volumes/">Kubernetes Volume</a> options passed to the Kubernetes with <code>OptionName</code> as key having specified value, must conform with Kubernetes option format. For example,
    <code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.options.claimName=spark-pvc-claim</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.path</code></td>
@@ -952,6 +1012,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add the <a href="https://kubernetes.io/docs/concepts/storage/volumes/">Kubernetes Volume</a> named <code>VolumeName</code> of the <code>VolumeType</code> type to the executor pod on the path specified in the value. For example,
    <code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.mount.path=/checkpoint</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.subPath</code></td>
@@ -960,6 +1021,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specifies a <a href="https://kubernetes.io/docs/concepts/storage/volumes/#using-subpath">subpath</a> to be mounted from the volume into the executor pod.
    <code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.mount.subPath=checkpoint</code>.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.readOnly</code></td>
@@ -968,6 +1030,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify if the mounted volume is read only or not. For example,
    <code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.mount.readOnly=false</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].options.[OptionName]</code></td>
@@ -976,6 +1039,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Configure <a href="https://kubernetes.io/docs/concepts/storage/volumes/">Kubernetes Volume</a> options passed to the Kubernetes with <code>OptionName</code> as key having specified value. For example,
    <code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.options.claimName=spark-pvc-claim</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.local.dirs.tmpfs</code></td>
@@ -984,6 +1048,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Configure the <code>emptyDir</code> volumes used to back <code>SPARK_LOCAL_DIRS</code> within the Spark driver and executor pods to use <code>tmpfs</code> backing i.e. RAM.  See <a href="#local-storage">Local Storage</a> earlier on this page
    for more discussion of this.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.memoryOverheadFactor</code></td>
@@ -992,6 +1057,7 @@ See the [configuration page](configuration.html) for information on Spark config
     This sets the Memory Overhead Factor that will allocate memory to non-JVM memory, which includes off-heap memory allocations, non-JVM tasks, and various systems processes. For JVM-based jobs this value will default to 0.10 and 0.40 for non-JVM jobs.
     This is done as non-JVM tasks need more non-JVM heap space and such tasks commonly fail with "Memory Overhead Exceeded" errors. This prempts this error with a higher default.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.pyspark.pythonVersion</code></td>
@@ -999,6 +1065,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
    This sets the major Python version of the docker image used to run the driver and executor containers. Can either be 2 or 3.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.kerberos.krb5.path</code></td>
@@ -1007,6 +1074,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify the local location of the krb5.conf file to be mounted on the driver and executors for Kerberos interaction.
    It is important to note that the KDC defined needs to be visible from inside the containers.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.kerberos.krb5.configMapName</code></td>
@@ -1016,6 +1084,7 @@ See the [configuration page](configuration.html) for information on Spark config
    for Kerberos interaction. The KDC defined needs to be visible from inside the containers. The ConfigMap must also
    be in the same namespace of the driver and executor pods.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.hadoop.configMapName</code></td>
@@ -1024,6 +1093,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify the name of the ConfigMap, containing the HADOOP_CONF_DIR files, to be mounted on the driver
     and executors for custom Hadoop configuration.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.kerberos.tokenSecret.name</code></td>
@@ -1032,6 +1102,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify the name of the secret where your existing delegation tokens are stored. This removes the need for the job user
     to provide any kerberos credentials for launching a job.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.kerberos.tokenSecret.itemKey</code></td>
@@ -1040,6 +1111,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify the item key of the data where your existing delegation tokens are stored. This removes the need for the job user
     to provide any kerberos credentials for launching a job.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.podTemplateFile</code></td>
@@ -1048,6 +1120,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify the local file that contains the driver <a href="#pod-template">pod template</a>. For example
    <code>spark.kubernetes.driver.podTemplateFile=/path/to/driver-pod-template.yaml</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.podTemplateContainerName</code></td>
@@ -1056,6 +1129,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify the container name to be used as a basis for the driver in the given <a href="#pod-template">pod template</a>.
    For example <code>spark.kubernetes.driver.podTemplateContainerName=spark-driver</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.podTemplateFile</code></td>
@@ -1064,6 +1138,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify the local file that contains the executor <a href="#pod-template">pod template</a>. For example
    <code>spark.kubernetes.executor.podTemplateFile=/path/to/executor-pod-template.yaml</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.podTemplateContainerName</code></td>
@@ -1072,6 +1147,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify the container name to be used as a basis for the executor in the given <a href="#pod-template">pod template</a>.
    For example <code>spark.kubernetes.executor.podTemplateContainerName=spark-executor</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.deleteOnTermination</code></td>
@@ -1079,6 +1155,15 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
   Specify whether executor pods should be deleted in case of failure or normal termination.
   </td>
+  <td>3.0.0</td>
+</tr>
+<tr>
+  <td><code>spark.kubernetes.executor.checkAllContainers</code></td>
+  <td>false</td>
+  <td>
+  Specify whether executor pods should be check all containers (including sidecars) or only the executor container when determining the pod status.
+  </td>
+  <td>3.1.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.submission.connectionTimeout</code></td>
@@ -1086,6 +1171,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Connection timeout in milliseconds for the kubernetes client to use for starting the driver.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.submission.requestTimeout</code></td>
@@ -1093,6 +1179,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Request timeout in milliseconds for the kubernetes client to use for starting the driver.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.connectionTimeout</code></td>
@@ -1100,6 +1187,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Connection timeout in milliseconds for the kubernetes client in driver to use when requesting executors.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.requestTimeout</code></td>
@@ -1107,6 +1195,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Request timeout in milliseconds for the kubernetes client in driver to use when requesting executors.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>  
   <td><code>spark.kubernetes.appKillPodDeletionGracePeriod</code></td>
@@ -1114,15 +1203,17 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
   Specify the grace period in seconds when deleting a Spark application using spark-submit.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.file.upload.path</code></td>
   <td>(none)</td>
   <td>
     Path to store files at the spark submit side in cluster mode. For example:
-    <code>spark.kubernetes.file.upload.path=s3a://<s3-bucket>/path</code>
+    <code>spark.kubernetes.file.upload.path=s3a://&lt;s3-bucket&gt;/path</code>
     File should specified as <code>file://path/to/file </code> or absolute path.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -1247,10 +1338,10 @@ The following affect the driver and executor containers. All other containers in
 </tr>
 <tr>
   <td>name</td>
-  <td>See description.</code></td>
+  <td>See description</td>
   <td>
     The container name will be assigned by spark ("spark-kubernetes-driver" for the driver container, and
-    "executor" for each executor container) if not defined by the pod template. If the container is defined by the
+    "spark-kubernetes-executor" for each executor container) if not defined by the pod template. If the container is defined by the
     template, the template's name will be used.
   </td>
 </tr>
diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md
index cf51620a700bc..6f6ae1c0ff264 100644
--- a/docs/running-on-mesos.md
+++ b/docs/running-on-mesos.md
@@ -371,7 +371,7 @@ See the [configuration page](configuration.html) for information on Spark config
 #### Spark Properties
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.mesos.coarse</code></td>
   <td>true</td>
@@ -380,6 +380,7 @@ See the [configuration page](configuration.html) for information on Spark config
     If set to <code>false</code>, runs over Mesos cluster in "fine-grained" sharing mode, where one Mesos task is created per Spark task.
     Detailed information in <a href="running-on-mesos.html#mesos-run-modes">'Mesos Run Modes'</a>.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.extra.cores</code></td>
@@ -391,6 +392,7 @@ See the [configuration page](configuration.html) for information on Spark config
     send it more tasks.  Use this to increase parallelism.  This
     setting is only used for Mesos coarse-grained mode.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.mesosExecutor.cores</code></td>
@@ -401,6 +403,7 @@ See the [configuration page](configuration.html) for information on Spark config
     is being run, each Mesos executor will occupy the number of cores configured here.
     The value can be a floating point number.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.docker.image</code></td>
@@ -411,6 +414,7 @@ See the [configuration page](configuration.html) for information on Spark config
     The installed path of Spark in the image can be specified with <code>spark.mesos.executor.home</code>;
     the installed path of the Mesos library can be specified with <code>spark.executorEnv.MESOS_NATIVE_JAVA_LIBRARY</code>.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.docker.forcePullImage</code></td>
@@ -419,6 +423,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Force Mesos agents to pull the image specified in <code>spark.mesos.executor.docker.image</code>.
     By default Mesos agents will not pull images they already have cached.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.docker.parameters</code></td>
@@ -429,6 +434,7 @@ See the [configuration page](configuration.html) for information on Spark config
 
     <pre>key1=val1,key2=val2,key3=val3</pre>
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.docker.volumes</code></td>
@@ -440,6 +446,7 @@ See the [configuration page](configuration.html) for information on Spark config
 
     <pre>[host_path:]container_path[:ro|:rw]</pre>
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.task.labels</code></td>
@@ -450,6 +457,7 @@ See the [configuration page](configuration.html) for information on Spark config
     list more than one.  If your label includes a colon or comma, you
     can escape it with a backslash.  Ex. key:value,key2:a\:b.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.home</code></td>
@@ -460,6 +468,7 @@ See the [configuration page](configuration.html) for information on Spark config
     them. Note that this is only relevant if a Spark binary package is not specified through
     <code>spark.executor.uri</code>.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.memoryOverhead</code></td>
@@ -469,6 +478,7 @@ See the [configuration page](configuration.html) for information on Spark config
     the overhead will be larger of either 384 or 10% of <code>spark.executor.memory</code>. If set,
     the final overhead will be this value.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.mesos.uris</code></td>
@@ -478,6 +488,7 @@ See the [configuration page](configuration.html) for information on Spark config
     when driver or executor is launched by Mesos.  This applies to
     both coarse-grained and fine-grained mode.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.principal</code></td>
@@ -485,6 +496,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Set the principal with which Spark framework will use to authenticate with Mesos.  You can also specify this via the environment variable `SPARK_MESOS_PRINCIPAL`.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.principal.file</code></td>
@@ -492,6 +504,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Set the file containing the principal with which Spark framework will use to authenticate with Mesos.  Allows specifying the principal indirectly in more security conscious deployments.  The file must be readable by the user launching the job and be UTF-8 encoded plaintext.  You can also specify this via the environment variable `SPARK_MESOS_PRINCIPAL_FILE`.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.secret</code></td>
@@ -500,6 +513,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the secret with which Spark framework will use to authenticate with Mesos. Used, for example, when
     authenticating with the registry.  You can also specify this via the environment variable `SPARK_MESOS_SECRET`.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.secret.file</code></td>
@@ -508,6 +522,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the file containing the secret with which Spark framework will use to authenticate with Mesos. Used, for example, when
     authenticating with the registry.  Allows for specifying the secret indirectly in more security conscious deployments.  The file must be readable by the user launching the job and be UTF-8 encoded plaintext.  You can also specify this via the environment variable `SPARK_MESOS_SECRET_FILE`.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.role</code></td>
@@ -516,6 +531,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the role of this Spark framework for Mesos. Roles are used in Mesos for reservations
     and resource weight sharing.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.constraints</code></td>
@@ -532,6 +548,7 @@ See the [configuration page](configuration.html) for information on Spark config
       <li>In case there is no value present as a part of the constraint any offer with the corresponding attribute will be accepted (without value check).</li>
     </ul>
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.driver.constraints</code></td>
@@ -540,6 +557,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Same as <code>spark.mesos.constraints</code> except applied to drivers when launched through the dispatcher. By default,
     all offers with sufficient resources will be accepted.
   </td>
+  <td>2.2.1</td>
 </tr>
 <tr>
   <td><code>spark.mesos.containerizer</code></td>
@@ -550,6 +568,7 @@ See the [configuration page](configuration.html) for information on Spark config
     containerizers for docker: the "docker" containerizer, and the preferred
     "mesos" containerizer.  Read more here: http://mesos.apache.org/documentation/latest/container-image/
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.driver.webui.url</code></td>
@@ -558,6 +577,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the Spark Mesos driver webui_url for interacting with the framework.
     If unset it will point to Spark's internal web UI.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.driver.labels</code></td>
@@ -566,8 +586,8 @@ See the [configuration page](configuration.html) for information on Spark config
     Mesos labels to add to the driver.  See <code>spark.mesos.task.labels</code>
     for formatting information.
   </td>
+  <td>2.3.0</td>
 </tr>
-
 <tr>
   <td>
     <code>spark.mesos.driver.secret.values</code>,
@@ -616,8 +636,8 @@ See the [configuration page](configuration.html) for information on Spark config
       <pre>spark.mesos.driver.secret.names=password1,password2</pre>
     </p>
   </td>
+  <td>2.3.0</td>
 </tr>
-
 <tr>
   <td>
     <code>spark.mesos.driver.secret.envkeys</code>,
@@ -670,8 +690,8 @@ See the [configuration page](configuration.html) for information on Spark config
       <pre>spark.mesos.driver.secret.filenames=pwdfile1,pwdfile2</pre>
     </p>
   </td>
+  <td>2.3.0</td>
 </tr>
-
 <tr>
   <td><code>spark.mesos.driverEnv.[EnvironmentVariableName]</code></td>
   <td><code>(none)</code></td>
@@ -681,6 +701,7 @@ See the [configuration page](configuration.html) for information on Spark config
     driver process. The user can specify multiple of these to set
     multiple environment variables.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.dispatcher.webui.url</code></td>
@@ -689,7 +710,8 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the Spark Mesos dispatcher webui_url for interacting with the framework.
     If unset it will point to Spark's internal web UI.
   </td>
-  </tr>
+  <td>2.0.0</td>
+</tr>
 <tr>
   <td><code>spark.mesos.dispatcher.driverDefault.[PropertyName]</code></td>
   <td><code>(none)</code></td>
@@ -699,7 +721,8 @@ See the [configuration page](configuration.html) for information on Spark config
     spark.mesos.dispatcher.driverProperty.spark.executor.memory=32g
     results in the executors for all drivers submitted in cluster mode
     to run in 32g containers.
-</td>
+  </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.dispatcher.historyServer.url</code></td>
@@ -709,6 +732,7 @@ See the [configuration page](configuration.html) for information on Spark config
     server</a>.  The dispatcher will then link each driver to its entry
     in the history server.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.gpus.max</code></td>
@@ -717,7 +741,8 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the maximum number GPU resources to acquire for this job. Note that executors will still launch when no GPU resources are found
     since this configuration is just an upper limit and not a guaranteed amount.
   </td>
-  </tr>
+  <td>2.1.0</td>
+</tr>
 <tr>
   <td><code>spark.mesos.network.name</code></td>
   <td><code>(none)</code></td>
@@ -728,6 +753,7 @@ See the [configuration page](configuration.html) for information on Spark config
     <a href="http://mesos.apache.org/documentation/latest/cni/">the Mesos CNI docs</a>
     for more details.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.network.labels</code></td>
@@ -742,6 +768,7 @@ See the [configuration page](configuration.html) for information on Spark config
     <a href="http://mesos.apache.org/documentation/latest/cni/#mesos-meta-data-to-cni-plugins">the Mesos CNI docs</a>
     for more details.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.fetcherCache.enable</code></td>
@@ -752,6 +779,7 @@ See the [configuration page](configuration.html) for information on Spark config
     href="http://mesos.apache.org/documentation/latest/fetcher/">Mesos
     Fetcher Cache</a>
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.driver.failoverTimeout</code></td>
@@ -763,6 +791,7 @@ See the [configuration page](configuration.html) for information on Spark config
     executors. The default value is zero, meaning no timeout: if the 
     driver disconnects, the master immediately tears down the framework.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.rejectOfferDuration</code></td>
@@ -772,6 +801,7 @@ See the [configuration page](configuration.html) for information on Spark config
     `spark.mesos.rejectOfferDurationForUnmetConstraints`,
     `spark.mesos.rejectOfferDurationForReachedMaxCores`
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.rejectOfferDurationForUnmetConstraints</code></td>
@@ -779,6 +809,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Time to consider unused resources refused with unmet constraints
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.rejectOfferDurationForReachedMaxCores</code></td>
@@ -787,6 +818,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Time to consider unused resources refused when maximum number of cores
     <code>spark.cores.max</code> is reached
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.appJar.local.resolution.mode</code></td>
@@ -799,6 +831,7 @@ See the [configuration page](configuration.html) for information on Spark config
     If the value is `container` then spark submit in the container will use the jar in the container's path:
     `/path/to/jar`.
   </td>
+  <td>2.4.0</td>
 </tr>
 </table>
 
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index ab4e96329a69b..7a03c91cd6320 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -60,7 +60,7 @@ For example:
         examples/jars/spark-examples*.jar \
         10
 
-The above starts a YARN client program which starts the default Application Master. Then SparkPi will be run as a child thread of Application Master. The client will periodically poll the Application Master for status updates and display them in the console. The client will exit once your application has finished running.  Refer to the "Debugging your Application" section below for how to see driver and executor logs.
+The above starts a YARN client program which starts the default Application Master. Then SparkPi will be run as a child thread of Application Master. The client will periodically poll the Application Master for status updates and display them in the console. The client will exit once your application has finished running.  Refer to the [Debugging your Application](#debugging-your-application) section below for how to see driver and executor logs.
 
 To launch a Spark application in `client` mode, do the same, but replace `cluster` with `client`. The following shows how you can run `spark-shell` in `client` mode:
 
@@ -84,7 +84,7 @@ Running Spark on YARN requires a binary distribution of Spark which is built wit
 Binary distributions can be downloaded from the [downloads page](https://spark.apache.org/downloads.html) of the project website.
 To build Spark yourself, refer to [Building Spark](building-spark.html).
 
-To make Spark runtime jars accessible from YARN side, you can specify `spark.yarn.archive` or `spark.yarn.jars`. For details please refer to [Spark Properties](running-on-yarn.html#spark-properties). If neither `spark.yarn.archive` nor `spark.yarn.jars` is specified, Spark will create a zip file with all jars under `$SPARK_HOME/jars` and upload it to the distributed cache.
+To make Spark runtime jars accessible from YARN side, you can specify `spark.yarn.archive` or `spark.yarn.jars`. For details please refer to [Spark Properties](#spark-properties). If neither `spark.yarn.archive` nor `spark.yarn.jars` is specified, Spark will create a zip file with all jars under `$SPARK_HOME/jars` and upload it to the distributed cache.
 
 # Configuration
 
@@ -130,7 +130,7 @@ To use a custom metrics.properties for the application master and executors, upd
 #### Spark Properties
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.yarn.am.memory</code></td>
   <td><code>512m</code></td>
@@ -140,6 +140,7 @@ To use a custom metrics.properties for the application master and executors, upd
     <p/>
     Use lower-case suffixes, e.g. <code>k</code>, <code>m</code>, <code>g</code>, <code>t</code>, and <code>p</code>, for kibi-, mebi-, gibi-, tebi-, and pebibytes, respectively.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.resource.{resource-type}.amount</code></td>
@@ -153,6 +154,7 @@ To use a custom metrics.properties for the application master and executors, upd
     Example: 
     To request GPU resources from YARN, use: <code>spark.yarn.am.resource.yarn.io/gpu.amount</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.driver.resource.{resource-type}.amount</code></td>
@@ -165,18 +167,20 @@ To use a custom metrics.properties for the application master and executors, upd
     Example: 
     To request GPU resources from YARN, use: <code>spark.yarn.driver.resource.yarn.io/gpu.amount</code>
   </td>
+  <td>3.0.0</td> 
 </tr>
 <tr>
   <td><code>spark.yarn.executor.resource.{resource-type}.amount</code></td>
   <td><code>(none)</code></td>
- <td>
-     Amount of resource to use per executor process.
-     Please note that this feature can be used only with YARN 3.0+
-     For reference, see YARN Resource Model documentation: https://hadoop.apache.org/docs/r3.0.1/hadoop-yarn/hadoop-yarn-site/ResourceModel.html
-     <p/>
-     Example: 
-     To request GPU resources from YARN, use: <code>spark.yarn.executor.resource.yarn.io/gpu.amount</code>
- </td>
+  <td>
+    Amount of resource to use per executor process.
+    Please note that this feature can be used only with YARN 3.0+
+    For reference, see YARN Resource Model documentation: https://hadoop.apache.org/docs/r3.0.1/hadoop-yarn/hadoop-yarn-site/ResourceModel.html
+    <p/>
+    Example: 
+    To request GPU resources from YARN, use: <code>spark.yarn.executor.resource.yarn.io/gpu.amount</code>
+  </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.cores</code></td>
@@ -185,6 +189,7 @@ To use a custom metrics.properties for the application master and executors, upd
     Number of cores to use for the YARN Application Master in client mode.
     In cluster mode, use <code>spark.driver.cores</code> instead.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.waitTime</code></td>
@@ -193,6 +198,7 @@ To use a custom metrics.properties for the application master and executors, upd
     Only used in <code>cluster</code> mode. Time for the YARN Application Master to wait for the
     SparkContext to be initialized.
   </td>
+ <td>1.3.0</td> 
 </tr>
 <tr>
   <td><code>spark.yarn.submit.file.replication</code></td>
@@ -200,6 +206,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     HDFS replication level for the files uploaded into HDFS for the application. These include things like the Spark jar, the app jar, and any distributed cache files/archives.
   </td>
+  <td>0.8.1</td>
 </tr>
 <tr>
   <td><code>spark.yarn.stagingDir</code></td>
@@ -207,6 +214,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Staging directory used while submitting applications.
   </td>
+ <td>2.0.0</td> 
 </tr>
 <tr>
   <td><code>spark.yarn.preserve.staging.files</code></td>
@@ -214,6 +222,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Set to <code>true</code> to preserve the staged files (Spark jar, app jar, distributed cache files) at the end of the job rather than delete them.
   </td>
+  <td>1.1.0</td> 
 </tr>
 <tr>
   <td><code>spark.yarn.scheduler.heartbeat.interval-ms</code></td>
@@ -223,6 +232,7 @@ To use a custom metrics.properties for the application master and executors, upd
     The value is capped at half the value of YARN's configuration for the expiry interval, i.e.
     <code>yarn.am.liveness-monitor.expiry-interval-ms</code>.
   </td>
+  <td>0.8.1</td>
 </tr>
 <tr>
   <td><code>spark.yarn.scheduler.initial-allocation.interval</code></td>
@@ -234,6 +244,7 @@ To use a custom metrics.properties for the application master and executors, upd
     successive eager heartbeats if pending containers still exist, until
     <code>spark.yarn.scheduler.heartbeat.interval-ms</code> is reached.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.max.executor.failures</code></td>
@@ -241,6 +252,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     The maximum number of executor failures before failing the application.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.historyServer.address</code></td>
@@ -249,6 +261,7 @@ To use a custom metrics.properties for the application master and executors, upd
     The address of the Spark history server, e.g. <code>host.com:18080</code>. The address should not contain a scheme (<code>http://</code>). Defaults to not being set since the history server is an optional service. This address is given to the YARN ResourceManager when the Spark application finishes to link the application from the ResourceManager UI to the Spark history server UI.
     For this property, YARN properties can be used as variables, and these are substituted by Spark at runtime. For example, if the Spark history server runs on the same node as the YARN ResourceManager, it can be set to <code>${hadoopconf-yarn.resourcemanager.hostname}:18080</code>.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.dist.archives</code></td>
@@ -256,6 +269,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Comma separated list of archives to be extracted into the working directory of each executor.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.dist.files</code></td>
@@ -263,6 +277,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Comma-separated list of files to be placed in the working directory of each executor.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.dist.jars</code></td>
@@ -270,6 +285,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Comma-separated list of jars to be placed in the working directory of each executor.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.dist.forceDownloadSchemes</code></td>
@@ -280,6 +296,7 @@ To use a custom metrics.properties for the application master and executors, upd
     support schemes that are supported by Spark, like http, https and ftp, or jars required to be in the
     local YARN client's classpath. Wildcard '*' is denoted to download resources for all the schemes.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.instances</code></td>
@@ -287,6 +304,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     The number of executors for static allocation. With <code>spark.dynamicAllocation.enabled</code>, the initial set of executors will be at least this large.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.memoryOverhead</code></td>
@@ -294,6 +312,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Same as <code>spark.driver.memoryOverhead</code>, but for the YARN Application Master in client mode.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.queue</code></td>
@@ -301,6 +320,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     The name of the YARN queue to which the application is submitted.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.jars</code></td>
@@ -312,6 +332,7 @@ To use a custom metrics.properties for the application master and executors, upd
     need to be distributed each time an application runs. To point to jars on HDFS, for example,
     set this configuration to <code>hdfs:///some/path</code>. Globs are allowed.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.archive</code></td>
@@ -323,6 +344,7 @@ To use a custom metrics.properties for the application master and executors, upd
     Like with the previous option, the archive can also be hosted on HDFS to speed up file
     distribution.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.appMasterEnv.[EnvironmentVariableName]</code></td>
@@ -334,6 +356,7 @@ To use a custom metrics.properties for the application master and executors, upd
      the environment of the Spark driver and in <code>client</code> mode it only controls
      the environment of the executor launcher.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.containerLauncherMaxThreads</code></td>
@@ -341,6 +364,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     The maximum number of threads to use in the YARN Application Master for launching executor containers.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.extraJavaOptions</code></td>
@@ -351,6 +375,7 @@ To use a custom metrics.properties for the application master and executors, upd
   to set maximum heap size (-Xmx) settings with this option. Maximum heap size settings can be set
   with <code>spark.yarn.am.memory</code>
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.extraLibraryPath</code></td>
@@ -358,6 +383,18 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Set a special library path to use when launching the YARN Application Master in client mode.
   </td>
+  <td>1.4.0</td>
+</tr>
+<tr>
+  <td><code>spark.yarn.populateHadoopClasspath</code></td>
+  <td>true</td>
+  <td>
+    Whether to populate Hadoop classpath from <code>yarn.application.classpath</code> and
+    <code>mapreduce.application.classpath</code> Note that if this is set to <code>false</code>, 
+    it requires a <code>with-Hadoop</code> Spark distribution that bundles Hadoop runtime or
+    user has to provide a Hadoop installation separately.
+  </td>
+  <td>2.4.6</td>
 </tr>
 <tr>
   <td><code>spark.yarn.maxAppAttempts</code></td>
@@ -366,6 +403,7 @@ To use a custom metrics.properties for the application master and executors, upd
   The maximum number of attempts that will be made to submit the application.
   It should be no larger than the global number of max attempts in the YARN configuration.
   </td>
+  <td>1.3.0</td> 
 </tr>
 <tr>
   <td><code>spark.yarn.am.attemptFailuresValidityInterval</code></td>
@@ -375,6 +413,7 @@ To use a custom metrics.properties for the application master and executors, upd
   If the AM has been running for at least the defined interval, the AM failure count will be reset.
   This feature is not enabled if not configured.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.executor.failuresValidityInterval</code></td>
@@ -383,6 +422,7 @@ To use a custom metrics.properties for the application master and executors, upd
   Defines the validity interval for executor failure tracking.
   Executor failures which are older than the validity interval will be ignored.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.submit.waitAppCompletion</code></td>
@@ -392,6 +432,7 @@ To use a custom metrics.properties for the application master and executors, upd
   If set to <code>true</code>, the client process will stay alive reporting the application's status.
   Otherwise, the client process will exit after submission.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.nodeLabelExpression</code></td>
@@ -401,6 +442,7 @@ To use a custom metrics.properties for the application master and executors, upd
   Only versions of YARN greater than or equal to 2.6 support node label expressions, so when
   running against earlier versions, this property will be ignored.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.executor.nodeLabelExpression</code></td>
@@ -410,6 +452,7 @@ To use a custom metrics.properties for the application master and executors, upd
   Only versions of YARN greater than or equal to 2.6 support node label expressions, so when
   running against earlier versions, this property will be ignored.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.tags</code></td>
@@ -418,6 +461,7 @@ To use a custom metrics.properties for the application master and executors, upd
   Comma-separated list of strings to pass through as YARN application tags appearing
   in YARN ApplicationReports, which can be used for filtering when querying YARN apps.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.priority</code></td>
@@ -427,6 +471,7 @@ To use a custom metrics.properties for the application master and executors, upd
   integer value have a better opportunity to be activated. Currently, YARN only supports application
   priority when using FIFO ordering policy.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.config.gatewayPath</code></td>
@@ -446,6 +491,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <code>$HADOOP_HOME</code> will make sure that paths used to launch remote processes properly
   reference the local YARN configuration.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.config.replacementPath</code></td>
@@ -453,6 +499,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
   See <code>spark.yarn.config.gatewayPath</code>.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.rolledLog.includePattern</code></td>
@@ -467,6 +514,7 @@ To use a custom metrics.properties for the application master and executors, upd
   on the file name configured in the log4j configuration (like spark.log), the user should set the
   regex (spark*) to include all the log files that need to be aggregated.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.rolledLog.excludePattern</code></td>
@@ -476,6 +524,7 @@ To use a custom metrics.properties for the application master and executors, upd
   and those log files will not be aggregated in a rolling fashion. If the log file
   name matches both the include and the exclude pattern, this file will be excluded eventually.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.blacklist.executor.launch.blacklisting.enabled</code></td>
@@ -485,6 +534,7 @@ To use a custom metrics.properties for the application master and executors, upd
   The error limit for blacklisting can be configured by
   <code>spark.blacklist.application.maxFailedExecutorsPerNode</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.exclude.nodes</code></td>
@@ -492,6 +542,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
   Comma-separated list of YARN node names which are excluded from resource allocation.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.metrics.namespace</code></td>
@@ -500,6 +551,7 @@ To use a custom metrics.properties for the application master and executors, upd
   The root namespace for AM metrics reporting. 
   If it is not set then the YARN application ID is used.
   </td>
+  <td>2.4.0</td>
 </tr>
 </table>
 
@@ -508,48 +560,48 @@ To use a custom metrics.properties for the application master and executors, upd
 <table class="table">
     <tr><th>Pattern</th><th>Meaning</th></tr>
     <tr>
-      <td>{{HTTP_SCHEME}}</td>
-      <td>`http://` or `https://` according to YARN HTTP policy. (Configured via `yarn.http.policy`)</td>
+      <td>&#123;&#123;HTTP_SCHEME&#125;&#125;</td>
+      <td><code>http://</code> or <code>https://</code> according to YARN HTTP policy. (Configured via <code>yarn.http.policy</code>)</td>
     </tr>
     <tr>
-      <td>{{NM_HOST}}</td>
+      <td>&#123;&#123;NM_HOST&#125;&#125;</td>
       <td>The "host" of node where container was run.</td>
     </tr>
     <tr>
-      <td>{{NM_PORT}}</td>
+      <td>&#123;&#123;NM_PORT&#125;&#125;</td>
       <td>The "port" of node manager where container was run.</td>
     </tr>
     <tr>
-      <td>{{NM_HTTP_PORT}}</td>
+      <td>&#123;&#123;NM_HTTP_PORT&#125;&#125;</td>
       <td>The "port" of node manager's http server where container was run.</td>
     </tr>
     <tr>
-      <td>{{NM_HTTP_ADDRESS}}</td>
+      <td>&#123;&#123;NM_HTTP_ADDRESS&#125;&#125;</td>
       <td>Http URI of the node on which the container is allocated.</td>
     </tr>
     <tr>
-      <td>{{CLUSTER_ID}}</td>
-      <td>The cluster ID of Resource Manager. (Configured via `yarn.resourcemanager.cluster-id`)</td>
+      <td>&#123;&#123;CLUSTER_ID&#125;&#125;</td>
+      <td>The cluster ID of Resource Manager. (Configured via <code>yarn.resourcemanager.cluster-id</code>)</td>
     </tr>
     <tr>
-      <td>{{CONTAINER_ID}}</td>
+      <td>&#123;&#123;CONTAINER_ID&#125;&#125;</td>
       <td>The ID of container.</td>
     </tr>
     <tr>
-      <td>{{USER}}</td>
-      <td>'SPARK_USER' on system environment.</td>
+      <td>&#123;&#123;USER&#125;&#125;</td>
+      <td><code>SPARK_USER</code> on system environment.</td>
     </tr>
     <tr>
-      <td>{{FILE_NAME}}</td>
-      <td>`stdout`, `stderr`.</td>
+      <td>&#123;&#123;FILE_NAME&#125;&#125;</td>
+      <td><code>stdout</code>, <code>stderr</code>.</td>
     </tr>
 </table>
 
 For example, suppose you would like to point log url link to Job History Server directly instead of let NodeManager http server redirects it, you can configure `spark.history.custom.executor.log.url` as below:
 
- `{{HTTP_SCHEME}}<JHS_HOST>:<JHS_PORT>/jobhistory/logs/{{NM_HOST}}:{{NM_PORT}}/{{CONTAINER_ID}}/{{CONTAINER_ID}}/{{USER}}/{{FILE_NAME}}?start=-4096`
+<code>&#123;&#123;HTTP_SCHEME&#125;&#125;&lt;JHS_HOST&gt;:&lt;JHS_PORT&gt;/jobhistory/logs/&#123;&#123;NM_HOST&#125;&#125;:&#123;&#123;NM_PORT&#125;&#125;/&#123;&#123;CONTAINER_ID&#125;&#125;/&#123;&#123;CONTAINER_ID&#125;&#125;/&#123;&#123;USER&#125;&#125;/&#123;&#123;FILE_NAME&#125;&#125;?start=-4096</code>
 
- NOTE: you need to replace `<JHS_POST>` and `<JHS_PORT>` with actual value.
+NOTE: you need to replace `<JHS_POST>` and `<JHS_PORT>` with actual value.
 
 # Resource Allocation and Configuration Overview
 
@@ -583,7 +635,7 @@ staging directory of the Spark application.
 ## YARN-specific Kerberos Configuration
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.kerberos.keytab</code></td>
   <td>(none)</td>
@@ -595,6 +647,7 @@ staging directory of the Spark application.
 
   <br /> (Works also with the "local" master.)
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kerberos.principal</code></td>
@@ -605,6 +658,7 @@ staging directory of the Spark application.
 
   <br /> (Works also with the "local" master.)
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.kerberos.relogin.period</code></td>
@@ -614,6 +668,7 @@ staging directory of the Spark application.
   that is shorter than the TGT renewal period (or the TGT lifetime if TGT renewal is not enabled).
   The default value should be enough for most deployments.
   </td>
+  <td>2.3.0</td>
 </tr>
 </table>
 
diff --git a/docs/security.md b/docs/security.md
index a4556e242cffc..aef6e69382a80 100644
--- a/docs/security.md
+++ b/docs/security.md
@@ -66,11 +66,12 @@ also see their authentication secret. Access control rules should be properly se
 Kubernetes admin to ensure that Spark authentication is secure.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.authenticate</code></td>
   <td>false</td>
   <td>Whether Spark authenticates its internal connections.</td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.authenticate.secret</code></td>
@@ -78,6 +79,7 @@ Kubernetes admin to ensure that Spark authentication is secure.
   <td>
     The secret key used authentication. See above for when this configuration should be set.
   </td>
+  <td>1.0.0</td>
 </tr>
 </table>
 
@@ -85,7 +87,7 @@ Alternatively, one can mount authentication secrets using files and Kubernetes s
 the user mounts into their pods.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.authenticate.secret.file</code></td>
   <td>None</td>
@@ -94,6 +96,7 @@ the user mounts into their pods.
     contents of the file have been securely generated. This file is loaded on both the driver
     and the executors unless other settings override this (see below).
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.authenticate.secret.driver.file</code></td>
@@ -106,6 +109,7 @@ the user mounts into their pods.
     and the executors can both use files to load the secret key. Ensure that the contents of the file
     on the driver is identical to the contents of the file on the executors.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.authenticate.secret.executor.file</code></td>
@@ -118,6 +122,7 @@ the user mounts into their pods.
     and the executors can both use files to load the secret key. Ensure that the contents of the file
     on the driver is identical to the contents of the file on the executors.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -138,13 +143,14 @@ is still required when talking to shuffle services from Spark versions older tha
 The following table describes the different options available for configuring this feature.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.network.crypto.enabled</code></td>
   <td>false</td>
   <td>
     Enable AES-based RPC encryption, including the new authentication protocol added in 2.2.0.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.keyLength</code></td>
@@ -152,6 +158,7 @@ The following table describes the different options available for configuring th
   <td>
     The length in bits of the encryption key to generate. Valid values are 128, 192 and 256.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.keyFactoryAlgorithm</code></td>
@@ -160,6 +167,7 @@ The following table describes the different options available for configuring th
     The key factory algorithm to use when generating encryption keys. Should be one of the
     algorithms supported by the javax.crypto.SecretKeyFactory class in the JRE being used.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.config.*</code></td>
@@ -169,6 +177,7 @@ The following table describes the different options available for configuring th
     use. The config name should be the name of commons-crypto configuration without the
     <code>commons.crypto</code> prefix.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.saslFallback</code></td>
@@ -179,6 +188,7 @@ The following table describes the different options available for configuring th
     do not support the internal Spark authentication protocol. On the shuffle service side,
     disabling this feature will block older clients from authenticating.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.authenticate.enableSaslEncryption</code></td>
@@ -186,6 +196,7 @@ The following table describes the different options available for configuring th
   <td>
     Enable SASL-based encrypted communication.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.network.sasl.serverAlwaysEncrypt</code></td>
@@ -194,6 +205,7 @@ The following table describes the different options available for configuring th
     Disable unencrypted connections for ports using SASL authentication. This will deny connections
     from clients that have authentication enabled, but do not request SASL-based encryption.
   </td>
+  <td>1.4.0</td>
 </tr>
 </table>
 
@@ -208,7 +220,7 @@ encrypting output data generated by applications with APIs such as `saveAsHadoop
 The following settings cover enabling encryption for data written to disk:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.io.encryption.enabled</code></td>
   <td>false</td>
@@ -216,6 +228,7 @@ The following settings cover enabling encryption for data written to disk:
     Enable local disk I/O encryption. Currently supported by all modes except Mesos. It's strongly
     recommended that RPC encryption be enabled when using this feature.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.io.encryption.keySizeBits</code></td>
@@ -223,6 +236,7 @@ The following settings cover enabling encryption for data written to disk:
   <td>
     IO encryption key size in bits. Supported values are 128, 192 and 256.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.io.encryption.keygen.algorithm</code></td>
@@ -232,6 +246,7 @@ The following settings cover enabling encryption for data written to disk:
     described in the KeyGenerator section of the Java Cryptography Architecture Standard Algorithm
     Name Documentation.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.io.encryption.commons.config.*</code></td>
@@ -241,6 +256,7 @@ The following settings cover enabling encryption for data written to disk:
     use. The config name should be the name of commons-crypto configuration without the
     <code>commons.crypto</code> prefix.
   </td>
+  <td>2.1.0</td>
 </tr>
 </table>
 
@@ -272,7 +288,7 @@ below.
 The following options control the authentication of Web UIs:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.ui.filters</code></td>
   <td>None</td>
@@ -280,6 +296,7 @@ The following options control the authentication of Web UIs:
     See the <a href="configuration.html#spark-ui">Spark UI</a> configuration for how to configure
     filters.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.acls.enable</code></td>
@@ -289,6 +306,7 @@ The following options control the authentication of Web UIs:
     permissions to view or modify the application. Note this requires the user to be authenticated,
     so if no authentication filter is installed, this option does not do anything.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.admin.acls</code></td>
@@ -296,6 +314,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of users that have view and modify access to the Spark application.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.admin.acls.groups</code></td>
@@ -303,6 +322,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of groups that have view and modify access to the Spark application.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.modify.acls</code></td>
@@ -310,6 +330,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of users that have modify access to the Spark application.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.modify.acls.groups</code></td>
@@ -317,6 +338,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of groups that have modify access to the Spark application.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.view.acls</code></td>
@@ -324,6 +346,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of users that have view access to the Spark application.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.view.acls.groups</code></td>
@@ -331,6 +354,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of groups that have view access to the Spark application.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.user.groups.mapping</code></td>
@@ -347,6 +371,7 @@ The following options control the authentication of Web UIs:
     Windows environment is currently <b>not</b> supported. However, a new platform/protocol can
     be supported by implementing the trait mentioned above.
   </td>
+  <td>2.0.0</td>
 </tr>
 </table>
 
@@ -361,7 +386,7 @@ servlet filters.
 To enable authorization in the SHS, a few extra options are used:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.history.ui.acls.enable</code></td>
   <td>false</td>
@@ -375,6 +400,7 @@ To enable authorization in the SHS, a few extra options are used:
     If disabled, no access control checks are made for any application UIs available through
     the history server.
   </td>
+  <td>1.0.1</td>
 </tr>
 <tr>
   <td><code>spark.history.ui.admin.acls</code></td>
@@ -383,6 +409,7 @@ To enable authorization in the SHS, a few extra options are used:
     Comma separated list of users that have view access to all the Spark applications in history
     server.
   </td>
+  <td>2.1.1</td>
 </tr>
 <tr>
   <td><code>spark.history.ui.admin.acls.groups</code></td>
@@ -391,6 +418,7 @@ To enable authorization in the SHS, a few extra options are used:
     Comma separated list of groups that have view access to all the Spark applications in history
     server.
   </td>
+  <td>2.1.1</td>
 </tr>
 </table>
 
@@ -606,7 +634,7 @@ Apache Spark can be configured to include HTTP headers to aid in preventing Cros
 Security.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.ui.xXssProtection</code></td>
   <td><code>1; mode=block</code></td>
@@ -621,6 +649,7 @@ Security.
         of the page if an attack is detected.)</li>
     </ul>
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.xContentTypeOptions.enabled</code></td>
@@ -628,7 +657,8 @@ Security.
   <td>
     When enabled, X-Content-Type-Options HTTP response header will be set to "nosniff".
   </td>
-  </tr>
+  <td>2.3.0</td>
+</tr>
 <tr>
   <td><code>spark.ui.strictTransportSecurity</code></td>
   <td>None</td>
@@ -642,6 +672,7 @@ Security.
       <li><code>max-age=&lt;expire-time&gt;; preload</code></li>
     </ul>
   </td>
+  <td>2.3.0</td>
 </tr>
 </table>
 
@@ -782,16 +813,17 @@ deployment-specific page for more information.
 The following options provides finer-grained control for this feature:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.security.credentials.${service}.enabled</code></td>
   <td><code>true</code></td>
   <td>
-  Controls whether to obtain credentials for services when security is enabled.
-  By default, credentials for all supported services are retrieved when those services are
-  configured, but it's possible to disable that behavior if it somehow conflicts with the
-  application being run.
+    Controls whether to obtain credentials for services when security is enabled.
+    By default, credentials for all supported services are retrieved when those services are
+    configured, but it's possible to disable that behavior if it somehow conflicts with the
+    application being run.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kerberos.access.hadoopFileSystems</code></td>
@@ -804,6 +836,7 @@ The following options provides finer-grained control for this feature:
     or in a trusted realm). Spark acquires security tokens for each of the filesystems so that
     the Spark application can access those remote Hadoop filesystems.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 1264951a2f270..566f081bcb4ff 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -185,13 +185,14 @@ You can optionally configure the cluster further by setting environment variable
 SPARK_MASTER_OPTS supports the following system properties:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.deploy.retainedApplications</code></td>
   <td>200</td>
   <td>
     The maximum number of completed applications to display. Older applications will be dropped from the UI to maintain this limit.<br/>
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.deploy.retainedDrivers</code></td>
@@ -199,6 +200,7 @@ SPARK_MASTER_OPTS supports the following system properties:
   <td>
    The maximum number of completed drivers to display. Older drivers will be dropped from the UI to maintain this limit.<br/>
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.deploy.spreadOut</code></td>
@@ -208,6 +210,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     to consolidate them onto as few nodes as possible. Spreading out is usually better for
     data locality in HDFS, but consolidating is more efficient for compute-intensive workloads. <br/>
   </td>
+  <td>0.6.1</td>
 </tr>
 <tr>
   <td><code>spark.deploy.defaultCores</code></td>
@@ -219,6 +222,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     Set this lower on a shared cluster to prevent users from grabbing
     the whole cluster by default. <br/>
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.deploy.maxExecutorRetries</code></td>
@@ -234,6 +238,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     <code>-1</code>.
     <br/>
   </td>
+  <td>1.6.3</td>
 </tr>
 <tr>
   <td><code>spark.worker.timeout</code></td>
@@ -242,6 +247,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     Number of seconds after which the standalone deploy master considers a worker lost if it
     receives no heartbeats.
   </td>
+  <td>0.6.2</td>
 </tr>
 <tr>
   <td><code>spark.worker.resource.{resourceName}.amount</code></td>
@@ -249,6 +255,7 @@ SPARK_MASTER_OPTS supports the following system properties:
   <td>
     Amount of a particular resource to use on the worker.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.resource.{resourceName}.discoveryScript</code></td>
@@ -256,9 +263,8 @@ SPARK_MASTER_OPTS supports the following system properties:
   <td>
     Path to resource discovery script, which is used to find a particular resource while worker starting up.
     And the output of the script should be formatted like the <code>ResourceInformation</code> class.
-    When <code>spark.resources.coordinate.enable</code> is off, the discovery script must assign different
-    resources for workers and drivers in client mode that run on the same host to avoid resource conflict.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.resourcesFile</code></td>
@@ -266,20 +272,19 @@ SPARK_MASTER_OPTS supports the following system properties:
   <td>
     Path to resources file which is used to find various resources while worker starting up.
     The content of resources file should be formatted like <code>
-    [[{"id":{"componentName": "spark.worker","resourceName":"gpu"},"addresses":["0","1","2"]}]]</code>.
-    When <code>spark.resources.coordinate.enable</code> is off, resources file must assign different
-    resources for workers and drivers in client mode that run on the same host to avoid resource conflict.
+    [{"id":{"componentName": "spark.worker","resourceName":"gpu"},"addresses":["0","1","2"]}]</code>.
     If a particular resource is not found in the resources file, the discovery script would be used to
     find that resource. If the discovery script also does not find the resources, the worker will fail
     to start up.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
 SPARK_WORKER_OPTS supports the following system properties:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.worker.cleanup.enabled</code></td>
   <td>false</td>
@@ -288,6 +293,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     mode, as YARN works differently. Only the directories of stopped applications are cleaned up.
     This should be enabled if spark.shuffle.service.db.enabled is "true"
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.cleanup.interval</code></td>
@@ -296,6 +302,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     Controls the interval, in seconds, at which the worker cleans up old application work dirs
     on the local machine.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.cleanup.appDataTtl</code></td>
@@ -306,6 +313,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     downloaded to each application work dir.  Over time, the work dirs can quickly fill up disk space,
     especially if you run jobs very frequently.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.service.db.enabled</code></td>
@@ -316,6 +324,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     enabled).  You should also enable <code>spark.worker.cleanup.enabled</code>, to ensure that the state
     eventually gets cleaned up.  This config may be removed in the future.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.storage.cleanupFilesAfterExecutorExit</code></td>
@@ -326,8 +335,9 @@ SPARK_WORKER_OPTS supports the following system properties:
     overlap with `spark.worker.cleanup.enabled`, as this enables cleanup of non-shuffle files in
     local directories of a dead executor, while `spark.worker.cleanup.enabled` enables cleanup of
     all files/subdirectories of a stopped and timeout application.
-    This only affects Standalone mode, support of other cluster manangers can be added in the future.
+    This only affects Standalone mode, support of other cluster managers can be added in the future.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.ui.compressedLogFileLengthCacheSize</code></td>
@@ -337,6 +347,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     Spark caches the uncompressed file size of compressed log files. This property controls the cache
     size.
   </td>
+  <td>2.0.2</td>
 </tr>
 </table>
 
@@ -346,9 +357,9 @@ Please make sure to have read the Custom Resource Scheduling and Configuration O
 
 Spark Standalone has 2 parts, the first is configuring the resources for the Worker, the second is the resource allocation for a specific application.
 
-The user must configure the Workers to have a set of resources available so that it can assign them out to Executors. The <code>spark.worker.resource.{resourceName}.amount</code> is used to control the amount of each resource the worker has allocated. The user must also specify either <code>spark.worker.resourcesFile</code> or <code>spark.worker.resource.{resourceName}.discoveryScript</code> to specify how the Worker discovers the resources its assigned. See the descriptions above for each of those to see which method works best for your setup. Please take note of <code>spark.resources.coordinate.enable</code> as it indicates whether Spark should handle coordinating resources or if the user has made sure each Worker has separate resources. Also note that if using the resources coordination <code>spark.resources.dir</code> can be used to specify the directory used to do that coordination.
+The user must configure the Workers to have a set of resources available so that it can assign them out to Executors. The <code>spark.worker.resource.{resourceName}.amount</code> is used to control the amount of each resource the worker has allocated. The user must also specify either <code>spark.worker.resourcesFile</code> or <code>spark.worker.resource.{resourceName}.discoveryScript</code> to specify how the Worker discovers the resources its assigned. See the descriptions above for each of those to see which method works best for your setup.
 
-The second part is running an application on Spark Standalone. The only special case from the standard Spark resource configs is when you are running the Driver in client mode. For a Driver in client mode, the user can specify the resources it uses via <code>spark.driver.resourcesfile</code> or <code>spark.driver.resources.{resourceName}.discoveryScript</code>. If the Driver is running on the same host as other Drivers or Workers there are 2 ways to make sure the they don't use the same resources. The user can either configure <code>spark.resources.coordinate.enable</code> on and give all the Driver/Workers the same set or resources and Spark will handle make sure each Driver/Worker has separate resources, or the user can make sure the resources file or discovery script only returns resources the do not conflict with other Drivers or Workers running on the same node.
+The second part is running an application on Spark Standalone. The only special case from the standard Spark resource configs is when you are running the Driver in client mode. For a Driver in client mode, the user can specify the resources it uses via <code>spark.driver.resourcesFile</code> or <code>spark.driver.resource.{resourceName}.discoveryScript</code>. If the Driver is running on the same host as other Drivers, please make sure the resources file or discovery script only returns resources that do not conflict with other Drivers running on the same node.
 
 Note, the user does not need to specify a discovery script when submitting an application as the Worker will start each Executor with the resources it allocates to it.
 
@@ -488,14 +499,16 @@ ZooKeeper is the best way to go for production-level high availability, but if y
 In order to enable this recovery mode, you can set SPARK_DAEMON_JAVA_OPTS in spark-env using this configuration:
 
 <table class="table">
-  <tr><th style="width:21%">System property</th><th>Meaning</th></tr>
+  <tr><th style="width:21%">System property</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.deploy.recoveryMode</code></td>
     <td>Set to FILESYSTEM to enable single-node recovery mode (default: NONE).</td>
+    <td>0.8.1</td>
   </tr>
   <tr>
     <td><code>spark.deploy.recoveryDirectory</code></td>
     <td>The directory in which Spark will store recovery state, accessible from the Master's perspective.</td>
+    <td>0.8.1</td>
   </tr>
 </table>
 
diff --git a/docs/sparkr-migration-guide.md b/docs/sparkr-migration-guide.md
index 6fbc4c03aefc1..32836cdac53d5 100644
--- a/docs/sparkr-migration-guide.md
+++ b/docs/sparkr-migration-guide.md
@@ -28,8 +28,7 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
 
 ## Upgrading from SparkR 2.4 to 3.0
 
- - The deprecated methods `sparkR.init`, `sparkRSQL.init`, `sparkRHive.init` have been removed. Use `sparkR.session` instead.
- - The deprecated methods `parquetFile`, `saveAsParquetFile`, `jsonFile`, `registerTempTable`, `createExternalTable`, and `dropTempTable` have been removed. Use `read.parquet`, `write.parquet`, `read.json`, `createOrReplaceTempView`, `createTable`, `dropTempView`, `union` instead.
+ - The deprecated methods `parquetFile`, `saveAsParquetFile`, `jsonFile`, `jsonRDD` have been removed. Use `read.parquet`, `write.parquet`, `read.json` instead.
 
 ## Upgrading from SparkR 2.3 to 2.4
 
diff --git a/docs/sparkr.md b/docs/sparkr.md
index 24fa3b4feac19..15578fdd41902 100644
--- a/docs/sparkr.md
+++ b/docs/sparkr.md
@@ -663,32 +663,27 @@ Apache Arrow is an in-memory columnar data format that is used in Spark to effic
 
 ## Ensure Arrow Installed
 
-Arrow R library is available on CRAN as of [ARROW-3204](https://issues.apache.org/jira/browse/ARROW-3204). It can be installed as below.
+Arrow R library is available on CRAN and it can be installed as below.
 
 ```bash
 Rscript -e 'install.packages("arrow", repos="https://cloud.r-project.org/")'
 ```
+Please refer [the official documentation of Apache Arrow](https://arrow.apache.org/docs/r/) for more detials.
 
-If you need to install old versions, it should be installed directly from Github. You can use `remotes::install_github` as below.
-
-```bash
-Rscript -e 'remotes::install_github("apache/arrow@apache-arrow-0.12.1", subdir = "r")'
-```
-
-`apache-arrow-0.12.1` is a version tag that can be checked in [Arrow at Github](https://github.com/apache/arrow/releases). You must ensure that Arrow R package is installed and available on all cluster nodes.
-The current supported minimum version is 0.12.1; however, this might change between the minor releases since Arrow optimization in SparkR is experimental.
+Note that you must ensure that Arrow R package is installed and available on all cluster nodes.
+The current supported minimum version is 0.15.1; however, this might change between the minor releases since Arrow optimization in SparkR is experimental.
 
 ## Enabling for Conversion to/from R DataFrame, `dapply` and `gapply`
 
 Arrow optimization is available when converting a Spark DataFrame to an R DataFrame using the call `collect(spark_df)`,
 when creating a Spark DataFrame from an R DataFrame with `createDataFrame(r_df)`, when applying an R native function to each partition
 via `dapply(...)` and when applying an R native function to grouped data via `gapply(...)`.
-To use Arrow when executing these calls, users need to first set the Spark configuration ‘spark.sql.execution.arrow.sparkr.enabled’
-to ‘true’. This is disabled by default.
+To use Arrow when executing these, users need to set the Spark configuration ‘spark.sql.execution.arrow.sparkr.enabled’
+to ‘true’ first. This is disabled by default.
 
-In addition, optimizations enabled by ‘spark.sql.execution.arrow.sparkr.enabled’ could fallback automatically to non-Arrow optimization
-implementation if an error occurs before the actual computation within Spark during converting a Spark DataFrame to/from an R
-DataFrame.
+Whether the optimization is enabled or not, SparkR produces the same results. In addition, the conversion
+between Spark DataFrame and R DataFrame falls back automatically to non-Arrow optimization implementation
+when the optimization fails for any reasons before the actual computation.
 
 <div data-lang="r" markdown="1">
 {% highlight r %}
@@ -715,9 +710,9 @@ collect(gapply(spark_df,
 {% endhighlight %}
 </div>
 
-Using the above optimizations with Arrow will produce the same results as when Arrow is not enabled. Note that even with Arrow,
-`collect(spark_df)` results in the collection of all records in the DataFrame to the driver program and should be done on a
-small subset of the data.
+Note that even with Arrow, `collect(spark_df)` results in the collection of all records in the DataFrame to
+the driver program and should be done on a small subset of the data. In addition, the specified output schema
+in `gapply(...)` and `dapply(...)` should be matched to the R DataFrame's returned by the given function.
 
 ## Supported SQL Types
 
diff --git a/docs/sql-data-sources-avro.md b/docs/sql-data-sources-avro.md
index b0076878e02da..69b165ed28bae 100644
--- a/docs/sql-data-sources-avro.md
+++ b/docs/sql-data-sources-avro.md
@@ -88,7 +88,7 @@ Kafka key-value record will be augmented with some metadata, such as the ingesti
 * If the "value" field that contains your data is in Avro, you could use `from_avro()` to extract your data, enrich it, clean it, and then push it downstream to Kafka again or write it out to a file.
 * `to_avro()` can be used to turn structs into Avro records. This method is particularly useful when you would like to re-encode multiple columns into a single one when writing data out to Kafka.
 
-Both functions are currently only available in Scala and Java.
+Both functions are currently only available in Scala, Java, and Python.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
@@ -230,7 +230,7 @@ Data source options of Avro can be set via:
   <tr>
     <td><code>ignoreExtension</code></td>
     <td>true</td>
-    <td>The option controls ignoring of files without <code>.avro</code> extensions in read.<br> If the option is enabled, all files (with and without <code>.avro</code> extension) are loaded.<br> The option has been deprecated, and it will be removed in the future releases. Please use the general data source option <code>pathGlobFilter</code> for filtering file names.</td>
+    <td>The option controls ignoring of files without <code>.avro</code> extensions in read.<br> If the option is enabled, all files (with and without <code>.avro</code> extension) are loaded.<br> The option has been deprecated, and it will be removed in the future releases. Please use the general data source option <a href="./sql-data-sources-generic-options.html#path-global-filter">pathGlobFilter</a> for filtering file names.</td>
     <td>read</td>
   </tr>
   <tr>
@@ -258,21 +258,34 @@ Data source options of Avro can be set via:
 ## Configuration
 Configuration of Avro can be done using the `setConf` method on SparkSession or by running `SET key=value` commands using SQL.
 <table class="table">
-  <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th></tr>
+  <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th><th><b>Since Version</b></th></tr>
   <tr>
     <td>spark.sql.legacy.replaceDatabricksSparkAvro.enabled</td>
     <td>true</td>
-    <td>If it is set to true, the data source provider <code>com.databricks.spark.avro</code> is mapped to the built-in but external Avro data source module for backward compatibility.</td>
+    <td>
+      If it is set to true, the data source provider <code>com.databricks.spark.avro</code> is mapped
+      to the built-in but external Avro data source module for backward compatibility.
+    </td>
+    <td>2.4.0</td>
   </tr>
   <tr>
     <td>spark.sql.avro.compression.codec</td>
     <td>snappy</td>
-    <td>Compression codec used in writing of AVRO files. Supported codecs: uncompressed, deflate, snappy, bzip2 and xz. Default codec is snappy.</td>
+    <td>
+      Compression codec used in writing of AVRO files. Supported codecs: uncompressed, deflate,
+      snappy, bzip2 and xz. Default codec is snappy.
+    </td>
+    <td>2.4.0</td>
   </tr>
   <tr>
     <td>spark.sql.avro.deflate.level</td>
     <td>-1</td>
-    <td>Compression level for the deflate codec used in writing of AVRO files. Valid value must be in the range of from 1 to 9 inclusive or -1. The default value is -1 which corresponds to 6 level in the current implementation.</td>
+    <td>
+      Compression level for the deflate codec used in writing of AVRO files. Valid value must be in
+      the range of from 1 to 9 inclusive or -1. The default value is -1 which corresponds to 6 level
+      in the current implementation.
+    </td>
+    <td>2.4.0</td>
   </tr>
 </table>
 
diff --git a/docs/sql-data-sources-generic-options.md b/docs/sql-data-sources-generic-options.md
new file mode 100644
index 0000000000000..6bcf48235bced
--- /dev/null
+++ b/docs/sql-data-sources-generic-options.md
@@ -0,0 +1,121 @@
+---
+layout: global
+title: Generic File Source Options
+displayTitle: Generic File Source Options
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+* Table of contents
+{:toc}
+
+These generic options/configurations are effective only when using file-based sources: parquet, orc, avro, json, csv, text.
+
+Please note that the hierarchy of directories used in examples below are:
+
+{% highlight text %}
+
+dir1/
+ ├── dir2/
+ │    └── file2.parquet (schema: <file: string>, content: "file2.parquet")
+ └── file1.parquet (schema: <file, string>, content: "file1.parquet")
+ └── file3.json (schema: <file, string>, content: "{'file':'corrupt.json'}")
+
+{% endhighlight %}
+
+### Ignore Corrupt Files
+
+Spark allows you to use `spark.sql.files.ignoreCorruptFiles` to ignore corrupt files while reading data
+from files. When set to true, the Spark jobs will continue to run when encountering corrupted files and
+the contents that have been read will still be returned.
+
+To ignore corrupt files while reading data files, you can use:
+
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+{% include_example ignore_corrupt_files scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+</div>
+
+<div data-lang="java"  markdown="1">
+{% include_example ignore_corrupt_files java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+</div>
+
+<div data-lang="python"  markdown="1">
+{% include_example ignore_corrupt_files python/sql/datasource.py %}
+</div>
+
+<div data-lang="r"  markdown="1">
+{% include_example ignore_corrupt_files r/RSparkSQLExample.R %}
+</div>
+</div>
+
+### Ignore Missing Files
+
+Spark allows you to use `spark.sql.files.ignoreMissingFiles` to ignore missing files while reading data
+from files. Here, missing file really means the deleted file under directory after you construct the
+`DataFrame`. When set to true, the Spark jobs will continue to run when encountering missing files and
+the contents that have been read will still be returned.
+
+### Path Global Filter
+
+`pathGlobFilter` is used to only include files with file names matching the pattern.
+The syntax follows <code>org.apache.hadoop.fs.GlobFilter</code>.
+It does not change the behavior of partition discovery.
+
+To load files with paths matching a given glob pattern while keeping the behavior of partition discovery,
+you can use:
+
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+{% include_example load_with_path_glob_filter scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+</div>
+
+<div data-lang="java"  markdown="1">
+{% include_example load_with_path_glob_filter java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+</div>
+
+<div data-lang="python"  markdown="1">
+{% include_example load_with_path_glob_filter python/sql/datasource.py %}
+</div>
+
+<div data-lang="r"  markdown="1">
+{% include_example load_with_path_glob_filter r/RSparkSQLExample.R %}
+</div>
+</div>
+
+### Recursive File Lookup
+`recursiveFileLookup` is used to recursively load files and it disables partition inferring. Its default value is `false`.
+If data source explicitly specifies the `partitionSpec` when `recursiveFileLookup` is true, exception will be thrown.
+
+To load all files recursively, you can use:
+
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+{% include_example recursive_file_lookup scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+</div>
+
+<div data-lang="java"  markdown="1">
+{% include_example recursive_file_lookup java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+</div>
+
+<div data-lang="python"  markdown="1">
+{% include_example recursive_file_lookup python/sql/datasource.py %}
+</div>
+
+<div data-lang="r"  markdown="1">
+{% include_example recursive_file_lookup r/RSparkSQLExample.R %}
+</div>
+</div>
diff --git a/docs/sql-data-sources-hive-tables.md b/docs/sql-data-sources-hive-tables.md
index 0054d466db987..ae3572c474e1a 100644
--- a/docs/sql-data-sources-hive-tables.md
+++ b/docs/sql-data-sources-hive-tables.md
@@ -124,14 +124,15 @@ will compile against built-in Hive and use those classes for internal execution
 The following options can be used to configure the version of Hive that is used to retrieve metadata:
 
 <table class="table">
-  <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+  <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.sql.hive.metastore.version</code></td>
-    <td><code>2.3.6</code></td>
+    <td><code>2.3.7</code></td>
     <td>
       Version of the Hive metastore. Available
-      options are <code>0.12.0</code> through <code>2.3.6</code> and <code>3.0.0</code> through <code>3.1.2</code>.
+      options are <code>0.12.0</code> through <code>2.3.7</code> and <code>3.0.0</code> through <code>3.1.2</code>.
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.hive.metastore.jars</code></td>
@@ -141,9 +142,9 @@ The following options can be used to configure the version of Hive that is used
       property can be one of three options:
       <ol>
         <li><code>builtin</code></li>
-        Use Hive 2.3.6, which is bundled with the Spark assembly when <code>-Phive</code> is
+        Use Hive 2.3.7, which is bundled with the Spark assembly when <code>-Phive</code> is
         enabled. When this option is chosen, <code>spark.sql.hive.metastore.version</code> must be
-        either <code>2.3.6</code> or not defined.
+        either <code>2.3.7</code> or not defined.
         <li><code>maven</code></li>
         Use Hive jars of specified version downloaded from Maven repositories. This configuration
         is not generally recommended for production deployments.
@@ -153,6 +154,7 @@ The following options can be used to configure the version of Hive that is used
         they are packaged with your application.</li>
       </ol>
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.hive.metastore.sharedPrefixes</code></td>
@@ -166,6 +168,7 @@ The following options can be used to configure the version of Hive that is used
         custom appenders that are used by log4j.
       </p>
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.hive.metastore.barrierPrefixes</code></td>
@@ -177,5 +180,6 @@ The following options can be used to configure the version of Hive that is used
         prefix that typically would be shared (i.e. <code>org.apache.spark.*</code>).
       </p>
     </td>
+    <td>1.4.0</td>
   </tr>
 </table>
diff --git a/docs/sql-data-sources-jdbc.md b/docs/sql-data-sources-jdbc.md
index b0d37b11c7117..eaee6706e5b30 100644
--- a/docs/sql-data-sources-jdbc.md
+++ b/docs/sql-data-sources-jdbc.md
@@ -23,7 +23,7 @@ license: |
 {:toc}
 
 Spark SQL also includes a data source that can read data from other databases using JDBC. This
-functionality should be preferred over using [JdbcRDD](api/scala/index.html#org.apache.spark.rdd.JdbcRDD).
+functionality should be preferred over using [JdbcRDD](api/scala/org/apache/spark/rdd/JdbcRDD.html).
 This is because the results are returned
 as a DataFrame and they can easily be processed in Spark SQL or joined with other data sources.
 The JDBC data source is also easier to use from Java or Python as it does not require the user to
@@ -217,7 +217,7 @@ the following case-insensitive options:
 {% include_example jdbc_dataset r/RSparkSQLExample.R %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
diff --git a/docs/sql-data-sources-json.md b/docs/sql-data-sources-json.md
index 4ce4897189846..0f1ca432b704a 100644
--- a/docs/sql-data-sources-json.md
+++ b/docs/sql-data-sources-json.md
@@ -77,7 +77,7 @@ For a regular multi-line JSON file, set a named parameter `multiLine` to `TRUE`.
 
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
@@ -93,4 +93,4 @@ SELECT * FROM jsonTable
 
 </div>
 
-</div>
\ No newline at end of file
+</div>
diff --git a/docs/sql-data-sources-load-save-functions.md b/docs/sql-data-sources-load-save-functions.md
index 07482137a28a3..0866f37fbdc71 100644
--- a/docs/sql-data-sources-load-save-functions.md
+++ b/docs/sql-data-sources-load-save-functions.md
@@ -102,27 +102,6 @@ To load a CSV file you can use:
 </div>
 </div>
 
-To load files with paths matching a given glob pattern while keeping the behavior of partition discovery,
-you can use:
-
-<div class="codetabs">
-<div data-lang="scala"  markdown="1">
-{% include_example load_with_path_glob_filter scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-</div>
-
-<div data-lang="java"  markdown="1">
-{% include_example load_with_path_glob_filter java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-</div>
-
-<div data-lang="python"  markdown="1">
-{% include_example load_with_path_glob_filter python/sql/datasource.py %}
-</div>
-
-<div data-lang="r"  markdown="1">
-{% include_example load_with_path_glob_filter r/RSparkSQLExample.R %}
-</div>
-</div>
-
 The extra options are also used during write operation.
 For example, you can control bloom filters and dictionary encodings for ORC data sources.
 The following ORC example will create bloom filter and use dictionary encoding only for `favorite_color`.
@@ -148,7 +127,7 @@ visit the official Apache ORC/Parquet websites.
 {% include_example manual_save_options_orc r/RSparkSQLExample.R %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 CREATE TABLE users_with_options (
@@ -278,7 +257,7 @@ Bucketing and sorting are applicable only to persistent tables:
 {% include_example write_sorting_and_bucketing python/sql/datasource.py %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
@@ -312,7 +291,7 @@ while partitioning can be used with both `save` and `saveAsTable` when using the
 {% include_example write_partitioning python/sql/datasource.py %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
@@ -344,7 +323,7 @@ It is possible to use both partitioning and bucketing for a single table:
 {% include_example write_partition_and_bucket python/sql/datasource.py %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
diff --git a/docs/sql-data-sources-orc.md b/docs/sql-data-sources-orc.md
index bddffe02602e8..4c4b3b1eee8c2 100644
--- a/docs/sql-data-sources-orc.md
+++ b/docs/sql-data-sources-orc.md
@@ -27,15 +27,25 @@ serde tables (e.g., the ones created using the clause `USING HIVE OPTIONS (fileF
 the vectorized reader is used when `spark.sql.hive.convertMetastoreOrc` is also set to `true`.
 
 <table class="table">
-  <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th></tr>
+  <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th><th><b>Since Version</b></th></tr>
   <tr>
     <td><code>spark.sql.orc.impl</code></td>
     <td><code>native</code></td>
-    <td>The name of ORC implementation. It can be one of <code>native</code> and <code>hive</code>. <code>native</code> means the native ORC support. <code>hive</code> means the ORC library in Hive.</td>
+    <td>
+      The name of ORC implementation. It can be one of <code>native</code> and <code>hive</code>.
+      <code>native</code> means the native ORC support. <code>hive</code> means the ORC library
+      in Hive.
+    </td>
+    <td>2.3.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.orc.enableVectorizedReader</code></td>
     <td><code>true</code></td>
-    <td>Enables vectorized orc decoding in <code>native</code> implementation. If <code>false</code>, a new non-vectorized ORC reader is used in <code>native</code> implementation. For <code>hive</code> implementation, this is ignored.</td>
+    <td>
+      Enables vectorized orc decoding in <code>native</code> implementation. If <code>false</code>,
+      a new non-vectorized ORC reader is used in <code>native</code> implementation.
+      For <code>hive</code> implementation, this is ignored.
+    </td>
+    <td>2.3.0</td>
   </tr>
 </table>
diff --git a/docs/sql-data-sources-parquet.md b/docs/sql-data-sources-parquet.md
index 53a1111cd8286..d34f8a698facb 100644
--- a/docs/sql-data-sources-parquet.md
+++ b/docs/sql-data-sources-parquet.md
@@ -22,7 +22,7 @@ license: |
 * Table of contents
 {:toc}
 
-[Parquet](http://parquet.io) is a columnar format that is supported by many other data processing systems.
+[Parquet](https://parquet.apache.org) is a columnar format that is supported by many other data processing systems.
 Spark SQL provides support for both reading and writing Parquet files that automatically preserves the schema
 of the original data. When reading Parquet files, all columns are automatically converted to be nullable for
 compatibility reasons.
@@ -52,7 +52,7 @@ Using the data from the above example:
 
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
@@ -242,7 +242,7 @@ refreshTable("my_table")
 
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 REFRESH TABLE my_table;
@@ -258,7 +258,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
 `SET key=value` commands using SQL.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.sql.parquet.binaryAsString</code></td>
   <td>false</td>
@@ -267,6 +267,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
     not differentiate between binary data and strings when writing out the Parquet schema. This
     flag tells Spark SQL to interpret binary data as a string to provide compatibility with these systems.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.sql.parquet.int96AsTimestamp</code></td>
@@ -275,6 +276,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
     Some Parquet-producing systems, in particular Impala and Hive, store Timestamp into INT96. This
     flag tells Spark SQL to interpret INT96 data as a timestamp to provide compatibility with these systems.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.parquet.compression.codec</code></td>
@@ -287,11 +289,13 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
     Note that <code>zstd</code> requires <code>ZStandardCodec</code> to be installed before Hadoop 2.9.0, <code>brotli</code> requires
     <code>BrotliCodec</code> to be installed.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.sql.parquet.filterPushdown</code></td>
   <td>true</td>
   <td>Enables Parquet filter push-down optimization when set to true.</td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.hive.convertMetastoreParquet</code></td>
@@ -300,6 +304,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
     When set to false, Spark SQL will use the Hive SerDe for parquet tables instead of the built in
     support.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.sql.parquet.mergeSchema</code></td>
@@ -310,6 +315,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
       schema is picked from the summary file or a random data file if no summary file is available.
     </p>
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.parquet.writeLegacyFormat</code></td>
@@ -321,5 +327,6 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
     example, decimals will be written in int-based format. If Parquet output is intended for use
     with systems that do not support this newer format, set to true.
   </td>
+  <td>1.6.0</td>
 </tr>
 </table>
diff --git a/docs/sql-data-sources.md b/docs/sql-data-sources.md
index 079c54060d15d..9396846041709 100644
--- a/docs/sql-data-sources.md
+++ b/docs/sql-data-sources.md
@@ -33,6 +33,11 @@ goes into specific options that are available for the built-in data sources.
   * [Save Modes](sql-data-sources-load-save-functions.html#save-modes)
   * [Saving to Persistent Tables](sql-data-sources-load-save-functions.html#saving-to-persistent-tables)
   * [Bucketing, Sorting and Partitioning](sql-data-sources-load-save-functions.html#bucketing-sorting-and-partitioning)
+* [Generic File Source Options](sql-data-sources-generic-options.html)
+  * [Ignore Corrupt Files](sql-data-sources-generic-options.html#ignore-corrupt-iles)
+  * [Ignore Missing Files](sql-data-sources-generic-options.html#ignore-missing-iles)
+  * [Path Global Filter](sql-data-sources-generic-options.html#path-global-filter)
+  * [Recursive File Lookup](sql-data-sources-generic-options.html#recursive-file-lookup)
 * [Parquet Files](sql-data-sources-parquet.html)
   * [Loading Data Programmatically](sql-data-sources-parquet.html#loading-data-programmatically)
   * [Partition Discovery](sql-data-sources-parquet.html#partition-discovery)
diff --git a/docs/sql-getting-started.md b/docs/sql-getting-started.md
index f637697a29ad7..5a6f18263ac6b 100644
--- a/docs/sql-getting-started.md
+++ b/docs/sql-getting-started.md
@@ -27,7 +27,7 @@ license: |
 <div class="codetabs">
 <div data-lang="scala"  markdown="1">
 
-The entry point into all functionality in Spark is the [`SparkSession`](api/scala/index.html#org.apache.spark.sql.SparkSession) class. To create a basic `SparkSession`, just use `SparkSession.builder()`:
+The entry point into all functionality in Spark is the [`SparkSession`](api/scala/org/apache/spark/sql/SparkSession.html) class. To create a basic `SparkSession`, just use `SparkSession.builder()`:
 
 {% include_example init_session scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
 </div>
@@ -104,7 +104,7 @@ As an example, the following creates a DataFrame based on the content of a JSON
 
 ## Untyped Dataset Operations (aka DataFrame Operations)
 
-DataFrames provide a domain-specific language for structured data manipulation in [Scala](api/scala/index.html#org.apache.spark.sql.Dataset), [Java](api/java/index.html?org/apache/spark/sql/Dataset.html), [Python](api/python/pyspark.sql.html#pyspark.sql.DataFrame) and [R](api/R/SparkDataFrame.html).
+DataFrames provide a domain-specific language for structured data manipulation in [Scala](api/scala/org/apache/spark/sql/Dataset.html), [Java](api/java/index.html?org/apache/spark/sql/Dataset.html), [Python](api/python/pyspark.sql.html#pyspark.sql.DataFrame) and [R](api/R/SparkDataFrame.html).
 
 As mentioned above, in Spark 2.0, DataFrames are just Dataset of `Row`s in Scala and Java API. These operations are also referred as "untyped transformations" in contrast to "typed transformations" come with strongly typed Scala/Java Datasets.
 
@@ -114,9 +114,9 @@ Here we include some basic examples of structured data processing using Datasets
 <div data-lang="scala"  markdown="1">
 {% include_example untyped_ops scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
 
-For a complete list of the types of operations that can be performed on a Dataset, refer to the [API Documentation](api/scala/index.html#org.apache.spark.sql.Dataset).
+For a complete list of the types of operations that can be performed on a Dataset, refer to the [API Documentation](api/scala/org/apache/spark/sql/Dataset.html).
 
-In addition to simple column references and expressions, Datasets also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/scala/index.html#org.apache.spark.sql.functions$).
+In addition to simple column references and expressions, Datasets also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/scala/org/apache/spark/sql/functions$.html).
 </div>
 
 <div data-lang="java" markdown="1">
@@ -205,7 +205,7 @@ refer it, e.g. `SELECT * FROM global_temp.view1`.
 {% include_example global_temp_view python/sql/basic.py %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
@@ -222,7 +222,7 @@ SELECT * FROM global_temp.temp_view
 ## Creating Datasets
 
 Datasets are similar to RDDs, however, instead of using Java serialization or Kryo they use
-a specialized [Encoder](api/scala/index.html#org.apache.spark.sql.Encoder) to serialize the objects
+a specialized [Encoder](api/scala/org/apache/spark/sql/Encoder.html) to serialize the objects
 for processing or transmitting over the network. While both encoders and standard serialization are
 responsible for turning an object into bytes, encoders are code generated dynamically and use a format
 that allows Spark to perform many operations like filtering, sorting and hashing without deserializing
@@ -347,40 +347,14 @@ For example:
 </div>
 
 ## Scalar Functions
-(to be filled soon)
 
-## Aggregations
+Scalar functions are functions that return a single value per row, as opposed to aggregation functions, which return a value for a group of rows. Spark SQL supports a variety of [Built-in Scalar Functions](sql-ref-functions.html#scalar-functions). It also supports [User Defined Scalar Functions](sql-ref-functions-udf-scalar.html).
 
-The [built-in DataFrames functions](api/scala/index.html#org.apache.spark.sql.functions$) provide common
-aggregations such as `count()`, `countDistinct()`, `avg()`, `max()`, `min()`, etc.
-While those functions are designed for DataFrames, Spark SQL also has type-safe versions for some of them in
-[Scala](api/scala/index.html#org.apache.spark.sql.expressions.scalalang.typed$) and
-[Java](api/java/org/apache/spark/sql/expressions/javalang/typed.html) to work with strongly typed Datasets.
-Moreover, users are not limited to the predefined aggregate functions and can create their own.
+## Aggregate Functions
 
-### Type-Safe User-Defined Aggregate Functions
+Aggregate functions are functions that return a single value on a group of rows. The [Built-in Aggregation Functions](sql-ref-functions-builtin.html#aggregate-functions) provide common aggregations such as `count()`, `countDistinct()`, `avg()`, `max()`, `min()`, etc.
+Users are not limited to the predefined aggregate functions and can create their own. For more details
+about user defined aggregate functions, please refer to the documentation of
+[User Defined Aggregate Functions](sql-ref-functions-udf-aggregate.html).
 
-User-defined aggregations for strongly typed Datasets revolve around the [Aggregator](api/scala/index.html#org.apache.spark.sql.expressions.Aggregator) abstract class.
-For example, a type-safe user-defined average can look like:
 
-<div class="codetabs">
-<div data-lang="scala"  markdown="1">
-{% include_example typed_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedTypedAggregation.scala%}
-</div>
-<div data-lang="java"  markdown="1">
-{% include_example typed_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedTypedAggregation.java%}
-</div>
-</div>
-
-### Untyped User-Defined Aggregate Functions
-Typed aggregations, as described above, may also be registered as untyped aggregating UDFs for use with DataFrames.
-For example, a user-defined average for untyped DataFrames can look like:
-
-<div class="codetabs">
-<div data-lang="scala"  markdown="1">
-{% include_example untyped_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedUntypedAggregation.scala%}
-</div>
-<div data-lang="java"  markdown="1">
-{% include_example untyped_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedUntypedAggregation.java%}
-</div>
-</div>
diff --git a/docs/sql-keywords.md b/docs/sql-keywords.md
deleted file mode 100644
index b18855366bb2b..0000000000000
--- a/docs/sql-keywords.md
+++ /dev/null
@@ -1,292 +0,0 @@
----
-layout: global
-title: Spark SQL Keywords
-displayTitle: Spark SQL Keywords
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-When `spark.sql.ansi.enabled` is true, Spark SQL will use the ANSI mode parser.
-In this mode, Spark SQL has two kinds of keywords:
-* Reserved keywords: Keywords that are reserved and can't be used as identifiers for table, view, column, function, alias, etc.
-* Non-reserved keywords: Keywords that have a special meaning only in particular contexts and can be used as identifiers in other contexts. For example, `EXPLAIN SELECT ...` is a command, but EXPLAIN can be used as identifiers in other places.
-
-When the ANSI mode is disabled, Spark SQL has two kinds of keywords:
-* Non-reserved keywords: Same definition as the one when the ANSI mode enabled.
-* Strict-non-reserved keywords: A strict version of non-reserved keywords, which can not be used as table alias.
-
-By default `spark.sql.ansi.enabled` is false.
-
-Below is a list of all the keywords in Spark SQL.
-
-<table class="table">
-  <tr><th rowspan="2" style="vertical-align: middle;"><b>Keyword</b></th><th colspan="2"><b>Spark SQL</b></th><th rowspan="2" style="vertical-align: middle;"><b>SQL-2011</b></th></tr>
-  <tr><th><b>ANSI mode</b></th><th><b>default mode</b></th></tr>
-  <tr><td>ADD</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>AFTER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ALL</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ALTER</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ANALYZE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>AND</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ANTI</td><td>reserved</td><td>strict-non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ANY</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ARCHIVE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ARRAY</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>AS</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ASC</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>AT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>AUTHORIZATION</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>BETWEEN</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>BOTH</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>BUCKET</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>BUCKETS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>BY</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CACHE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CASCADE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CASE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CAST</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CHANGE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CHECK</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CLEAR</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CLUSTER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CLUSTERED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CODEGEN</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COLLATE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>COLLECTION</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COLUMN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>COLUMNS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COMMENT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COMMIT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>COMPACT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COMPACTIONS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COMPUTE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CONCATENATE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CONSTRAINT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>COST</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CREATE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CROSS</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>CUBE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CURRENT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CURRENT_DATE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CURRENT_TIME</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CURRENT_TIMESTAMP</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CURRENT_USER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>DATA</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DATABASE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DATABASES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DAY</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>DBPROPERTIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DEFINED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DELETE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>DELIMITED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DESC</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DESCRIBE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>DFS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DIRECTORIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DIRECTORY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DISTINCT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>DISTRIBUTE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DIV</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DROP</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ELSE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>END</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ESCAPE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ESCAPED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>EXCEPT</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>EXCHANGE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>EXISTS</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>EXPLAIN</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>EXPORT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>EXTENDED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>EXTERNAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>EXTRACT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FALSE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FETCH</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FIELDS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FILTER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FILEFORMAT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FIRST</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FIRST_VALUE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FOLLOWING</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FOR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FOREIGN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FORMAT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FORMATTED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FROM</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FULL</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>FUNCTION</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FUNCTIONS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>GLOBAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>GRANT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>GROUP</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>GROUPING</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>HAVING</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>HOUR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>IF</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>IGNORE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>IMPORT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>IN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>INDEX</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>INDEXES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>INNER</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>INPATH</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>INPUTFORMAT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>INSERT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>INTERSECT</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>INTERVAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>INTO</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>IS</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ITEMS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>JOIN</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>KEYS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LAST</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LAST_VALUE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>LATERAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>LAZY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LEADING</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>LEFT</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>LIKE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>LIMIT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LINES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LIST</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LOAD</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LOCAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>LOCATION</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LOCK</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LOCKS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LOGICAL</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MACRO</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MAP</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MATCHED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MERGE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MINUS</td><td>reserved</td><td>strict-non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MINUTE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>MONTH</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>MSCK</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>NAMESPACE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>NAMESPACES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>NATURAL</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>NO</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>NOT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>NULL</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>NULLS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OF</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ON</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>ONLY</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>OPTION</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OPTIONS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ORDER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>OUT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>OUTER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>OUTPUTFORMAT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OVER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OVERLAPS</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>OVERLAY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OVERWRITE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PARTITION</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>PARTITIONED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PARTITIONS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PERCENT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PIVOT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PLACING</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>POSITION</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>PRECEDING</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PRIMARY</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>PRINCIPALS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PROPERTIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PURGE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>QUERY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RANGE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>RECORDREADER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RECORDWRITER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RECOVER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>REDUCE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>REFERENCES</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>REFRESH</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RENAME</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>REPAIR</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>REPLACE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RESET</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RESPECT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RESTRICT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>REVOKE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>RIGHT</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>RLIKE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ROLE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ROLES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ROLLBACK</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ROLLUP</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ROW</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ROWS</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SCHEMA</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SECOND</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SELECT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SEMI</td><td>reserved</td><td>strict-non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SEPARATED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SERDE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SERDEPROPERTIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SESSION_USER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SET</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SETS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SHOW</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SKEWED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SOME</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SORT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SORTED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>START</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>STATISTICS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>STORED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>STRATIFY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>STRUCT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SUBSTR</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SUBSTRING</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TABLE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TABLES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TABLESAMPLE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TBLPROPERTIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TEMPORARY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TERMINATED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>THEN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TO</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TOUCH</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TRAILING</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TRANSACTION</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TRANSACTIONS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TRANSFORM</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TRIM</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TRUE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TRUNCATE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>UNARCHIVE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>UNBOUNDED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>UNCACHE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>UNION</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>UNIQUE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>UNKNOWN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>UNLOCK</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>UNSET</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>UPDATE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>USE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>USER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>USING</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>VALUES</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>VIEW</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>WHEN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>WHERE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>WINDOW</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>WITH</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>YEAR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-</table>
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 533c96a0832de..fe2f62978cb62 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -22,313 +22,216 @@ license: |
 * Table of contents
 {:toc}
 
-## Upgrading from Spark SQL 2.4 to 3.0
-  - Since Spark 3.0, when inserting a value into a table column with a different data type, the type coercion is performed as per ANSI SQL standard. Certain unreasonable type conversions such as converting `string` to `int` and `double` to `boolean` are disallowed. A runtime exception will be thrown if the value is out-of-range for the data type of the column. In Spark version 2.4 and earlier, type conversions during table insertion are allowed as long as they are valid `Cast`. When inserting an out-of-range value to a integral field, the low-order bits of the value is inserted(the same as Java/Scala numeric type casting). For example, if 257 is inserted to a field of byte type, the result is 1. The behavior is controlled by the option `spark.sql.storeAssignmentPolicy`, with a default value as "ANSI". Setting the option as "Legacy" restores the previous behavior.
+## Upgrading from Spark SQL 3.0.1 to 3.0.2
+
+  - In Spark 3.0.2, `IllegalArgumentException` is returned for the incomplete interval literals, e.g. `INTERVAL '1'`, `INTERVAL '1 DAY 2'`, which are invalid. In Spark 3.0.1, these literals result in `NULL`s.
+
+  - In Spark 3.0.2, `AnalysisException` is replaced by its sub-classes that are thrown for tables from Hive external catalog in the following situations:
+    * `ALTER TABLE .. ADD PARTITION` throws `PartitionsAlreadyExistException` if new partition exists already
+    * `ALTER TABLE .. DROP PARTITION` throws `NoSuchPartitionsException` for not existing partitions
 
-  - In Spark 3.0, the deprecated methods `SQLContext.createExternalTable` and `SparkSession.createExternalTable` have been removed in favor of its replacement, `createTable`.
+## Upgrading from Spark SQL 3.0 to 3.0.1
 
-  - In Spark 3.0, the deprecated `HiveContext` class has been removed. Use `SparkSession.builder.enableHiveSupport()` instead.
+- In Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Since version 3.0.1, the timestamp type inference is disabled by default. Set the JSON option `inferTimestamp` to `true` to enable such type inference.
 
-  - Since Spark 3.0, configuration `spark.sql.crossJoin.enabled` become internal configuration, and is true by default, so by default spark won't raise exception on sql with implicit cross join.
+## Upgrading from Spark SQL 2.4 to 3.0
+
+### Dataset/DataFrame APIs
 
-  - Since Spark 3.0, we reversed argument order of the trim function from `TRIM(trimStr, str)` to `TRIM(str, trimStr)` to be compatible with other databases.
+  - In Spark 3.0, the Dataset and DataFrame API `unionAll` is no longer deprecated. It is an alias for `union`.
 
-  - In Spark version 2.4 and earlier, SQL queries such as `FROM <table>` or `FROM <table> UNION ALL FROM <table>` are supported by accident. In hive-style `FROM <table> SELECT <expr>`, the `SELECT` clause is not negligible. Neither Hive nor Presto support this syntax. Therefore we will treat these queries as invalid since Spark 3.0.
+  - In Spark 2.4 and below, `Dataset.groupByKey` results to a grouped dataset with key attribute is wrongly named as "value", if the key is non-struct type, for example, int, string, array, etc. This is counterintuitive and makes the schema of aggregation queries unexpected. For example, the schema of `ds.groupByKey(...).count()` is `(value, count)`. Since Spark 3.0, we name the grouping attribute to "key". The old behavior is preserved under a newly added configuration `spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue` with a default value of `false`.
 
-  - Since Spark 3.0, the Dataset and DataFrame API `unionAll` is not deprecated any more. It is an alias for `union`.
+  - In Spark 3.0, the column metadata will always be propagated in the API `Column.name` and `Column.as`. In Spark version 2.4 and earlier, the metadata of `NamedExpression` is set as the `explicitMetadata` for the new column at the time the API is called, it won't change even if the underlying `NamedExpression` changes metadata. To restore the behavior before Spark 2.4, you can use the API `as(alias: String, metadata: Metadata)` with explicit metadata.
 
-  - In Spark version 2.4 and earlier, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`. For `FloatType` and `DoubleType`, it fails on empty strings and throws exceptions. Since Spark 3.0, we disallow empty strings and will throw exceptions for data types except for `StringType` and `BinaryType`.
+### DDL Statements
 
-  - Since Spark 3.0, the `from_json` functions supports two modes - `PERMISSIVE` and `FAILFAST`. The modes can be set via the `mode` option. The default mode became `PERMISSIVE`. In previous versions, behavior of `from_json` did not conform to either `PERMISSIVE` nor `FAILFAST`, especially in processing of malformed JSON records. For example, the JSON string `{"a" 1}` with the schema `a INT` is converted to `null` by previous versions but Spark 3.0 converts it to `Row(null)`.
+  - In Spark 3.0, when inserting a value into a table column with a different data type, the type coercion is performed as per ANSI SQL standard. Certain unreasonable type conversions such as converting `string` to `int` and `double` to `boolean` are disallowed. A runtime exception is thrown if the value is out-of-range for the data type of the column. In Spark version 2.4 and below, type conversions during table insertion are allowed as long as they are valid `Cast`. When inserting an out-of-range value to an integral field, the low-order bits of the value is inserted(the same as Java/Scala numeric type casting). For example, if 257 is inserted to a field of byte type, the result is 1. The behavior is controlled by the option `spark.sql.storeAssignmentPolicy`, with a default value as "ANSI". Setting the option as "Legacy" restores the previous behavior.
 
   - The `ADD JAR` command previously returned a result set with the single value 0. It now returns an empty result set.
 
-  - In Spark version 2.4 and earlier, users can create map values with map type key via built-in function like `CreateMap`, `MapFromArrays`, etc. Since Spark 3.0, it's not allowed to create map values with map type key with these built-in functions. Users can still read map values with map type key from data source or Java/Scala collections, though they are not very useful.
+  - Spark 2.4 and below: the `SET` command works without any warnings even if the specified key is for `SparkConf` entries and it has no effect because the command does not update `SparkConf`, but the behavior might confuse users. In 3.0, the command fails if a `SparkConf` key is used. You can disable such a check by setting `spark.sql.legacy.setCommandRejectsSparkCoreConfs` to `false`.
 
-  - In Spark version 2.4 and earlier, `Dataset.groupByKey` results to a grouped dataset with key attribute wrongly named as "value", if the key is non-struct type, e.g. int, string, array, etc. This is counterintuitive and makes the schema of aggregation queries weird. For example, the schema of `ds.groupByKey(...).count()` is `(value, count)`. Since Spark 3.0, we name the grouping attribute to "key". The old behaviour is preserved under a newly added configuration `spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue` with a default value of `false`.
+  - Refreshing a cached table would trigger a table uncache operation and then a table cache (lazily) operation. In Spark version 2.4 and below, the cache name and storage level are not preserved before the uncache operation. Therefore, the cache name and storage level could be changed unexpectedly. In Spark 3.0, cache name and storage level are first preserved for cache recreation. It helps to maintain a consistent cache behavior upon table refreshing.
 
-  - In Spark version 2.4 and earlier, float/double -0.0 is semantically equal to 0.0, but -0.0 and 0.0 are considered as different values when used in aggregate grouping keys, window partition keys and join keys. Since Spark 3.0, this bug is fixed. For example, `Seq(-0.0, 0.0).toDF("d").groupBy("d").count()` returns `[(0.0, 2)]` in Spark 3.0, and `[(0.0, 1), (-0.0, 1)]` in Spark 2.4 and earlier.
+  - In Spark 3.0, the properties listing below become reserved; commands fail if you specify reserved properties in places like `CREATE DATABASE ... WITH DBPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. You need their specific clauses to specify them, for example, `CREATE DATABASE test COMMENT 'any comment' LOCATION 'some path'`. You can set `spark.sql.legacy.notReserveProperties` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, for example: `SET DBPROPERTIES('location'='/tmp')` will have no effect. In Spark version 2.4 and below, these properties are neither reserved nor have side effects, for example, `SET DBPROPERTIES('location'='/tmp')` do not change the location of the database but only create a headless property just like `'a'='b'`.
 
-  - In Spark version 2.4 and earlier, users can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, e.g. map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. Since Spark 3.0, these built-in functions will remove duplicated map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (e.g. Parquet), the behavior will be undefined.
+    | Property (case sensitive) | Database Reserved | Table Reserved | Remarks |
+    | ------------------------- | ----------------- | -------------- | ------- |
+    | provider                 | no                | yes            | For tables, use the `USING` clause to specify it. Once set, it can't be changed. |
+    | location                 | yes               | yes            | For databases and tables, use the `LOCATION` clause to specify it. |
+    | owner                    | yes               | yes            | For databases and tables, it is determined by the user who runs spark and create the table. |
 
-  - In Spark version 2.4 and earlier, partition column value is converted as null if it can't be casted to corresponding user provided schema. Since 3.0, partition column value is validated with user provided schema. An exception is thrown if the validation fails. You can disable such validation by setting `spark.sql.sources.validatePartitionColumns` to `false`.
+ 
+  - In Spark 3.0, you can use `ADD FILE` to add file directories as well. Earlier you could add only single files using this command. To restore the behavior of earlier versions, set `spark.sql.legacy.addSingleFileInAddFile` to `true`.
 
-  - In Spark version 2.4 and earlier, the `SET` command works without any warnings even if the specified key is for `SparkConf` entries and it has no effect because the command does not update `SparkConf`, but the behavior might confuse users. Since 3.0, the command fails if a `SparkConf` key is used. You can disable such a check by setting `spark.sql.legacy.setCommandRejectsSparkCoreConfs` to `false`.
+  - In Spark 3.0, `SHOW TBLPROPERTIES` throws `AnalysisException` if the table does not exist. In Spark version 2.4 and below, this scenario caused `NoSuchTableException`.
 
-  - In Spark version 2.4 and earlier, CSV datasource converts a malformed CSV string to a row with all `null`s in the PERMISSIVE mode. Since Spark 3.0, the returned row can contain non-`null` fields if some of CSV column values were parsed and converted to desired types successfully.
+  - In Spark 3.0, `SHOW CREATE TABLE` always returns Spark DDL, even when the given table is a Hive SerDe table. For generating Hive DDL, use `SHOW CREATE TABLE AS SERDE` command instead.
 
-  - In Spark version 2.4 and earlier, JSON datasource and JSON functions like `from_json` convert a bad JSON record to a row with all `null`s in the PERMISSIVE mode when specified schema is `StructType`. Since Spark 3.0, the returned row can contain non-`null` fields if some of JSON column values were parsed and converted to desired types successfully.
+  - In Spark 3.0, column of CHAR type is not allowed in non-Hive-Serde tables, and CREATE/ALTER TABLE commands will fail if CHAR type is detected. Please use STRING type instead. In Spark version 2.4 and below, CHAR type is treated as STRING type and the length parameter is simply ignored.
 
-  - Refreshing a cached table would trigger a table uncache operation and then a table cache (lazily) operation. In Spark version 2.4 and earlier, the cache name and storage level are not preserved before the uncache operation. Therefore, the cache name and storage level could be changed unexpectedly. Since Spark 3.0, cache name and storage level will be first preserved for cache recreation. It helps to maintain a consistent cache behavior upon table refreshing.
+### UDFs and Built-in Functions
 
-  - Since Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Set JSON option `inferTimestamp` to `false` to disable such type inferring.
+  - In Spark 3.0, the `date_add` and `date_sub` functions accepts only int, smallint, tinyint as the 2nd argument; fractional and non-literal strings are not valid anymore, for example: `date_add(cast('1964-05-23' as date), '12.34')` causes `AnalysisException`. Note that, string literals are still allowed, but Spark will throw `AnalysisException` if the string content is not a valid integer. In Spark version 2.4 and below, if the 2nd argument is fractional or string value, it is coerced to int value, and the result is a date value of `1964-06-04`.
 
-  - In Spark version 2.4 and earlier, if `org.apache.spark.sql.functions.udf(Any, DataType)` gets a Scala closure with primitive-type argument, the returned UDF will return null if the input values is null. Since Spark 3.0, the UDF will return the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` will return null in Spark 2.4 and earlier if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
+  - In Spark 3.0, the function `percentile_approx` and its alias `approx_percentile` only accept integral value with range in `[1, 2147483647]` as its 3rd argument `accuracy`, fractional and string types are disallowed, for example, `percentile_approx(10.0, 0.2, 1.8D)` causes `AnalysisException`. In Spark version 2.4 and below, if `accuracy` is fractional or string value, it is coerced to an int value, `percentile_approx(10.0, 0.2, 1.8D)` is operated as `percentile_approx(10.0, 0.2, 1)` which results in `10.0`.
 
-  - Since Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and etc. Spark 3.0 uses Java 8 API classes from the java.time packages that based on ISO chronology (https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and earlier, those operations are performed by using the hybrid calendar (Julian + Gregorian, see https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
+  - In Spark 3.0, an analysis exception is thrown when hash expressions are applied on elements of `MapType`. To restore the behavior before Spark 3.0, set `spark.sql.legacy.allowHashOnMapType` to `true`.
 
-    - CSV/JSON datasources use java.time API for parsing and generating CSV/JSON content. In Spark version 2.4 and earlier, java.text.SimpleDateFormat is used for the same purpose with fallbacks to the parsing mechanisms of Spark 2.0 and 1.x. For example, `2018-12-08 10:39:21.123` with the pattern `yyyy-MM-dd'T'HH:mm:ss.SSS` cannot be parsed since Spark 3.0 because the timestamp does not match to the pattern but it can be parsed by earlier Spark versions due to a fallback to `Timestamp.valueOf`. To parse the same timestamp since Spark 3.0, the pattern should be `yyyy-MM-dd HH:mm:ss.SSS`.
+  - In Spark 3.0, when the `array`/`map` function is called without any parameters, it returns an empty collection with `NullType` as element type. In Spark version 2.4 and below, it returns an empty collection with `StringType` as element type. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createEmptyCollectionUsingStringType` to `true`.
 
-    - The `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions. New implementation supports pattern formats as described here https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html and performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`.
+  - In Spark 3.0, the `from_json` functions supports two modes - `PERMISSIVE` and `FAILFAST`. The modes can be set via the `mode` option. The default mode became `PERMISSIVE`. In previous versions, behavior of `from_json` did not conform to either `PERMISSIVE` nor `FAILFAST`, especially in processing of malformed JSON records. For example, the JSON string `{"a" 1}` with the schema `a INT` is converted to `null` by previous versions but Spark 3.0 converts it to `Row(null)`.
 
-    - The `weekofyear`, `weekday`, `dayofweek`, `date_trunc`, `from_utc_timestamp`, `to_utc_timestamp`, and `unix_timestamp` functions use java.time API for calculation week number of year, day number of week as well for conversion from/to TimestampType values in UTC time zone.
+  - In Spark version 2.4 and below, you can create map values with map type key via built-in function such as `CreateMap`, `MapFromArrays`, etc. In Spark 3.0, it's not allowed to create map values with map type key with these built-in functions. Users can use `map_entries` function to convert map to array<struct<key, value>> as a workaround. In addition, users can still read map values with map type key from data source or Java/Scala collections, though it is discouraged.
 
-    - the JDBC options `lowerBound` and `upperBound` are converted to TimestampType/DateType values in the same way as casting strings to TimestampType/DateType values. The conversion is based on Proleptic Gregorian calendar, and time zone defined by the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and earlier, the conversion is based on the hybrid calendar (Julian + Gregorian) and on default system time zone.
+  - In Spark version 2.4 and below, you can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, for example, map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. In Spark 3.0, Spark throws `RuntimeException` when duplicated keys are found. You can set `spark.sql.mapKeyDedupPolicy` to `LAST_WIN` to deduplicate map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (for example, Parquet), the behavior is undefined.
 
-    - Formatting of `TIMESTAMP` and `DATE` literals.
+  - In Spark 3.0, using `org.apache.spark.sql.functions.udf(AnyRef, DataType)` is not allowed by default. Remove the return type parameter to automatically switch to typed Scala udf is recommended, or set `spark.sql.legacy.allowUntypedScalaUDF` to true to keep using it. In Spark version 2.4 and below, if `org.apache.spark.sql.functions.udf(AnyRef, DataType)` gets a Scala closure with primitive-type argument, the returned UDF returns null if the input values is null. However, in Spark 3.0, the UDF returns the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` returns null in Spark 2.4 and below if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
 
-    - Creating of typed `TIMESTAMP` and `DATE` literals from strings. Since Spark 3.0, string conversion to typed `TIMESTAMP`/`DATE` literals is performed via casting to `TIMESTAMP`/`DATE` values. For example, `TIMESTAMP '2019-12-23 12:59:30'` is semantically equal to `CAST('2019-12-23 12:59:30' AS TIMESTAMP)`. In Spark version 2.4 and earlier, the `java.sql.Timestamp.valueOf()` and `java.sql.Date.valueOf()` functions are used for the same purpose.
+  - In Spark 3.0, a higher-order function `exists` follows the three-valued boolean logic, that is, if the `predicate` returns any `null`s and no `true` is obtained, then `exists` returns `null` instead of `false`. For example, `exists(array(1, null, 3), x -> x % 2 == 0)` is `null`. The previous behaviorcan be restored by setting `spark.sql.legacy.followThreeValuedLogicInArrayExists` to `false`.
 
-  - In Spark version 2.4 and earlier, invalid time zone ids are silently ignored and replaced by GMT time zone, for example, in the from_utc_timestamp function. Since Spark 3.0, such time zone ids are rejected, and Spark throws `java.time.DateTimeException`.
+  - In Spark 3.0, the `add_months` function does not adjust the resulting date to a last day of month if the original date is a last day of months. For example, `select add_months(DATE'2019-02-28', 1)` results `2019-03-28`. In Spark version 2.4 and below, the resulting date is adjusted when the original date is a last day of months. For example, adding a month to `2019-02-28` results in `2019-03-31`.
 
-  - In Spark version 2.4 and earlier, the `current_timestamp` function returns a timestamp with millisecond resolution only. Since Spark 3.0, the function can return the result with microsecond resolution if the underlying clock available on the system offers such resolution.
+  - In Spark version 2.4 and below, the `current_timestamp` function returns a timestamp with millisecond resolution only. In Spark 3.0, the function can return the result with microsecond resolution if the underlying clock available on the system offers such resolution.
 
-  - In Spark version 2.4 and earlier, when reading a Hive Serde table with Spark native data sources(parquet/orc), Spark will infer the actual file schema and update the table schema in metastore. Since Spark 3.0, Spark doesn't infer the schema anymore. This should not cause any problems to end users, but if it does, please set `spark.sql.hive.caseSensitiveInferenceMode` to `INFER_AND_SAVE`.
+  - In Spark 3.0, a 0-argument Java UDF is executed in the executor side identically with other UDFs. In Spark version 2.4 and below, the 0-argument Java UDF alone was executed in the driver side, and the result was propagated to executors, which might be more performant in some cases but caused inconsistency with a correctness issue in some cases.
 
-  - Since Spark 3.0, `TIMESTAMP` literals are converted to strings using the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and earlier, the conversion uses the default time zone of the Java virtual machine.
+  - The result of `java.lang.Math`'s `log`, `log1p`, `exp`, `expm1`, and `pow` may vary across platforms. In Spark 3.0, the result of the equivalent SQL functions (including related SQL functions like `LOG10`) return values consistent with `java.lang.StrictMath`. In virtually all cases this makes no difference in the return value, and the difference is very small, but may not exactly match `java.lang.Math` on x86 platforms in cases like, for example, `log(3.0)`, whose value varies between `Math.log()` and `StrictMath.log()`.
 
-  - In Spark version 2.4, when a spark session is created via `cloneSession()`, the newly created spark session inherits its configuration from its parent `SparkContext` even though the same configuration may exist with a different value in its parent spark session. Since Spark 3.0, the configurations of a parent `SparkSession` have a higher precedence over the parent `SparkContext`. The old behavior can be restored by setting `spark.sql.legacy.sessionInitWithConfigDefaults` to `true`.
+  - In Spark 3.0, the `Cast` function processes string literals such as 'Infinity', '+Infinity', '-Infinity', 'NaN', 'Inf', '+Inf', '-Inf' in a case-insensitive manner when casting the literals to `Double` or `Float` type to ensure greater compatibility with other database systems. This behavior change is illustrated in the table below:
 
-  - Since Spark 3.0, parquet logical type `TIMESTAMP_MICROS` is used by default while saving `TIMESTAMP` columns. In Spark version 2.4 and earlier, `TIMESTAMP` columns are saved as `INT96` in parquet files. To set `INT96` to `spark.sql.parquet.outputTimestampType` restores the previous behavior.
+    | Operation | Result before Spark 3.0 | Result in Spark 3.0 |
+    | --------- | ----------------------- | ------------------- |
+    | CAST('infinity' AS DOUBLE) | NULL | Double.PositiveInfinity |
+    | CAST('+infinity' AS DOUBLE) | NULL | Double.PositiveInfinity |
+    | CAST('inf' AS DOUBLE) | NULL | Double.PositiveInfinity |
+    | CAST('inf' AS DOUBLE) | NULL | Double.PositiveInfinity |
+    | CAST('-infinity' AS DOUBLE) | NULL | Double.NegativeInfinity |
+    | CAST('-inf' AS DOUBLE) | NULL | Double.NegativeInfinity |
+    | CAST('infinity' AS FLOAT) | NULL | Float.PositiveInfinity |
+    | CAST('+infinity' AS FLOAT) | NULL | Float.PositiveInfinity |
+    | CAST('inf' AS FLOAT) | NULL | Float.PositiveInfinity |
+    | CAST('+inf' AS FLOAT) | NULL | Float.PositiveInfinity |
+    | CAST('-infinity' AS FLOAT) | NULL | Float.NegativeInfinity |
+    | CAST('-inf' AS FLOAT) | NULL | Float.NegativeInfinity |
+    | CAST('nan' AS DOUBLE) | NULL | Double.Nan |
+    | CAST('nan' AS FLOAT) | NULL | Float.NaN |
 
-  - Since Spark 3.0, if `hive.default.fileformat` is not found in `Spark SQL configuration` then it will fallback to hive-site.xml present in the `Hadoop configuration` of `SparkContext`.
+  - In Spark 3.0, when casting interval values to string type, there is no "interval" prefix, for example, `1 days 2 hours`. In Spark version 2.4 and below, the string contains the "interval" prefix like `interval 1 days 2 hours`.
 
-  - Since Spark 3.0, Spark will cast `String` to `Date/TimeStamp` in binary comparisons with dates/timestamps. The previous behaviour of casting `Date/Timestamp` to `String` can be restored by setting `spark.sql.legacy.typeCoercion.datetimeToString.enabled` to `true`.
+  - In Spark 3.0, when casting string value to integral types(tinyint, smallint, int and bigint), datetime types(date, timestamp and interval) and boolean type, the leading and trailing whitespaces (<= ASCII 32) will be trimmed before converted to these type values, for example, `cast(' 1\t' as int)` results `1`, `cast(' 1\t' as boolean)` results `true`, `cast('2019-10-10\t as date)` results the date value `2019-10-10`. In Spark version 2.4 and below, when casting string to integrals and booleans, it does not trim the whitespaces from both ends; the foregoing results is `null`, while to datetimes, only the trailing spaces (= ASCII 32) are removed.
 
-  - Since Spark 3.0, when Avro files are written with user provided schema, the fields will be matched by field names between catalyst schema and avro schema instead of positions.
+### Query Engine
 
-  - Since Spark 3.0, when Avro files are written with user provided non-nullable schema, even the catalyst schema is nullable, Spark is still able to write the files. However, Spark will throw runtime NPE if any of the records contains null.
+  - In Spark version 2.4 and below, SQL queries such as `FROM <table>` or `FROM <table> UNION ALL FROM <table>` are supported by accident. In hive-style `FROM <table> SELECT <expr>`, the `SELECT` clause is not negligible. Neither Hive nor Presto support this syntax. These queries are treated as invalid in Spark 3.0.
 
-  - Since Spark 3.0, a higher-order function `exists` follows the three-valued boolean logic, i.e., if the `predicate` returns any `null`s and no `true` is obtained, then `exists` will return `null` instead of `false`. For example, `exists(array(1, null, 3), x -> x % 2 == 0)` will be `null`. The previous behaviour can be restored by setting `spark.sql.legacy.arrayExistsFollowsThreeValuedLogic` to `false`.
+  - In Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
 
-  - Since Spark 3.0, if files or subdirectories disappear during recursive directory listing (i.e. they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time.
+  - In Spark 3.0, numbers written in scientific notation(for example, `1E2`) would be parsed as Double. In Spark version 2.4 and below, they're parsed as Decimal. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.exponentLiteralAsDecimal.enabled` to `true`.
 
-  - Since Spark 3.0, substitution order of nested WITH clauses is changed and an inner CTE definition takes precedence over an outer. In version 2.4 and earlier, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `1` while in version 3.0 it returns `2`. The previous behaviour can be restored by setting `spark.sql.legacy.ctePrecedence.enabled` to `true`.
+  - In Spark 3.0, day-time interval strings are converted to intervals with respect to the `from` and `to` bounds. If an input string does not match to the pattern defined by specified bounds, the `ParseException` exception is thrown. For example, `interval '2 10:20' hour to minute` raises the exception because the expected format is `[+|-]h[h]:[m]m`. In Spark version 2.4, the `from` bound was not taken into account, and the `to` bound was used to truncate the resulted interval. For instance, the day-time interval string from the showed example is converted to `interval 10 hours 20 minutes`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.fromDayTimeString.enabled` to `true`.
 
-  - Since Spark 3.0, the `add_months` function does not adjust the resulting date to a last day of month if the original date is a last day of months. For example, `select add_months(DATE'2019-02-28', 1)` results `2019-03-28`. In Spark version 2.4 and earlier, the resulting date is adjusted when the original date is a last day of months. For example, adding a month to `2019-02-28` results in `2019-03-31`.
+  - In Spark 3.0, negative scale of decimal is not allowed by default, for example, data type of literal like `1E10BD` is `DecimalType(11, 0)`. In Spark version 2.4 and below, it was `DecimalType(2, -9)`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.allowNegativeScaleOfDecimal` to `true`.
 
-  - Since Spark 3.0, 0-argument Java UDF is executed in the executor side identically with other UDFs. In Spark version 2.4 and earlier, 0-argument Java UDF alone was executed in the driver side, and the result was propagated to executors, which might be more performant in some cases but caused inconsistency with a correctness issue in some cases.
+  - In Spark 3.0, the unary arithmetic operator plus(`+`) only accepts string, numeric and interval type values as inputs. Besides, `+` with an integral string representation is coerced to a double value, for example, `+'1'` returns `1.0`. In Spark version 2.4 and below, this operator is ignored. There is no type checking for it, thus, all type values with a `+` prefix are valid, for example, `+ array(1, 2)` is valid and results `[1, 2]`. Besides, there is no type coercion for it at all, for example, in Spark 2.4, the result of `+'1'` is string `1`.
 
-  - The result of `java.lang.Math`'s `log`, `log1p`, `exp`, `expm1`, and `pow` may vary across platforms. In Spark 3.0, the result of the equivalent SQL functions (including related SQL functions like `LOG10`) return values consistent with `java.lang.StrictMath`. In virtually all cases this makes no difference in the return value, and the difference is very small, but may not exactly match `java.lang.Math` on x86 platforms in cases like, for example, `log(3.0)`, whose value varies between `Math.log()` and `StrictMath.log()`.
+  - In Spark 3.0, Dataset query fails if it contains ambiguous column reference that is caused by self join. A typical example: `val df1 = ...; val df2 = df1.filter(...);`, then `df1.join(df2, df1("a") > df2("a"))` returns an empty result which is quite confusing. This is because Spark cannot resolve Dataset column references that point to tables being self joined, and `df1("a")` is exactly the same as `df2("a")` in Spark. To restore the behavior before Spark 3.0, you can set `spark.sql.analyzer.failAmbiguousSelfJoin` to `false`.
 
-  - Since Spark 3.0, Dataset query fails if it contains ambiguous column reference that is caused by self join. A typical example: `val df1 = ...; val df2 = df1.filter(...);`, then `df1.join(df2, df1("a") > df2("a"))` returns an empty result which is quite confusing. This is because Spark cannot resolve Dataset column references that point to tables being self joined, and `df1("a")` is exactly the same as `df2("a")` in Spark. To restore the behavior before Spark 3.0, you can set `spark.sql.analyzer.failAmbiguousSelfJoin.enabled` to `false`.
+  - In Spark 3.0, `spark.sql.legacy.ctePrecedencePolicy` is introduced to control the behavior for name conflicting in the nested WITH clause. By default value `EXCEPTION`, Spark throws an AnalysisException, it forces users to choose the specific substitution order they wanted. If set to `CORRECTED` (which is recommended), inner CTE definitions take precedence over outer definitions. For example, set the config to `false`, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `2`, while setting it to `LEGACY`, the result is `1` which is the behavior in version 2.4 and below.
 
-  - Since Spark 3.0, `Cast` function processes string literals such as 'Infinity', '+Infinity', '-Infinity', 'NaN', 'Inf', '+Inf', '-Inf' in case insensitive manner when casting the literals to `Double` or `Float` type to ensure greater compatibility with other database systems. This behaviour change is illustrated in the table below:
-    <table class="table">
-        <tr>
-          <th>
-            <b>Operation</b>
-          </th>
-          <th>
-            <b>Result prior to Spark 3.0</b>
-          </th>
-          <th>
-            <b>Result starting Spark 3.0</b>
-          </th>
-        </tr>
-        <tr>
-          <td>
-            CAST('infinity' AS DOUBLE)<br>
-            CAST('+infinity' AS DOUBLE)<br>
-            CAST('inf' AS DOUBLE)<br>
-            CAST('+inf' AS DOUBLE)<br>
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Double.PositiveInfinity
-          </td>
-        </tr>
-        <tr>
-          <td>
-            CAST('-infinity' AS DOUBLE)<br>
-            CAST('-inf' AS DOUBLE)<br>
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Double.NegativeInfinity
-          </td>
-        </tr>
-        <tr>
-          <td>
-            CAST('infinity' AS FLOAT)<br>
-            CAST('+infinity' AS FLOAT)<br>
-            CAST('inf' AS FLOAT)<br>
-            CAST('+inf' AS FLOAT)<br>
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Float.PositiveInfinity
-          </td>
-        </tr>
-        <tr>
-          <td>
-            CAST('-infinity' AS FLOAT)<br>
-            CAST('-inf' AS FLOAT)<br>
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Float.NegativeInfinity
-          </td>
-        </tr>
-        <tr>
-          <td>
-            CAST('nan' AS DOUBLE)
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Double.NaN
-          </td>
-        </tr>
-        <tr>
-          <td>
-            CAST('nan' AS FLOAT)
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Float.NaN
-          </td>
-        </tr>
-    </table>
+  - In Spark 3.0, configuration `spark.sql.crossJoin.enabled` become internal configuration, and is true by default, so by default spark won't raise exception on sql with implicit cross join.
 
-  - Since Spark 3.0, special values are supported in conversion from strings to dates and timestamps. Those values are simply notational shorthands that will be converted to ordinary date or timestamp values when read. The following string values are supported for dates:
-    - `epoch [zoneId]` - 1970-01-01
-    - `today [zoneId]` - the current date in the time zone specified by `spark.sql.session.timeZone`
-    - `yesterday [zoneId]` - the current date - 1
-    - `tomorrow [zoneId]` - the current date + 1
-    - `now` - the date of running the current query. It has the same notion as today
-  For example `SELECT date 'tomorrow' - date 'yesterday';` should output `2`. Here are special timestamp values:
-    - `epoch [zoneId]` - 1970-01-01 00:00:00+00 (Unix system time zero)
-    - `today [zoneId]` - midnight today
-    - `yesterday [zoneId]` - midnight yesterday
-    - `tomorrow [zoneId]` - midnight tomorrow
-    - `now` - current query start time
-  For example `SELECT timestamp 'tomorrow';`.
+  - In Spark version 2.4 and below, float/double -0.0 is semantically equal to 0.0, but -0.0 and 0.0 are considered as different values when used in aggregate grouping keys, window partition keys, and join keys. In Spark 3.0, this bug is fixed. For example, `Seq(-0.0, 0.0).toDF("d").groupBy("d").count()` returns `[(0.0, 2)]` in Spark 3.0, and `[(0.0, 1), (-0.0, 1)]` in Spark 2.4 and below.
 
-  - Since Spark 3.0, the `size` function returns `NULL` for the `NULL` input. In Spark version 2.4 and earlier, this function gives `-1` for the same input. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.sizeOfNull` to `true`.
+  - In Spark version 2.4 and below, invalid time zone ids are silently ignored and replaced by GMT time zone, for example, in the from_utc_timestamp function. In Spark 3.0, such time zone ids are rejected, and Spark throws `java.time.DateTimeException`.
 
-  - Since Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
+  - In Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and so on. Spark 3.0 uses Java 8 API classes from the `java.time` packages that are based on [ISO chronology](https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and below, those operations are performed using the hybrid calendar ([Julian + Gregorian](https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
 
-  - Since Spark 3.0, when casting interval values to string type, there is no "interval" prefix, e.g. `1 days 2 hours`. In Spark version 2.4 and earlier, the string contains the "interval" prefix like `interval 1 days 2 hours`.
+    * Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. In Spark 3.0, we define our own pattern strings in [Datetime Patterns for Formatting and Parsing](sql-ref-datetime-pattern.html),
+     which is implemented via [DateTimeFormatter](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html) under the hood. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and below, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in [SimpleDateFormat](https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html). The old behavior can be restored by setting `spark.sql.legacy.timeParserPolicy` to `LEGACY`.
 
-  - Since Spark 3.0, when casting string value to integral types(tinyint, smallint, int and bigint), datetime types(date, timestamp and interval) and boolean type, the leading and trailing whitespaces (<= ASCII 32) will be trimmed before converted to these type values, e.g. `cast(' 1\t' as int)` results `1`, `cast(' 1\t' as boolean)` results `true`, `cast('2019-10-10\t as date)` results the date value `2019-10-10`. In Spark version 2.4 and earlier, while casting string to integrals and booleans, it will not trim the whitespaces from both ends, the foregoing results will be `null`, while to datetimes, only the trailing spaces (= ASCII 32) will be removed.
+    * The `weekofyear`, `weekday`, `dayofweek`, `date_trunc`, `from_utc_timestamp`, `to_utc_timestamp`, and `unix_timestamp` functions use java.time API for calculation week number of year, day number of week as well for conversion from/to TimestampType values in UTC time zone.
 
-  - Since Spark 3.0, numbers written in scientific notation(e.g. `1E2`) would be parsed as Double. In Spark version 2.4 and earlier, they're parsed as Decimal. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.exponentLiteralAsDecimal.enabled` to `true`.
+    * The JDBC options `lowerBound` and `upperBound` are converted to TimestampType/DateType values in the same way as casting strings to TimestampType/DateType values. The conversion is based on Proleptic Gregorian calendar, and time zone defined by the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and below, the conversion is based on the hybrid calendar (Julian + Gregorian) and on default system time zone.
 
-  - Since Spark 3.0, we pad decimal numbers with trailing zeros to the scale of the column for `spark-sql` interface, for example:
-    <table class="table">
-        <tr>
-          <th>
-            <b>Query</b>
-          </th>
-          <th>
-            <b>Spark 2.4 or Prior</b>
-          </th>
-          <th>
-            <b>Spark 3.0</b>
-          </th>
-        </tr>
-        <tr>
-          <td>
-            <code>SELECT CAST(1 AS decimal(38, 18));</code>
-          </td>
-          <td>
-            <code>1</code>
-          </td>
-          <td>
-            <code>1.000000000000000000</code>
-          </td>
-        </tr>
-    </table>
-    
-  - Since Spark 3.0, CREATE TABLE without a specific provider will use the value of `spark.sql.sources.default` as its provider. In Spark version 2.4 and earlier, it was hive. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createHiveTableByDefault.enabled` to `true`.
+    * Formatting `TIMESTAMP` and `DATE` literals.
 
-  - Since Spark 3.0, the unary arithmetic operator plus(`+`) only accepts string, numeric and interval type values as inputs. Besides, `+` with a integral string representation will be coerced to double value, e.g. `+'1'` results `1.0`. In Spark version 2.4 and earlier, this operator is ignored. There is no type checking for it, thus, all type values with a `+` prefix are valid, e.g. `+ array(1, 2)` is valid and results `[1, 2]`. Besides, there is no type coercion for it at all, e.g. in Spark 2.4, the result of `+'1'` is string `1`.
+    * Creating typed `TIMESTAMP` and `DATE` literals from strings. In Spark 3.0, string conversion to typed `TIMESTAMP`/`DATE` literals is performed via casting to `TIMESTAMP`/`DATE` values. For example, `TIMESTAMP '2019-12-23 12:59:30'` is semantically equal to `CAST('2019-12-23 12:59:30' AS TIMESTAMP)`. When the input string does not contain information about time zone, the time zone from the SQL config `spark.sql.session.timeZone` is used in that case. In Spark version 2.4 and below, the conversion is based on JVM system time zone. The different sources of the default time zone may change the behavior of typed `TIMESTAMP` and `DATE` literals.
 
-  - Since Spark 3.0, day-time interval strings are converted to intervals with respect to the `from` and `to` bounds. If an input string does not match to the pattern defined by specified bounds, the `ParseException` exception is thrown. For example, `interval '2 10:20' hour to minute` raises the exception because the expected format is `[+|-]h[h]:[m]m`. In Spark version 2.4, the `from` bound was not taken into account, and the `to` bound was used to truncate the resulted interval. For instance, the day-time interval string from the showed example is converted to `interval 10 hours 20 minutes`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.fromDayTimeString.enabled` to `true`.
-  
-  - Since Spark 3.0, negative scale of decimal is not allowed by default, e.g. data type of literal like `1E10BD` is `DecimalType(11, 0)`. In Spark version 2.4 and earlier, it was `DecimalType(2, -9)`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.allowNegativeScaleOfDecimal.enabled` to `true`.
+  - In Spark 3.0, `TIMESTAMP` literals are converted to strings using the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and below, the conversion uses the default time zone of the Java virtual machine.
 
-  - Since Spark 3.0, the `date_add` and `date_sub` functions only accepts int, smallint, tinyint as the 2nd argument, fractional and string types are not valid anymore, e.g. `date_add(cast('1964-05-23' as date), '12.34')` will cause `AnalysisException`. In Spark version 2.4 and earlier, if the 2nd argument is fractional or string value, it will be coerced to int value, and the result will be a date value of `1964-06-04`.
+  - In Spark 3.0, Spark casts `String` to `Date/Timestamp` in binary comparisons with dates/timestamps. The previous behavior of casting `Date/Timestamp` to `String` can be restored by setting `spark.sql.legacy.typeCoercion.datetimeToString.enabled` to `true`.
 
-  - Since Spark 3.0, the function `percentile_approx` and its alias `approx_percentile` only accept integral value with range in `[1, 2147483647]` as its 3rd argument `accuracy`, fractional and string types are disallowed, e.g. `percentile_approx(10.0, 0.2, 1.8D)` will cause `AnalysisException`. In Spark version 2.4 and earlier, if `accuracy` is fractional or string value, it will be coerced to an int value, `percentile_approx(10.0, 0.2, 1.8D)` is operated as `percentile_approx(10.0, 0.2, 1)` which results in `10.0`.
+  - In Spark 3.0, special values are supported in conversion from strings to dates and timestamps. Those values are simply notational shorthands that are converted to ordinary date or timestamp values when read. The following string values are supported for dates:
+    * `epoch [zoneId]` - 1970-01-01
+    * `today [zoneId]` - the current date in the time zone specified by `spark.sql.session.timeZone`
+    * `yesterday [zoneId]` - the current date - 1
+    * `tomorrow [zoneId]` - the current date + 1
+    * `now` - the date of running the current query. It has the same notion as today
 
-  - Since Spark 3.0, the properties listing below become reserved, commands will fail if we specify reserved properties in places like `CREATE DATABASE ... WITH DBPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. We need their specific clauses to specify them, e.g. `CREATE DATABASE test COMMENT 'any comment' LOCATION 'some path'`. We can set `spark.sql.legacy.property.nonReserved` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, e.g `SET DBPROTERTIES('location'='/tmp')` will affect nothing. In Spark version 2.4 and earlier, these properties are neither reserved nor have side effects, e.g. `SET DBPROTERTIES('location'='/tmp')` will not change the location of the database but only create a headless property just like `'a'='b'`.
-    <table class="table">
-        <tr>
-          <th>
-            <b>Property(case sensitive)</b>
-          </th>
-          <th>
-            <b>Database Reserved</b>
-          </th>
-          <th>
-            <b>Table Reserved</b>
-          </th>
-          <th>
-            <b>Remarks</b>
-          </th>
-        </tr>
-        <tr>
-          <td>
-            provider
-          </td>
-          <td>
-            no
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            For tables, please use the USING clause to specify it. Once set, it can't be changed.
-          </td>
-        </tr>
-        <tr>
-          <td>
-            location
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            For databases and tables, please use the LOCATION clause to specify it.
-          </td>
-        </tr>
-        <tr>
-          <td>
-            owner
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            For databases and tables, it is determined by the user who runs spark and create the table.
-          </td>
-        </tr>
-    </table>
+    For example `SELECT date 'tomorrow' - date 'yesterday';` should output `2`. Here are special timestamp values:
+    * `epoch [zoneId]` - 1970-01-01 00:00:00+00 (Unix system time zero)
+    * `today [zoneId]` - midnight today
+    * `yesterday [zoneId]` - midnight yesterday
+    * `tomorrow [zoneId]` - midnight tomorrow
+    * `now` - current query start time
+
+    For example `SELECT timestamp 'tomorrow';`.
+
+  - Since Spark 3.0, when using `EXTRACT` expression to extract the second field from date/timestamp values, the result will be a `DecimalType(8, 6)` value with 2 digits for second part, and 6 digits for the fractional part with microsecond precision. e.g. `extract(second from to_timestamp('2019-09-20 10:10:10.1'))` results `10.100000`.  In Spark version 2.4 and earlier, it returns an `IntegerType` value and the result for the former example is `10`.
+
+  - In Spark 3.0, datetime pattern letter `F` is **aligned day of week in month** that represents the concept of the count of days within the period of a week where the weeks are aligned to the start of the month. In Spark version 2.4 and earlier, it is **week of month** that represents the concept of the count of weeks within the month where weeks start on a fixed day-of-week, e.g. `2020-07-30` is 30 days (4 weeks and 2 days) after the first day of the month, so `date_format(date '2020-07-30', 'F')` returns 2 in Spark 3.0, but as a week count in Spark 2.x, it returns 5 because it locates in the 5th week of July 2020, where week one is 2020-07-01 to 07-04.
+
+### Data Sources
+
+  - In Spark version 2.4 and below, when reading a Hive SerDe table with Spark native data sources(parquet/orc), Spark infers the actual file schema and update the table schema in metastore. In Spark 3.0, Spark doesn't infer the schema anymore. This should not cause any problems to end users, but if it does, set `spark.sql.hive.caseSensitiveInferenceMode` to `INFER_AND_SAVE`.
+
+  - In Spark version 2.4 and below, partition column value is converted as null if it can't be casted to corresponding user provided schema. In 3.0, partition column value is validated with user provided schema. An exception is thrown if the validation fails. You can disable such validation by setting  `spark.sql.sources.validatePartitionColumns` to `false`.
+
+  - In Spark 3.0, if files or subdirectories disappear during recursive directory listing (that is, they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time.
+
+  - In Spark version 2.4 and below, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`. For `FloatType`, `DoubleType`, `DateType` and `TimestampType`, it fails on empty strings and throws exceptions. Spark 3.0 disallows empty strings and will throw an exception for data types except for `StringType` and `BinaryType`. The previous behavior of allowing an empty string can be restored by setting `spark.sql.legacy.json.allowEmptyString.enabled` to `true`.
+
+  - In Spark version 2.4 and below, JSON datasource and JSON functions like `from_json` convert a bad JSON record to a row with all `null`s in the PERMISSIVE mode when specified schema is `StructType`. In Spark 3.0, the returned row can contain non-`null` fields if some of JSON column values were parsed and converted to desired types successfully.
+
+  - In Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Set JSON option `inferTimestamp` to `false` to disable such type inference.
+
+  - In Spark version 2.4 and below, CSV datasource converts a malformed CSV string to a row with all `null`s in the PERMISSIVE mode. In Spark 3.0, the returned row can contain non-`null` fields if some of CSV column values were parsed and converted to desired types successfully.
+
+  - In Spark 3.0, when Avro files are written with user provided schema, the fields are matched by field names between catalyst schema and Avro schema instead of positions.
+
+  - In Spark 3.0, when Avro files are written with user provided non-nullable schema, even the catalyst schema is nullable, Spark is still able to write the files. However, Spark throws runtime NullPointerException if any of the records contains null.
 
-  - Since Spark 3.0, `ADD FILE` can be used to add file directories as well. Earlier only single files can be added using this command. To restore the behaviour of earlier versions, set `spark.sql.legacy.addDirectory.recursive.enabled` to false.
+### Others
 
-  - Since Spark 3.0, `SHOW TBLPROPERTIES` will cause `AnalysisException` if the table does not exist. In Spark version 2.4 and earlier, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view will cause `AnalysisException`. In Spark version 2.4 and earlier, it returned an empty result.
+  - In Spark version 2.4, when a Spark session is created via `cloneSession()`, the newly created Spark session inherits its configuration from its parent `SparkContext` even though the same configuration may exist with a different value in its parent Spark session. In Spark 3.0, the configurations of a parent `SparkSession` have a higher precedence over the parent `SparkContext`. You can restore the old behavior by setting `spark.sql.legacy.sessionInitWithConfigDefaults` to `true`.
 
-  - Since Spark 3.0, `SHOW CREATE TABLE` will always return Spark DDL, even when the given table is a Hive serde table. For Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead.
+  - In Spark 3.0, if `hive.default.fileformat` is not found in `Spark SQL configuration` then it falls back to the `hive-site.xml` file present in the `Hadoop configuration` of `SparkContext`.
+
+  - In Spark 3.0, we pad decimal numbers with trailing zeros to the scale of the column for `spark-sql` interface, for example:
+
+    | Query | Spark 2.4 | Spark 3.0 |
+    | ----- | --------- | --------- |
+    |`SELECT CAST(1 AS decimal(38, 18));` | 1 | 1.000000000000000000 |
+
+  - In Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3 and it brings following impacts:
+
+    * You may need to set `spark.sql.hive.metastore.version` and `spark.sql.hive.metastore.jars` according to the version of the Hive metastore you want to connect to. For example: set `spark.sql.hive.metastore.version` to `1.2.1` and `spark.sql.hive.metastore.jars` to `maven` if your Hive metastore version is 1.2.1.
+
+    * You need to migrate your custom SerDes to Hive 2.3 or build your own Spark with `hive-1.2` profile. See [HIVE-15167](https://issues.apache.org/jira/browse/HIVE-15167) for more details.
+
+    * The decimal string representation can be different between Hive 1.2 and Hive 2.3 when using `TRANSFORM` operator in SQL for script transformation, which depends on hive's behavior. In Hive 1.2, the string representation omits trailing zeroes. But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
+
+## Upgrading from Spark SQL 2.4.7 to 2.4.8
+
+  - In Spark 2.4.8, `AnalysisException` is replaced by its sub-classes that are thrown for tables from Hive external catalog in the following situations:
+    * `ALTER TABLE .. ADD PARTITION` throws `PartitionsAlreadyExistException` if new partition exists already
+    * `ALTER TABLE .. DROP PARTITION` throws `NoSuchPartitionsException` for not existing partitions
+    
+## Upgrading from Spark SQL 2.4.5 to 2.4.6
+
+  - In Spark 2.4.6, the `RESET` command does not reset the static SQL configuration values to the default. It only clears the runtime SQL configuration values.
 
 ## Upgrading from Spark SQL 2.4.4 to 2.4.5
 
@@ -734,11 +637,11 @@ and writing data out (`DataFrame.write`),
 and deprecated the old APIs (e.g., `SQLContext.parquetFile`, `SQLContext.jsonFile`).
 
 See the API docs for `SQLContext.read` (
-  <a href="api/scala/index.html#org.apache.spark.sql.SQLContext@read:DataFrameReader">Scala</a>,
+  <a href="api/scala/org/apache/spark/sql/SQLContext.html#read:DataFrameReader">Scala</a>,
   <a href="api/java/org/apache/spark/sql/SQLContext.html#read()">Java</a>,
   <a href="api/python/pyspark.sql.html#pyspark.sql.SQLContext.read">Python</a>
 ) and `DataFrame.write` (
-  <a href="api/scala/index.html#org.apache.spark.sql.DataFrame@write:DataFrameWriter">Scala</a>,
+  <a href="api/scala/org/apache/spark/sql/DataFrame.html#write:DataFrameWriter">Scala</a>,
   <a href="api/java/org/apache/spark/sql/Dataset.html#write()">Java</a>,
   <a href="api/python/pyspark.sql.html#pyspark.sql.DataFrame.write">Python</a>
 ) more information.
@@ -903,7 +806,7 @@ Python UDF registration is unchanged.
 Spark SQL is designed to be compatible with the Hive Metastore, SerDes and UDFs.
 Currently, Hive SerDes and UDFs are based on built-in Hive,
 and Spark SQL can be connected to different versions of Hive Metastore
-(from 0.12.0 to 2.3.6 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
+(from 0.12.0 to 2.3.7 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
 
 #### Deploying in Existing Hive Warehouses
 {:.no_toc}
@@ -944,29 +847,29 @@ Spark SQL supports the vast majority of Hive features, such as:
 * Unions
 * Sub-queries
   * Sub-queries in the FROM Clause
-  
+
     ```SELECT col FROM (SELECT a + b AS col FROM t1) t2```
   * Sub-queries in WHERE Clause
     * Correlated or non-correlated IN and NOT IN statement in WHERE Clause
-    
+
       ```
       SELECT col FROM t1 WHERE col IN (SELECT a FROM t2 WHERE t1.a = t2.a)
       SELECT col FROM t1 WHERE col IN (SELECT a FROM t2)
       ```
     * Correlated or non-correlated EXISTS and NOT EXISTS statement in WHERE Clause
-    
+
       ```
       SELECT col FROM t1 WHERE EXISTS (SELECT t2.a FROM t2 WHERE t1.a = t2.a AND t2.a > 10)
       SELECT col FROM t1 WHERE EXISTS (SELECT t2.a FROM t2 WHERE t2.a > 10)
       ```
     * Non-correlated IN and NOT IN statement in JOIN Condition
-    
+
       ```SELECT t1.col FROM t1 JOIN t2 ON t1.a = t2.a AND t1.a IN (SELECT a FROM t3)```
-   
+
     * Non-correlated EXISTS and NOT EXISTS statement in JOIN Condition
-       
-      ```SELECT t1.col FROM t1 JOIN t2 ON t1.a = t2.a AND EXISTS (SELECT * FROM t3 WHERE t3.a > 10)``` 
-       
+
+      ```SELECT t1.col FROM t1 JOIN t2 ON t1.a = t2.a AND EXISTS (SELECT * FROM t3 WHERE t3.a > 10)```
+
 * Sampling
 * Explain
 * Partitioned tables including dynamic partition insertion
@@ -1073,3 +976,4 @@ Below are the scenarios in which Hive and Spark generate different results:
 * `SQRT(n)` If n < 0, Hive returns null, Spark SQL returns NaN.
 * `ACOS(n)` If n < -1 or n > 1, Hive returns null, Spark SQL returns NaN.
 * `ASIN(n)` If n < -1 or n > 1, Hive returns null, Spark SQL returns NaN.
+* `CAST(n AS TIMESTAMP)` If n is integral numbers, Hive treats n as milliseconds, Spark SQL treats n as seconds.
diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index e289854c7acc7..49b32e7562e0b 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -35,7 +35,7 @@ Configuration of in-memory caching can be done using the `setConf` method on `Sp
 `SET key=value` commands using SQL.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.sql.inMemoryColumnarStorage.compressed</code></td>
   <td>true</td>
@@ -43,6 +43,7 @@ Configuration of in-memory caching can be done using the `setConf` method on `Sp
     When set to true Spark SQL will automatically select a compression codec for each column based
     on statistics of the data.
   </td>
+  <td>1.0.1</td>
 </tr>
 <tr>
   <td><code>spark.sql.inMemoryColumnarStorage.batchSize</code></td>
@@ -51,6 +52,7 @@ Configuration of in-memory caching can be done using the `setConf` method on `Sp
     Controls the size of batches for columnar caching. Larger batch sizes can improve memory utilization
     and compression, but risk OOMs when caching data.
   </td>
+  <td>1.1.1</td>
 </tr>
 
 </table>
@@ -61,13 +63,15 @@ The following options can also be used to tune the performance of query executio
 that these options will be deprecated in future release as more optimizations are performed automatically.
 
 <table class="table">
-  <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+  <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.sql.files.maxPartitionBytes</code></td>
     <td>134217728 (128 MB)</td>
     <td>
       The maximum number of bytes to pack into a single partition when reading files.
+      This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.
     </td>
+    <td>2.0.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.files.openCostInBytes</code></td>
@@ -76,17 +80,20 @@ that these options will be deprecated in future release as more optimizations ar
       The estimated cost to open a file, measured by the number of bytes could be scanned in the same
       time. This is used when putting multiple files into a partition. It is better to over-estimated,
       then the partitions with small files will be faster than partitions with bigger files (which is
-      scheduled first).
+      scheduled first). This configuration is effective only when using file-based sources such as Parquet,
+      JSON and ORC.
     </td>
+    <td>2.0.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.broadcastTimeout</code></td>
     <td>300</td>
     <td>
-    <p>
-      Timeout in seconds for the broadcast wait time in broadcast joins
-    </p>
+      <p>
+        Timeout in seconds for the broadcast wait time in broadcast joins
+      </p>
     </td>
+    <td>1.3.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.autoBroadcastJoinThreshold</code></td>
@@ -97,6 +104,7 @@ that these options will be deprecated in future release as more optimizations ar
       statistics are only supported for Hive Metastore tables where the command
       <code>ANALYZE TABLE &lt;tableName&gt; COMPUTE STATISTICS noscan</code> has been run.
     </td>
+    <td>1.1.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.shuffle.partitions</code></td>
@@ -104,6 +112,29 @@ that these options will be deprecated in future release as more optimizations ar
     <td>
       Configures the number of partitions to use when shuffling data for joins or aggregations.
     </td>
+    <td>1.1.0</td>
+  </tr>
+  <tr>
+    <td><code>spark.sql.sources.parallelPartitionDiscovery.threshold</code></td>
+    <td>32</td>
+    <td>
+      Configures the threshold to enable parallel listing for job input paths. If the number of
+      input paths is larger than this threshold, Spark will list the files by using Spark distributed job.
+      Otherwise, it will fallback to sequential listing. This configuration is only effective when
+      using file-based data sources such as Parquet, ORC and JSON.
+    </td>
+    <td>1.5.0</td>
+  </tr>
+  <tr>
+    <td><code>spark.sql.sources.parallelPartitionDiscovery.parallelism</code></td>
+    <td>10000</td>
+    <td>
+      Configures the maximum listing parallelism for job input paths. In case the number of input
+      paths is larger than this value, it will be throttled down to use this value. Same as above,
+      this configuration is only effective when using file-based data sources such as Parquet, ORC
+      and JSON.
+    </td>
+    <td>2.1.1</td>
   </tr>
 </table>
 
@@ -160,7 +191,7 @@ head(join(src, hint(records, "broadcast"), src$key == records$key))
 
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 -- We accept BROADCAST, BROADCASTJOIN and MAPJOIN for broadcast hint
@@ -170,6 +201,8 @@ SELECT /*+ BROADCAST(r) */ * FROM records r JOIN src s ON r.key = s.key
 </div>
 </div>
 
+For more details please refer to the documentation of [Join Hints](sql-ref-syntax-qry-select-hints.html#join-hints).
+
 ## Coalesce Hints for SQL Queries
 
 Coalesce hints allows the Spark SQL users to control the number of output files just like the
@@ -184,3 +217,79 @@ The "REPARTITION_BY_RANGE" hint must have column names and a partition number is
     SELECT /*+ REPARTITION(3, c) */ * FROM t
     SELECT /*+ REPARTITION_BY_RANGE(c) */ * FROM t
     SELECT /*+ REPARTITION_BY_RANGE(3, c) */ * FROM t
+
+For more details please refer to the documentation of [Partitioning Hints](sql-ref-syntax-qry-select-hints.html#partitioning-hints).
+
+## Adaptive Query Execution
+Adaptive Query Execution (AQE) is an optimization technique in Spark SQL that makes use of the runtime statistics to choose the most efficient query execution plan. AQE is disabled by default. Spark SQL can use the umbrella configuration of `spark.sql.adaptive.enabled` to control whether turn it on/off. As of Spark 3.0, there are three major features in AQE, including coalescing post-shuffle partitions, converting sort-merge join to broadcast join, and skew join optimization.
+
+### Coalescing Post Shuffle Partitions
+This feature coalesces the post shuffle partitions based on the map output statistics when both `spark.sql.adaptive.enabled` and `spark.sql.adaptive.coalescePartitions.enabled` configurations are true. This feature simplifies the tuning of shuffle partition number when running queries. You do not need to set a proper shuffle partition number to fit your dataset. Spark can pick the proper shuffle partition number at runtime once you set a large enough initial number of shuffle partitions via `spark.sql.adaptive.coalescePartitions.initialPartitionNum` configuration.
+ <table class="table">
+   <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
+   <tr>
+     <td><code>spark.sql.adaptive.coalescePartitions.enabled</code></td>
+     <td>true</td>
+     <td>
+       When true and <code>spark.sql.adaptive.enabled</code> is true, Spark will coalesce contiguous shuffle partitions according to the target size (specified by <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code>), to avoid too many small tasks.
+     </td>
+     <td>3.0.0</td>
+   </tr>
+   <tr>
+     <td><code>spark.sql.adaptive.coalescePartitions.minPartitionNum</code></td>
+     <td>Default Parallelism</td>
+     <td>
+       The minimum number of shuffle partitions after coalescing. If not set, the default value is the default parallelism of the Spark cluster. This configuration only has an effect when <code>spark.sql.adaptive.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> are both enabled.
+     </td>
+     <td>3.0.0</td>
+   </tr>
+   <tr>
+     <td><code>spark.sql.adaptive.coalescePartitions.initialPartitionNum</code></td>
+     <td>200</td>
+     <td>
+       The initial number of shuffle partitions before coalescing. By default it equals to <code>spark.sql.shuffle.partitions</code>. This configuration only has an effect when <code>spark.sql.adaptive.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> are both enabled.
+     </td>
+     <td>3.0.0</td>
+   </tr>
+   <tr>
+     <td><code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code></td>
+     <td>64 MB</td>
+     <td>
+       The advisory size in bytes of the shuffle partition during adaptive optimization (when <code>spark.sql.adaptive.enabled</code> is true). It takes effect when Spark coalesces small shuffle partitions or splits skewed shuffle partition.
+     </td>
+     <td>3.0.0</td>
+   </tr>
+ </table>
+ 
+### Converting sort-merge join to broadcast join
+AQE converts sort-merge join to broadcast hash join when the runtime statistics of any join side is smaller than the broadcast hash join threshold. This is not as efficient as planning a broadcast hash join in the first place, but it's better than keep doing the sort-merge join, as we can save the sorting of both the join sides, and read shuffle files locally to save network traffic(if `spark.sql.adaptive.localShuffleReader.enabled` is true)
+
+### Optimizing Skew Join
+Data skew can severely downgrade the performance of join queries. This feature dynamically handles skew in sort-merge join by splitting (and replicating if needed) skewed tasks into roughly evenly sized tasks. It takes effect when both `spark.sql.adaptive.enabled` and `spark.sql.adaptive.skewJoin.enabled` configurations are enabled.
+  <table class="table">
+     <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
+     <tr>
+       <td><code>spark.sql.adaptive.skewJoin.enabled</code></td>
+       <td>true</td>
+       <td>
+         When true and <code>spark.sql.adaptive.enabled</code> is true, Spark dynamically handles skew in sort-merge join by splitting (and replicating if needed) skewed partitions.
+       </td>
+       <td>3.0.0</td>
+     </tr>
+     <tr>
+       <td><code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code></td>
+       <td>10</td>
+       <td>
+         A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code>.
+       </td>
+       <td>3.0.0</td>
+     </tr>
+     <tr>
+       <td><code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code></td>
+       <td>256MB</td>
+       <td>
+         A partition is considered as skewed if its size in bytes is larger than this threshold and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> multiplying the median partition size. Ideally this config should be set larger than <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code>.
+       </td>
+       <td>3.0.0</td>
+     </tr>
+   </table>
diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md
index 0a4d07ea37b69..06bf553e4f704 100644
--- a/docs/sql-programming-guide.md
+++ b/docs/sql-programming-guide.md
@@ -61,7 +61,7 @@ In Scala and Java, a DataFrame is represented by a Dataset of `Row`s.
 In [the Scala API][scala-datasets], `DataFrame` is simply a type alias of `Dataset[Row]`.
 While, in [Java API][java-datasets], users need to use `Dataset<Row>` to represent a `DataFrame`.
 
-[scala-datasets]: api/scala/index.html#org.apache.spark.sql.Dataset
+[scala-datasets]: api/scala/org/apache/spark/sql/Dataset.html
 [java-datasets]: api/java/index.html?org/apache/spark/sql/Dataset.html
 
 Throughout this document, we will often refer to Scala/Java Datasets of `Row`s as DataFrames.
diff --git a/docs/sql-pyspark-pandas-with-arrow.md b/docs/sql-pyspark-pandas-with-arrow.md
index 7eb8a74547f70..a11162963b380 100644
--- a/docs/sql-pyspark-pandas-with-arrow.md
+++ b/docs/sql-pyspark-pandas-with-arrow.md
@@ -33,9 +33,11 @@ working with Arrow-enabled data.
 
 ### Ensure PyArrow Installed
 
+To use Apache Arrow in PySpark, [the recommended version of PyArrow](#recommended-pandas-and-pyarrow-versions)
+should be installed.
 If you install PySpark using pip, then PyArrow can be brought in as an extra dependency of the
 SQL module with the command `pip install pyspark[sql]`. Otherwise, you must ensure that PyArrow
-is installed and available on all cluster nodes. The current supported version is 0.12.1.
+is installed and available on all cluster nodes.
 You can install using pip or conda from the conda-forge channel. See PyArrow
 [installation](https://arrow.apache.org/docs/python/install.html) for details.
 
@@ -65,132 +67,221 @@ Spark will fall back to create the DataFrame without Arrow.
 
 ## Pandas UDFs (a.k.a. Vectorized UDFs)
 
-Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and
-Pandas to work with the data. A Pandas UDF is defined using the keyword `pandas_udf` as a decorator
-or to wrap the function, no additional configuration is required. Currently, there are two types of
-Pandas UDF: Scalar and Grouped Map.
+Pandas UDFs are user defined functions that are executed by Spark using
+Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas
+UDF is defined using the `pandas_udf` as a decorator or to wrap the function, and no additional
+configuration is required. A Pandas UDF behaves as a regular PySpark function API in general.
 
-### Scalar
+Before Spark 3.0, Pandas UDFs used to be defined with `PandasUDFType`. From Spark 3.0
+with Python 3.6+, you can also use [Python type hints](https://www.python.org/dev/peps/pep-0484).
+Using Python type hints are preferred and using `PandasUDFType` will be deprecated in
+the future release.
 
-Scalar Pandas UDFs are used for vectorizing scalar operations. They can be used with functions such
-as `select` and `withColumn`. The Python function should take `pandas.Series` as inputs and return
-a `pandas.Series` of the same length. Internally, Spark will execute a Pandas UDF by splitting
-columns into batches and calling the function for each batch as a subset of the data, then
-concatenating the results together.
+Note that the type hint should use `pandas.Series` in all cases but there is one variant
+that `pandas.DataFrame` should be used for its input or output type hint instead when the input
+or output column is of `StructType`. The following example shows a Pandas UDF which takes long
+column, string column and struct column, and outputs a struct column. It requires the function to
+specify the type hints of `pandas.Series` and `pandas.DataFrame` as below:
 
-The following example shows how to create a scalar Pandas UDF that computes the product of 2 columns.
+<p>
+<div class="codetabs">
+<div data-lang="python" markdown="1">
+{% include_example ser_to_frame_pandas_udf python/sql/arrow.py %}
+</div>
+</div>
+</p>
+
+In the following sections, it describes the combinations of the supported type hints. For simplicity,
+`pandas.DataFrame` variant is omitted.
+
+### Series to Series
+
+The type hint can be expressed as `pandas.Series`, ... -> `pandas.Series`.
+
+By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the given
+function takes one or more `pandas.Series` and outputs one `pandas.Series`. The output of the function should
+always be of the same length as the input. Internally, PySpark will execute a Pandas UDF by splitting
+columns into batches and calling the function for each batch as a subset of the data, then concatenating
+the results together.
+
+The following example shows how to create this Pandas UDF that computes the product of 2 columns.
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example scalar_pandas_udf python/sql/arrow.py %}
+{% include_example ser_to_ser_pandas_udf python/sql/arrow.py %}
 </div>
 </div>
 
-### Scalar Iterator
+For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
+
+### Iterator of Series to Iterator of Series
+
+The type hint can be expressed as `Iterator[pandas.Series]` -> `Iterator[pandas.Series]`.
+
+By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the given
+function takes an iterator of `pandas.Series` and outputs an iterator of `pandas.Series`. The
+length of the entire output from the function should be the same length of the entire input; therefore, it can
+prefetch the data from the input iterator as long as the lengths are the same.
+In this case, the created Pandas UDF requires one input column when the Pandas UDF is called. To use
+multiple input columns, a different type hint is required. See Iterator of Multiple Series to Iterator
+of Series.
+
+It is also useful when the UDF execution requires initializing some states although internally it works
+identically as Series to Series case. The pseudocode below illustrates the example.
+
+{% highlight python %}
+@pandas_udf("long")
+def calculate(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
+    # Do some expensive initialization with a state
+    state = very_expensive_initialization()
+    for x in iterator:
+        # Use that state for whole iterator.
+        yield calculate_with_state(x, state)
 
-Scalar iterator (`SCALAR_ITER`) Pandas UDF is the same as scalar Pandas UDF above except that the
-underlying Python function takes an iterator of batches as input instead of a single batch and,
-instead of returning a single output batch, it yields output batches or returns an iterator of
-output batches.
-It is useful when the UDF execution requires initializing some states, e.g., loading an machine
-learning model file to apply inference to every input batch.
+df.select(calculate("value")).show()
+{% endhighlight %}
 
-The following example shows how to create scalar iterator Pandas UDFs:
+The following example shows how to create this Pandas UDF:
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example scalar_iter_pandas_udf python/sql/arrow.py %}
+{% include_example iter_ser_to_iter_ser_pandas_udf python/sql/arrow.py %}
 </div>
 </div>
 
-### Grouped Map
-Grouped map Pandas UDFs are used with `groupBy().apply()` which implements the "split-apply-combine" pattern.
-Split-apply-combine consists of three steps:
-* Split the data into groups by using `DataFrame.groupBy`.
-* Apply a function on each group. The input and output of the function are both `pandas.DataFrame`. The
-  input data contains all the rows and columns for each group.
-* Combine the results into a new `DataFrame`.
+For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
 
-To use `groupBy().apply()`, the user needs to define the following:
-* A Python function that defines the computation for each group.
-* A `StructType` object or a string that defines the schema of the output `DataFrame`.
+### Iterator of Multiple Series to Iterator of Series
 
-The column labels of the returned `pandas.DataFrame` must either match the field names in the
-defined output schema if specified as strings, or match the field data types by position if not
-strings, e.g. integer indices. See [pandas.DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html#pandas.DataFrame)
-on how to label columns when constructing a `pandas.DataFrame`.
+The type hint can be expressed as `Iterator[Tuple[pandas.Series, ...]]` -> `Iterator[pandas.Series]`.
 
-Note that all data for a group will be loaded into memory before the function is applied. This can
-lead to out of memory exceptions, especially if the group sizes are skewed. The configuration for
-[maxRecordsPerBatch](#setting-arrow-batch-size) is not applied on groups and it is up to the user
-to ensure that the grouped data will fit into the available memory.
+By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the
+given function takes an iterator of a tuple of multiple `pandas.Series` and outputs an iterator of `pandas.Series`.
+In this case, the created pandas UDF requires multiple input columns as many as the series in the tuple
+when the Pandas UDF is called. Otherwise, it has the same characteristics and restrictions as Iterator of Series
+to Iterator of Series case.
 
-The following example shows how to use `groupby().apply()` to subtract the mean from each value in the group.
+The following example shows how to create this Pandas UDF:
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example grouped_map_pandas_udf python/sql/arrow.py %}
+{% include_example iter_sers_to_iter_ser_pandas_udf python/sql/arrow.py %}
 </div>
 </div>
 
-For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf) and
-[`pyspark.sql.GroupedData.apply`](api/python/pyspark.sql.html#pyspark.sql.GroupedData.apply).
+For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
 
-### Grouped Aggregate
+### Series to Scalar
 
-Grouped aggregate Pandas UDFs are similar to Spark aggregate functions. Grouped aggregate Pandas UDFs are used with `groupBy().agg()` and
-[`pyspark.sql.Window`](api/python/pyspark.sql.html#pyspark.sql.Window). It defines an aggregation from one or more `pandas.Series`
-to a scalar value, where each `pandas.Series` represents a column within the group or window.
+The type hint can be expressed as `pandas.Series`, ... -> `Any`.
 
-Note that this type of UDF does not support partial aggregation and all data for a group or window will be loaded into memory. Also,
-only unbounded window is supported with Grouped aggregate Pandas UDFs currently.
+By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF similar
+to PySpark's aggregate functions. The given function takes `pandas.Series` and returns a scalar value.
+The return type should be a primitive data type, and the returned scalar can be either a python
+primitive type, e.g., `int` or `float` or a numpy data type, e.g., `numpy.int64` or `numpy.float64`.
+`Any` should ideally be a specific scalar type accordingly.
 
-The following example shows how to use this type of UDF to compute mean with groupBy and window operations:
+This UDF can be also used with `groupBy().agg()` and [`pyspark.sql.Window`](api/python/pyspark.sql.html#pyspark.sql.Window).
+It defines an aggregation from one or more `pandas.Series` to a scalar value, where each `pandas.Series`
+represents a column within the group or window.
+
+Note that this type of UDF does not support partial aggregation and all data for a group or window
+will be loaded into memory. Also, only unbounded window is supported with Grouped aggregate Pandas
+UDFs currently. The following example shows how to use this type of UDF to compute mean with a group-by
+and window operations:
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example grouped_agg_pandas_udf python/sql/arrow.py %}
+{% include_example ser_to_scalar_pandas_udf python/sql/arrow.py %}
 </div>
 </div>
 
 For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
 
 
-### Map Iterator
+## Pandas Function APIs
+
+Pandas Function APIs can directly apply a Python native function against the whole `DataFrame` by
+using Pandas instances. Internally it works similarly with Pandas UDFs by using Arrow to transfer
+data and Pandas to work with the data, which allows vectorized operations. However, A Pandas Function
+API behaves as a regular API under PySpark `DataFrame` instead of `Column`, and Python type hints in Pandas
+Functions APIs are optional and do not affect how it works internally at this moment although they
+might be required in the future.
 
-Map iterator Pandas UDFs are used to transform data with an iterator of batches. Map iterator
-Pandas UDFs can be used with 
-[`pyspark.sql.DataFrame.mapInPandas`](api/python/pyspark.sql.html#pyspark.sql.DataFrame.mapInPandas).
-It defines a map function that transforms an iterator of `pandas.DataFrame` to another.
+From Spark 3.0, grouped map pandas UDF is now categorized as a separate Pandas Function API,
+`DataFrame.groupby().applyInPandas()`. It is still possible to use it with `PandasUDFType`
+and `DataFrame.groupby().apply()` as it was; however, it is preferred to use
+`DataFrame.groupby().applyInPandas()` directly. Using `PandasUDFType` will be deprecated
+in the future.
+
+### Grouped Map
 
-It can return the output of arbitrary length in contrast to the scalar Pandas UDF. It maps an iterator of `pandas.DataFrame`s,
-that represents the current `DataFrame`, using the map iterator UDF and returns the result as a `DataFrame`.
+Grouped map operations with Pandas instances are supported by `DataFrame.groupby().applyInPandas()`
+which requires a Python function that takes a `pandas.DataFrame` and return another `pandas.DataFrame`.
+It maps each group to each `pandas.DataFrame` in the Python function.
 
-The following example shows how to create map iterator Pandas UDFs:
+This API implements the "split-apply-combine" pattern which consists of three steps:
+* Split the data into groups by using `DataFrame.groupBy`.
+* Apply a function on each group. The input and output of the function are both `pandas.DataFrame`. The
+  input data contains all the rows and columns for each group.
+* Combine the results into a new PySpark `DataFrame`.
+
+To use `groupBy().applyInPandas()`, the user needs to define the following:
+* A Python function that defines the computation for each group.
+* A `StructType` object or a string that defines the schema of the output PySpark `DataFrame`.
+
+The column labels of the returned `pandas.DataFrame` must either match the field names in the
+defined output schema if specified as strings, or match the field data types by position if not
+strings, e.g. integer indices. See [pandas.DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html#pandas.DataFrame)
+on how to label columns when constructing a `pandas.DataFrame`.
+
+Note that all data for a group will be loaded into memory before the function is applied. This can
+lead to out of memory exceptions, especially if the group sizes are skewed. The configuration for
+[maxRecordsPerBatch](#setting-arrow-batch-size) is not applied on groups and it is up to the user
+to ensure that the grouped data will fit into the available memory.
+
+The following example shows how to use `groupby().applyInPandas()` to subtract the mean from each value
+in the group.
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example map_iter_pandas_udf python/sql/arrow.py %}
+{% include_example grouped_apply_in_pandas python/sql/arrow.py %}
 </div>
 </div>
 
-For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf) and
-[`pyspark.sql.DataFrame.mapsInPandas`](api/python/pyspark.sql.html#pyspark.sql.DataFrame.mapInPandas).
+For detailed usage, please see [`pyspark.sql.GroupedData.applyInPandas`](api/python/pyspark.sql.html#pyspark.sql.GroupedData.applyInPandas).
+
+### Map
+
+Map operations with Pandas instances are supported by `DataFrame.mapInPandas()` which maps an iterator
+of `pandas.DataFrame`s to another iterator of `pandas.DataFrame`s that represents the current
+PySpark `DataFrame` and returns the result as a PySpark `DataFrame`. The functions takes and outputs
+an iterator of `pandas.DataFrame`. It can return the output of arbitrary length in contrast to some
+Pandas UDFs although internally it works similarly with Series to Series Pandas UDF.
 
+The following example shows how to use `mapInPandas()`:
 
-### Cogrouped Map
+<div class="codetabs">
+<div data-lang="python" markdown="1">
+{% include_example map_in_pandas python/sql/arrow.py %}
+</div>
+</div>
 
-Cogrouped map Pandas UDFs allow two DataFrames to be cogrouped by a common key and then a python function applied to
-each cogroup.  They are used with `groupBy().cogroup().apply()` which consists of the following steps:
+For detailed usage, please see [`pyspark.sql.DataFrame.mapsInPandas`](api/python/pyspark.sql.html#pyspark.sql.DataFrame.mapInPandas).
 
+### Co-grouped Map
+
+Co-grouped map operations with Pandas instances are supported by `DataFrame.groupby().cogroup().applyInPandas()` which
+allows two PySpark `DataFrame`s to be cogrouped by a common key and then a Python function applied to each
+cogroup. It consists of the following steps:
 * Shuffle the data such that the groups of each dataframe which share a key are cogrouped together.
-* Apply a function to each cogroup.  The input of the function is two `pandas.DataFrame` (with an optional Tuple
-representing the key).  The output of the function is a `pandas.DataFrame`.
-* Combine the pandas.DataFrames from all groups into a new `DataFrame`. 
+* Apply a function to each cogroup. The input of the function is two `pandas.DataFrame` (with an optional tuple
+representing the key). The output of the function is a `pandas.DataFrame`.
+* Combine the `pandas.DataFrame`s from all groups into a new PySpark `DataFrame`. 
 
-To use `groupBy().cogroup().apply()`, the user needs to define the following:
+To use `groupBy().cogroup().applyInPandas()`, the user needs to define the following:
 * A Python function that defines the computation for each cogroup.
-* A `StructType` object or a string that defines the schema of the output `DataFrame`.
+* A `StructType` object or a string that defines the schema of the output PySpark `DataFrame`.
 
 The column labels of the returned `pandas.DataFrame` must either match the field names in the
 defined output schema if specified as strings, or match the field data types by position if not
@@ -201,16 +292,15 @@ Note that all data for a cogroup will be loaded into memory before the function
 memory exceptions, especially if the group sizes are skewed. The configuration for [maxRecordsPerBatch](#setting-arrow-batch-size)
 is not applied and it is up to the user to ensure that the cogrouped data will fit into the available memory.
 
-The following example shows how to use `groupby().cogroup().apply()` to perform an asof join between two datasets.
+The following example shows how to use `groupby().cogroup().applyInPandas()` to perform an asof join between two datasets.
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example cogrouped_map_pandas_udf python/sql/arrow.py %}
+{% include_example cogrouped_apply_in_pandas python/sql/arrow.py %}
 </div>
 </div>
 
-For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf) and
-[`pyspark.sql.CoGroupedData.apply`](api/python/pyspark.sql.html#pyspark.sql.CoGroupedData.apply).
+For detailed usage, please see [`pyspark.sql.PandasCogroupedOps.applyInPandas()`](api/python/pyspark.sql.html#pyspark.sql.PandasCogroupedOps.applyInPandas).
 
 
 ## Usage Notes
@@ -255,6 +345,12 @@ different than a Pandas timestamp. It is recommended to use Pandas time series f
 working with timestamps in `pandas_udf`s to get the best performance, see
 [here](https://pandas.pydata.org/pandas-docs/stable/timeseries.html) for details.
 
+### Recommended Pandas and PyArrow Versions
+
+For usage with pyspark.sql, the supported versions of Pandas is 0.24.2 and PyArrow is 0.15.1. Higher
+versions may be used, however, compatibility and data correctness can not be guaranteed and should
+be verified by the user.
+
 ### Compatibility Setting for PyArrow >= 0.15.0 and Spark 2.3.x, 2.4.x
 
 Since Arrow 0.15.0, a change in the binary IPC format requires an environment variable to be
diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
new file mode 100644
index 0000000000000..948a36e15f3ce
--- /dev/null
+++ b/docs/sql-ref-ansi-compliance.md
@@ -0,0 +1,381 @@
+---
+layout: global
+title: ANSI Compliance
+displayTitle: ANSI Compliance
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+Since Spark 3.0, Spark SQL introduces two experimental options to comply with the SQL standard: `spark.sql.ansi.enabled` and `spark.sql.storeAssignmentPolicy` (See a table below for details).
+
+When `spark.sql.ansi.enabled` is set to `true`, Spark SQL follows the standard in basic behaviours (e.g., arithmetic operations, type conversion, SQL functions and SQL parsing).
+Moreover, Spark SQL has an independent option to control implicit casting behaviours when inserting rows in a table.
+The casting behaviours are defined as store assignment rules in the standard.
+
+When `spark.sql.storeAssignmentPolicy` is set to `ANSI`, Spark SQL complies with the ANSI store assignment rules. This is a separate configuration because its default value is `ANSI`, while the configuration `spark.sql.ansi.enabled` is disabled by default.
+
+|Property Name|Default|Meaning|Since Version|
+|-------------|-------|-------|-------------|
+|`spark.sql.ansi.enabled`|false|(Experimental) When true, Spark tries to conform to the ANSI SQL specification: <br/> 1. Spark will throw a runtime exception if an overflow occurs in any operation on integral/decimal field. <br/> 2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in the SQL parser.|3.0.0|
+|`spark.sql.storeAssignmentPolicy`|ANSI|(Experimental) When inserting a value into a column with different data type, Spark will perform type coercion.  Currently, we support 3 policies for the type coercion rules: ANSI, legacy and strict. With ANSI policy, Spark performs the type coercion as per ANSI SQL. In practice, the behavior is mostly the same as PostgreSQL.  It disallows certain unreasonable type conversions such as converting string to int or double to boolean.  With legacy policy, Spark allows the type coercion as long as it is a valid Cast, which is very loose.  e.g. converting string to int or double to boolean is allowed.  It is also the only behavior in Spark 2.x and it is compatible with Hive.  With strict policy, Spark doesn't allow any possible precision loss or data truncation in type coercion, e.g. converting double to int or decimal to double is not allowed.|3.0.0|
+
+The following subsections present behaviour changes in arithmetic operations, type conversions, and SQL parsing when the ANSI mode enabled.
+
+### Arithmetic Operations
+
+In Spark SQL, arithmetic operations performed on numeric types (with the exception of decimal) are not checked for overflows by default.
+This means that in case an operation causes overflows, the result is the same with the corresponding operation in a Java/Scala program (e.g., if the sum of 2 integers is higher than the maximum value representable, the result is a negative number).
+On the other hand, Spark SQL returns null for decimal overflows.
+When `spark.sql.ansi.enabled` is set to `true` and an overflow occurs in numeric and interval arithmetic operations, it throws an arithmetic exception at runtime.
+
+```sql
+-- `spark.sql.ansi.enabled=true`
+SELECT 2147483647 + 1;
+java.lang.ArithmeticException: integer overflow
+
+-- `spark.sql.ansi.enabled=false`
+SELECT 2147483647 + 1;
++----------------+
+|(2147483647 + 1)|
++----------------+
+|     -2147483648|
++----------------+
+```
+
+### Type Conversion
+
+Spark SQL has three kinds of type conversions: explicit casting, type coercion, and store assignment casting.
+When `spark.sql.ansi.enabled` is set to `true`, explicit casting by `CAST` syntax throws a runtime exception for illegal cast patterns defined in the standard, e.g. casts from a string to an integer.
+On the other hand, `INSERT INTO` syntax throws an analysis exception when the ANSI mode enabled via `spark.sql.storeAssignmentPolicy=ANSI`.
+
+Currently, the ANSI mode affects explicit casting and assignment casting only.
+In future releases, the behaviour of type coercion might change along with the other two type conversion rules.
+
+```sql
+-- Examples of explicit casting
+
+-- `spark.sql.ansi.enabled=true`
+SELECT CAST('a' AS INT);
+java.lang.NumberFormatException: invalid input syntax for type numeric: a
+
+SELECT CAST(2147483648L AS INT);
+java.lang.ArithmeticException: Casting 2147483648 to int causes overflow
+
+-- `spark.sql.ansi.enabled=false` (This is a default behaviour)
+SELECT CAST('a' AS INT);
++--------------+
+|CAST(a AS INT)|
++--------------+
+|          null|
++--------------+
+
+SELECT CAST(2147483648L AS INT);
++-----------------------+
+|CAST(2147483648 AS INT)|
++-----------------------+
+|            -2147483648|
++-----------------------+
+
+-- Examples of store assignment rules
+CREATE TABLE t (v INT);
+
+-- `spark.sql.storeAssignmentPolicy=ANSI`
+INSERT INTO t VALUES ('1');
+org.apache.spark.sql.AnalysisException: Cannot write incompatible data to table '`default`.`t`':
+- Cannot safely cast 'v': string to int;
+
+-- `spark.sql.storeAssignmentPolicy=LEGACY` (This is a legacy behaviour until Spark 2.x)
+INSERT INTO t VALUES ('1');
+SELECT * FROM t;
++---+
+|  v|
++---+
+|  1|
++---+
+```
+
+### SQL Functions
+
+The behavior of some SQL functions can be different under ANSI mode (`spark.sql.ansi.enabled=true`).
+  - `size`: This function returns null for null input under ANSI mode.
+
+### SQL Keywords
+
+When `spark.sql.ansi.enabled` is true, Spark SQL will use the ANSI mode parser.
+In this mode, Spark SQL has two kinds of keywords:
+* Reserved keywords: Keywords that are reserved and can't be used as identifiers for table, view, column, function, alias, etc.
+* Non-reserved keywords: Keywords that have a special meaning only in particular contexts and can be used as identifiers in other contexts. For example, `EXPLAIN SELECT ...` is a command, but EXPLAIN can be used as identifiers in other places.
+
+When the ANSI mode is disabled, Spark SQL has two kinds of keywords:
+* Non-reserved keywords: Same definition as the one when the ANSI mode enabled.
+* Strict-non-reserved keywords: A strict version of non-reserved keywords, which can not be used as table alias.
+
+By default `spark.sql.ansi.enabled` is false.
+
+Below is a list of all the keywords in Spark SQL.
+
+|Keyword|Spark SQL<br/>ANSI Mode|Spark SQL<br/>Default Mode|SQL-2016|
+|-------|----------------------|-------------------------|--------|
+|ADD|non-reserved|non-reserved|non-reserved|
+|AFTER|non-reserved|non-reserved|non-reserved|
+|ALL|reserved|non-reserved|reserved|
+|ALTER|non-reserved|non-reserved|reserved|
+|ANALYZE|non-reserved|non-reserved|non-reserved|
+|AND|reserved|non-reserved|reserved|
+|ANTI|non-reserved|strict-non-reserved|non-reserved|
+|ANY|reserved|non-reserved|reserved|
+|ARCHIVE|non-reserved|non-reserved|non-reserved|
+|ARRAY|non-reserved|non-reserved|reserved|
+|AS|reserved|non-reserved|reserved|
+|ASC|non-reserved|non-reserved|non-reserved|
+|AT|non-reserved|non-reserved|reserved|
+|AUTHORIZATION|reserved|non-reserved|reserved|
+|BETWEEN|non-reserved|non-reserved|reserved|
+|BOTH|reserved|non-reserved|reserved|
+|BUCKET|non-reserved|non-reserved|non-reserved|
+|BUCKETS|non-reserved|non-reserved|non-reserved|
+|BY|non-reserved|non-reserved|reserved|
+|CACHE|non-reserved|non-reserved|non-reserved|
+|CASCADE|non-reserved|non-reserved|non-reserved|
+|CASE|reserved|non-reserved|reserved|
+|CAST|reserved|non-reserved|reserved|
+|CHANGE|non-reserved|non-reserved|non-reserved|
+|CHECK|reserved|non-reserved|reserved|
+|CLEAR|non-reserved|non-reserved|non-reserved|
+|CLUSTER|non-reserved|non-reserved|non-reserved|
+|CLUSTERED|non-reserved|non-reserved|non-reserved|
+|CODEGEN|non-reserved|non-reserved|non-reserved|
+|COLLATE|reserved|non-reserved|reserved|
+|COLLECTION|non-reserved|non-reserved|non-reserved|
+|COLUMN|reserved|non-reserved|reserved|
+|COLUMNS|non-reserved|non-reserved|non-reserved|
+|COMMENT|non-reserved|non-reserved|non-reserved|
+|COMMIT|non-reserved|non-reserved|reserved|
+|COMPACT|non-reserved|non-reserved|non-reserved|
+|COMPACTIONS|non-reserved|non-reserved|non-reserved|
+|COMPUTE|non-reserved|non-reserved|non-reserved|
+|CONCATENATE|non-reserved|non-reserved|non-reserved|
+|CONSTRAINT|reserved|non-reserved|reserved|
+|COST|non-reserved|non-reserved|non-reserved|
+|CREATE|reserved|non-reserved|reserved|
+|CROSS|reserved|strict-non-reserved|reserved|
+|CUBE|non-reserved|non-reserved|reserved|
+|CURRENT|non-reserved|non-reserved|reserved|
+|CURRENT_DATE|reserved|non-reserved|reserved|
+|CURRENT_TIME|reserved|non-reserved|reserved|
+|CURRENT_TIMESTAMP|reserved|non-reserved|reserved|
+|CURRENT_USER|reserved|non-reserved|reserved|
+|DATA|non-reserved|non-reserved|non-reserved|
+|DATABASE|non-reserved|non-reserved|non-reserved|
+|DATABASES|non-reserved|non-reserved|non-reserved|
+|DBPROPERTIES|non-reserved|non-reserved|non-reserved|
+|DEFINED|non-reserved|non-reserved|non-reserved|
+|DELETE|non-reserved|non-reserved|reserved|
+|DELIMITED|non-reserved|non-reserved|non-reserved|
+|DESC|non-reserved|non-reserved|non-reserved|
+|DESCRIBE|non-reserved|non-reserved|reserved|
+|DFS|non-reserved|non-reserved|non-reserved|
+|DIRECTORIES|non-reserved|non-reserved|non-reserved|
+|DIRECTORY|non-reserved|non-reserved|non-reserved|
+|DISTINCT|reserved|non-reserved|reserved|
+|DISTRIBUTE|non-reserved|non-reserved|non-reserved|
+|DIV|non-reserved|non-reserved|not a keyword|
+|DROP|non-reserved|non-reserved|reserved|
+|ELSE|reserved|non-reserved|reserved|
+|END|reserved|non-reserved|reserved|
+|ESCAPE|reserved|non-reserved|reserved|
+|ESCAPED|non-reserved|non-reserved|non-reserved|
+|EXCEPT|reserved|strict-non-reserved|reserved|
+|EXCHANGE|non-reserved|non-reserved|non-reserved|
+|EXISTS|non-reserved|non-reserved|reserved|
+|EXPLAIN|non-reserved|non-reserved|non-reserved|
+|EXPORT|non-reserved|non-reserved|non-reserved|
+|EXTENDED|non-reserved|non-reserved|non-reserved|
+|EXTERNAL|non-reserved|non-reserved|reserved|
+|EXTRACT|non-reserved|non-reserved|reserved|
+|FALSE|reserved|non-reserved|reserved|
+|FETCH|reserved|non-reserved|reserved|
+|FIELDS|non-reserved|non-reserved|non-reserved|
+|FILTER|reserved|non-reserved|reserved|
+|FILEFORMAT|non-reserved|non-reserved|non-reserved|
+|FIRST|non-reserved|non-reserved|non-reserved|
+|FOLLOWING|non-reserved|non-reserved|non-reserved|
+|FOR|reserved|non-reserved|reserved|
+|FOREIGN|reserved|non-reserved|reserved|
+|FORMAT|non-reserved|non-reserved|non-reserved|
+|FORMATTED|non-reserved|non-reserved|non-reserved|
+|FROM|reserved|non-reserved|reserved|
+|FULL|reserved|strict-non-reserved|reserved|
+|FUNCTION|non-reserved|non-reserved|reserved|
+|FUNCTIONS|non-reserved|non-reserved|non-reserved|
+|GLOBAL|non-reserved|non-reserved|reserved|
+|GRANT|reserved|non-reserved|reserved|
+|GROUP|reserved|non-reserved|reserved|
+|GROUPING|non-reserved|non-reserved|reserved|
+|HAVING|reserved|non-reserved|reserved|
+|IF|non-reserved|non-reserved|not a keyword|
+|IGNORE|non-reserved|non-reserved|non-reserved|
+|IMPORT|non-reserved|non-reserved|non-reserved|
+|IN|reserved|non-reserved|reserved|
+|INDEX|non-reserved|non-reserved|non-reserved|
+|INDEXES|non-reserved|non-reserved|non-reserved|
+|INNER|reserved|strict-non-reserved|reserved|
+|INPATH|non-reserved|non-reserved|non-reserved|
+|INPUTFORMAT|non-reserved|non-reserved|non-reserved|
+|INSERT|non-reserved|non-reserved|reserved|
+|INTERSECT|reserved|strict-non-reserved|reserved|
+|INTERVAL|non-reserved|non-reserved|reserved|
+|INTO|reserved|non-reserved|reserved|
+|IS|reserved|non-reserved|reserved|
+|ITEMS|non-reserved|non-reserved|non-reserved|
+|JOIN|reserved|strict-non-reserved|reserved|
+|KEYS|non-reserved|non-reserved|non-reserved|
+|LAST|non-reserved|non-reserved|non-reserved|
+|LATERAL|non-reserved|non-reserved|reserved|
+|LAZY|non-reserved|non-reserved|non-reserved|
+|LEADING|reserved|non-reserved|reserved|
+|LEFT|reserved|strict-non-reserved|reserved|
+|LIKE|non-reserved|non-reserved|reserved|
+|LIMIT|non-reserved|non-reserved|non-reserved|
+|LINES|non-reserved|non-reserved|non-reserved|
+|LIST|non-reserved|non-reserved|non-reserved|
+|LOAD|non-reserved|non-reserved|non-reserved|
+|LOCAL|non-reserved|non-reserved|reserved|
+|LOCATION|non-reserved|non-reserved|non-reserved|
+|LOCK|non-reserved|non-reserved|non-reserved|
+|LOCKS|non-reserved|non-reserved|non-reserved|
+|LOGICAL|non-reserved|non-reserved|non-reserved|
+|MACRO|non-reserved|non-reserved|non-reserved|
+|MAP|non-reserved|non-reserved|non-reserved|
+|MATCHED|non-reserved|non-reserved|non-reserved|
+|MERGE|non-reserved|non-reserved|non-reserved|
+|MINUS|non-reserved|strict-non-reserved|non-reserved|
+|MSCK|non-reserved|non-reserved|non-reserved|
+|NAMESPACE|non-reserved|non-reserved|non-reserved|
+|NAMESPACES|non-reserved|non-reserved|non-reserved|
+|NATURAL|reserved|strict-non-reserved|reserved|
+|NO|non-reserved|non-reserved|reserved|
+|NOT|reserved|non-reserved|reserved|
+|NULL|reserved|non-reserved|reserved|
+|NULLS|non-reserved|non-reserved|non-reserved|
+|OF|non-reserved|non-reserved|reserved|
+|ON|reserved|strict-non-reserved|reserved|
+|ONLY|reserved|non-reserved|reserved|
+|OPTION|non-reserved|non-reserved|non-reserved|
+|OPTIONS|non-reserved|non-reserved|non-reserved|
+|OR|reserved|non-reserved|reserved|
+|ORDER|reserved|non-reserved|reserved|
+|OUT|non-reserved|non-reserved|reserved|
+|OUTER|reserved|non-reserved|reserved|
+|OUTPUTFORMAT|non-reserved|non-reserved|non-reserved|
+|OVER|non-reserved|non-reserved|non-reserved|
+|OVERLAPS|reserved|non-reserved|reserved|
+|OVERLAY|non-reserved|non-reserved|non-reserved|
+|OVERWRITE|non-reserved|non-reserved|non-reserved|
+|PARTITION|non-reserved|non-reserved|reserved|
+|PARTITIONED|non-reserved|non-reserved|non-reserved|
+|PARTITIONS|non-reserved|non-reserved|non-reserved|
+|PERCENT|non-reserved|non-reserved|non-reserved|
+|PIVOT|non-reserved|non-reserved|non-reserved|
+|PLACING|non-reserved|non-reserved|non-reserved|
+|POSITION|non-reserved|non-reserved|reserved|
+|PRECEDING|non-reserved|non-reserved|non-reserved|
+|PRIMARY|reserved|non-reserved|reserved|
+|PRINCIPALS|non-reserved|non-reserved|non-reserved|
+|PROPERTIES|non-reserved|non-reserved|non-reserved|
+|PURGE|non-reserved|non-reserved|non-reserved|
+|QUERY|non-reserved|non-reserved|non-reserved|
+|RANGE|non-reserved|non-reserved|reserved|
+|RECORDREADER|non-reserved|non-reserved|non-reserved|
+|RECORDWRITER|non-reserved|non-reserved|non-reserved|
+|RECOVER|non-reserved|non-reserved|non-reserved|
+|REDUCE|non-reserved|non-reserved|non-reserved|
+|REFERENCES|reserved|non-reserved|reserved|
+|REFRESH|non-reserved|non-reserved|non-reserved|
+|REGEXP|non-reserved|non-reserved|not a keyword|
+|RENAME|non-reserved|non-reserved|non-reserved|
+|REPAIR|non-reserved|non-reserved|non-reserved|
+|REPLACE|non-reserved|non-reserved|non-reserved|
+|RESET|non-reserved|non-reserved|non-reserved|
+|RESTRICT|non-reserved|non-reserved|non-reserved|
+|REVOKE|non-reserved|non-reserved|reserved|
+|RIGHT|reserved|strict-non-reserved|reserved|
+|RLIKE|non-reserved|non-reserved|non-reserved|
+|ROLE|non-reserved|non-reserved|non-reserved|
+|ROLES|non-reserved|non-reserved|non-reserved|
+|ROLLBACK|non-reserved|non-reserved|reserved|
+|ROLLUP|non-reserved|non-reserved|reserved|
+|ROW|non-reserved|non-reserved|reserved|
+|ROWS|non-reserved|non-reserved|reserved|
+|SCHEMA|non-reserved|non-reserved|non-reserved|
+|SCHEMAS|non-reserved|non-reserved|not a keyword|
+|SELECT|reserved|non-reserved|reserved|
+|SEMI|non-reserved|strict-non-reserved|non-reserved|
+|SEPARATED|non-reserved|non-reserved|non-reserved|
+|SERDE|non-reserved|non-reserved|non-reserved|
+|SERDEPROPERTIES|non-reserved|non-reserved|non-reserved|
+|SESSION_USER|reserved|non-reserved|reserved|
+|SET|non-reserved|non-reserved|reserved|
+|SETS|non-reserved|non-reserved|non-reserved|
+|SHOW|non-reserved|non-reserved|non-reserved|
+|SKEWED|non-reserved|non-reserved|non-reserved|
+|SOME|reserved|non-reserved|reserved|
+|SORT|non-reserved|non-reserved|non-reserved|
+|SORTED|non-reserved|non-reserved|non-reserved|
+|START|non-reserved|non-reserved|reserved|
+|STATISTICS|non-reserved|non-reserved|non-reserved|
+|STORED|non-reserved|non-reserved|non-reserved|
+|STRATIFY|non-reserved|non-reserved|non-reserved|
+|STRUCT|non-reserved|non-reserved|non-reserved|
+|SUBSTR|non-reserved|non-reserved|non-reserved|
+|SUBSTRING|non-reserved|non-reserved|non-reserved|
+|TABLE|reserved|non-reserved|reserved|
+|TABLES|non-reserved|non-reserved|non-reserved|
+|TABLESAMPLE|non-reserved|non-reserved|reserved|
+|TBLPROPERTIES|non-reserved|non-reserved|non-reserved|
+|TEMP|non-reserved|non-reserved|not a keyword|
+|TEMPORARY|non-reserved|non-reserved|non-reserved|
+|TERMINATED|non-reserved|non-reserved|non-reserved|
+|THEN|reserved|non-reserved|reserved|
+|TO|reserved|non-reserved|reserved|
+|TOUCH|non-reserved|non-reserved|non-reserved|
+|TRAILING|reserved|non-reserved|reserved|
+|TRANSACTION|non-reserved|non-reserved|non-reserved|
+|TRANSACTIONS|non-reserved|non-reserved|non-reserved|
+|TRANSFORM|non-reserved|non-reserved|non-reserved|
+|TRIM|non-reserved|non-reserved|non-reserved|
+|TRUE|non-reserved|non-reserved|reserved|
+|TRUNCATE|non-reserved|non-reserved|reserved|
+|TYPE|non-reserved|non-reserved|non-reserved|
+|UNARCHIVE|non-reserved|non-reserved|non-reserved|
+|UNBOUNDED|non-reserved|non-reserved|non-reserved|
+|UNCACHE|non-reserved|non-reserved|non-reserved|
+|UNION|reserved|strict-non-reserved|reserved|
+|UNIQUE|reserved|non-reserved|reserved|
+|UNKNOWN|reserved|non-reserved|reserved|
+|UNLOCK|non-reserved|non-reserved|non-reserved|
+|UNSET|non-reserved|non-reserved|non-reserved|
+|UPDATE|non-reserved|non-reserved|reserved|
+|USE|non-reserved|non-reserved|non-reserved|
+|USER|reserved|non-reserved|reserved|
+|USING|reserved|strict-non-reserved|reserved|
+|VALUES|non-reserved|non-reserved|reserved|
+|VIEW|non-reserved|non-reserved|non-reserved|
+|VIEWS|non-reserved|non-reserved|non-reserved|
+|WHEN|reserved|non-reserved|reserved|
+|WHERE|reserved|non-reserved|reserved|
+|WINDOW|non-reserved|non-reserved|reserved|
+|WITH|reserved|non-reserved|reserved|
diff --git a/docs/sql-ref-arithmetic-ops.md b/docs/sql-ref-arithmetic-ops.md
deleted file mode 100644
index 7bc8ffe31c990..0000000000000
--- a/docs/sql-ref-arithmetic-ops.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Arithmetic Operations
-displayTitle: Arithmetic Operations
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-Operations performed on numeric types (with the exception of decimal) are not checked for overflow. This means that in case an operation causes an overflow, the result is the same that the same operation returns in a Java/Scala program (eg. if the sum of 2 integers is higher than the maximum value representable, the result is a negative number).
diff --git a/docs/sql-ref-datatypes.md b/docs/sql-ref-datatypes.md
index 0add62b10ed6b..144594bd5a222 100644
--- a/docs/sql-ref-datatypes.md
+++ b/docs/sql-ref-datatypes.md
@@ -19,6 +19,7 @@ license: |
   limitations under the License.
 ---
 
+### Supported Data Types
 
 Spark SQL and DataFrames support the following data types:
 
@@ -70,130 +71,24 @@ You can access them by doing
 
 {% include_example data_types scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
 
-<table class="table">
-<tr>
-  <th style="width:20%">Data type</th>
-  <th style="width:40%">Value type in Scala</th>
-  <th>API to access or create a data type</th></tr>
-<tr>
-  <td> <b>ByteType</b> </td>
-  <td> Byte </td>
-  <td>
-  ByteType
-  </td>
-</tr>
-<tr>
-  <td> <b>ShortType</b> </td>
-  <td> Short </td>
-  <td>
-  ShortType
-  </td>
-</tr>
-<tr>
-  <td> <b>IntegerType</b> </td>
-  <td> Int </td>
-  <td>
-  IntegerType
-  </td>
-</tr>
-<tr>
-  <td> <b>LongType</b> </td>
-  <td> Long </td>
-  <td>
-  LongType
-  </td>
-</tr>
-<tr>
-  <td> <b>FloatType</b> </td>
-  <td> Float </td>
-  <td>
-  FloatType
-  </td>
-</tr>
-<tr>
-  <td> <b>DoubleType</b> </td>
-  <td> Double </td>
-  <td>
-  DoubleType
-  </td>
-</tr>
-<tr>
-  <td> <b>DecimalType</b> </td>
-  <td> java.math.BigDecimal </td>
-  <td>
-  DecimalType
-  </td>
-</tr>
-<tr>
-  <td> <b>StringType</b> </td>
-  <td> String </td>
-  <td>
-  StringType
-  </td>
-</tr>
-<tr>
-  <td> <b>BinaryType</b> </td>
-  <td> Array[Byte] </td>
-  <td>
-  BinaryType
-  </td>
-</tr>
-<tr>
-  <td> <b>BooleanType</b> </td>
-  <td> Boolean </td>
-  <td>
-  BooleanType
-  </td>
-</tr>
-<tr>
-  <td> <b>TimestampType</b> </td>
-  <td> java.sql.Timestamp </td>
-  <td>
-  TimestampType
-  </td>
-</tr>
-<tr>
-  <td> <b>DateType</b> </td>
-  <td> java.sql.Date </td>
-  <td>
-  DateType
-  </td>
-</tr>
-<tr>
-  <td> <b>ArrayType</b> </td>
-  <td> scala.collection.Seq </td>
-  <td>
-  ArrayType(<i>elementType</i>, [<i>containsNull</i>])<br />
-  <b>Note:</b> The default value of <i>containsNull</i> is <i>true</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>MapType</b> </td>
-  <td> scala.collection.Map </td>
-  <td>
-  MapType(<i>keyType</i>, <i>valueType</i>, [<i>valueContainsNull</i>])<br />
-  <b>Note:</b> The default value of <i>valueContainsNull</i> is <i>true</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructType</b> </td>
-  <td> org.apache.spark.sql.Row </td>
-  <td>
-  StructType(<i>fields</i>)<br />
-  <b>Note:</b> <i>fields</i> is a Seq of StructFields. Also, two fields with the same
-  name are not allowed.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructField</b> </td>
-  <td> The value type in Scala of the data type of this field
-  (For example, Int for a StructField with the data type IntegerType) </td>
-  <td>
-  StructField(<i>name</i>, <i>dataType</i>, [<i>nullable</i>])<br />
-  <b>Note:</b> The default value of <i>nullable</i> is <i>true</i>.
-  </td>
-</tr>
-</table>
+|Data type|Value type in Scala|API to access or create a data type|
+|---------|-------------------|-----------------------------------|
+|**ByteType**|Byte|ByteType|
+|**ShortType**|Short|ShortType|
+|**IntegerType**|Int|IntegerType|
+|**LongType**|Long|LongType|
+|**FloatType**|Float|FloatType|
+|**DoubleType**|Double|DoubleType|
+|**DecimalType**|java.math.BigDecimal|DecimalType|
+|**StringType**|String|StringType|
+|**BinaryType**|Array[Byte]|BinaryType|
+|**BooleanType**|Boolean|BooleanType|
+|**TimestampType**|java.sql.Timestamp|TimestampType|
+|**DateType**|java.sql.Date|DateType|
+|**ArrayType**|scala.collection.Seq|ArrayType(*elementType*, [*containsNull]*)<br/>**Note:** The default value of *containsNull* is true.|
+|**MapType**|scala.collection.Map|MapType(*keyType*, *valueType*, [*valueContainsNull]*)<br/>**Note:** The default value of *valueContainsNull* is true.|
+|**StructType**|org.apache.spark.sql.Row|StructType(*fields*)<br/>**Note:** *fields* is a Seq of StructFields. Also, two fields with the same name are not allowed.|
+|**StructField**|The value type in Scala of the data type of this field(For example, Int for a StructField with the data type IntegerType)|StructField(*name*, *dataType*, [*nullable*])<br/>**Note:** The default value of *nullable* is true.|
 
 </div>
 
@@ -204,132 +99,24 @@ All data types of Spark SQL are located in the package of
 please use factory methods provided in
 `org.apache.spark.sql.types.DataTypes`.
 
-<table class="table">
-<tr>
-  <th style="width:20%">Data type</th>
-  <th style="width:40%">Value type in Java</th>
-  <th>API to access or create a data type</th></tr>
-<tr>
-  <td> <b>ByteType</b> </td>
-  <td> byte or Byte </td>
-  <td>
-  DataTypes.ByteType
-  </td>
-</tr>
-<tr>
-  <td> <b>ShortType</b> </td>
-  <td> short or Short </td>
-  <td>
-  DataTypes.ShortType
-  </td>
-</tr>
-<tr>
-  <td> <b>IntegerType</b> </td>
-  <td> int or Integer </td>
-  <td>
-  DataTypes.IntegerType
-  </td>
-</tr>
-<tr>
-  <td> <b>LongType</b> </td>
-  <td> long or Long </td>
-  <td>
-  DataTypes.LongType
-  </td>
-</tr>
-<tr>
-  <td> <b>FloatType</b> </td>
-  <td> float or Float </td>
-  <td>
-  DataTypes.FloatType
-  </td>
-</tr>
-<tr>
-  <td> <b>DoubleType</b> </td>
-  <td> double or Double </td>
-  <td>
-  DataTypes.DoubleType
-  </td>
-</tr>
-<tr>
-  <td> <b>DecimalType</b> </td>
-  <td> java.math.BigDecimal </td>
-  <td>
-  DataTypes.createDecimalType()<br />
-  DataTypes.createDecimalType(<i>precision</i>, <i>scale</i>).
-  </td>
-</tr>
-<tr>
-  <td> <b>StringType</b> </td>
-  <td> String </td>
-  <td>
-  DataTypes.StringType
-  </td>
-</tr>
-<tr>
-  <td> <b>BinaryType</b> </td>
-  <td> byte[] </td>
-  <td>
-  DataTypes.BinaryType
-  </td>
-</tr>
-<tr>
-  <td> <b>BooleanType</b> </td>
-  <td> boolean or Boolean </td>
-  <td>
-  DataTypes.BooleanType
-  </td>
-</tr>
-<tr>
-  <td> <b>TimestampType</b> </td>
-  <td> java.sql.Timestamp </td>
-  <td>
-  DataTypes.TimestampType
-  </td>
-</tr>
-<tr>
-  <td> <b>DateType</b> </td>
-  <td> java.sql.Date </td>
-  <td>
-  DataTypes.DateType
-  </td>
-</tr>
-<tr>
-  <td> <b>ArrayType</b> </td>
-  <td> java.util.List </td>
-  <td>
-  DataTypes.createArrayType(<i>elementType</i>)<br />
-  <b>Note:</b> The value of <i>containsNull</i> will be <i>true</i><br />
-  DataTypes.createArrayType(<i>elementType</i>, <i>containsNull</i>).
-  </td>
-</tr>
-<tr>
-  <td> <b>MapType</b> </td>
-  <td> java.util.Map </td>
-  <td>
-  DataTypes.createMapType(<i>keyType</i>, <i>valueType</i>)<br />
-  <b>Note:</b> The value of <i>valueContainsNull</i> will be <i>true</i>.<br />
-  DataTypes.createMapType(<i>keyType</i>, <i>valueType</i>, <i>valueContainsNull</i>)<br />
-  </td>
-</tr>
-<tr>
-  <td> <b>StructType</b> </td>
-  <td> org.apache.spark.sql.Row </td>
-  <td>
-  DataTypes.createStructType(<i>fields</i>)<br />
-  <b>Note:</b> <i>fields</i> is a List or an array of StructFields.
-  Also, two fields with the same name are not allowed.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructField</b> </td>
-  <td> The value type in Java of the data type of this field
-  (For example, int for a StructField with the data type IntegerType) </td>
-  <td>
-  DataTypes.createStructField(<i>name</i>, <i>dataType</i>, <i>nullable</i>)
-  </td>
-</tr>
-</table>
+|Data type|Value type in Java|API to access or create a data type|
+|---------|------------------|-----------------------------------|
+|**ByteType**|byte or Byte|DataTypes.ByteType|
+|**ShortType**|short or Short|DataTypes.ShortType|
+|**IntegerType**|int or Integer|DataTypes.IntegerType|
+|**LongType**|long or Long|DataTypes.LongType|
+|**FloatType**|float or Float|DataTypes.FloatType|
+|**DoubleType**|double or Double|DataTypes.DoubleType|
+|**DecimalType**|java.math.BigDecimal|DataTypes.createDecimalType()<br/>DataTypes.createDecimalType(*precision*, *scale*).|
+|**StringType**|String|DataTypes.StringType|
+|**BinaryType**|byte[]|DataTypes.BinaryType|
+|**BooleanType**|boolean or Boolean|DataTypes.BooleanType|
+|**TimestampType**|java.sql.Timestamp|DataTypes.TimestampType|
+|**DateType**|java.sql.Date|DataTypes.DateType|
+|**ArrayType**|java.util.List|DataTypes.createArrayType(*elementType*)<br/>**Note:** The value of *containsNull* will be true.<br/>DataTypes.createArrayType(*elementType*, *containsNull*).|
+|**MapType**|java.util.Map|DataTypes.createMapType(*keyType*, *valueType*)<br/>**Note:** The value of *valueContainsNull* will be true.<br/>DataTypes.createMapType(*keyType*, *valueType*, *valueContainsNull*)|
+|**StructType**|org.apache.spark.sql.Row|DataTypes.createStructType(*fields*)<br/>**Note:** *fields* is a List or an array of StructFields.Also, two fields with the same name are not allowed.|
+|**StructField**|The value type in Java of the data type of this field (For example, int for a StructField with the data type IntegerType)|DataTypes.createStructField(*name*, *dataType*, *nullable*)| 
 
 </div>
 
@@ -341,294 +128,189 @@ You can access them by doing
 from pyspark.sql.types import *
 {% endhighlight %}
 
-<table class="table">
-<tr>
-  <th style="width:20%">Data type</th>
-  <th style="width:40%">Value type in Python</th>
-  <th>API to access or create a data type</th></tr>
-<tr>
-  <td> <b>ByteType</b> </td>
-  <td>
-  int or long <br />
-  <b>Note:</b> Numbers will be converted to 1-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of -128 to 127.
-  </td>
-  <td>
-  ByteType()
-  </td>
-</tr>
-<tr>
-  <td> <b>ShortType</b> </td>
-  <td>
-  int or long <br />
-  <b>Note:</b> Numbers will be converted to 2-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of -32768 to 32767.
-  </td>
-  <td>
-  ShortType()
-  </td>
-</tr>
-<tr>
-  <td> <b>IntegerType</b> </td>
-  <td> int or long </td>
-  <td>
-  IntegerType()
-  </td>
-</tr>
-<tr>
-  <td> <b>LongType</b> </td>
-  <td>
-  long <br />
-  <b>Note:</b> Numbers will be converted to 8-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of
-  -9223372036854775808 to 9223372036854775807.
-  Otherwise, please convert data to decimal.Decimal and use DecimalType.
-  </td>
-  <td>
-  LongType()
-  </td>
-</tr>
-<tr>
-  <td> <b>FloatType</b> </td>
-  <td>
-  float <br />
-  <b>Note:</b> Numbers will be converted to 4-byte single-precision floating
-  point numbers at runtime.
-  </td>
-  <td>
-  FloatType()
-  </td>
-</tr>
-<tr>
-  <td> <b>DoubleType</b> </td>
-  <td> float </td>
-  <td>
-  DoubleType()
-  </td>
-</tr>
-<tr>
-  <td> <b>DecimalType</b> </td>
-  <td> decimal.Decimal </td>
-  <td>
-  DecimalType()
-  </td>
-</tr>
-<tr>
-  <td> <b>StringType</b> </td>
-  <td> string </td>
-  <td>
-  StringType()
-  </td>
-</tr>
-<tr>
-  <td> <b>BinaryType</b> </td>
-  <td> bytearray </td>
-  <td>
-  BinaryType()
-  </td>
-</tr>
-<tr>
-  <td> <b>BooleanType</b> </td>
-  <td> bool </td>
-  <td>
-  BooleanType()
-  </td>
-</tr>
-<tr>
-  <td> <b>TimestampType</b> </td>
-  <td> datetime.datetime </td>
-  <td>
-  TimestampType()
-  </td>
-</tr>
-<tr>
-  <td> <b>DateType</b> </td>
-  <td> datetime.date </td>
-  <td>
-  DateType()
-  </td>
-</tr>
-<tr>
-  <td> <b>ArrayType</b> </td>
-  <td> list, tuple, or array </td>
-  <td>
-  ArrayType(<i>elementType</i>, [<i>containsNull</i>])<br />
-  <b>Note:</b> The default value of <i>containsNull</i> is <i>True</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>MapType</b> </td>
-  <td> dict </td>
-  <td>
-  MapType(<i>keyType</i>, <i>valueType</i>, [<i>valueContainsNull</i>])<br />
-  <b>Note:</b> The default value of <i>valueContainsNull</i> is <i>True</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructType</b> </td>
-  <td> list or tuple </td>
-  <td>
-  StructType(<i>fields</i>)<br />
-  <b>Note:</b> <i>fields</i> is a Seq of StructFields. Also, two fields with the same
-  name are not allowed.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructField</b> </td>
-  <td> The value type in Python of the data type of this field
-  (For example, Int for a StructField with the data type IntegerType) </td>
-  <td>
-  StructField(<i>name</i>, <i>dataType</i>, [<i>nullable</i>])<br />
-  <b>Note:</b> The default value of <i>nullable</i> is <i>True</i>.
-  </td>
-</tr>
-</table>
+|Data type|Value type in Python|API to access or create a data type|
+|---------|--------------------|-----------------------------------|
+|**ByteType**|int or long<br/>**Note:** Numbers will be converted to 1-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -128 to 127.|ByteType()|
+|**ShortType**|int or long<br/>**Note:** Numbers will be converted to 2-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -32768 to 32767.|ShortType()|
+|**IntegerType**|int or long|IntegerType()|
+|**LongType**|long<br/>**Note:** Numbers will be converted to 8-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -9223372036854775808 to 9223372036854775807.Otherwise, please convert data to decimal.Decimal and use DecimalType.|LongType()|
+|**FloatType**|float<br/>**Note:** Numbers will be converted to 4-byte single-precision floating point numbers at runtime.|FloatType()|
+|**DoubleType**|float|DoubleType()|
+|**DecimalType**|decimal.Decimal|DecimalType()|
+|**StringType**|string|StringType()|
+|**BinaryType**|bytearray|BinaryType()|
+|**BooleanType**|bool|BooleanType()|
+|**TimestampType**|datetime.datetime|TimestampType()|
+|**DateType**|datetime.date|DateType()|
+|**ArrayType**|list, tuple, or array|ArrayType(*elementType*, [*containsNull*])<br/>**Note:**The default value of *containsNull* is True.|
+|**MapType**|dict|MapType(*keyType*, *valueType*, [*valueContainsNull]*)<br/>**Note:**The default value of *valueContainsNull* is True.|
+|**StructType**|list or tuple|StructType(*fields*)<br/>**Note:** *fields* is a Seq of StructFields. Also, two fields with the same name are not allowed.|
+|**StructField**|The value type in Python of the data type of this field<br/>(For example, Int for a StructField with the data type IntegerType)|StructField(*name*, *dataType*, [*nullable*])<br/>**Note:** The default value of *nullable* is True.|
 
 </div>
 
 <div data-lang="r"  markdown="1">
 
-<table class="table">
-<tr>
-  <th style="width:20%">Data type</th>
-  <th style="width:40%">Value type in R</th>
-  <th>API to access or create a data type</th></tr>
-<tr>
-  <td> <b>ByteType</b> </td>
-  <td>
-  integer <br />
-  <b>Note:</b> Numbers will be converted to 1-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of -128 to 127.
-  </td>
-  <td>
-  "byte"
-  </td>
-</tr>
-<tr>
-  <td> <b>ShortType</b> </td>
-  <td>
-  integer <br />
-  <b>Note:</b> Numbers will be converted to 2-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of -32768 to 32767.
-  </td>
-  <td>
-  "short"
-  </td>
-</tr>
-<tr>
-  <td> <b>IntegerType</b> </td>
-  <td> integer </td>
-  <td>
-  "integer"
-  </td>
-</tr>
-<tr>
-  <td> <b>LongType</b> </td>
-  <td>
-  integer <br />
-  <b>Note:</b> Numbers will be converted to 8-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of
-  -9223372036854775808 to 9223372036854775807.
-  Otherwise, please convert data to decimal.Decimal and use DecimalType.
-  </td>
-  <td>
-  "long"
-  </td>
-</tr>
-<tr>
-  <td> <b>FloatType</b> </td>
-  <td>
-  numeric <br />
-  <b>Note:</b> Numbers will be converted to 4-byte single-precision floating
-  point numbers at runtime.
-  </td>
-  <td>
-  "float"
-  </td>
-</tr>
-<tr>
-  <td> <b>DoubleType</b> </td>
-  <td> numeric </td>
-  <td>
-  "double"
-  </td>
-</tr>
-<tr>
-  <td> <b>DecimalType</b> </td>
-  <td> Not supported </td>
-  <td>
-   Not supported
-  </td>
-</tr>
-<tr>
-  <td> <b>StringType</b> </td>
-  <td> character </td>
-  <td>
-  "string"
-  </td>
-</tr>
-<tr>
-  <td> <b>BinaryType</b> </td>
-  <td> raw </td>
-  <td>
-  "binary"
-  </td>
-</tr>
-<tr>
-  <td> <b>BooleanType</b> </td>
-  <td> logical </td>
-  <td>
-  "bool"
-  </td>
-</tr>
-<tr>
-  <td> <b>TimestampType</b> </td>
-  <td> POSIXct </td>
-  <td>
-  "timestamp"
-  </td>
-</tr>
-<tr>
-  <td> <b>DateType</b> </td>
-  <td> Date </td>
-  <td>
-  "date"
-  </td>
-</tr>
-<tr>
-  <td> <b>ArrayType</b> </td>
-  <td> vector or list </td>
-  <td>
-  list(type="array", elementType=<i>elementType</i>, containsNull=[<i>containsNull</i>])<br />
-  <b>Note:</b> The default value of <i>containsNull</i> is <i>TRUE</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>MapType</b> </td>
-  <td> environment </td>
-  <td>
-  list(type="map", keyType=<i>keyType</i>, valueType=<i>valueType</i>, valueContainsNull=[<i>valueContainsNull</i>])<br />
-  <b>Note:</b> The default value of <i>valueContainsNull</i> is <i>TRUE</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructType</b> </td>
-  <td> named list</td>
-  <td>
-  list(type="struct", fields=<i>fields</i>)<br />
-  <b>Note:</b> <i>fields</i> is a Seq of StructFields. Also, two fields with the same
-  name are not allowed.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructField</b> </td>
-  <td> The value type in R of the data type of this field
-  (For example, integer for a StructField with the data type IntegerType) </td>
-  <td>
-  list(name=<i>name</i>, type=<i>dataType</i>, nullable=[<i>nullable</i>])<br />
-  <b>Note:</b> The default value of <i>nullable</i> is <i>TRUE</i>.
-  </td>
-</tr>
-</table>
+|Data type|Value type in R|API to access or create a data type|
+|---------|---------------|-----------------------------------|
+|**ByteType**|integer <br/>**Note:** Numbers will be converted to 1-byte signed integer numbers at runtime.  Please make sure that numbers are within the range of -128 to 127.|"byte"|
+|**ShortType**|integer <br/>**Note:** Numbers will be converted to 2-byte signed integer numbers at runtime.  Please make sure that numbers are within the range of -32768 to 32767.|"short"|
+|**IntegerType**|integer|"integer"|
+|**LongType**|integer <br/>**Note:** Numbers will be converted to 8-byte signed integer numbers at runtime.  Please make sure that numbers are within the range of -9223372036854775808 to 9223372036854775807.  Otherwise, please convert data to decimal.Decimal and use DecimalType.|"long"|
+|**FloatType**|numeric <br/>**Note:** Numbers will be converted to 4-byte single-precision floating point numbers at runtime.|"float"|
+|**DoubleType**|numeric|"double"|
+|**DecimalType**|Not supported|Not supported|
+|**StringType**|character|"string"|
+|**BinaryType**|raw|"binary"|
+|**BooleanType**|logical|"bool"|
+|**TimestampType**|POSIXct|"timestamp"|
+|**DateType**|Date|"date"|
+|**ArrayType**|vector or list|list(type="array", elementType=*elementType*, containsNull=[*containsNull*])<br/>**Note:** The default value of *containsNull* is TRUE.|
+|**MapType**|environment|list(type="map", keyType=*keyType*, valueType=*valueType*, valueContainsNull=[*valueContainsNull*])<br/> **Note:** The default value of *valueContainsNull* is TRUE.|
+|**StructType**|named list|list(type="struct", fields=*fields*)<br/> **Note:** *fields* is a Seq of StructFields. Also, two fields with the same name are not allowed.|
+|**StructField**|The value type in R of the data type of this field (For example, integer for a StructField with the data type IntegerType)|list(name=*name*, type=*dataType*, nullable=[*nullable*])<br/> **Note:** The default value of *nullable* is TRUE.|
+
+</div>
+
+<div data-lang="SQL"  markdown="1">
+
+The following table shows the type names as well as aliases used in Spark SQL parser for each data type.
+
+|Data type|SQL name|
+|---------|--------|
+|**BooleanType**|BOOLEAN|
+|**ByteType**|BYTE, TINYINT|
+|**ShortType**|SHORT, SMALLINT|
+|**IntegerType**|INT, INTEGER|
+|**LongType**|LONG, BIGINT|
+|**FloatType**|FLOAT, REAL|
+|**DoubleType**|DOUBLE|
+|**DateType**|DATE|
+|**TimestampType**|TIMESTAMP|
+|**StringType**|STRING|
+|**BinaryType**|BINARY|
+|**DecimalType**|DECIMAL, DEC, NUMERIC|
+|**CalendarIntervalType**|INTERVAL|
+|**ArrayType**|ARRAY\<element_type>|
+|**StructType**|STRUCT<field1_name: field1_type, field2_name: field2_type, ...>|
+|**MapType**|MAP<key_type, value_type>|
+
 </div>
 </div>
+
+### Floating Point Special Values
+
+Spark SQL supports several special floating point values in a case-insensitive manner:
+
+ * Inf/+Inf/Infinity/+Infinity: positive infinity
+   * ```FloatType```: equivalent to Scala <code>Float.PositiveInfinity</code>.
+   * ```DoubleType```: equivalent to Scala <code>Double.PositiveInfinity</code>.
+ * -Inf/-Infinity: negative infinity
+   * ```FloatType```: equivalent to Scala <code>Float.NegativeInfinity</code>.
+   * ```DoubleType```: equivalent to Scala <code>Double.NegativeInfinity</code>.
+ * NaN: not a number
+   * ```FloatType```: equivalent to Scala <code>Float.NaN</code>.
+   * ```DoubleType```:  equivalent to Scala <code>Double.NaN</code>.
+
+#### Positive/Negative Infinity Semantics
+
+There is special handling for positive and negative infinity. They have the following semantics:
+
+ * Positive infinity multiplied by any positive value returns positive infinity.
+ * Negative infinity multiplied by any positive value returns negative infinity.
+ * Positive infinity multiplied by any negative value returns negative infinity.
+ * Negative infinity multiplied by any negative value returns positive infinity.
+ * Positive/negative infinity multiplied by 0 returns NaN.
+ * Positive/negative infinity is equal to itself.
+ * In aggregations, all positive infinity values are grouped together. Similarly, all negative infinity values are grouped together.
+ * Positive infinity and negative infinity are treated as normal values in join keys.
+ * Positive infinity sorts lower than NaN and higher than any other values.
+ * Negative infinity sorts lower than any other values.
+
+#### NaN Semantics
+
+There is special handling for not-a-number (NaN) when dealing with `float` or `double` types that
+do not exactly match standard floating point semantics.
+Specifically:
+
+ * NaN = NaN returns true.
+ * In aggregations, all NaN values are grouped together.
+ * NaN is treated as a normal value in join keys.
+ * NaN values go last when in ascending order, larger than any other numeric value.
+
+#### Examples
+
+```sql
+SELECT double('infinity') AS col;
++--------+
+|     col|
++--------+
+|Infinity|
++--------+
+
+SELECT float('-inf') AS col;
++---------+
+|      col|
++---------+
+|-Infinity|
++---------+
+
+SELECT float('NaN') AS col;
++---+
+|col|
++---+
+|NaN|
++---+
+
+SELECT double('infinity') * 0 AS col;
++---+
+|col|
++---+
+|NaN|
++---+
+
+SELECT double('-infinity') * (-1234567) AS col;
++--------+
+|     col|
++--------+
+|Infinity|
++--------+
+
+SELECT double('infinity') < double('NaN') AS col;
++----+
+| col|
++----+
+|true|
++----+
+
+SELECT double('NaN') = double('NaN') AS col;
++----+
+| col|
++----+
+|true|
++----+
+
+SELECT double('inf') = double('infinity') AS col;
++----+
+| col|
++----+
+|true|
++----+
+
+CREATE TABLE test (c1 int, c2 double);
+INSERT INTO test VALUES (1, double('infinity'));
+INSERT INTO test VALUES (2, double('infinity'));
+INSERT INTO test VALUES (3, double('inf'));
+INSERT INTO test VALUES (4, double('-inf'));
+INSERT INTO test VALUES (5, double('NaN'));
+INSERT INTO test VALUES (6, double('NaN'));
+INSERT INTO test VALUES (7, double('-infinity'));
+SELECT COUNT(*), c2 FROM test GROUP BY c2;
++---------+---------+
+| count(1)|       c2|
++---------+---------+
+|        2|      NaN|
+|        2|-Infinity|
+|        3| Infinity|
++---------+---------+
+```
diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
new file mode 100644
index 0000000000000..4b02cdad36109
--- /dev/null
+++ b/docs/sql-ref-datetime-pattern.md
@@ -0,0 +1,136 @@
+---
+layout: global
+title: Datetime patterns
+displayTitle: Datetime Patterns for Formatting and Parsing
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+There are several common scenarios for datetime usage in Spark:
+
+- CSV/JSON datasources use the pattern string for parsing and formatting datetime content.
+
+- Datetime functions related to convert `StringType` to/from `DateType` or `TimestampType`.
+  For example, `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp`, `from_utc_timestamp`, `to_utc_timestamp`, etc.
+
+Spark uses pattern letters in the following table for date and timestamp parsing and formatting:
+
+|Symbol|Meaning|Presentation|Examples|
+|------|-------|------------|--------|
+|**G**|era|text|AD; Anno Domini|
+|**y**|year|year|2020; 20|
+|**D**|day-of-year|number(3)|189|
+|**M/L**|month-of-year|month|7; 07; Jul; July|
+|**d**|day-of-month|number(3)|28|
+|**Q/q**|quarter-of-year|number/text|3; 03; Q3; 3rd quarter|
+|**E**|day-of-week|text|Tue; Tuesday|
+|**F**|aligned day of week in month|number(1)|3|
+|**a**|am-pm-of-day|am-pm|PM|
+|**h**|clock-hour-of-am-pm (1-12)|number(2)|12|
+|**K**|hour-of-am-pm (0-11)|number(2)|0|
+|**k**|clock-hour-of-day (1-24)|number(2)|0|
+|**H**|hour-of-day (0-23)|number(2)|0|
+|**m**|minute-of-hour|number(2)|30|
+|**s**|second-of-minute|number(2)|55|
+|**S**|fraction-of-second|fraction|978|
+|**V**|time-zone ID|zone-id|America/Los_Angeles; Z; -08:30|
+|**z**|time-zone name|zone-name|Pacific Standard Time; PST|
+|**O**|localized zone-offset|offset-O|GMT+8; GMT+08:00; UTC-08:00;|
+|**X**|zone-offset 'Z' for zero|offset-X|Z; -08; -0830; -08:30; -083015; -08:30:15;|
+|**x**|zone-offset|offset-x|+0000; -08; -0830; -08:30; -083015; -08:30:15;|
+|**Z**|zone-offset|offset-Z|+0000; -0800; -08:00;|
+|**'**|escape for text|delimiter| |
+|**''**|single quote|literal|'|
+|**[**|optional section start| | |
+|**]**|optional section end| | |
+
+The count of pattern letters determines the format.
+
+- Text: The text style is determined based on the number of pattern letters used. Less than 4 pattern letters will use the short text form, typically an abbreviation, e.g. day-of-week Monday might output "Mon". Exactly 4 pattern letters will use the full text form, typically the full description, e.g, day-of-week Monday might output "Monday". 5 or more letters will fail.
+
+- Number(n): The n here represents the maximum count of letters this type of datetime pattern can be used. If the count of letters is one, then the value is output using the minimum number of digits and without padding. Otherwise, the count of digits is used as the width of the output field, with the value zero-padded as necessary.
+
+- Number/Text: If the count of pattern letters is 3 or greater, use the Text rules above. Otherwise use the Number rules above.
+
+- Fraction: Use one or more (up to 9) contiguous `'S'` characters, e,g `SSSSSS`, to parse and format fraction of second.
+  For parsing, the acceptable fraction length can be [1, the number of contiguous 'S'].
+  For formatting, the fraction length would be padded to the number of contiguous 'S' with zeros.
+  Spark supports datetime of micro-of-second precision, which has up to 6 significant digits, but can parse nano-of-second with exceeded part truncated.
+
+- Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present. 7 or more letters will fail.
+
+- Month: It follows the rule of Number/Text. The text form is depend on letters - 'M' denotes the 'standard' form, and 'L' is for 'stand-alone' form. These two forms are different only in some certain languages. For example, in Russian, 'Июль' is the stand-alone form of July, and 'Июля' is the standard form. Here are examples for all supported pattern letters:
+  - `'M'` or `'L'`: Month number in a year starting from 1. There is no difference between 'M' and 'L'. Month from 1 to 9 are printed without padding.
+    ```sql
+    spark-sql> select date_format(date '1970-01-01', "M");
+    1
+    spark-sql> select date_format(date '1970-12-01', "L");
+    12
+    ```
+  - `'MM'` or `'LL'`: Month number in a year starting from 1. Zero padding is added for month 1-9.
+      ```sql
+      spark-sql> select date_format(date '1970-1-01', "LL");
+      01
+      spark-sql> select date_format(date '1970-09-01', "MM");
+      09
+      ```
+  - `'MMM'`: Short textual representation in the standard form. The month pattern should be a part of a date pattern not just a stand-alone month except locales where there is no difference between stand and stand-alone forms like in English.
+    ```sql
+    spark-sql> select date_format(date '1970-01-01', "d MMM");
+    1 Jan
+    spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'dd MMM', 'locale', 'RU'));
+    01 янв.
+    ```
+  - `'LLL'`: Short textual representation in the stand-alone form. It should be used to format/parse only months without any other date fields.
+    ```sql
+    spark-sql> select date_format(date '1970-01-01', "LLL");
+    Jan
+    spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'LLL', 'locale', 'RU'));
+    янв.
+    ```
+  - `'MMMM'`: full textual month representation in the standard form. It is used for parsing/formatting months as a part of dates/timestamps.
+    ```sql
+    spark-sql> select date_format(date '1970-01-01', "d MMMM");
+    1 January
+    spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'd MMMM', 'locale', 'RU'));
+    1 января
+    ```
+  - `'LLLL'`: full textual month representation in the stand-alone form. The pattern can be used to format/parse only months.
+    ```sql
+    spark-sql> select date_format(date '1970-01-01', "LLLL");
+    January
+    spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'LLLL', 'locale', 'RU'));
+    январь
+    ```
+
+- am-pm: This outputs the am-pm-of-day. Pattern letter count must be 1.
+
+- Zone ID(V): This outputs the display the time-zone ID. Pattern letter count must be 2.
+
+- Zone names(z): This outputs the display textual name of the time-zone ID. If the count of letters is one, two or three, then the short name is output. If the count of letters is four, then the full name is output. Five or more letters will fail.
+
+- Offset X and x: This formats the offset based on the number of pattern letters. One letter outputs just the hour, such as '+01', unless the minute is non-zero in which case the minute is also output, such as '+0130'. Two letters outputs the hour and minute, without a colon, such as '+0130'. Three letters outputs the hour and minute, with a colon, such as '+01:30'. Four letters outputs the hour and minute and optional second, without a colon, such as '+013015'. Five letters outputs the hour and minute and optional second, with a colon, such as '+01:30:15'. Six or more letters will fail. Pattern letter 'X' (upper case) will output 'Z' when the offset to be output would be zero, whereas pattern letter 'x' (lower case) will output '+00', '+0000', or '+00:00'.
+
+- Offset O: This formats the localized offset based on the number of pattern letters. One letter outputs the short form of the localized offset, which is localized offset text, such as 'GMT', with hour without leading zero, optional 2-digit minute and second if non-zero, and colon, for example 'GMT+8'. Four letters outputs the full form, which is localized offset text, such as 'GMT, with 2-digit hour and minute field, optional second field if non-zero, and colon, for example 'GMT+08:00'. Any other count of letters will fail.
+
+- Offset Z: This formats the offset based on the number of pattern letters. One, two or three letters outputs the hour and minute, without a colon, such as '+0130'. The output will be '+0000' when the offset is zero. Four letters outputs the full form of localized offset, equivalent to four letters of Offset-O. The output will be the corresponding localized offset text if the offset is zero. Five letters outputs the hour, minute, with optional second if non-zero, with colon. It outputs 'Z' if the offset is zero. Six or more letters will fail.
+
+- Optional section start and end: Use `[]` to define an optional section and maybe nested.
+  During formatting, all valid data will be output even it is in the optional section.
+  During parsing, the whole section may be missing from the parsed string.
+  An optional section is started by `[` and ended using `]` (or at the end of the pattern).
+  
+- Symbols of 'E', 'F', 'q' and 'Q' can only be used for datetime formatting, e.g. `date_format`. They are not allowed used for datetime parsing, e.g. `to_timestamp`.
diff --git a/docs/sql-ref-functions-builtin-aggregate.md b/docs/sql-ref-functions-builtin-aggregate.md
deleted file mode 100644
index 3fcd78245ca04..0000000000000
--- a/docs/sql-ref-functions-builtin-aggregate.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Builtin Aggregate Functions
-displayTitle: Builtin Aggregate Functions
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-**This page is under construction**
diff --git a/docs/sql-ref-functions-builtin-scalar.md b/docs/sql-ref-functions-builtin-scalar.md
deleted file mode 100644
index 1d818a25c4ac1..0000000000000
--- a/docs/sql-ref-functions-builtin-scalar.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Builtin Scalar Functions
-displayTitle: Builtin Scalar Functions
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-**This page is under construction**
diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md
index 6c57b0d6fdf6f..cabb83e09fde9 100644
--- a/docs/sql-ref-functions-builtin.md
+++ b/docs/sql-ref-functions-builtin.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Reference
-displayTitle: Reference
+title: Built-in Functions
+displayTitle: Built-in Functions
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,7 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
+  http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,7 +17,61 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL is Apache Spark's module for working with structured data.
-This guide is a reference for Structured Query Language (SQL) for Apache 
-Spark. This document describes the SQL constructs supported by Spark in detail
-along with usage examples when applicable.
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-agg-funcs-table.html' %}
+### Aggregate Functions
+{% include_relative generated-agg-funcs-table.html %}
+#### Examples
+{% include_relative generated-agg-funcs-examples.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-window-funcs-table.html' %}
+### Window Functions
+{% include_relative generated-window-funcs-table.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-array-funcs-table.html' %}
+### Array Functions
+{% include_relative generated-array-funcs-table.html %}
+#### Examples
+{% include_relative generated-array-funcs-examples.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-map-funcs-table.html' %}
+### Map Functions
+{% include_relative generated-map-funcs-table.html %}
+#### Examples
+{% include_relative generated-map-funcs-examples.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-datetime-funcs-table.html' %}
+### Date and Timestamp Functions
+{% include_relative generated-datetime-funcs-table.html %}
+#### Examples
+{% include_relative generated-datetime-funcs-examples.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-json-funcs-table.html' %}
+### JSON Functions
+{% include_relative generated-json-funcs-table.html %}
+#### Examples
+{% include_relative generated-json-funcs-examples.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
diff --git a/docs/sql-ref-functions-udf-aggregate.md b/docs/sql-ref-functions-udf-aggregate.md
index 49c7b5824dfd6..da3182149410b 100644
--- a/docs/sql-ref-functions-udf-aggregate.md
+++ b/docs/sql-ref-functions-udf-aggregate.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: User defined Aggregate Functions (UDAF)
-displayTitle: User defined Aggregate Functions (UDAF)
+title: User Defined Aggregate Functions (UDAFs)
+displayTitle: User Defined Aggregate Functions (UDAFs)
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,4 +19,109 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+User-Defined Aggregate Functions (UDAFs) are user-programmable routines that act on multiple rows at once and return a single aggregated value as a result. This documentation lists the classes that are required for creating and registering UDAFs. It also contains examples that demonstrate how to define and register UDAFs in Scala and invoke them in Spark SQL.
+
+### Aggregator[-IN, BUF, OUT]
+
+A base class for user-defined aggregations, which can be used in Dataset operations to take all of the elements of a group and reduce them to a single value.
+
+  ***IN*** - The input type for the aggregation.
+
+  ***BUF*** - The type of the intermediate value of the reduction.
+
+  ***OUT*** - The type of the final output result.
+
+* **bufferEncoder: Encoder[BUF]**
+
+    Specifies the Encoder for the intermediate value type.
+
+* **finish(reduction: BUF): OUT**
+
+    Transform the output of the reduction.
+
+* **merge(b1: BUF, b2: BUF): BUF**
+
+    Merge two intermediate values.
+
+* **outputEncoder: Encoder[OUT]**
+
+    Specifies the Encoder for the final output value type.
+
+* **reduce(b: BUF, a: IN): BUF**
+
+     Aggregate input value `a` into current intermediate value. For performance, the function may modify `b` and return it instead of constructing new object for `b`.
+
+* **zero: BUF**
+
+    The initial value of the intermediate result for this aggregation.
+
+### Examples
+
+#### Type-Safe User-Defined Aggregate Functions
+
+User-defined aggregations for strongly typed Datasets revolve around the [Aggregator](api/scala/org/apache/spark/sql/expressions/Aggregator.html) abstract class.
+For example, a type-safe user-defined average can look like:
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+  {% include_example typed_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedTypedAggregation.scala%}
+</div>
+<div data-lang="java"  markdown="1">
+  {% include_example typed_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedTypedAggregation.java%}
+</div>
+</div>
+
+#### Untyped User-Defined Aggregate Functions
+
+Typed aggregations, as described above, may also be registered as untyped aggregating UDFs for use with DataFrames.
+For example, a user-defined average for untyped DataFrames can look like:
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+  {% include_example untyped_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedUntypedAggregation.scala%}
+</div>
+<div data-lang="java"  markdown="1">
+  {% include_example untyped_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedUntypedAggregation.java%}
+</div>
+<div data-lang="SQL"  markdown="1">
+```sql
+-- Compile and place UDAF MyAverage in a JAR file called `MyAverage.jar` in /tmp.
+CREATE FUNCTION myAverage AS 'MyAverage' USING JAR '/tmp/MyAverage.jar';
+
+SHOW USER FUNCTIONS;
++------------------+
+|          function|
++------------------+
+| default.myAverage|
++------------------+
+
+CREATE TEMPORARY VIEW employees
+USING org.apache.spark.sql.json
+OPTIONS (
+    path "examples/src/main/resources/employees.json"
+);
+
+SELECT * FROM employees;
++-------+------+
+|   name|salary|
++-------+------+
+|Michael|  3000|
+|   Andy|  4500|
+| Justin|  3500|
+|  Berta|  4000|
++-------+------+
+
+SELECT myAverage(salary) as average_salary FROM employees;
++--------------+
+|average_salary|
++--------------+
+|        3750.0|
++--------------+
+```
+</div>
+</div>
+
+### Related Statements
+
+* [Scalar User Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+* [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
diff --git a/docs/sql-ref-functions-udf-hive.md b/docs/sql-ref-functions-udf-hive.md
new file mode 100644
index 0000000000000..819c446c411d2
--- /dev/null
+++ b/docs/sql-ref-functions-udf-hive.md
@@ -0,0 +1,108 @@
+---
+layout: global
+title: Integration with Hive UDFs/UDAFs/UDTFs
+displayTitle: Integration with Hive UDFs/UDAFs/UDTFs
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+Spark SQL supports integration of Hive UDFs, UDAFs and UDTFs. Similar to Spark UDFs and UDAFs, Hive UDFs work on a single row as input and generate a single row as output, while Hive UDAFs operate on multiple rows and return a single aggregated row as a result. In addition, Hive also supports UDTFs (User Defined Tabular Functions) that act on one row as input and return multiple rows as output. To use Hive UDFs/UDAFs/UTFs, the user should register them in Spark, and then use them in Spark SQL queries.
+
+### Examples
+
+Hive has two UDF interfaces: [UDF](https://github.com/apache/hive/blob/master/udf/src/java/org/apache/hadoop/hive/ql/exec/UDF.java) and [GenericUDF](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java).
+An example below uses [GenericUDFAbs](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java) derived from `GenericUDF`.
+
+```sql
+-- Register `GenericUDFAbs` and use it in Spark SQL.
+-- Note that, if you use your own programmed one, you need to add a JAR containing it
+-- into a classpath,
+-- e.g., ADD JAR yourHiveUDF.jar;
+CREATE TEMPORARY FUNCTION testUDF AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFAbs';
+
+SELECT * FROM t;
++-----+
+|value|
++-----+
+| -1.0|
+|  2.0|
+| -3.0|
++-----+
+
+SELECT testUDF(value) FROM t;
++--------------+
+|testUDF(value)|
++--------------+
+|           1.0|
+|           2.0|
+|           3.0|
++--------------+
+```
+
+
+An example below uses [GenericUDTFExplode](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFExplode.java) derived from [GenericUDTF](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java).
+
+```sql
+-- Register `GenericUDTFExplode` and use it in Spark SQL
+CREATE TEMPORARY FUNCTION hiveUDTF
+    AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode';
+
+SELECT * FROM t;
++------+
+| value|
++------+
+|[1, 2]|
+|[3, 4]|
++------+
+
+SELECT hiveUDTF(value) FROM t;
++---+
+|col|
++---+
+|  1|
+|  2|
+|  3|
+|  4|
++---+
+```
+
+Hive has two UDAF interfaces: [UDAF](https://github.com/apache/hive/blob/master/udf/src/java/org/apache/hadoop/hive/ql/exec/UDAF.java) and [GenericUDAFResolver](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFResolver.java).
+An example below uses [GenericUDAFSum](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java) derived from `GenericUDAFResolver`.
+
+```sql
+-- Register `GenericUDAFSum` and use it in Spark SQL
+CREATE TEMPORARY FUNCTION hiveUDAF
+    AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum';
+
+SELECT * FROM t;
++---+-----+
+|key|value|
++---+-----+
+|  a|    1|
+|  a|    2|
+|  b|    3|
++---+-----+
+
+SELECT key, hiveUDAF(value) FROM t GROUP BY key;
++---+---------------+
+|key|hiveUDAF(value)|
++---+---------------+
+|  b|              3|
+|  a|              3|
++---+---------------+
+```
\ No newline at end of file
diff --git a/docs/sql-ref-functions-udf-scalar.md b/docs/sql-ref-functions-udf-scalar.md
index cee135b572aea..97f5a89d3fb19 100644
--- a/docs/sql-ref-functions-udf-scalar.md
+++ b/docs/sql-ref-functions-udf-scalar.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: User defined Scalar Functions (UDF)
-displayTitle: User defined Scalar Functions (UDF)
+title: Scalar User Defined Functions (UDFs)
+displayTitle: Scalar User Defined Functions (UDFs)
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,4 +19,37 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+User-Defined Functions (UDFs) are user-programmable routines that act on one row. This documentation lists the classes that are required for creating and registering UDFs. It also contains examples that demonstrate how to define and register UDFs and invoke them in Spark SQL.
+
+### UserDefinedFunction
+
+To define the properties of a user-defined function, the user can use some of the methods defined in this class.
+
+* **asNonNullable(): UserDefinedFunction**
+
+    Updates UserDefinedFunction to non-nullable.
+
+* **asNondeterministic(): UserDefinedFunction**
+
+    Updates UserDefinedFunction to nondeterministic.
+
+* **withName(name: String): UserDefinedFunction**
+
+    Updates UserDefinedFunction with a given name.
+
+### Examples
+
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+{% include_example udf_scalar scala/org/apache/spark/examples/sql/UserDefinedScalar.scala%}
+</div>
+<div data-lang="java"  markdown="1">
+  {% include_example udf_scalar java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java%}
+</div>
+</div>
+
+### Related Statements
+* [User Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+* [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
diff --git a/docs/sql-ref-functions-udf.md b/docs/sql-ref-functions-udf.md
deleted file mode 100644
index 6c57b0d6fdf6f..0000000000000
--- a/docs/sql-ref-functions-udf.md
+++ /dev/null
@@ -1,25 +0,0 @@
----
-layout: global
-title: Reference
-displayTitle: Reference
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-Spark SQL is Apache Spark's module for working with structured data.
-This guide is a reference for Structured Query Language (SQL) for Apache 
-Spark. This document describes the SQL constructs supported by Spark in detail
-along with usage examples when applicable.
diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md
index 6c57b0d6fdf6f..67951a9695f5e 100644
--- a/docs/sql-ref-functions.md
+++ b/docs/sql-ref-functions.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Reference
-displayTitle: Reference
+title: Functions
+displayTitle: Functions
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,7 +19,28 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL is Apache Spark's module for working with structured data.
-This guide is a reference for Structured Query Language (SQL) for Apache 
-Spark. This document describes the SQL constructs supported by Spark in detail
-along with usage examples when applicable.
+Spark SQL provides two function features to meet a wide range of user needs: built-in functions and user-defined functions (UDFs).
+Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task.
+
+### Built-in Functions
+
+Spark SQL has some categories of frequently-used built-in functions for aggregation, arrays/maps, date/timestamp, and JSON data.
+This subsection presents the usages and descriptions of these functions.
+
+#### Scalar Functions
+ * [Array Functions](sql-ref-functions-builtin.html#array-functions)
+ * [Map Functions](sql-ref-functions-builtin.html#map-functions)
+ * [Date and Timestamp Functions](sql-ref-functions-builtin.html#date-and-timestamp-functions)
+ * [JSON Functions](sql-ref-functions-builtin.html#json-functions)
+
+#### Aggregate-like Functions
+ * [Aggregate Functions](sql-ref-functions-builtin.html#aggregate-functions)
+ * [Window Functions](sql-ref-functions-builtin.html#window-functions)
+
+### UDFs (User-Defined Functions)
+
+User-Defined Functions (UDFs) are a feature of Spark SQL that allows users to define their own functions when the system's built-in functions are not enough to perform the desired task. To use UDFs in Spark SQL, users must first define the function, then register the function with Spark, and finally call the registered function. The User-Defined Functions can act on a single row or act on multiple rows at once. Spark SQL also supports integration of existing Hive implementations of UDFs, UDAFs and UDTFs.
+
+ * [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+ * [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+ * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
diff --git a/docs/sql-ref-identifier.md b/docs/sql-ref-identifier.md
new file mode 100644
index 0000000000000..f65d491cc2fc4
--- /dev/null
+++ b/docs/sql-ref-identifier.md
@@ -0,0 +1,73 @@
+---
+layout: global
+title: Identifiers
+displayTitle: Identifiers
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+An identifier is a string used to identify a database object such as a table, view, schema, column, etc. Spark SQL has regular identifiers and delimited identifiers, which are enclosed within backticks. Both regular identifiers and delimited identifiers are case-insensitive.
+
+### Syntax
+
+#### Regular Identifier
+
+```sql
+{ letter | digit | '_' } [ , ... ]
+```
+**Note:** If `spark.sql.ansi.enabled` is set to true, ANSI SQL reserved keywords cannot be used as identifiers. For more details, please refer to [ANSI Compliance](sql-ref-ansi-compliance.html).
+
+#### Delimited Identifier
+
+```sql
+`c [ ... ]`
+```
+
+### Parameters
+
+* **letter**
+
+    Any letter from A-Z or a-z.
+
+* **digit**
+
+    Any numeral from 0 to 9.
+
+* **c**
+
+    Any character from the character set. Use <code>`</code> to escape special characters (e.g., <code>`</code>).
+
+### Examples
+
+```sql
+-- This CREATE TABLE fails with ParseException because of the illegal identifier name a.b
+CREATE TABLE test (a.b int);
+org.apache.spark.sql.catalyst.parser.ParseException:
+no viable alternative at input 'CREATE TABLE test (a.'(line 1, pos 20)
+
+-- This CREATE TABLE works
+CREATE TABLE test (`a.b` int);
+
+-- This CREATE TABLE fails with ParseException because special character ` is not escaped
+CREATE TABLE test1 (`a`b` int);
+org.apache.spark.sql.catalyst.parser.ParseException:
+no viable alternative at input 'CREATE TABLE test (`a`b`'(line 1, pos 23)
+
+-- This CREATE TABLE works
+CREATE TABLE test (`a``b` int);
+```
diff --git a/docs/sql-ref-literals.md b/docs/sql-ref-literals.md
new file mode 100644
index 0000000000000..b83f7f0a97c24
--- /dev/null
+++ b/docs/sql-ref-literals.md
@@ -0,0 +1,501 @@
+---
+layout: global
+title: Literals
+displayTitle: Literals
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+A literal (also known as a constant) represents a fixed data value. Spark SQL supports the following literals:
+
+ * [String Literal](#string-literal)
+ * [Binary Literal](#binary-literal)
+ * [Null Literal](#null-literal)
+ * [Boolean Literal](#boolean-literal)
+ * [Numeric Literal](#numeric-literal)
+ * [Datetime Literal](#datetime-literal)
+ * [Interval Literal](#interval-literal)
+
+### String Literal
+
+A string literal is used to specify a character string value.
+
+#### Syntax
+
+```sql
+'char [ ... ]' | "char [ ... ]"
+```
+
+#### Parameters
+
+* **char**
+
+    One character from the character set. Use `\` to escape special characters (e.g., `'` or `\`).
+
+#### Examples
+
+```sql
+SELECT 'Hello, World!' AS col;
++-------------+
+|          col|
++-------------+
+|Hello, World!|
++-------------+
+
+SELECT "SPARK SQL" AS col;
++---------+
+|      col|
++---------+
+|Spark SQL|
++---------+
+
+SELECT 'it\'s $10.' AS col;
++---------+
+|      col|
++---------+
+|It's $10.|
++---------+
+```
+
+### Binary Literal
+
+A binary literal is used to specify a byte sequence value.
+
+#### Syntax
+
+```sql
+X { 'num [ ... ]' | "num [ ... ]" }
+```
+
+#### Parameters
+
+* **num**
+
+    Any hexadecimal number from 0 to F.
+
+#### Examples
+
+```sql
+SELECT X'123456' AS col;
++----------+
+|       col|
++----------+
+|[12 34 56]|
++----------+
+```
+
+### Null Literal
+
+A null literal is used to specify a null value.
+
+#### Syntax
+
+```sql
+NULL
+```
+
+#### Examples
+
+```sql
+SELECT NULL AS col;
++----+
+| col|
++----+
+|NULL|
++----+
+```
+
+### Boolean Literal
+
+A boolean literal is used to specify a boolean value.
+
+#### Syntax
+
+```sql
+TRUE | FALSE
+```
+
+#### Examples
+
+```sql
+SELECT TRUE AS col;
++----+
+| col|
++----+
+|true|
++----+
+```
+
+### Numeric Literal
+
+A numeric literal is used to specify a fixed or floating-point number.
+
+#### Integral Literal
+
+##### Syntax
+
+```sql
+[ + | - ] digit [ ... ] [ L | S | Y ]
+```
+
+##### Parameters
+
+* **digit**
+
+    Any numeral from 0 to 9.
+
+* **L**
+
+    Case insensitive, indicates `BIGINT`, which is an 8-byte signed integer number.
+
+* **S**
+
+    Case insensitive, indicates `SMALLINT`, which is a 2-byte signed integer number.
+
+* **Y**
+
+    Case insensitive, indicates `TINYINT`, which is a 1-byte signed integer number.
+
+* **default (no postfix)**
+
+    Indicates a 4-byte signed integer number.
+
+##### Examples
+
+```sql
+SELECT -2147483648 AS col;
++-----------+
+|        col|
++-----------+
+|-2147483648|
++-----------+
+
+SELECT 9223372036854775807l AS col;
++-------------------+
+|                col|
++-------------------+
+|9223372036854775807|
++-------------------+
+
+SELECT -32Y AS col;
++---+
+|col|
++---+
+|-32|
++---+
+
+SELECT 482S AS col;
++---+
+|col|
++---+
+|482|
++---+
+```
+
+#### Fractional Literals
+
+##### Syntax
+
+decimal literals:
+```sql
+decimal_digits { [ BD ] | [ exponent BD ] } | digit [ ... ] [ exponent ] BD
+```
+
+double literals:
+```sql
+decimal_digits  { D | exponent [ D ] }  | digit [ ... ] { exponent [ D ] | [ exponent ] D }
+```
+
+While decimal_digits is defined as
+```sql
+[ + | - ] { digit [ ... ] . [ digit [ ... ] ] | . digit [ ... ] }
+```
+
+and exponent is defined as
+```sql
+E [ + | - ] digit [ ... ]
+```
+
+##### Parameters
+
+* **digit**
+
+    Any numeral from 0 to 9.
+
+* **D**
+
+    Case insensitive, indicates `DOUBLE`, which is an 8-byte double-precision floating point number.
+
+* **BD**
+
+    Case insensitive, indicates `DECIMAL`, with the total number of digits as precision and the number of digits to right of decimal point as scale.
+
+##### Examples
+
+```sql
+SELECT 12.578 AS col;
++------+
+|   col|
++------+
+|12.578|
++------+
+
+SELECT -0.1234567 AS col;
++----------+
+|       col|
++----------+
+|-0.1234567|
++----------+
+
+SELECT -.1234567 AS col;
++----------+
+|       col|
++----------+
+|-0.1234567|
++----------+
+
+SELECT 123. AS col;
++---+
+|col|
++---+
+|123|
++---+
+
+SELECT 123.BD AS col;
++---+
+|col|
++---+
+|123|
++---+
+
+SELECT 5E2 AS col;
++-----+
+|  col|
++-----+
+|500.0|
++-----+
+
+SELECT 5D AS col;
++---+
+|col|
++---+
+|5.0|
++---+
+
+SELECT -5BD AS col;
++---+
+|col|
++---+
+| -5|
++---+
+
+SELECT 12.578e-2d AS col;
++-------+
+|    col|
++-------+
+|0.12578|
++-------+
+
+SELECT -.1234567E+2BD AS col;
++---------+
+|      col|
++---------+
+|-12.34567|
++---------+
+
+SELECT +3.e+3 AS col;
++------+
+|   col|
++------+
+|3000.0|
++------+
+
+SELECT -3.E-3D AS col;
++------+
+|   col|
++------+
+|-0.003|
++------+
+```
+
+### Datetime Literal
+
+A Datetime literal is used to specify a datetime value.
+
+#### Date Literal
+
+##### Syntax
+
+```sql
+DATE { 'yyyy' |
+       'yyyy-[m]m' |
+       'yyyy-[m]m-[d]d' |
+       'yyyy-[m]m-[d]d[T]' }
+```
+**Note:** defaults to `01` if month or day is not specified.
+
+##### Examples
+
+```sql
+SELECT DATE '1997' AS col;
++----------+
+|       col|
++----------+
+|1997-01-01|
++----------+
+
+SELECT DATE '1997-01' AS col;
++----------+
+|       col|
++----------+
+|1997-01-01|
++----------+
+
+SELECT DATE '2011-11-11' AS col;
++----------+
+|       col|
++----------+
+|2011-11-11|
++----------+
+```
+
+#### Timestamp Literal
+
+##### Syntax
+
+```sql
+TIMESTAMP { 'yyyy' |
+            'yyyy-[m]m' |
+            'yyyy-[m]m-[d]d' |
+            'yyyy-[m]m-[d]d ' |
+            'yyyy-[m]m-[d]d[T][h]h[:]' |
+            'yyyy-[m]m-[d]d[T][h]h:[m]m[:]' |
+            'yyyy-[m]m-[d]d[T][h]h:[m]m:[s]s[.]' |
+            'yyyy-[m]m-[d]d[T][h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]'}
+```
+**Note:** defaults to `00` if hour, minute or second is not specified.
+`zone_id` should have one of the forms:
+* Z - Zulu time zone UTC+0
+* `+|-[h]h:[m]m`
+* An id with one of the prefixes UTC+, UTC-, GMT+, GMT-, UT+ or UT-, and a suffix in the formats:
+  * `+|-h[h]`
+  * `+|-hh[:]mm`
+  * `+|-hh:mm:ss`
+  * `+|-hhmmss`
+* Region-based zone IDs in the form `area/city`, such as `Europe/Paris`
+
+**Note:** defaults to the session local timezone (set via `spark.sql.session.timeZone`) if `zone_id` is not specified.
+
+##### Examples
+
+```sql
+SELECT TIMESTAMP '1997-01-31 09:26:56.123' AS col;
++-----------------------+
+|                    col|
++-----------------------+
+|1997-01-31 09:26:56.123|
++-----------------------+
+
+SELECT TIMESTAMP '1997-01-31 09:26:56.66666666UTC+08:00' AS col;
++--------------------------+
+|                      col |
++--------------------------+
+|1997-01-30 17:26:56.666666|
++--------------------------+
+
+SELECT TIMESTAMP '1997-01' AS col;
++-------------------+
+|                col|
++-------------------+
+|1997-01-01 00:00:00|
++-------------------+
+```
+
+### Interval Literal
+
+An interval literal is used to specify a fixed period of time.
+
+```sql
+INTERVAL interval_value interval_unit [ interval_value interval_unit ... ] |
+INTERVAL 'interval_value interval_unit [ interval_value interval_unit ... ]' |
+INTERVAL interval_string_value interval_unit TO interval_unit
+```
+
+#### Parameters
+
+* **interval_value**
+
+    **Syntax:**
+
+      [ + | - ] number_value | '[ + | - ] number_value'
+
+* **interval_string_value**
+
+     year-month/day-time interval string.
+
+* **interval_unit**
+
+    **Syntax:**
+
+      YEAR[S] | MONTH[S] | WEEK[S] | DAY[S] | HOUR[S] | MINUTE[S] | SECOND[S] |
+      MILLISECOND[S] | MICROSECOND[S]
+
+#### Examples
+
+```sql
+SELECT INTERVAL 3 YEAR AS col;
++-------+
+|    col|
++-------+
+|3 years|
++-------+
+
+SELECT INTERVAL -2 HOUR '3' MINUTE AS col;
++--------------------+
+|                 col|
++--------------------+
+|-1 hours -57 minutes|
++--------------------+
+
+SELECT INTERVAL '1 YEAR 2 DAYS 3 HOURS';
++----------------------+
+|                   col|
++----------------------+
+|1 years 2 days 3 hours|
++----------------------+
+
+SELECT INTERVAL 1 YEARS 2 MONTH 3 WEEK 4 DAYS 5 HOUR 6 MINUTES 7 SECOND 8
+    MILLISECOND 9 MICROSECONDS AS col;
++-----------------------------------------------------------+
+|                                                        col|
++-----------------------------------------------------------+
+|1 years 2 months 25 days 5 hours 6 minutes 7.008009 seconds|
++-----------------------------------------------------------+
+
+SELECT INTERVAL '2-3' YEAR TO MONTH AS col;
++----------------+
+|             col|
++----------------+
+|2 years 3 months|
++----------------+
+
+SELECT INTERVAL '20 15:40:32.99899999' DAY TO SECOND AS col;
++---------------------------------------------+
+|                                          col|
++---------------------------------------------+
+|20 days 15 hours 40 minutes 32.998999 seconds|
++---------------------------------------------+
+```
diff --git a/docs/sql-ref-nan-semantics.md b/docs/sql-ref-nan-semantics.md
deleted file mode 100644
index f6a85728263a7..0000000000000
--- a/docs/sql-ref-nan-semantics.md
+++ /dev/null
@@ -1,29 +0,0 @@
----
-layout: global
-title: Nan Semantics
-displayTitle: NaN Semantics
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-There is specially handling for not-a-number (NaN) when dealing with `float` or `double` types that
-does not exactly match standard floating point semantics.
-Specifically:
- 
- - NaN = NaN returns true.
- - In aggregations, all NaN values are grouped together.
- - NaN is treated as a normal value in join keys.
- - NaN values go last when in ascending order, larger than any other numeric value.
diff --git a/docs/sql-ref-null-semantics.md b/docs/sql-ref-null-semantics.md
index 3cbc15c600cee..3c12e7a28b64e 100644
--- a/docs/sql-ref-null-semantics.md
+++ b/docs/sql-ref-null-semantics.md
@@ -20,6 +20,7 @@ license: |
 ---
 
 ### Description
+
 A table consists of a set of rows and each row contains a set of columns.
 A column is associated with a data type and represents
 a specific attribute of an entity (for example, `age` is a column of an
@@ -48,20 +49,20 @@ other `SQL` constructs.
 </style>
 
 The following illustrates the schema layout and data of a table named `person`. The data contains `NULL` values in
-the `age` column and this table will be used in various examples in the sections below.
+the `age` column and this table will be used in various examples in the sections below.<br/>
 **<u>TABLE: person</u>**
-<table class="tsclass" border="1">
-<tr><th>Id</th><th>Name</th><th>Age</th></tr>
-<tr><td>100</td><td>Joe</td><td>30</td></tr>
-<tr><td>200</td><td>Marry</td><td>NULL</td></tr>
-<tr><td>300</td><td>Mike</td><td>18</td></tr>
-<tr><td>400</td><td>Fred</td><td>50</td></tr>
-<tr><td>500</td><td>Albert</td><td>NULL</td></tr>
-<tr><td>600</td><td>Michelle</td><td>30</td></tr>
-<tr><td>700</td><td>Dan</td><td>50</td></tr>
-</table>
-
-### Comparison operators <a name="comp-operators"></a>
+
+|Id |Name|Age|
+|---|----|---|
+|100|Joe|30|
+|200|Marry|NULL|
+|300|Mike|18|
+|400|Fred|50|
+|500|Albert|NULL|
+|600|Michelle|30|
+|700|Dan|50|
+
+### Comparison Operators <a name="comp-operators"></a>
 
 Apache spark supports the standard comparison operators such as '>', '>=', '=', '<' and '<='.
 The result of these operators is unknown or `NULL` when one of the operands or both the operands are
@@ -70,168 +71,97 @@ equal operator ('<=>'), which returns `False` when one of the operand is `NULL`
 both the operands are `NULL`. The following table illustrates the behaviour of comparison operators when
 one or both operands are `NULL`:
 
-<table class="tsclass" border="1">
-  <tr>
-    <th>Left Operand</th>
-    <th>Right  Operand</th>
-    <th>></th>
-    <th>>=</th>
-    <th>=</th>
-    <th><</th>
-    <th><=</th>
-    <th><=></th>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>Any value</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>False</td>
-  </tr>
-  <tr>
-    <td>Any value</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>False</td>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>True</td>
-  </tr>
-</table>
+|Left Operand|Right  Operand|>  |>= |=  |<  |<= |<=>|
+|------------|--------------|---|---|---|---|---|---|
+|NULL|Any value|NULL|NULL|NULL|NULL|NULL|False|
+|Any value|NULL|NULL|NULL|NULL|NULL|NULL|False|
+|NULL|NULL|NULL|NULL|NULL|NULL|NULL|True|
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Normal comparison operators return `NULL` when one of the operand is `NULL`.
 SELECT 5 > null AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |null             |
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 
 -- Normal comparison operators return `NULL` when both the operands are `NULL`.
 SELECT null = null AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |null             |
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 
 -- Null-safe equal operator return `False` when one of the operand is `NULL`
 SELECT 5 <=> null AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |false            |
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|            false|
++-----------------+
 
 -- Null-safe equal operator return `True` when one of the operand is `NULL`
 SELECT NULL <=> NULL;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |true             |
-  +-----------------+
-{% endhighlight %}
-
-### Logical operators <a name="logical-operators"></a>
++-----------------+
+|expression_output|
++-----------------+
+|             true|
++-----------------+
+```
+
+### Logical Operators <a name="logical-operators"></a>
+
 Spark supports standard logical operators such as `AND`, `OR` and `NOT`. These operators take `Boolean` expressions
 as the arguments and return a `Boolean` value.  
 
 The following tables illustrate the behavior of logical operators when one or both operands are `NULL`.
 
-<table class="tsclass" border="1">
-  <tr>
-    <th>Left Operand</th>
-    <th>Right Operand</th>
-    <th>OR</th>
-    <th>AND</th>
-  </tr>
-  <tr>
-    <td>True</td>
-    <td>NULL</td>
-    <td>True</td>
-    <td>NULL</td>
-  </tr>
-  <tr>
-    <td>False</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>False</td>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>True</td>
-    <td>True</td>
-    <td>NULL</td>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>False</td>
-    <td>NULL</td>
-    <td>NULL</td>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-  </tr>
-</table>
-<br>
-<table class="tsclass" border="1">
-  <tr>
-    <th>operand</th>
-    <th>NOT</th>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>NULL</td>
-  </tr>
-</table>
+|Left Operand|Right Operand|OR |AND|
+|------------|-------------|---|---|
+|True|NULL|True|NULL|
+|False|NULL|NULL|False|
+|NULL|True|True|NULL|
+|NULL|False|NULL|False|
+|NULL|NULL|NULL|NULL|
+
+|operand|NOT|
+|-------|---|
+|NULL|NULL|
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Normal comparison operators return `NULL` when one of the operands is `NULL`.
 SELECT (true OR null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |true             |
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             true|
++-----------------+
 
 -- Normal comparison operators return `NULL` when both the operands are `NULL`.
 SELECT (null OR false) AS expression_output
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |null             |
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 
 -- Null-safe equal operator returns `False` when one of the operands is `NULL`
 SELECT NOT(null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |null             |
-  +-----------------+
-{% endhighlight %}
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
+```
 
 ### Expressions <a name="expressions"></a>
+
 The comparison operators and logical operators are treated as expressions in
 Spark. Other than these two kinds of expressions, Spark supports other form of
 expressions such as function expressions, cast expressions, etc. The expressions
@@ -240,35 +170,37 @@ in Spark can be broadly classified as :
 - Expressions that can process `NULL` value operands
   - The result of these expressions depends on the expression itself.
 
-#### Null intolerant expressions <a name="null-intolerant"></a>
+#### Null Intolerant Expressions <a name="null-intolerant"></a>
+
 Null intolerant expressions return `NULL` when one or more arguments of 
 expression are `NULL` and most of the expressions fall in this category.
 
 ##### Examples
-{% highlight sql %}
-SELECT concat('John', null) as expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |null             |
-  +-----------------+
-
-SELECT positive(null) as expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |null             |
-  +-----------------+
-
-SELECT to_date(null) as expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |null             |
-  +-----------------+
-{% endhighlight %}
-
-#### Expressions that can process null value operands. <a name="can-process-null"></a>
+
+```sql
+SELECT concat('John', null) AS expression_output;
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
+
+SELECT positive(null) AS expression_output;
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
+
+SELECT to_date(null) AS expression_output;
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
+```
+
+#### Expressions That Can Process Null Value Operands <a name="can-process-null"></a>
 
 This class of expressions are designed to handle `NULL` values. The result of the 
 expressions depends on the expression itself. As an example, function expression `isnull`
@@ -287,47 +219,48 @@ returns the first non `NULL` value in its list of operands. However, `coalesce`
   - ATLEASTNNONNULLS 
   - IN
 
-
 ##### Examples
-{% highlight sql %}
+
+```sql
 SELECT isnull(null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |true             |
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             true|
++-----------------+
 
 -- Returns the first occurrence of non `NULL` value.
 SELECT coalesce(null, null, 3, null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |3                |
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|                3|
++-----------------+
 
 -- Returns `NULL` as all its operands are `NULL`. 
 SELECT coalesce(null, null, null, null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |null             |
-  +-----------------+
-
-SELECT isnan(null) as expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |false            |
-  +-----------------+
-{% endhighlight %}
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
+
+SELECT isnan(null) AS expression_output;
++-----------------+
+|expression_output|
++-----------------+
+|            false|
++-----------------+
+```
 
 #### Builtin Aggregate Expressions <a name="built-in-aggregate"></a>
+
 Aggregate functions compute a single result by processing a set of input rows. Below are
 the rules of how `NULL` values are handled by aggregate functions.
 - `NULL` values are ignored from processing by all the aggregate functions.
   - Only exception to this rule is COUNT(*) function.
 - Some aggregate functions return `NULL` when all input values are `NULL` or the input data set
-  is empty.<br> The list of these functions is:
+  is empty.<br/> The list of these functions is:
   - MAX
   - MIN
   - SUM
@@ -337,114 +270,116 @@ the rules of how `NULL` values are handled by aggregate functions.
   - SOME
    
 #### Examples
-{% highlight sql %}
+
+```sql
 -- `count(*)` does not skip `NULL` values.
 SELECT count(*) FROM person;
-  +--------+
-  |count(1)|
-  +--------+
-  |7       |
-  +--------+
++--------+
+|count(1)|
++--------+
+|       7|
++--------+
 
 -- `NULL` values in column `age` are skipped from processing.
 SELECT count(age) FROM person;
-  +----------+
-  |count(age)|
-  +----------+
-  |5         |
-  +----------+
++----------+
+|count(age)|
++----------+
+|         5|
++----------+
 
 -- `count(*)` on an empty input set returns 0. This is unlike the other
 -- aggregate functions, such as `max`, which return `NULL`.
 SELECT count(*) FROM person where 1 = 0;
-  +--------+
-  |count(1)|
-  +--------+
-  |0       |
-  +--------+
++--------+
+|count(1)|
++--------+
+|       0|
++--------+
 
 -- `NULL` values are excluded from computation of maximum value.
 SELECT max(age) FROM person;
-  +--------+
-  |max(age)|
-  +--------+
-  |50      |
-  +--------+
++--------+
+|max(age)|
++--------+
+|      50|
++--------+
 
 -- `max` returns `NULL` on an empty input set.
 SELECT max(age) FROM person where 1 = 0;
-  +--------+
-  |max(age)|
-  +--------+
-  |null    |
-  +--------+
++--------+
+|max(age)|
++--------+
+|    null|
++--------+
+```
 
-{% endhighlight %}
+### Condition Expressions in WHERE, HAVING and JOIN Clauses <a name="condition-expressions"></a>
 
-### Condition expressions in WHERE, HAVING and JOIN clauses. <a name="condition-expressions"></a>
 `WHERE`, `HAVING` operators filter rows based on the user specified condition.
 A `JOIN` operator is used to combine rows from two tables based on a join condition.
 For all the three operators, a condition expression is a boolean expression and can return
  <code>True, False or Unknown (NULL)</code>. They are "satisfied" if the result of the condition is `True`.
 
 #### Examples
-{% highlight sql %}
+
+```sql
 -- Persons whose age is unknown (`NULL`) are filtered out from the result set.
 SELECT * FROM person WHERE age > 0;
-  +--------+---+
-  |name    |age|
-  +--------+---+
-  |Michelle|30 |
-  |Fred    |50 |
-  |Mike    |18 |
-  |Dan     |50 |
-  |Joe     |30 |
-  +--------+---+
++--------+---+
+|    name|age|
++--------+---+
+|Michelle| 30|
+|    Fred| 50|
+|    Mike| 18|
+|     Dan| 50|
+|     Joe| 30|
++--------+---+
 
 -- `IS NULL` expression is used in disjunction to select the persons
 -- with unknown (`NULL`) records.
 SELECT * FROM person WHERE age > 0 OR age IS NULL;
-  +--------+----+
-  |name    |age |
-  +--------+----+
-  |Albert  |null|
-  |Michelle|30  |
-  |Fred    |50  |
-  |Mike    |18  |
-  |Dan     |50  |
-  |Marry   |null|
-  |Joe     |30  |
-  +--------+----+
++--------+----+
+|    name| age|
++--------+----+
+|  Albert|null|
+|Michelle|  30|
+|    Fred|  50|
+|    Mike|  18|
+|     Dan|  50|
+|   Marry|null|
+|     Joe|  30|
++--------+----+
 
 -- Person with unknown(`NULL`) ages are skipped from processing.
 SELECT * FROM person GROUP BY age HAVING max(age) > 18;
-  +---+--------+                                                                  
-  |age|count(1)|
-  +---+--------+
-  |50 |2       |
-  |30 |2       |
-  +---+--------+
++---+--------+
+|age|count(1)|
++---+--------+
+| 50|       2|
+| 30|       2|
++---+--------+
 
 -- A self join case with a join condition `p1.age = p2.age AND p1.name = p2.name`.
 -- The persons with unknown age (`NULL`) are filtered out by the join operator.
 SELECT * FROM person p1, person p2
-WHERE p1.age = p2.age
-  AND p1.name = p2.name;
-  +--------+---+--------+---+
-  |name    |age|name    |age|
-  +--------+---+--------+---+
-  |Michelle|30 |Michelle|30 |
-  |Fred    |50 |Fred    |50 |
-  |Mike    |18 |Mike    |18 |
-  |Dan     |50 |Dan     |50 |
-  |Joe     |30 |Joe     |30 |
-  +--------+---+--------+---+
+    WHERE p1.age = p2.age
+    AND p1.name = p2.name;
++--------+---+--------+---+
+|    name|age|    name|age|
++--------+---+--------+---+
+|Michelle| 30|Michelle| 30|
+|    Fred| 50|    Fred| 50|
+|    Mike| 18|    Mike| 18|
+|     Dan| 50|     Dan| 50|
+|     Joe| 30|     Joe| 30|
++--------+---+--------+---+
 
 -- The age column from both legs of join are compared using null-safe equal which
 -- is why the persons with unknown age (`NULL`) are qualified by the join.
 SELECT * FROM person p1, person p2
-WHERE p1.age <=> p2.age
-  AND p1.name = p2.name;
+    WHERE p1.age <=> p2.age
+    AND p1.name = p2.name;
 +--------+----+--------+----+
 |    name| age|    name| age|
 +--------+----+--------+----+
@@ -456,156 +391,160 @@ WHERE p1.age <=> p2.age
 |   Marry|null|   Marry|null|
 |     Joe|  30|     Joe|  30|
 +--------+----+--------+----+
+```
 
-{% endhighlight %}
+### Aggregate Operator (GROUP BY, DISTINCT) <a name="aggregate-operator"></a>
 
-### Aggregate operator (GROUP BY, DISTINCT) <a name="aggregate-operator"></a>
 As discussed in the previous section [comparison operator](sql-ref-null-semantics.html#comparison-operators),
 two `NULL` values are not equal. However, for the purpose of grouping and distinct processing, the two or more
 values with `NULL data`are grouped together into the same bucket. This behaviour is conformant with SQL
 standard and with other enterprise database management systems.
 
 #### Examples
-{% highlight sql %}
+
+```sql
 -- `NULL` values are put in one bucket in `GROUP BY` processing.
 SELECT age, count(*) FROM person GROUP BY age;
-  +----+--------+                                                                 
-  |age |count(1)|
-  +----+--------+
-  |null|2       |
-  |50  |2       |
-  |30  |2       |
-  |18  |1       |
-  +----+--------+
++----+--------+
+| age|count(1)|
++----+--------+
+|null|       2|
+|  50|       2|
+|  30|       2|
+|  18|       1|
++----+--------+
 
 -- All `NULL` ages are considered one distinct value in `DISTINCT` processing.
 SELECT DISTINCT age FROM person;
-  +----+
-  |age |
-  +----+
-  |null|
-  |50  |
-  |30  |
-  |18  |
-  +----+
-
-{% endhighlight %}
-
-### Sort operator (ORDER BY Clause) <a name="order-by"></a>
++----+
+| age|
++----+
+|null|
+|  50|
+|  30|
+|  18|
++----+
+```
+
+### Sort Operator (ORDER BY Clause) <a name="order-by"></a>
+
 Spark SQL supports null ordering specification in `ORDER BY` clause. Spark processes the `ORDER BY` clause by
 placing all the `NULL` values at first or at last depending on the null ordering specification. By default, all
 the `NULL` values are placed at first.
 
 #### Examples
-{% highlight sql %}
+
+```sql
 -- `NULL` values are shown at first and other values
 -- are sorted in ascending way.
 SELECT age, name FROM person ORDER BY age;
-  +----+--------+
-  |age |name    |
-  +----+--------+
-  |null|Marry   |
-  |null|Albert  |
-  |18  |Mike    |
-  |30  |Michelle|
-  |30  |Joe     |
-  |50  |Fred    |
-  |50  |Dan     |
-  +----+--------+
++----+--------+
+| age|    name|
++----+--------+
+|null|   Marry|
+|null|  Albert|
+|  18|    Mike|
+|  30|Michelle|
+|  30|     Joe|
+|  50|    Fred|
+|  50|     Dan|
++----+--------+
 
 -- Column values other than `NULL` are sorted in ascending
 -- way and `NULL` values are shown at the last.
 SELECT age, name FROM person ORDER BY age NULLS LAST;
-  +----+--------+
-  |age |name    |
-  +----+--------+
-  |18  |Mike    |
-  |30  |Michelle|
-  |30  |Joe     |
-  |50  |Dan     |
-  |50  |Fred    |
-  |null|Marry   |
-  |null|Albert  |
-  +----+--------+
++----+--------+
+| age|    name|
++----+--------+
+|  18|    Mike|
+|  30|Michelle|
+|  30|     Joe|
+|  50|     Dan|
+|  50|    Fred|
+|null|   Marry|
+|null|  Albert|
++----+--------+
 
 -- Columns other than `NULL` values are sorted in descending
 -- and `NULL` values are shown at the last.
 SELECT age, name FROM person ORDER BY age DESC NULLS LAST;
-  +----+--------+
-  |age |name    |
-  +----+--------+
-  |50  |Fred    |
-  |50  |Dan     |
-  |30  |Michelle|
-  |30  |Joe     |
-  |18  |Mike    |
-  |null|Marry   |
-  |null|Albert  |
-  +----+--------+
-{% endhighlight %}
-
-### Set operators (UNION, INTERSECT, EXCEPT) <a name="set-operators"></a>
++----+--------+
+| age|    name|
++----+--------+
+|  50|    Fred|
+|  50|     Dan|
+|  30|Michelle|
+|  30|     Joe|
+|  18|    Mike|
+|null|   Marry|
+|null|  Albert|
++----+--------+
+```
+
+### Set Operators (UNION, INTERSECT, EXCEPT) <a name="set-operators"></a>
+
 `NULL` values are compared in a null-safe manner for equality in the context of
 set operations. That means when comparing rows, two `NULL` values are considered 
 equal unlike the regular `EqualTo`(`=`) operator.
 
 #### Examples
-{% highlight sql %}
+
+```sql
 CREATE VIEW unknown_age SELECT * FROM person WHERE age IS NULL;
 
 -- Only common rows between two legs of `INTERSECT` are in the 
 -- result set. The comparison between columns of the row are done
 -- in a null-safe manner.
 SELECT name, age FROM person
-INTERSECT
-SELECT name, age from unknown_age;
-  +------+----+                                                                   
-  |name  |age |
-  +------+----+
-  |Albert|null|
-  |Marry |null|
-  +------+----+
+    INTERSECT
+    SELECT name, age from unknown_age;
++------+----+
+|  name| age|
++------+----+
+|Albert|null|
+| Marry|null|
++------+----+
 
 -- `NULL` values from two legs of the `EXCEPT` are not in output. 
 -- This basically shows that the comparison happens in a null-safe manner.
 SELECT age, name FROM person
-EXCEPT
-SELECT age FROM unknown_age;
-  +---+--------+                                                                  
-  |age|name    |
-  +---+--------+
-  |30 |Joe     |
-  |50 |Fred    |
-  |30 |Michelle|
-  |18 |Mike    |
-  |50 |Dan     |
-  +---+--------+
+    EXCEPT
+    SELECT age FROM unknown_age;
++---+--------+
+|age|    name|
++---+--------+
+| 30|     Joe|
+| 50|    Fred|
+| 30|Michelle|
+| 18|    Mike|
+| 50|     Dan|
++---+--------+
 
 -- Performs `UNION` operation between two sets of data. 
 -- The comparison between columns of the row ae done in
 -- null-safe manner.
 SELECT name, age FROM person
-UNION 
-SELECT name, age FROM unknown_age;
-  +--------+----+                                                                 
-  |name    |age |
-  +--------+----+
-  |Albert  |null|
-  |Joe     |30  |
-  |Michelle|30  |
-  |Marry   |null|
-  |Fred    |50  |
-  |Mike    |18  |
-  |Dan     |50  |
-  +--------+----+
-{% endhighlight %}
-
+    UNION 
+    SELECT name, age FROM unknown_age;
++--------+----+
+|    name| age|
++--------+----+
+|  Albert|null|
+|     Joe|  30|
+|Michelle|  30|
+|   Marry|null|
+|    Fred|  50|
+|    Mike|  18|
+|     Dan|  50|
++--------+----+
+```
 
 ### EXISTS/NOT EXISTS Subquery <a name="exists-not-exists"></a>
+
 In Spark, EXISTS and NOT EXISTS expressions are allowed inside a WHERE clause. 
 These are boolean expressions which return either `TRUE` or
 `FALSE`. In other words, EXISTS is a membership condition and returns `TRUE`
-when the subquery it refers to returns one or more rows. Similary, NOT EXISTS
+when the subquery it refers to returns one or more rows. Similarly, NOT EXISTS
 is a non-membership condition and returns TRUE when no rows or zero rows are
 returned from the subquery.
 
@@ -614,46 +553,48 @@ the subquery. They are normally faster because they can be converted to
 semijoins / anti-semijoins without special provisions for null awareness.
 
 #### Examples
-{% highlight sql %}
+
+```sql
 -- Even if subquery produces rows with `NULL` values, the `EXISTS` expression
 -- evaluates to `TRUE` as the subquery produces 1 row.
 SELECT * FROM person WHERE EXISTS (SELECT null);
-  +--------+----+                                                                 
-  |name    |age |
-  +--------+----+
-  |Albert  |null|
-  |Michelle|30  |
-  |Fred    |50  |
-  |Mike    |18  |
-  |Dan     |50  |
-  |Marry   |null|
-  |Joe     |30  |
-  +--------+----+
++--------+----+
+|    name| age|
++--------+----+
+|  Albert|null|
+|Michelle|  30|
+|    Fred|  50|
+|    Mike|  18|
+|     Dan|  50|
+|   Marry|null|
+|     Joe|  30|
++--------+----+
 
 -- `NOT EXISTS` expression returns `FALSE`. It returns `TRUE` only when
 -- subquery produces no rows. In this case, it returns 1 row.
 SELECT * FROM person WHERE NOT EXISTS (SELECT null);
-  +----+---+
-  |name|age|
-  +----+---+
-  +----+---+
++----+---+
+|name|age|
++----+---+
++----+---+
 
 -- `NOT EXISTS` expression returns `TRUE`.
 SELECT * FROM person WHERE NOT EXISTS (SELECT 1 WHERE 1 = 0);
-  +--------+----+
-  |name    |age |
-  +--------+----+
-  |Albert  |null|
-  |Michelle|30  |
-  |Fred    |50  |
-  |Mike    |18  |
-  |Dan     |50  |
-  |Marry   |null|
-  |Joe     |30  |
-  +--------+----+
-{% endhighlight %}
++--------+----+
+|    name| age|
++--------+----+
+|  Albert|null|
+|Michelle|  30|
+|    Fred|  50|
+|    Mike|  18|
+|     Dan|  50|
+|   Marry|null|
+|     Joe|  30|
++--------+----+
+```
 
 ### IN/NOT IN Subquery <a name="in-not-in"></a>
+
 In Spark, `IN` and `NOT IN` expressions are allowed inside a WHERE clause of
 a query. Unlike the `EXISTS` expression, `IN` expression can return a `TRUE`,
 `FALSE` or `UNKNOWN (NULL)` value. Conceptually a `IN` expression is semantically
@@ -675,34 +616,34 @@ This is because IN returns UNKNOWN if the value is not in the list containing `N
 and because NOT UNKNOWN is again UNKNOWN.
  
 #### Examples
-{% highlight sql %}
+
+```sql
 -- The subquery has only `NULL` value in its result set. Therefore,
 -- the result of `IN` predicate is UNKNOWN.
 SELECT * FROM person WHERE age IN (SELECT null);
-  +----+---+
-  |name|age|
-  +----+---+
-  +----+---+
++----+---+
+|name|age|
++----+---+
++----+---+
 
 -- The subquery has `NULL` value in the result set as well as a valid 
 -- value `50`. Rows with age = 50 are returned. 
 SELECT * FROM person
-WHERE age IN (SELECT age FROM VALUES (50), (null) sub(age));
-  +----+---+
-  |name|age|
-  +----+---+
-  |Fred|50 |
-  |Dan |50 |
-  +----+---+
+    WHERE age IN (SELECT age FROM VALUES (50), (null) sub(age));
++----+---+
+|name|age|
++----+---+
+|Fred| 50|
+| Dan| 50|
++----+---+
 
 -- Since subquery has `NULL` value in the result set, the `NOT IN`
 -- predicate would return UNKNOWN. Hence, no rows are
 -- qualified for this query.
 SELECT * FROM person
-WHERE age NOT IN (SELECT age FROM VALUES (50), (null) sub(age));
-  +----+---+
-  |name|age|
-  +----+---+
-  +----+---+
-
-{% endhighlight %}
+    WHERE age NOT IN (SELECT age FROM VALUES (50), (null) sub(age));
++----+---+
+|name|age|
++----+---+
++----+---+
+```
diff --git a/docs/sql-ref-syntax-aux-analyze-table.md b/docs/sql-ref-syntax-aux-analyze-table.md
index e448ca38c849e..8f43d7388d7db 100644
--- a/docs/sql-ref-syntax-aux-analyze-table.md
+++ b/docs/sql-ref-syntax-aux-analyze-table.md
@@ -24,86 +24,101 @@ license: |
 The `ANALYZE TABLE` statement collects statistics about the table to be used by the query optimizer to find a better query execution plan.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 ANALYZE TABLE table_identifier [ partition_spec ]
     COMPUTE STATISTICS [ NOSCAN | FOR COLUMNS col [ , ... ] | FOR ALL COLUMNS ]
-
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions. When specified, partition statistics is returned.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>[ NOSCAN | FOR COLUMNS col [ , ... ] | FOR ALL COLUMNS ]</em></code></dt>
-    <dd>
-      <ul>
-        <li> If no analyze option is specified, <code>ANALYZE TABLE</code> collects the table's number of rows and size in bytes. </li>
-        <li> <b>NOSCAN</b>
-          <br> Collect only the table's size in bytes ( which does not require scanning the entire table ). </li>
-        <li> <b>FOR COLUMNS col [ , ... ] <code> | </code> FOR ALL COLUMNS</b>
-          <br> Collect column statistics for each column specified, or alternatively for every column, as well as table statistics.
-        </li>
-      </ul>
-     </dd>
-</dl>
+    for partitions. When specified, partition statistics is returned.
+
+    **Syntax:** `PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )`
+
+* **[ NOSCAN `|` FOR COLUMNS col [ , ... ] `|` FOR ALL COLUMNS ]**
+
+     * If no analyze option is specified, `ANALYZE TABLE` collects the table's number of rows and size in bytes.
+     * **NOSCAN**
+
+       Collects only the table's size in bytes ( which does not require scanning the entire table ).
+     * **FOR COLUMNS col [ , ... ] `|` FOR ALL COLUMNS**
+
+       Collects column statistics for each column specified, or alternatively for every column, as well as table statistics.
 
 ### Examples
-{% highlight sql %}
- ANALYZE TABLE students COMPUTE STATISTICS NOSCAN;
-
- DESC EXTENDED students;
-     ......
-     Statistics	2820 bytes
-     ......
-
- ANALYZE TABLE students COMPUTE STATISTICS;
-
- DESC EXTENDED students;
-     ......
-     Statistics	2820 bytes, 3 rows
-     ......
-
- ANALYZE TABLE students PARTITION (student_id = 111111) COMPUTE STATISTICS;
-
- DESC EXTENDED students PARTITION (student_id = 111111);
-     ......
-     Partition Statistics	919 bytes, 1 rows
-     ......
-
- ANALYZE TABLE students COMPUTE STATISTICS FOR COLUMNS name;
-
- DESC EXTENDED students name;
-     =default tbl=students
-     col_name	name
-     data_type	string
-     comment	NULL
-     min	NULL
-     max	NULL
-     num_nulls	0
-     distinct_count	3
-     avg_col_len	11
-     max_col_len	13
-     histogram	NULL
-
-{% endhighlight %}
\ No newline at end of file
+
+```sql
+CREATE TABLE students (name STRING, student_id INT) PARTITIONED BY (student_id);
+INSERT INTO students PARTITION (student_id = 111111) VALUES ('Mark');
+INSERT INTO students PARTITION (student_id = 222222) VALUES ('John');
+
+ANALYZE TABLE students COMPUTE STATISTICS NOSCAN;
+
+DESC EXTENDED students;
++--------------------+--------------------+-------+
+|            col_name|           data_type|comment|
++--------------------+--------------------+-------+
+|                name|              string|   null|
+|          student_id|                 int|   null|
+|                 ...|                 ...|    ...|
+|          Statistics|           864 bytes|       |
+|                 ...|                 ...|    ...|
+|  Partition Provider|             Catalog|       |
++--------------------+--------------------+-------+
+
+ANALYZE TABLE students COMPUTE STATISTICS;
+
+DESC EXTENDED students;
++--------------------+--------------------+-------+
+|            col_name|           data_type|comment|
++--------------------+--------------------+-------+
+|                name|              string|   null|
+|          student_id|                 int|   null|
+|                 ...|                 ...|    ...|
+|          Statistics|   864 bytes, 2 rows|       |
+|                 ...|                 ...|    ...|
+|  Partition Provider|             Catalog|       |
++--------------------+--------------------+-------+
+
+ANALYZE TABLE students PARTITION (student_id = 111111) COMPUTE STATISTICS;
+
+DESC EXTENDED students PARTITION (student_id = 111111);
++--------------------+--------------------+-------+
+|            col_name|           data_type|comment|
++--------------------+--------------------+-------+
+|                name|              string|   null|
+|          student_id|                 int|   null|
+|                 ...|                 ...|    ...|
+|Partition Statistics|   432 bytes, 1 rows|       |
+|                 ...|                 ...|    ...|
+|        OutputFormat|org.apache.hadoop...|       |
++--------------------+--------------------+-------+
+
+ANALYZE TABLE students COMPUTE STATISTICS FOR COLUMNS name;
+
+DESC EXTENDED students name;
++--------------+----------+
+|     info_name|info_value|
++--------------+----------+
+|      col_name|      name|
+|     data_type|    string|
+|       comment|      NULL|
+|           min|      NULL|
+|           max|      NULL|
+|     num_nulls|         0|
+|distinct_count|         2|
+|   avg_col_len|         4|
+|   max_col_len|         4|
+|     histogram|      NULL|
++--------------+----------+
+```
diff --git a/docs/sql-ref-syntax-aux-analyze.md b/docs/sql-ref-syntax-aux-analyze.md
index b1bdc73657724..4c68e6b9ec974 100644
--- a/docs/sql-ref-syntax-aux-analyze.md
+++ b/docs/sql-ref-syntax-aux-analyze.md
@@ -19,4 +19,4 @@ license: |
   limitations under the License.
 ---
 
-* [ANALYZE TABLE statement](sql-ref-syntax-aux-analyze-table.html)
+ * [ANALYZE TABLE statement](sql-ref-syntax-aux-analyze-table.html)
diff --git a/docs/sql-ref-syntax-aux-cache-cache-table.md b/docs/sql-ref-syntax-aux-cache-cache-table.md
index ed6ef973466dd..fdef3d657dfa3 100644
--- a/docs/sql-ref-syntax-aux-cache-cache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-cache-table.md
@@ -20,71 +20,63 @@ license: |
 ---
 
 ### Description
-`CACHE TABLE` statement caches contents of a table or output of a query with the given storage level. This reduces scanning of the original files in future queries.
+
+`CACHE TABLE` statement caches contents of a table or output of a query with the given storage level. If a query is cached, then a temp view will be created for this query.
+This reduces scanning of the original files in future queries. 
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 CACHE [ LAZY ] TABLE table_identifier
     [ OPTIONS ( 'storageLevel' [ = ] value ) ] [ [ AS ] query ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>LAZY</em></code></dt>
-  <dd>Only cache the table when it is first used, instead of immediately.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies the table or view name to be cached. The table or view name may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>OPTIONS ( 'storageLevel' [ = ] value )</em></code></dt>
-  <dd>
-  <code>OPTIONS</code> clause with <code>storageLevel</code> key and value pair. A Warning is issued when a key other than <code>storageLevel</code> is used. The valid options for <code>storageLevel</code> are:
-    <ul>
-      <li><code>NONE</code></li>
-      <li><code>DISK_ONLY</code></li>
-      <li><code>DISK_ONLY_2</code></li>
-      <li><code>MEMORY_ONLY</code></li>
-      <li><code>MEMORY_ONLY_2</code></li>
-      <li><code>MEMORY_ONLY_SER</code></li>
-      <li><code>MEMORY_ONLY_SER_2</code></li>
-      <li><code>MEMORY_AND_DISK</code></li>
-      <li><code>MEMORY_AND_DISK_2</code></li>
-      <li><code>MEMORY_AND_DISK_SER</code></li>
-      <li><code>MEMORY_AND_DISK_SER_2</code></li>
-      <li><code>OFF_HEAP</code></li>
-    </ul>
-    An Exception is thrown when an invalid value is set for <code>storageLevel</code>. If <code>storageLevel</code> is not explicitly set using <code>OPTIONS</code> clause, the default <code>storageLevel</code> is set to <code>MEMORY_AND_DISK</code>.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A query that produces the rows to be cached. It can be in one of following formats:
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-   </dd>
-</dl>
+
+* **LAZY**
+
+    Only cache the table when it is first used, instead of immediately.
+
+* **table_identifier**
+
+    Specifies the table or view name to be cached. The table or view name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **OPTIONS ( 'storageLevel' [ = ] value )**
+
+    `OPTIONS` clause with `storageLevel` key and value pair. A Warning is issued when a key other than `storageLevel` is used. The valid options for `storageLevel` are:
+     * `NONE`
+     * `DISK_ONLY`
+     * `DISK_ONLY_2`
+     * `MEMORY_ONLY`
+     * `MEMORY_ONLY_2`
+     * `MEMORY_ONLY_SER`
+     * `MEMORY_ONLY_SER_2`
+     * `MEMORY_AND_DISK`
+     * `MEMORY_AND_DISK_2`
+     * `MEMORY_AND_DISK_SER`
+     * `MEMORY_AND_DISK_SER_2`
+     * `OFF_HEAP`
+
+    An Exception is thrown when an invalid value is set for `storageLevel`. If `storageLevel` is not explicitly set using `OPTIONS` clause, the default `storageLevel` is set to `MEMORY_AND_DISK`.
+
+* **query**
+
+    A query that produces the rows to be cached. It can be in one of following formats:
+    * a `SELECT` statement
+    * a `TABLE` statement
+    * a `FROM` statement
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CACHE TABLE testCache OPTIONS ('storageLevel' 'DISK_ONLY') SELECT * FROM testData;
-{% endhighlight %}
+```
 
 ### Related Statements
-  * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-  * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-  * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
 
+* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
+* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-cache-clear-cache.md b/docs/sql-ref-syntax-aux-cache-clear-cache.md
index 57ba77d59ea7b..a27cd83c146a3 100644
--- a/docs/sql-ref-syntax-aux-cache-clear-cache.md
+++ b/docs/sql-ref-syntax-aux-cache-clear-cache.md
@@ -20,20 +20,24 @@ license: |
 ---
 
 ### Description
+
 `CLEAR CACHE` removes the entries and associated data from the in-memory and/or on-disk cache for all cached tables and views.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 CLEAR CACHE
-{% endhighlight %}
+```
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CLEAR CACHE;
-{% endhighlight %}
+```
 
 ### Related Statements
- * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
- * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
- * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
 
+* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
+* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-refresh-table.md b/docs/sql-ref-syntax-aux-cache-refresh-table.md
similarity index 79%
rename from docs/sql-ref-syntax-aux-refresh-table.md
rename to docs/sql-ref-syntax-aux-cache-refresh-table.md
index 4b3fcdd65ba75..8d4a804f88671 100644
--- a/docs/sql-ref-syntax-aux-refresh-table.md
+++ b/docs/sql-ref-syntax-aux-cache-refresh-table.md
@@ -20,29 +20,28 @@ license: |
 ---
 
 ### Description
+
 `REFRESH TABLE` statement invalidates the cached entries, which include data
 and metadata of the given table or view. The invalidated cache is populated in
 lazy manner when the cached table or the query associated with it is executed again.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 REFRESH [TABLE] table_identifier
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which is either a qualified or unqualified name that designates a table/view. If no database identifier is provided, it refers to a temporary view or a table/view in the current database.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
+
+* **table_identifier**
+
+    Specifies a table name, which is either a qualified or unqualified name that designates a table/view. If no database identifier is provided, it refers to a temporary view or a table/view in the current database.
+
+    **Syntax:** `[ database_name. ] table_name`
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- The cached entries of the table will be refreshed  
 -- The table is resolved from the current database as the table name is unqualified.
 REFRESH TABLE tbl1;
@@ -50,9 +49,11 @@ REFRESH TABLE tbl1;
 -- The cached entries of the view will be refreshed or invalidated
 -- The view is resolved from tempDB database, as the view name is qualified.
 REFRESH TABLE tempDB.view1;   
-{% endhighlight %}
+```
 
 ### Related Statements
-- [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
-- [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-- [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
\ No newline at end of file
+
+* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-cache-refresh.md b/docs/sql-ref-syntax-aux-cache-refresh.md
index fea3100d96f01..b10e6fb47aaf7 100644
--- a/docs/sql-ref-syntax-aux-cache-refresh.md
+++ b/docs/sql-ref-syntax-aux-cache-refresh.md
@@ -20,35 +20,37 @@ license: |
 ---
 
 ### Description
+
 `REFRESH` is used to invalidate and refresh all the cached data (and the associated metadata) for
 all Datasets that contains the given data source path. Path matching is by prefix, i.e. "/" would
 invalidate everything that is cached. 
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 REFRESH resource_path
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
- <dt><code><em>resource_path</em></code></dt>
- <dd>The path of the resource that is to be refreshed.</dd>
-</dl>
+
+* **resource_path**
+
+    The path of the resource that is to be refreshed.
 
 ### Examples
-{% highlight sql %}
- -- The Path is resolved using the datasource's File Index.
- 
+
+```sql
+-- The Path is resolved using the datasource's File Index.
 CREATE TABLE test(ID INT) using parquet;
 INSERT INTO test SELECT 1000;
 CACHE TABLE test;
 INSERT INTO test SELECT 100;
 REFRESH "hdfs://path/to/table";
- 
-{% endhighlight %}
+```
 
 ### Related Statements
-- [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
-- [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-- [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-- [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
\ No newline at end of file
+
+* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
diff --git a/docs/sql-ref-syntax-aux-cache-uncache-table.md b/docs/sql-ref-syntax-aux-cache-uncache-table.md
index e0581d0d213df..96a691e4c3931 100644
--- a/docs/sql-ref-syntax-aux-cache-uncache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-uncache-table.md
@@ -20,28 +20,33 @@ license: |
 ---
 
 ### Description
+
 `UNCACHE TABLE` removes the entries and associated data from the in-memory and/or on-disk cache for a given table or view. The
-underlying entries should already have been brought to cache by previous `CACHE TABLE` operation. `UNCACHE TABLE` on a non-existent table throws Exception if `IF EXISTS` is not specified.
+underlying entries should already have been brought to cache by previous `CACHE TABLE` operation. `UNCACHE TABLE` on a non-existent table throws an exception if `IF EXISTS` is not specified.
+
 ### Syntax
-{% highlight sql %}
+
+```sql
 UNCACHE TABLE [ IF EXISTS ] table_identifier
-{% endhighlight %}
+```
+
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies the table or view name to be uncached. The table or view name may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
+
+* **table_identifier**
+
+    Specifies the table or view name to be uncached. The table or view name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
 ### Examples
-{% highlight sql %}
+
+```sql
 UNCACHE TABLE t1;
-{% endhighlight %}
+```
+
 ### Related Statements
- * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
- * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
- * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+
+* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
+* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-cache.md b/docs/sql-ref-syntax-aux-cache.md
index e4a640a6edbe5..0ccb1c61a0da5 100644
--- a/docs/sql-ref-syntax-aux-cache.md
+++ b/docs/sql-ref-syntax-aux-cache.md
@@ -19,7 +19,8 @@ license: |
   limitations under the License.
 ---
 
-* [CACHE TABLE statement](sql-ref-syntax-aux-cache-cache-table.html)
-* [UNCACHE TABLE statement](sql-ref-syntax-aux-cache-uncache-table.html)
-* [CLEAR CACHE statement](sql-ref-syntax-aux-cache-clear-cache.html)
-* [REFRESH TABLE statement](sql-ref-syntax-aux-refresh-table.html)
+ * [CACHE TABLE statement](sql-ref-syntax-aux-cache-cache-table.html)
+ * [UNCACHE TABLE statement](sql-ref-syntax-aux-cache-uncache-table.html)
+ * [CLEAR CACHE statement](sql-ref-syntax-aux-cache-clear-cache.html)
+ * [REFRESH TABLE statement](sql-ref-syntax-aux-cache-refresh-table.html)
+ * [REFRESH statement](sql-ref-syntax-aux-cache-refresh.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt-reset.md b/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
index 8ee61514ee4ef..4caf57a232f89 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
@@ -20,19 +20,22 @@ license: |
 ---
 
 ### Description
-Reset all the properties specific to the current session to their default values. After RESET command, executing SET command will output empty.
+
+Reset any runtime configurations specific to the current session which were set via the [SET](sql-ref-syntax-aux-conf-mgmt-set.html) command to their default values.
 
 ### Syntax
-{% highlight sql %}
-RESET
-{% endhighlight %}
 
+```sql
+RESET
+```
 
 ### Examples
-{% highlight sql %}
--- Reset all the properties specific to the current session to their default values.
+
+```sql
+-- Reset any runtime configurations specific to the current session which were set via the SET command to their default values.
 RESET;
-{% endhighlight %}
+```
 
 ### Related Statements
-- [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
+
+* [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt-set.md b/docs/sql-ref-syntax-aux-conf-mgmt-set.md
index f05dde3f567ee..f97b7f2a8efed 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt-set.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt-set.md
@@ -20,35 +20,36 @@ license: |
 ---
 
 ### Description
+
 The SET command sets a property, returns the value of an existing property or returns all SQLConf properties with value and meaning.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 SET
 SET [ -v ]
 SET property_key[ = property_value ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>-v</em></code></dt>
-  <dd>Outputs the key, value and meaning of existing SQLConf properties.</dd>
-</dl>
 
-<dl>
-  <dt><code><em>property_key</em></code></dt>
-  <dd>Returns the value of specified property key.</dd>
-</dl>
+* **-v**
+
+    Outputs the key, value and meaning of existing SQLConf properties.
+
+* **property_key**
 
-<dl>
-  <dt><code><em>property_key=property_value</em></code></dt>
-  <dd>Sets the value for a given property key. If an old value exists for a given property key, then it gets overridden by the new value.</dd>
-</dl>
+    Returns the value of specified property key.
+
+* **property_key=property_value**
+
+     Sets the value for a given property key. If an old value exists for a given property key, then it gets overridden by the new value.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Set a property.
-SET  spark.sql.variable.substitute=false;
+SET spark.sql.variable.substitute=false;
 
 -- List all SQLConf properties with value and meaning.
 SET -v;
@@ -57,13 +58,14 @@ SET -v;
 SET;
 
 -- List the value of specified property key.
-SET  spark.sql.variable.substitute;
-    +--------------------------------+--------+
-    |              key               | value  |
-    +--------------------------------+--------+
-    | spark.sql.variable.substitute  | false  |
-    +--------------------------------+--------+
-{% endhighlight %}
+SET spark.sql.variable.substitute;
++-----------------------------+-----+
+|                          key|value|
++-----------------------------+-----+
+|spark.sql.variable.substitute|false|
++-----------------------------+-----+
+```
 
 ### Related Statements
-- [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
+
+* [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt.md b/docs/sql-ref-syntax-aux-conf-mgmt.md
index 7c5d9cc895c10..1900fb7f1cb9a 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt.md
@@ -19,5 +19,5 @@ license: |
   limitations under the License.
 ---
 
-* [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
-* [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)
+ * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
+ * [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
diff --git a/docs/sql-ref-syntax-aux-describe-database.md b/docs/sql-ref-syntax-aux-describe-database.md
index 05a64ab2060b4..143fa78b205ca 100644
--- a/docs/sql-ref-syntax-aux-describe-database.md
+++ b/docs/sql-ref-syntax-aux-describe-database.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 ​
 `DESCRIBE DATABASE` statement returns the metadata of an existing database. The metadata information includes database
@@ -26,21 +27,21 @@ returns the basic metadata information along with the database properties. The `
 interchangeable.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 { DESC | DESCRIBE } DATABASE [ EXTENDED ] db_name
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>db_name</em></code></dt>
-  <dd>
+
+* **db_name**
+
     Specifies a name of an existing database or an existing schema in the system. If the name does not exist, an
     exception is thrown.
-  </dd>
-</dl>
 
-### Example
-{% highlight sql %}
+### Examples
+
+```sql
 -- Create employees DATABASE
 CREATE DATABASE employees COMMENT 'For software companies';
 
@@ -48,13 +49,13 @@ CREATE DATABASE employees COMMENT 'For software companies';
 -- Returns Database Name, Description and Root location of the filesystem
 -- for the employees DATABASE.
 DESCRIBE DATABASE employees;
-  +-------------------------+-----------------------------+
-  |database_description_item|database_description_value   |
-  +-------------------------+-----------------------------+
-  |Database Name            |employees                    |
-  |Description              |For software companies       |
-  |Location                 |file:/Users/Temp/employees.db|
-  +-------------------------+-----------------------------+
++-------------------------+-----------------------------+
+|database_description_item|   database_description_value|
++-------------------------+-----------------------------+
+|            Database Name|                    employees|
+|              Description|       For software companies|
+|                 Location|file:/Users/Temp/employees.db|
++-------------------------+-----------------------------+
 
 -- Create employees DATABASE
 CREATE DATABASE employees COMMENT 'For software companies';
@@ -64,31 +65,31 @@ ALTER DATABASE employees SET DBPROPERTIES ('Create-by' = 'Kevin', 'Create-date'
 
 -- Describe employees DATABASE with EXTENDED option to return additional database properties
 DESCRIBE DATABASE EXTENDED employees;
-  +-------------------------+---------------------------------------------+
-  |database_description_item|database_description_value                   |
-  +-------------------------+---------------------------------------------+
-  |Database Name            |employees                                    |
-  |Description              |For software companies                       |
-  |Location                 |file:/Users/Temp/employees.db                |
-  |Properties               |((Create-by,kevin), (Create-date,09/01/2019))|
-  +-------------------------+---------------------------------------------+
++-------------------------+---------------------------------------------+
+|database_description_item|                   database_description_value|
++-------------------------+---------------------------------------------+
+|            Database Name|                                    employees|
+|              Description|                       For software companies|
+|                 Location|                file:/Users/Temp/employees.db|
+|               Properties|((Create-by,kevin), (Create-date,09/01/2019))|
++-------------------------+---------------------------------------------+
 
 -- Create deployment SCHEMA
 CREATE SCHEMA deployment COMMENT 'Deployment environment';
 
 -- Describe deployment, the DATABASE and SCHEMA are interchangeable, their meaning are the same.
 DESC DATABASE deployment;
-  +-------------------------+------------------------------+
-  |database_description_item|database_description_value    |
-  +-------------------------+------------------------------+
-  |Database Name            |deployment                    |
-  |Description              |Deployment environment        |
-  |Location                 |file:/Users/Temp/deployment.db|
-  +-------------------------+------------------------------+
-
-{% endhighlight %}
++-------------------------+------------------------------+
+|database_description_item|database_description_value    |
++-------------------------+------------------------------+
+|            Database Name|                    deployment|
+|              Description|        Deployment environment|
+|                 Location|file:/Users/Temp/deployment.db|
++-------------------------+------------------------------+
+```
 
 ### Related Statements
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
diff --git a/docs/sql-ref-syntax-aux-describe-function.md b/docs/sql-ref-syntax-aux-describe-function.md
index f3c9c625b97b8..a871fb5bfd406 100644
--- a/docs/sql-ref-syntax-aux-describe-function.md
+++ b/docs/sql-ref-syntax-aux-describe-function.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 `DESCRIBE FUNCTION` statement returns the basic metadata information of an
@@ -26,87 +27,85 @@ class and the usage details.  If the optional `EXTENDED` option is specified, th
 metadata information is returned along with the extended usage information.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 { DESC | DESCRIBE } FUNCTION [ EXTENDED ] function_name
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>function_name</em></code></dt>
-  <dd>
+
+* **function_name**
+
     Specifies a name of an existing function in the system. The function name may be
     optionally qualified with a database name. If `function_name` is qualified with
     a database then the function is resolved from the user specified database, otherwise
-    it is resolved from the current database.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] function_name
-      </code>
-  </dd>
-</dl>
+    it is resolved from the current database.
+
+    **Syntax:** `[ database_name. ] function_name`
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Describe a builtin scalar function.
 -- Returns function name, implementing class and usage
 DESC FUNCTION abs;
-  +-------------------------------------------------------------------+
-  |function_desc                                                      |
-  +-------------------------------------------------------------------+
-  |Function: abs                                                      |
-  |Class: org.apache.spark.sql.catalyst.expressions.Abs               |
-  |Usage: abs(expr) - Returns the absolute value of the numeric value.|
-  +-------------------------------------------------------------------+
++-------------------------------------------------------------------+
+|function_desc                                                      |
++-------------------------------------------------------------------+
+|Function: abs                                                      |
+|Class: org.apache.spark.sql.catalyst.expressions.Abs               |
+|Usage: abs(expr) - Returns the absolute value of the numeric value.|
++-------------------------------------------------------------------+
 
 -- Describe a builtin scalar function.
 -- Returns function name, implementing class and usage and examples.
 DESC FUNCTION EXTENDED abs;
-  +-------------------------------------------------------------------+
-  |function_desc                                                      |
-  +-------------------------------------------------------------------+
-  |Function: abs                                                      |
-  |Class: org.apache.spark.sql.catalyst.expressions.Abs               |
-  |Usage: abs(expr) - Returns the absolute value of the numeric value.|
-  |Extended Usage:                                                    |
-  |    Examples:                                                      |
-  |      > SELECT abs(-1);                                            |
-  |       1                                                           |
-  |                                                                   |
-  +-------------------------------------------------------------------+
++-------------------------------------------------------------------+
+|function_desc                                                      |
++-------------------------------------------------------------------+
+|Function: abs                                                      |
+|Class: org.apache.spark.sql.catalyst.expressions.Abs               |
+|Usage: abs(expr) - Returns the absolute value of the numeric value.|
+|Extended Usage:                                                    |
+|    Examples:                                                      |
+|      > SELECT abs(-1);                                            |
+|       1                                                           |
+|                                                                   |
++-------------------------------------------------------------------+
 
 -- Describe a builtin aggregate function
 DESC FUNCTION max;
-  +--------------------------------------------------------------+
-  |function_desc                                                 |
-  +--------------------------------------------------------------+
-  |Function: max                                                 |
-  |Class: org.apache.spark.sql.catalyst.expressions.aggregate.Max|
-  |Usage: max(expr) - Returns the maximum value of `expr`.       |
-  +--------------------------------------------------------------+
++--------------------------------------------------------------+
+|function_desc                                                 |
++--------------------------------------------------------------+
+|Function: max                                                 |
+|Class: org.apache.spark.sql.catalyst.expressions.aggregate.Max|
+|Usage: max(expr) - Returns the maximum value of `expr`.       |
++--------------------------------------------------------------+
 
 -- Describe a builtin user defined aggregate function
 -- Returns function name, implementing class and usage and examples.
 DESC FUNCTION EXTENDED explode
-  +---------------------------------------------------------------+
-  |function_desc                                                  |
-  +---------------------------------------------------------------+
-  |Function: explode                                              |
-  |Class: org.apache.spark.sql.catalyst.expressions.Explode       | 
-  |Usage: explode(expr) - Separates the elements of array `expr`  |
-  | into multiple rows, or the elements of map `expr` into        |
-  | multiple rows and columns. Unless specified otherwise, uses   |
-  | the default column name `col` for elements of the array or    |
-  | `key` and `value` for the elements of the map.                |
-  |Extended Usage:                                                |
-  |    Examples:                                                  |
-  |      > SELECT explode(array(10, 20));                         |
-  |       10                                                      |
-  |       20                                                      |
-  +---------------------------------------------------------------+
-
-{% endhighlight %}
++---------------------------------------------------------------+
+|function_desc                                                  |
++---------------------------------------------------------------+
+|Function: explode                                              |
+|Class: org.apache.spark.sql.catalyst.expressions.Explode       |
+|Usage: explode(expr) - Separates the elements of array `expr`  |
+| into multiple rows, or the elements of map `expr` into        |
+| multiple rows and columns. Unless specified otherwise, uses   |
+| the default column name `col` for elements of the array or    |
+| `key` and `value` for the elements of the map.                |
+|Extended Usage:                                                |
+|    Examples:                                                  |
+|      > SELECT explode(array(10, 20));                         |
+|       10                                                      |
+|       20                                                      |
++---------------------------------------------------------------+
+```
 
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
diff --git a/docs/sql-ref-syntax-aux-describe-query.md b/docs/sql-ref-syntax-aux-describe-query.md
index b07ebe78193d1..b2a74cbd06078 100644
--- a/docs/sql-ref-syntax-aux-describe-query.md
+++ b/docs/sql-ref-syntax-aux-describe-query.md
@@ -20,88 +20,91 @@ license: |
 ---
 
 ### Description
+
 The `DESCRIBE QUERY` statement is used to return the metadata of output
 of a query. A shorthand `DESC` may be used instead of `DESCRIBE` to
 describe the query output.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 { DESC | DESCRIBE } [ QUERY ] input_statement
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>QUERY</em></code></dt>
-  <dd>This clause is optional and may be omitted.</dd>
-  <dt><code><em>input_statement</em></code></dt>
-  <dd>
+
+* **QUERY**
+    This clause is optional and may be omitted.
+
+* **input_statement**
+
     Specifies a result set producing statement and may be one of the following: 
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>CTE(Common table expression)</code> statement</li>
-      <li>an <code>INLINE TABLE</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-    Please refer to <a href="sql-ref-syntax-qry-select.html">select-statement</a>
+
+    * a `SELECT` statement
+    * a `CTE(Common table expression)` statement
+    * an `INLINE TABLE` statement
+    * a `TABLE` statement
+    * a `FROM` statement`
+
+    Please refer to [select-statement](sql-ref-syntax-qry-select.html)
     for a detailed syntax of the query parameter.
-  </dd>
-</dl>
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Create table `person`
 CREATE TABLE person (name STRING , age INT COMMENT 'Age column', address STRING);
 
 -- Returns column metadata information for a simple select query
-DESCRIBE QUERY select age, sum(age) FROM person GROUP BY age;
-  +--------+---------+----------+
-  |col_name|data_type|comment   |
-  +--------+---------+----------+
-  |age     |int      |Age column|
-  |sum(age)|bigint   |null      |
-  +--------+---------+----------+
+DESCRIBE QUERY SELECT age, sum(age) FROM person GROUP BY age;
++--------+---------+----------+
+|col_name|data_type|   comment|
++--------+---------+----------+
+|     age|      int|Age column|
+|sum(age)|   bigint|      null|
++--------+---------+----------+
 
 -- Returns column metadata information for common table expression (`CTE`).
 DESCRIBE QUERY WITH all_names_cte
-  AS (SELECT name from person) SELECT * FROM all_names_cte;
-  +--------+---------+-------+
-  |col_name|data_type|comment|
-  +--------+---------+-------+
-  |name    |string   |null   |
-  +--------+---------+-------+
-
--- Returns column metadata information for a inline table.
+    AS (SELECT name from person) SELECT * FROM all_names_cte;
++--------+---------+-------+
+|col_name|data_type|comment|
++--------+---------+-------+
+|    name|   string|   null|
++--------+---------+-------+
+
+-- Returns column metadata information for an inline table.
 DESC QUERY VALUES(100, 'John', 10000.20D) AS employee(id, name, salary);
-  +--------+---------+-------+
-  |col_name|data_type|comment|
-  +--------+---------+-------+
-  |id      |int      |null   |
-  |name    |string   |null   |
-  |salary  |double   |null   |
-  +--------+---------+-------+
++--------+---------+-------+
+|col_name|data_type|comment|
++--------+---------+-------+
+|      id|      int|   null|
+|    name|   string|   null|
+|  salary|   double|   null|
++--------+---------+-------+
 
 -- Returns column metadata information for `TABLE` statement.
 DESC QUERY TABLE person;
-  +--------+---------+----------+
-  |col_name|data_type|comment   |
-  +--------+---------+----------+
-  |name    |string   |null      |
-  |age     |int      |Age column|
-  |address |string   |null      |
-  +--------+---------+----------+
++--------+---------+----------+
+|col_name|data_type|   comment|
++--------+---------+----------+
+|    name|   string|      null|
+|     age|      int| Agecolumn|
+| address|   string|      null|
++--------+---------+----------+
 
 -- Returns column metadata information for a `FROM` statement.
 -- `QUERY` clause is optional and can be omitted.
 DESCRIBE FROM person SELECT age;
-  +--------+---------+----------+
-  |col_name|data_type|comment   |
-  +--------+---------+----------+
-  |age     |int      |Age column|
-  +--------+---------+----------+
-{% endhighlight %}
++--------+---------+----------+
+|col_name|data_type|   comment|
++--------+---------+----------+
+|     age|      int| Agecolumn|
++--------+---------+----------+
+```
 
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
diff --git a/docs/sql-ref-syntax-aux-describe-table.md b/docs/sql-ref-syntax-aux-describe-table.md
index 4e6aeb5b6f349..4b6e1e8c3461e 100644
--- a/docs/sql-ref-syntax-aux-describe-table.md
+++ b/docs/sql-ref-syntax-aux-describe-table.md
@@ -18,166 +18,165 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
+
 `DESCRIBE TABLE` statement returns the basic metadata information of a
 table. The metadata information includes column name, column type
 and column comment. Optionally a partition spec or column name may be specified
 to return the metadata pertaining to a partition or column respectively.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 { DESC | DESCRIBE } [ TABLE ] [ format ] table_identifier [ partition_spec ] [ col_name ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>format</em></code></dt>
-  <dd>
+
+* **format**
+
     Specifies the optional format of describe output. If `EXTENDED` is specified
     then additional metadata information (such as parent database, owner, and access time)
     is returned. 
-  </dd>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions. When specified, additional partition metadata is returned.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>  
-  <dt><code><em>col_name</em></code></dt>
-  <dd>
+    for partitions. When specified, additional partition metadata is returned.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
+
+* **col_name**
+
     An optional parameter that specifies the column name that needs to be described.
     The supplied column name may be optionally qualified. Parameters `partition_spec`
     and `col_name` are  mutually exclusive and can not be specified together. Currently
-    nested columns are not allowed to be specified.<br><br>
+    nested columns are not allowed to be specified.
     
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] [ table_name. ] column_name
-      </code>
-   </dd>
-</dl>
+    **Syntax:** `[ database_name. ] [ table_name. ] column_name`
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Creates a table `customer`. Assumes current database is `salesdb`.
 CREATE TABLE customer(
-    cust_id INT,
-    state VARCHAR(20),
-    name STRING COMMENT 'Short name'
-  )
-  USING parquet
-  PARTITION BY state;
-  ;
-
+        cust_id INT,
+        state VARCHAR(20),
+        name STRING COMMENT 'Short name'
+    )
+    USING parquet
+    PARTITIONED BY (state);
+    
+INSERT INTO customer PARTITION (state = 'AR') VALUES (100, 'Mike');
+    
 -- Returns basic metadata information for unqualified table `customer`
 DESCRIBE TABLE customer;
-  +-----------------------+---------+----------+
-  |col_name               |data_type|comment   |
-  +-----------------------+---------+----------+
-  |cust_id                |int      |null      |
-  |name                   |string   |Short name|
-  |state                  |string   |null      |
-  |# Partition Information|         |          |
-  |# col_name             |data_type|comment   |
-  |state                  |string   |null      |
-  +-----------------------+---------+----------+
++-----------------------+---------+----------+
+|               col_name|data_type|   comment|
++-----------------------+---------+----------+
+|                cust_id|      int|      null|
+|                   name|   string|Short name|
+|                  state|   string|      null|
+|# Partition Information|         |          |
+|             # col_name|data_type|   comment|
+|                  state|   string|      null|
++-----------------------+---------+----------+
 
 -- Returns basic metadata information for qualified table `customer`
 DESCRIBE TABLE salesdb.customer;
-  +-----------------------+---------+----------+
-  |col_name               |data_type|comment   |
-  +-----------------------+---------+----------+
-  |cust_id                |int      |null      |
-  |name                   |string   |Short name|
-  |state                  |string   |null      |
-  |# Partition Information|         |          |
-  |# col_name             |data_type|comment   |
-  |state                  |string   |null      |
-  +-----------------------+---------+----------+
++-----------------------+---------+----------+
+|               col_name|data_type|   comment|
++-----------------------+---------+----------+
+|                cust_id|      int|      null|
+|                   name|   string|Short name|
+|                  state|   string|      null|
+|# Partition Information|         |          |
+|             # col_name|data_type|   comment|
+|                  state|   string|      null|
++-----------------------+---------+----------+
 
 -- Returns additional metadata such as parent database, owner, access time etc.
 DESCRIBE TABLE EXTENDED customer;
-  +----------------------------+------------------------------+----------+
-  |col_name                    |data_type                     |comment   |
-  +----------------------------+------------------------------+----------+
-  |cust_id                     |int                           |null      |
-  |name                        |string                        |Short name|
-  |state                       |string                        |null      |
-  |# Partition Information     |                              |          |
-  |# col_name                  |data_type                     |comment   |
-  |state                       |string                        |null      |
-  |                            |                              |          |
-  |# Detailed Table Information|                              |          |
-  |Database                    |salesdb                       |          |
-  |Table                       |customer                      |          |
-  |Owner                       |<table owner>                 |          |
-  |Created Time                |Fri Aug 30 09:26:04 PDT 2019  |          |
-  |Last Access                 |Wed Dec 31 16:00:00 PST 1969  |          |
-  |Created By                  |<spark version>               |          |
-  |Type                        |MANAGED                       |          |
-  |Provider                    |parquet                       |          |
-  |Location                    |file:.../salesdb.db/customer  |          |
-  |Serde Library               |...serde.ParquetHiveSerDe     |          |
-  |InputFormat                 |...MapredParquetInputFormat   |          |
-  |OutputFormat                |...MapredParquetOutputFormat  |          |
-  +----------------------------+------------------------------+----------+
++----------------------------+------------------------------+----------+
+|                    col_name|                     data_type|   comment|
++----------------------------+------------------------------+----------+
+|                     cust_id|                           int|      null|
+|                        name|                        string|Short name|
+|                       state|                        string|      null|
+|     # Partition Information|                              |          |
+|                  # col_name|                     data_type|   comment|
+|                       state|                        string|      null|
+|                            |                              |          |
+|# Detailed Table Information|                              |          |
+|                    Database|                       default|          |
+|                       Table|                      customer|          |
+|                       Owner|                 <TABLE OWNER>|          |
+|                Created Time|  Tue Apr 07 22:56:34 JST 2020|          |
+|                 Last Access|                       UNKNOWN|          |
+|                  Created By|               <SPARK VERSION>|          |
+|                        Type|                       MANAGED|          |
+|                    Provider|                       parquet|          |
+|                    Location|file:/tmp/salesdb.db/custom...|          |
+|               Serde Library|org.apache.hadoop.hive.ql.i...|          |
+|                 InputFormat|org.apache.hadoop.hive.ql.i...|          |
+|                OutputFormat|org.apache.hadoop.hive.ql.i...|          |
+|          Partition Provider|                       Catalog|          |
++----------------------------+------------------------------+----------+
 
 -- Returns partition metadata such as partitioning column name, column type and comment.
-DESCRIBE TABLE customer PARTITION (state = 'AR');
-
-  +--------------------------------+-----------------------------------------+----------+
-  |col_name                        |data_type                                |comment   |
-  +--------------------------------+-----------------------------------------+----------+
-  |cust_id                         |int                                      |null      |
-  |name                            |string                                   |Short name|
-  |state                           |string                                   |null      |
-  |# Partition Information         |                                         |          |
-  |# col_name                      |data_type                                |comment   |
-  |state                           |string                                   |null      |
-  |                                |                                         |          |
-  |# Detailed Partition Information|                                         |          |
-  |Database                        |salesdb                                  |          |
-  |Table                           |customer                                 |          |
-  |Partition Values                |[state=AR]                               |          |
-  |Location                        |file:.../salesdb.db/customer/state=AR    |          |
-  |Serde Library                   |...serde.ParquetHiveSerDe                |          |
-  |InputFormat                     |...parquet.MapredParquetInputFormat      |          |
-  |OutputFormat                    |...parquet.MapredParquetOutputFormat     |          |
-  |Storage Properties              |[path=file:.../salesdb.db/customer,      |          |
-  |                                | serialization.format=1]                 |          |
-  |Partition Parameters            |{rawDataSize=-1, numFiles=1l,            |          |
-  |                                | transient_lastDdlTime=1567185245,       |          |
-  |                                | totalSize=688,                          |          |
-  |                                | COLUMN_STATS_ACCURATE=false, numRows=-1}|          |
-  |Created Time                    |Fri Aug 30 10:14:05 PDT 2019             |          |
-  |Last Access                     |Wed Dec 31 16:00:00 PST 1969             |          |
-  |Partition Statistics            |688 bytes                                |          |
-  +--------------------------------+-----------------------------------------+----------+
+DESCRIBE TABLE EXTENDED customer PARTITION (state = 'AR');
++------------------------------+------------------------------+----------+
+|                      col_name|                     data_type|   comment|
++------------------------------+------------------------------+----------+
+|                       cust_id|                           int|      null|
+|                          name|                        string|Short name|
+|                         state|                        string|      null|
+|       # Partition Information|                              |          |
+|                    # col_name|                     data_type|   comment|
+|                         state|                        string|      null|
+|                              |                              |          |
+|# Detailed Partition Inform...|                              |          |
+|                      Database|                       default|          |
+|                         Table|                      customer|          |
+|              Partition Values|                    [state=AR]|          |
+|                      Location|file:/tmp/salesdb.db/custom...|          |
+|                 Serde Library|org.apache.hadoop.hive.ql.i...|          |
+|                   InputFormat|org.apache.hadoop.hive.ql.i...|          |
+|                  OutputFormat|org.apache.hadoop.hive.ql.i...|          |
+|            Storage Properties|[serialization.format=1, pa...|          |
+|          Partition Parameters|{transient_lastDdlTime=1586...|          |
+|                  Created Time|  Tue Apr 07 23:05:43 JST 2020|          |
+|                   Last Access|                       UNKNOWN|          |
+|          Partition Statistics|                     659 bytes|          |
+|                              |                              |          |
+|         # Storage Information|                              |          |
+|                      Location|file:/tmp/salesdb.db/custom...|          |
+|                 Serde Library|org.apache.hadoop.hive.ql.i...|          |
+|                   InputFormat|org.apache.hadoop.hive.ql.i...|          |
+|                  OutputFormat|org.apache.hadoop.hive.ql.i...|          |
++------------------------------+------------------------------+----------+
 
 -- Returns the metadata for `name` column.
 -- Optional `TABLE` clause is omitted and column is fully qualified.
 DESCRIBE customer salesdb.customer.name;
-  +---------+----------+
-  |info_name|info_value|
-  +---------+----------+
-  |col_name |name      |
-  |data_type|string    |
-  |comment  |Short name|
-  +---------+----------+
-{% endhighlight %}
++---------+----------+
+|info_name|info_value|
++---------+----------+
+| col_name|      name|
+|data_type|    string|
+|  comment|Short name|
++---------+----------+
+```
 
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
diff --git a/docs/sql-ref-syntax-aux-describe.md b/docs/sql-ref-syntax-aux-describe.md
index 9f17746316480..723943f97aa07 100644
--- a/docs/sql-ref-syntax-aux-describe.md
+++ b/docs/sql-ref-syntax-aux-describe.md
@@ -19,7 +19,7 @@ license: |
   limitations under the License.
 ---
 
-* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
-* [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+ * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+ * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+ * [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md b/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
index 7e485cbafe709..9203293d0c981 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
@@ -20,30 +20,34 @@ license: |
 ---
 
 ### Description
+
 `ADD FILE` can be used to add a single file as well as a directory to the list of resources. The added resource can be listed using [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html).
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 ADD FILE resource_name
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
- <dt><code><em>resource_name</em></code></dt>
- <dd>The name of the file or directory to be added.</dd>
-</dl>
+
+* **resource_name**
+
+    The name of the file or directory to be added.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 ADD FILE /tmp/test;
 ADD FILE "/path/to/file/abc.txt";
 ADD FILE '/another/test.txt';
 ADD FILE "/path with space/abc.txt";
 ADD FILE "/path/to/some/directory";
-{% endhighlight %}
+```
 
 ### Related Statements
- * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
- * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
- * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+
+* [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+* [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
 
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md b/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
index db0a85013321d..4694bff99daf5 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
@@ -20,28 +20,32 @@ license: |
 ---
 
 ### Description
+
 `ADD JAR` adds a JAR file to the list of resources. The added JAR file can be listed using [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html).
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 ADD JAR file_name
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
- <dt><code><em>file_name</em></code></dt>
- <dd>The name of the JAR file to be added. It could be either on a local file system or a distributed file system.</dd>
-</dl>
+
+* **file_name**
+
+    The name of the JAR file to be added. It could be either on a local file system or a distributed file system.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 ADD JAR /tmp/test.jar;
 ADD JAR "/path/to/some.jar";
 ADD JAR '/some/other.jar';
 ADD JAR "/path with space/abc.jar";
-{% endhighlight %}
+```
 
 ### Related Statements
- * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
- * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
- * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+
+* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+* [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+* [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md b/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
index c42bf7ae8dd41..9b9a7df7f612f 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
@@ -20,15 +20,18 @@ license: |
 ---
 
 ### Description
+
 `LIST FILE` lists the resources added by [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html).
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 LIST FILE
-{% endhighlight %}
+```
 
 ### Examples
-{% highlight sql %}
+
+```sql
 ADD FILE /tmp/test;
 ADD FILE /tmp/test_2;
 LIST FILE;
@@ -39,10 +42,11 @@ file:/private/tmp/test_2
 LIST FILE /tmp/test /some/random/file /another/random/file
 --output
 file:/private/tmp/test
-{% endhighlight %}
+```
 
 ### Related Statements
- * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
- * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
- * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+
+* [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+* [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
 
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md b/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
index 9d1739753099e..04aa52c2ad8af 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
@@ -20,15 +20,18 @@ license: |
 ---
 
 ### Description
+
 `LIST JAR` lists the JARs added by [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html).
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 LIST JAR
-{% endhighlight %}
+```
 
 ### Examples
-{% highlight sql %}
+
+```sql
 ADD JAR /tmp/test.jar;
 ADD JAR /tmp/test_2.jar;
 LIST JAR;
@@ -39,10 +42,11 @@ spark://192.168.1.112:62859/jars/test_2.jar
 LIST JAR /tmp/test.jar /some/random.jar /another/random.jar;
 -- output
 spark://192.168.1.112:62859/jars/test.jar
-{% endhighlight %}
+```
 
 ### Related Statements
- * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
- * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
- * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+
+* [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+* [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+* [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
 
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt.md b/docs/sql-ref-syntax-aux-resource-mgmt.md
index 5eddb979388aa..50c12ef7c2beb 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt.md
@@ -19,7 +19,7 @@ license: |
   limitations under the License.
 ---
 
-* [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
-* [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
-* [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
-* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
\ No newline at end of file
+ * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+ * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+ * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+ * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
diff --git a/docs/sql-ref-syntax-aux-show-columns.md b/docs/sql-ref-syntax-aux-show-columns.md
index 0c8aba83a8403..b76db252f1a0f 100644
--- a/docs/sql-ref-syntax-aux-show-columns.md
+++ b/docs/sql-ref-syntax-aux-show-columns.md
@@ -18,80 +18,80 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
-Return the list of columns in a table. If the table does not exist, an exception is thrown.
+
+Returns the list of columns in a table. If the table does not exist, an exception is thrown.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 SHOW COLUMNS table_identifier [ database ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
+
+* **table_identifier**
+
     Specifies the table name of an existing table. The table may be optionally qualified
-    with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        { IN | FROM } [ database_name . ] table_name
-      </code><br><br>
-    <b>Note:</b>
-    Keywords <code>IN</code> and <code>FROM</code> are interchangeable.
-  </dd>
-  <dt><code><em>database</em></code></dt>
-  <dd>
+    with a database name.
+
+    **Syntax:** `{ IN | FROM } [ database_name . ] table_name`
+
+    **Note:** Keywords `IN` and `FROM` are interchangeable.
+
+* **database**
+
     Specifies an optional database name. The table is resolved from this database when it
-    is specified. Please note that when this parameter is specified then table
-    name should not be qualified with a different database name. <br><br>
-    <b>Syntax:</b>
-      <code>
-        { IN | FROM } database_name
-      </code><br><br>
-    <b>Note:</b>
-    Keywords <code>IN</code> and <code>FROM</code> are interchangeable.
-  </dd>
-</dl>
+    is specified. When this parameter is specified then table
+    name should not be qualified with a different database name. 
+
+    **Syntax:** `{ IN | FROM } database_name`
+
+    **Note:** Keywords `IN` and `FROM` are interchangeable.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Create `customer` table in `salesdb` database;
 USE salesdb;
-CREATE TABLE customer(cust_cd INT,
-  name VARCHAR(100),
-  cust_addr STRING);
+CREATE TABLE customer(
+    cust_cd INT,
+    name VARCHAR(100),
+    cust_addr STRING);
 
 -- List the columns of `customer` table in current database.
 SHOW COLUMNS IN customer;
-  +---------+
-  |col_name |
-  +---------+
-  |cust_cd  |
-  |name     |
-  |cust_addr|
-  +---------+
++---------+
+| col_name|
++---------+
+|  cust_cd|
+|     name|
+|cust_addr|
++---------+
 
 -- List the columns of `customer` table in `salesdb` database.
 SHOW COLUMNS IN salesdb.customer;
-  +---------+
-  |col_name |
-  +---------+
-  |cust_cd  |
-  |name     |
-  |cust_addr|
-  +---------+
++---------+
+| col_name|
++---------+
+|  cust_cd|
+|     name|
+|cust_addr|
++---------+
 
 -- List the columns of `customer` table in `salesdb` database
 SHOW COLUMNS IN customer IN salesdb;
-  +---------+
-  |col_name |
-  +---------+
-  |cust_cd  |
-  |name     |
-  |cust_addr|
-  +---------+
-{% endhighlight %}
++---------+
+| col_name|
++---------+
+|  cust_cd|
+|     name|
+|cust_addr|
++---------+
+```
 
 ### Related Statements
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
+
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-create-table.md b/docs/sql-ref-syntax-aux-show-create-table.md
index 24aba602ab3cf..ae8c10e2d0178 100644
--- a/docs/sql-ref-syntax-aux-show-create-table.md
+++ b/docs/sql-ref-syntax-aux-show-create-table.md
@@ -20,51 +20,44 @@ license: |
 ---
 
 ### Description
+
 `SHOW CREATE TABLE` returns the [CREATE TABLE statement](sql-ref-syntax-ddl-create-table.html) or [CREATE VIEW statement](sql-ref-syntax-ddl-create-view.html) that was used to create a given table or view. `SHOW CREATE TABLE` on a non-existent table or a temporary view throws an exception.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 SHOW CREATE TABLE table_identifier
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table or view name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
+
+* **table_identifier**
+
+    Specifies a table or view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CREATE TABLE test (c INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
     STORED AS TEXTFILE
     TBLPROPERTIES ('prop1' = 'value1', 'prop2' = 'value2');
 
-show create table test;
-
--- the result of SHOW CREATE TABLE test
-CREATE TABLE `test`(`c` INT)
-ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-WITH SERDEPROPERTIES (
-  'field.delim' = ',',
-  'serialization.format' = ','
-)
-STORED AS
-  INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
-  OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
-TBLPROPERTIES (
-  'transient_lastDdlTime' = '1569350233',
-  'prop1' = 'value1',
-  'prop2' = 'value2'
-)
-
-{% endhighlight %}
+SHOW CREATE TABLE test;
++----------------------------------------------------+
+|                                      createtab_stmt|
++----------------------------------------------------+
+|CREATE TABLE `default`.`test` (`c` INT)
+ USING text
+ TBLPROPERTIES (
+   'transient_lastDdlTime' = '1586269021',
+   'prop1' = 'value1',
+   'prop2' = 'value2')
++----------------------------------------------------+
+```
 
 ### Related Statements
- * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
- * [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
diff --git a/docs/sql-ref-syntax-aux-show-databases.md b/docs/sql-ref-syntax-aux-show-databases.md
index 9d4be21daeabc..44c0fbbef3929 100644
--- a/docs/sql-ref-syntax-aux-show-databases.md
+++ b/docs/sql-ref-syntax-aux-show-databases.md
@@ -20,27 +20,32 @@ license: |
 ---
 
 ### Description
-Lists the databases that match an optionally supplied string pattern. If no
+
+Lists the databases that match an optionally supplied regular expression pattern. If no
 pattern is supplied then the command lists all the databases in the system.
 Please note that the usage of `SCHEMAS` and `DATABASES` are interchangeable
 and mean the same thing.
 
 ### Syntax
-{% highlight sql %}
-SHOW { DATABASES | SCHEMAS } [ LIKE string_pattern ]
-{% endhighlight %}
+
+```sql
+SHOW { DATABASES | SCHEMAS } [ LIKE regex_pattern ]
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>LIKE string_pattern</em></code></dt>
-  <dd>
-    Specifies a string pattern that is used to match the databases in the system. In 
-    the specified string pattern <code>'*'</code> matches any number of characters.
-  </dd>
-</dl>
+
+* **regex_pattern**
+
+    Specifies a regular expression pattern that is used to filter the results of the
+    statement.
+    * Except for `*` and `|` character, the pattern works like a regular expression.
+    * `*` alone matches 0 or more characters and `|` is used to separate multiple different regular expressions,
+       any of which can match.
+    * The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Create database. Assumes a database named `default` already exists in
 -- the system. 
 CREATE DATABASE payroll_db;
@@ -48,32 +53,36 @@ CREATE DATABASE payments_db;
 
 -- Lists all the databases. 
 SHOW DATABASES;
-  +------------+
-  |databaseName|
-  +------------+
-  |     default|
-  | payments_db|
-  |  payroll_db|
-  +------------+
++------------+
+|databaseName|
++------------+
+|     default|
+| payments_db|
+|  payroll_db|
++------------+
+  
 -- Lists databases with name starting with string pattern `pay`
 SHOW DATABASES LIKE 'pay*';
-  +------------+
-  |databaseName|
-  +------------+
-  | payments_db|
-  |  payroll_db|
-  +------------+
++------------+
+|databaseName|
++------------+
+| payments_db|
+|  payroll_db|
++------------+
+  
 -- Lists all databases. Keywords SCHEMAS and DATABASES are interchangeable. 
 SHOW SCHEMAS;
-  +------------+
-  |databaseName|
-  +------------+
-  |     default|
-  | payments_db|
-  |  payroll_db|
-  +------------+
-{% endhighlight %}
++------------+
+|databaseName|
++------------+
+|     default|
+| payments_db|
+|  payroll_db|
++------------+
+```
+
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
+
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
diff --git a/docs/sql-ref-syntax-aux-show-functions.md b/docs/sql-ref-syntax-aux-show-functions.md
index 701d427039aa5..b4dd72801202e 100644
--- a/docs/sql-ref-syntax-aux-show-functions.md
+++ b/docs/sql-ref-syntax-aux-show-functions.md
@@ -9,7 +9,7 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
  
   Unless required by applicable law or agreed to in writing, software
@@ -20,120 +20,120 @@ license: |
 ---
 
 ### Description
+
 Returns the list of functions after applying an optional regex pattern.
 Given number of functions supported by Spark is quite large, this statement
-in conjuction with [describe function](sql-ref-syntax-aux-describe-function.html)
+in conjunction with [describe function](sql-ref-syntax-aux-describe-function.html)
 may be used to quickly find the function and understand its usage. The `LIKE` 
 clause is optional and supported only for compatibility with other systems.
 
 ### Syntax
-{% highlight sql %}
-SHOW [ function_kind ] FUNCTIONS ( [ LIKE ] function_name | regex_pattern )
-{% endhighlight %}
+
+```sql
+SHOW [ function_kind ] FUNCTIONS [ [ LIKE ] { function_name | regex_pattern } ]
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>function_kind</em></code></dt>
-  <dd>
+
+* **function_kind**
+
     Specifies the name space of the function to be searched upon. The valid name spaces are :
-    <ul>
-      <li><b>USER</b> - Looks up the function(s) among the user defined functions.</li>
-      <li><b>SYSTEM</b> - Looks up the function(s) among the system defined functions.</li>
-      <li><b>ALL</b> -  Looks up the function(s) among both user and system defined functions.</li>
-    </ul>
-  </dd>
-  <dt><code><em>function_name</em></code></dt>
-  <dd>
+
+    * **USER** - Looks up the function(s) among the user defined functions.
+    * **SYSTEM** - Looks up the function(s) among the system defined functions.
+    * **ALL** -  Looks up the function(s) among both user and system defined functions.
+
+* **function_name**
+
     Specifies a name of an existing function in the system. The function name may be
     optionally qualified with a database name. If `function_name` is qualified with
     a database then the function is resolved from the user specified database, otherwise
-    it is resolved from the current database.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [database_name.]function_name
-      </code>
-  </dd>
-  <dt><code><em>regex_pattern</em></code></dt>
-  <dd>
-    Specifies a regular expression pattern that is used to limit the results of the
+    it is resolved from the current database.
+
+    **Syntax:** `[ database_name. ] function_name`
+
+* **regex_pattern**
+
+    Specifies a regular expression pattern that is used to filter the results of the
     statement.
-    <ul>
-      <li>Only `*` and `|` are allowed as wildcard pattern.</li>
-      <li>Excluding `*` and `|` the remaining pattern follows the regex semantics.</li>
-      <li>The leading and trailing blanks are trimmed in the input pattern before processing.</li> 
-    </ul>
-  </dd>
-</dl>
+
+    * Except for `*` and `|` character, the pattern works like a regular expression.
+    * `*` alone matches 0 or more characters and `|` is used to separate multiple different regular expressions,
+       any of which can match.
+    * The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- List a system function `trim` by searching both user defined and system
 -- defined functions.
 SHOW FUNCTIONS trim;
-  +--------+
-  |function|
-  +--------+
-  |trim    |
-  +--------+
++--------+
+|function|
++--------+
+|    trim|
++--------+
 
 -- List a system function `concat` by searching system defined functions.
 SHOW SYSTEM FUNCTIONS concat;
-  +--------+
-  |function|
-  +--------+
-  |concat  |
-  +--------+
++--------+
+|function|
++--------+
+|  concat|
++--------+
 
 -- List a qualified function `max` from database `salesdb`. 
 SHOW SYSTEM FUNCTIONS salesdb.max;
-  +--------+
-  |function|
-  +--------+
-  |max     |
-  +--------+
++--------+
+|function|
++--------+
+|     max|
++--------+
 
 -- List all functions starting with `t`
 SHOW FUNCTIONS LIKE 't*';
-  +-----------------+
-  |function         |
-  +-----------------+
-  |tan              |
-  |tanh             |
-  |timestamp        |
-  |tinyint          |
-  |to_csv           |
-  |to_date          |
-  |to_json          |
-  |to_timestamp     |
-  |to_unix_timestamp|
-  |to_utc_timestamp |
-  |transform        |
-  |transform_keys   |
-  |transform_values |
-  |translate        |
-  |trim             |
-  |trunc            |
-  +-----------------+
++-----------------+
+|         function|
++-----------------+
+|              tan|
+|             tanh|
+|        timestamp|
+|          tinyint|
+|           to_csv|
+|          to_date|
+|          to_json|
+|     to_timestamp|
+|to_unix_timestamp|
+| to_utc_timestamp|
+|        transform|
+|   transform_keys|
+| transform_values|
+|        translate|
+|             trim|
+|            trunc|
+|           typeof|
++-----------------+
 
 -- List all functions starting with `yea` or `windo`
 SHOW FUNCTIONS LIKE 'yea*|windo*';
-  +--------+
-  |function|
-  +--------+
-  |window  |
-  |year    |
-  +--------+
++--------+
+|function|
++--------+
+|  window|
+|    year|
++--------+
 
 -- Use normal regex pattern to list function names that has 4 characters
 -- with `t` as the starting character.
 SHOW FUNCTIONS LIKE 't[a-z][a-z][a-z]';
-  +--------+
-  |function|
-  +--------+
-  |tanh    |
-  |trim    |
-  +--------+
-{% endhighlight %}
-
-### Related statements
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
++--------+
+|function|
++--------+
+|    tanh|
+|    trim|
++--------+
+```
+
+### Related Statements
+
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
diff --git a/docs/sql-ref-syntax-aux-show-partitions.md b/docs/sql-ref-syntax-aux-show-partitions.md
index 6c8401e8e4718..d93825550413f 100644
--- a/docs/sql-ref-syntax-aux-show-partitions.md
+++ b/docs/sql-ref-syntax-aux-show-partitions.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 The `SHOW PARTITIONS` statement is used to list partitions of a table. An optional
@@ -25,35 +26,29 @@ partition spec may be specified to return the partitions matching the supplied
 partition spec.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 SHOW PARTITIONS table_identifier [ partition_spec ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions. When specified, the partitions that match the partition spec are returned.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )
-      </code>
-  </dd>
-</dl>
+    for partitions. When specified, the partitions that match the partition specification are returned.
+
+    **Syntax:** `PARTITION ( partition_col_name = partition_col_val [ , ... ] )`
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- create a partitioned table and insert a few rows.
 USE salesdb;
 CREATE TABLE customer(id INT, name STRING) PARTITIONED BY (state STRING, city STRING);
@@ -63,52 +58,53 @@ INSERT INTO customer PARTITION (state = 'AZ', city = 'Peoria') VALUES (300, 'Dan
 
 -- Lists all partitions for table `customer`
 SHOW PARTITIONS customer;
-  +----------------------+
-  |partition             |
-  +----------------------+
-  |state=AZ/city=Peoria  |
-  |state=CA/city=Fremont |
-  |state=CA/city=San Jose|
-  +----------------------+
++----------------------+
+|             partition|
++----------------------+
+|  state=AZ/city=Peoria|
+| state=CA/city=Fremont|
+|state=CA/city=San Jose|
++----------------------+
 
 -- Lists all partitions for the qualified table `customer`
 SHOW PARTITIONS salesdb.customer;
-  +----------------------+
-  |partition             |
-  +----------------------+
-  |state=AZ/city=Peoria  |
-  |state=CA/city=Fremont |
-  |state=CA/city=San Jose|
-  +----------------------+
++----------------------+
+|             partition|
++----------------------+
+|  state=AZ/city=Peoria|
+| state=CA/city=Fremont|
+|state=CA/city=San Jose|
++----------------------+
 
 -- Specify a full partition spec to list specific partition
 SHOW PARTITIONS customer PARTITION (state = 'CA', city = 'Fremont');
-  +---------------------+
-  |partition            |
-  +---------------------+
-  |state=CA/city=Fremont|
-  +---------------------+
++---------------------+
+|            partition|
++---------------------+
+|state=CA/city=Fremont|
++---------------------+
 
 -- Specify a partial partition spec to list the specific partitions
 SHOW PARTITIONS customer PARTITION (state = 'CA');
-  +----------------------+
-  |partition             |
-  +----------------------+
-  |state=CA/city=Fremont |
-  |state=CA/city=San Jose|
-  +----------------------+
++----------------------+
+|             partition|
++----------------------+
+| state=CA/city=Fremont|
+|state=CA/city=San Jose|
++----------------------+
 
 -- Specify a partial spec to list specific partition
 SHOW PARTITIONS customer PARTITION (city =  'San Jose');
-  +----------------------+
-  |partition             |
-  +----------------------+
-  |state=CA/city=San Jose|
-  +----------------------+
-{% endhighlight %}
-
-### Related statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [INSERT STATEMENT](sql-ref-syntax-dml-insert.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
++----------------------+
+|             partition|
++----------------------+
+|state=CA/city=San Jose|
++----------------------+
+```
+
+### Related Statements
+
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [INSERT STATEMENT](sql-ref-syntax-dml-insert.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-table.md b/docs/sql-ref-syntax-aux-show-table.md
index 49696585ba581..3314402ea3e2b 100644
--- a/docs/sql-ref-syntax-aux-show-table.md
+++ b/docs/sql-ref-syntax-aux-show-table.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 `SHOW TABLE EXTENDED` will show information for all tables matching the given regular expression.
@@ -30,52 +31,47 @@ information such as `Partition Parameters` and `Partition Statistics`. Note that
 cannot be used with a partition specification.
 
 ### Syntax
-{% highlight sql %}
-SHOW TABLE EXTENDED [ IN | FROM database_name ] LIKE 'identifier_with_wildcards'
+
+```sql
+SHOW TABLE EXTENDED [ { IN | FROM } database_name ] LIKE regex_pattern
     [ partition_spec ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
- <dt><code><em>IN|FROM database_name</em></code></dt>
-  <dd>
+
+* **{ IN`|`FROM } database_name**
+
     Specifies database name. If not provided, will use the current database.
-  </dd>
-  <dt><code><em>LIKE string_pattern</em></code></dt>
-  <dd>
+
+* **regex_pattern**
+
     Specifies the regular expression pattern that is used to filter out unwanted tables.
-    <ul> 
-       <li> Except for `*` and `|` character, the pattern works like a regex.</li>
-       <li> `*` alone matches 0 or more characters and `|` is used to separate multiple different regexes,
-             any of which can match. </li>
-       <li> The leading and trailing blanks are trimmed in the input pattern before processing.</li>
-    </ul> 
-  </dd>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+
+    * Except for `*` and `|` character, the pattern works like a regular expression.
+    * `*` alone matches 0 or more characters and `|` is used to separate multiple different regular expressions,
+      any of which can match.
+    * The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions. Note that a table regex cannot be used with a partition specification.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )
-      </code>
-  </dd>
-</dl>
+    for partitions. Note that a table regex cannot be used with a partition specification.
+
+    **Syntax:** `PARTITION ( partition_col_name = partition_col_val [ , ... ] )`
+
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Assumes `employee` table created with partitioned by column `grade`
--- +-------+--------+--+
--- | name  | grade  |
--- +-------+--------+--+
--- | sam   | 1      |
--- | suj   | 2      |
--- +-------+--------+--+
+CREATE TABLE employee(name STRING, grade INT) PARTITIONED BY (grade);
+INSERT INTO employee PARTITION (grade = 1) VALUES ('sam');
+INSERT INTO employee PARTITION (grade = 2) VALUES ('suj');
 
  -- Show the details of the table
-SHOW TABLE EXTENDED  LIKE `employee`;
-+--------+---------+-----------+---------------------------------------------------------------
-|database|tableName|isTemporary|                         information
-+--------+---------+-----------+---------------------------------------------------------------
+SHOW TABLE EXTENDED LIKE 'employee';
++--------+---------+-----------+--------------------------------------------------------------+
+|database|tableName|isTemporary|                         information                          |
++--------+---------+-----------+--------------------------------------------------------------+
 |default |employee |false      |Database: default
                                 Table: employee
                                 Owner: root
@@ -86,7 +82,7 @@ SHOW TABLE EXTENDED  LIKE `employee`;
                                 Provider: hive
                                 Table Properties: [transient_lastDdlTime=1567158021]
                                 Location: file:/opt/spark1/spark/spark-warehouse/employee
-                                Serde Library: org.apache.hadoop.hive.serde2.lazy   
+                                Serde Library: org.apache.hadoop.hive.serde2.lazy
                                 .LazySimpleSerDe
                                 InputFormat: org.apache.hadoop.mapred.TextInputFormat
                                 OutputFormat: org.apache.hadoop.hive.ql.io
@@ -97,36 +93,36 @@ SHOW TABLE EXTENDED  LIKE `employee`;
                                 Schema: root
                                  |-- name: string (nullable = true)
                                  |-- grade: integer (nullable = true)
-                                                                                                          
-+--------+---------+-----------+---------------------------------------------------------------
+                                                                                                            
++--------+---------+-----------+--------------------------------------------------------------+
 
 -- showing the multiple table details with pattern matching
-SHOW TABLE EXTENDED  LIKE `employe*`;
-+--------+---------+-----------+---------------------------------------------------------------
-|database|tableName|isTemporary|                         information
-+--------+---------+-----------+---------------------------------------------------------------
+SHOW TABLE EXTENDED  LIKE 'employe*';
++--------+---------+-----------+--------------------------------------------------------------+
+|database|tableName|isTemporary|                         information                          |
++--------+---------+-----------+--------------------------------------------------------------+
 |default |employee |false      |Database: default
-                                 Table: employee
-                                 Owner: root
-                                 Created Time: Fri Aug 30 15:10:21 IST 2019
-                                 Last Access: Thu Jan 01 05:30:00 IST 1970
-                                 Created By: Spark 3.0.0-SNAPSHOT
-                                 Type: MANAGED
-                                 Provider: hive
-                                 Table Properties: [transient_lastDdlTime=1567158021]
-                                 Location: file:/opt/spark1/spark/spark-warehouse/employee
-                                 Serde Library: org.apache.hadoop.hive.serde2.lazy
-                                 .LazySimpleSerDe
-                                 InputFormat: org.apache.hadoop.mapred.TextInputFormat
-                                 OutputFormat: org.apache.hadoop.hive.ql.io
-                                 .HiveIgnoreKeyTextOutputFormat
-                                 Storage Properties: [serialization.format=1]
-                                 Partition Provider: Catalog
-                                 Partition Columns: [`grade`]
-                                 Schema: root
-                                  |-- name: string (nullable = true)
-                                  |-- grade: integer (nullable = true)
-
+                                Table: employee
+                                Owner: root
+                                Created Time: Fri Aug 30 15:10:21 IST 2019
+                                Last Access: Thu Jan 01 05:30:00 IST 1970
+                                Created By: Spark 3.0.0-SNAPSHOT
+                                Type: MANAGED
+                                Provider: hive
+                                Table Properties: [transient_lastDdlTime=1567158021]
+                                Location: file:/opt/spark1/spark/spark-warehouse/employee
+                                Serde Library: org.apache.hadoop.hive.serde2.lazy
+                                .LazySimpleSerDe
+                                InputFormat: org.apache.hadoop.mapred.TextInputFormat
+                                OutputFormat: org.apache.hadoop.hive.ql.io
+                                .HiveIgnoreKeyTextOutputFormat
+                                Storage Properties: [serialization.format=1]
+                                Partition Provider: Catalog
+                                Partition Columns: [`grade`]
+                                Schema: root
+                                 |-- name: string (nullable = true)
+                                 |-- grade: integer (nullable = true)
+  
 |default |employee1|false      |Database: default
                                 Table: employee1
                                 Owner: root
@@ -145,39 +141,40 @@ SHOW TABLE EXTENDED  LIKE `employe*`;
                                 Storage Properties: [serialization.format=1]
                                 Partition Provider: Catalog
                                 Schema: root
-                                |-- name: string (nullable = true)
-                                                                                                             
-+--------+---------+----------+----------------------------------------------------------------
+                                 |-- name: string (nullable = true)
+                                                                                                               
++--------+---------+----------+---------------------------------------------------------------+
   
 -- show partition file system details
-SHOW TABLE EXTENDED  IN `default` LIKE `employee` PARTITION (`grade=1`);
-+--------+---------+-----------+---------------------------------------------------------------
-|database|tableName|isTemporary|                         information                           
-+--------+---------+-----------+---------------------------------------------------------------
-|default |employee |false      | Partition Values: [grade=1]
-                               Location: file:/opt/spark1/spark/spark-warehouse/employee
-                               /grade=1
-                               Serde Library: org.apache.hadoop.hive.serde2.lazy
-                               .LazySimpleSerDe
-                               InputFormat: org.apache.hadoop.mapred.TextInputFormat
-                               OutputFormat: org.apache.hadoop.hive.ql.io
-                               .HiveIgnoreKeyTextOutputFormat
-                               Storage Properties: [serialization.format=1]
-                               Partition Parameters: {rawDataSize=-1, numFiles=1, 
-                               transient_lastDdlTime=1567158221, totalSize=4, 
-                               COLUMN_STATS_ACCURATE=false, numRows=-1}
-                               Created Time: Fri Aug 30 15:13:41 IST 2019
-                               Last Access: Thu Jan 01 05:30:00 IST 1970
-                               Partition Statistics: 4 bytes
-                                                                                                                                                                             |
-+--------+---------+-----------+---------------------------------------------------------------
+SHOW TABLE EXTENDED  IN default LIKE 'employee' PARTITION (grade=1);
++--------+---------+-----------+--------------------------------------------------------------+
+|database|tableName|isTemporary|                         information                          |
++--------+---------+-----------+--------------------------------------------------------------+
+|default |employee |false      |Partition Values: [grade=1]
+                                Location: file:/opt/spark1/spark/spark-warehouse/employee
+                                /grade=1
+                                Serde Library: org.apache.hadoop.hive.serde2.lazy
+                                .LazySimpleSerDe
+                                InputFormat: org.apache.hadoop.mapred.TextInputFormat
+                                OutputFormat: org.apache.hadoop.hive.ql.io
+                                .HiveIgnoreKeyTextOutputFormat
+                                Storage Properties: [serialization.format=1]
+                                Partition Parameters: {rawDataSize=-1, numFiles=1,
+                                transient_lastDdlTime=1567158221, totalSize=4,
+                                COLUMN_STATS_ACCURATE=false, numRows=-1}
+                                Created Time: Fri Aug 30 15:13:41 IST 2019
+                                Last Access: Thu Jan 01 05:30:00 IST 1970
+                                Partition Statistics: 4 bytes
+                                                                                                                                                                          |
++--------+---------+-----------+--------------------------------------------------------------+
 
 -- show partition file system details with regex fails as shown below
-SHOW TABLE EXTENDED  IN `default` LIKE `empl*` PARTITION (`grade=1`);
+SHOW TABLE EXTENDED  IN default LIKE 'empl*' PARTITION (grade=1);
 Error: Error running query: org.apache.spark.sql.catalyst.analysis.NoSuchTableException:
  Table or view 'emplo*' not found in database 'default'; (state=,code=0)
+```
 
-{% endhighlight %}
 ### Related Statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-tables.md b/docs/sql-ref-syntax-aux-show-tables.md
index 41af8e7d607d0..fef9722a444f8 100644
--- a/docs/sql-ref-syntax-aux-show-tables.md
+++ b/docs/sql-ref-syntax-aux-show-tables.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 The `SHOW TABLES` statement returns all the tables for an optionally specified database.
@@ -26,82 +27,80 @@ pattern. If no database is specified then the tables are returned from the
 current database.
 
 ### Syntax
-{% highlight sql %}
-SHOW TABLES [ { FROM | IN } database_name ] [ LIKE 'regex_pattern' ]
-{% endhighlight %}
+
+```sql
+SHOW TABLES [ { FROM | IN } database_name ] [ LIKE regex_pattern ]
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>{ FROM | IN } database_name</em></code></dt>
-  <dd>
+
+* **{ FROM `|` IN } database_name**
+
      Specifies the database name from which tables are listed.
-  </dd>
-  <dt><code><em>LIKE regex_pattern</em></code></dt>
-  <dd>
+
+* **regex_pattern**
+
      Specifies the regular expression pattern that is used to filter out unwanted tables. 
-     <ul> 
-          <li> Except for `*` and `|` character, the pattern works like a regex.</li>
-          <li> `*` alone matches 0 or more characters and `|` is used to separate multiple different regexes,
-           any of which can match. </li>
-          <li> The leading and trailing blanks are trimmed in the input pattern before processing.</li>
-     </ul>
-    
-  </dd>
-</dl>
-
-### Example
-{% highlight sql %}
+
+     * Except for `*` and `|` character, the pattern works like a regular expression.
+     * `*` alone matches 0 or more characters and `|` is used to separate multiple different regular expressions,
+       any of which can match.
+     * The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.
+
+### Examples
+
+```sql
 -- List all tables in default database
 SHOW TABLES;
-  +-----------+------------+--------------+--+
-  | database  | tableName  | isTemporary  |
-  +-----------+------------+--------------+--+
-  | default   | sam        | false        |
-  | default   | sam1       | false        |
-  | default   | suj        | false        |
-  +-----------+------------+--------------+--+
++--------+---------+-----------+
+|database|tableName|isTemporary|
++--------+---------+-----------+
+| default|      sam|      false|
+| default|     sam1|      false|
+| default|      suj|      false|
++--------+---------+-----------+
 
 -- List all tables from userdb database 
 SHOW TABLES FROM userdb;
-  +-----------+------------+--------------+--+
-  | database  | tableName  | isTemporary  |
-  +-----------+------------+--------------+--+
-  | userdb    | user1      | false        |
-  | userdb    | user2      | false        |
-  +-----------+------------+--------------+--+
++--------+---------+-----------+
+|database|tableName|isTemporary|
++--------+---------+-----------+
+|  userdb|    user1|      false|
+|  userdb|    user2|      false|
++--------+---------+-----------+
 
 -- List all tables in userdb database
 SHOW TABLES IN userdb;
-  +-----------+------------+--------------+--+
-  | database  | tableName  | isTemporary  |
-  +-----------+------------+--------------+--+
-  | userdb    | user1      | false        |
-  | userdb    | user2      | false        |
-  +-----------+------------+--------------+--+
++--------+---------+-----------+
+|database|tableName|isTemporary|
++--------+---------+-----------+
+|  userdb|    user1|      false|
+|  userdb|    user2|      false|
++--------+---------+-----------+
 
 -- List all tables from default database matching the pattern `sam*`
 SHOW TABLES FROM default LIKE 'sam*';
-  +-----------+------------+--------------+--+
-  | database  | tableName  | isTemporary  |
-  +-----------+------------+--------------+--+
-  | default   | sam        | false        |
-  | default   | sam1       | false        |
-  +-----------+------------+--------------+--+
++--------+---------+-----------+
+|database|tableName|isTemporary|
++--------+---------+-----------+
+| default|      sam|      false|
+| default|     sam1|      false|
++--------+---------+-----------+
   
 -- List all tables matching the pattern `sam*|suj`
-SHOW TABLES  LIKE 'sam*|suj';
-  +-----------+------------+--------------+--+
-  | database  | tableName  | isTemporary  |
-  +-----------+------------+--------------+--+
-  | default   | sam        | false        |
-  | default   | sam1       | false        |
-  | default   | suj        | false        |
-  +-----------+------------+--------------+--+
-
-{% endhighlight %}
-
-### Related statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
\ No newline at end of file
+SHOW TABLES LIKE 'sam*|suj';
++--------+---------+-----------+
+|database|tableName|isTemporary|
++--------+---------+-----------+
+| default|      sam|      false|
+| default|     sam1|      false|
+| default|      suj|      false|
++--------+---------+-----------+
+```
+
+### Related Statements
+
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-aux-show-tblproperties.md b/docs/sql-ref-syntax-aux-show-tblproperties.md
index 451fd4abc85bb..5b7ddcbcd9534 100644
--- a/docs/sql-ref-syntax-aux-show-tblproperties.md
+++ b/docs/sql-ref-syntax-aux-show-tblproperties.md
@@ -20,40 +20,36 @@ license: |
 ---
 
 ### Description
+
 This statement returns the value of a table property given an optional value for
 a property key. If no key is specified then all the properties are returned. 
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 SHOW TBLPROPERTIES table_identifier 
    [ ( unquoted_property_key | property_key_as_string_literal ) ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
+
+* **table_identifier**
+
     Specifies the table name of an existing table. The table may be optionally qualified
-    with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-  <dt><code><em>unquoted_property_key</em></code></dt>
-  <dd>
+    with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **unquoted_property_key**
+
     Specifies the property key in unquoted form. The key may consists of multiple
-    parts separated by dot.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ key_part1 ] [ .key_part2 ] [ ... ]
-      </code>
-  </dd>   
-  <dt><code><em>property_key_as_string_literal</em></code></dt>
-  <dd>
+    parts separated by dot.
+
+    **Syntax:** `[ key_part1 ] [ .key_part2 ] [ ... ]`
+
+* **property_key_as_string_literal**
+
     Specifies a property key value as a string literal.
-  </dd>
-</dl>
 
 **Note**
 - Property value returned by this statement excludes some properties 
@@ -64,52 +60,54 @@ SHOW TBLPROPERTIES table_identifier
     properties are: `numFiles`, `numPartitions`, `numRows`.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- create a table `customer` in database `salesdb`
 USE salesdb;
 CREATE TABLE customer(cust_code INT, name VARCHAR(100), cust_addr STRING)
-  TBLPROPERTIES ('created.by.user' = 'John', 'created.date' = '01-01-2001');
+    TBLPROPERTIES ('created.by.user' = 'John', 'created.date' = '01-01-2001');
 
 -- show all the user specified properties for table `customer`
 SHOW TBLPROPERTIES customer;
-  +---------------------+----------+
-  |key                  |value     |
-  +---------------------+----------+
-  |created.by.user      |John      |
-  |created.date         |01-01-2001|
-  |transient_lastDdlTime|1567554931|
-  +---------------------+----------+
++---------------------+----------+
+|                  key|     value|
++---------------------+----------+
+|      created.by.user|      John|
+|         created.date|01-01-2001|
+|transient_lastDdlTime|1567554931|
++---------------------+----------+
 
 -- show all the user specified properties for a qualified table `customer`
 -- in database `salesdb`
 SHOW TBLPROPERTIES salesdb.customer;
-  +---------------------+----------+
-  |key                  |value     |
-  +---------------------+----------+
-  |created.by.user      |John      |
-  |created.date         |01-01-2001|
-  |transient_lastDdlTime|1567554931|
-  +---------------------+----------+
++---------------------+----------+
+|                  key|     value|
++---------------------+----------+
+|      created.by.user|      John|
+|         created.date|01-01-2001|
+|transient_lastDdlTime|1567554931|
++---------------------+----------+
 
 -- show value for unquoted property key `created.by.user`
 SHOW TBLPROPERTIES customer (created.by.user);
-  +-----+
-  |value|
-  +-----+
-  |John |
-  +-----+
++-----+
+|value|
++-----+
+| John|
++-----+
 
 -- show value for property `created.date`` specified as string literal
 SHOW TBLPROPERTIES customer ('created.date');
-  +----------+
-  |value     |
-  +----------+
-  |01-01-2001|
-  +----------+
-{% endhighlight %}
++----------+
+|     value|
++----------+
+|01-01-2001|
++----------+
+```
 
 ### Related Statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [ALTER TABLE SET TBLPROPERTIES](sql-ref-syntax-ddl-alter-table.html)
-- [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
-- [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
+
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [ALTER TABLE SET TBLPROPERTIES](sql-ref-syntax-ddl-alter-table.html)
+* [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
+* [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-views.md b/docs/sql-ref-syntax-aux-show-views.md
new file mode 100644
index 0000000000000..5003c092cabce
--- /dev/null
+++ b/docs/sql-ref-syntax-aux-show-views.md
@@ -0,0 +1,117 @@
+---
+layout: global
+title: SHOW VIEWS
+displayTitle: SHOW VIEWS
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `SHOW VIEWS` statement returns all the views for an optionally specified database.
+Additionally, the output of this statement may be filtered by an optional matching
+pattern. If no database is specified then the views are returned from the 
+current database. If the specified database is global temporary view database, we will
+list global temporary views. Note that the command also lists local temporary views 
+regardless of a given database.
+
+### Syntax
+```sql
+SHOW VIEWS [ { FROM | IN } database_name ] [ LIKE regex_pattern ]
+```
+
+### Parameters
+* **{ FROM `|` IN } database_name**
+
+     Specifies the database name from which views are listed.
+
+* **regex_pattern**
+
+     Specifies the regular expression pattern that is used to filter out unwanted views.
+
+     * Except for `*` and `|` character, the pattern works like a regular expression.
+     * `*` alone matches 0 or more characters and `|` is used to separate multiple different regular expressions,
+       any of which can match.
+     * The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.
+
+### Examples
+```sql
+-- Create views in different databases, also create global/local temp views.
+CREATE VIEW sam AS SELECT id, salary FROM employee WHERE name = 'sam';
+CREATE VIEW sam1 AS SELECT id, salary FROM employee WHERE name = 'sam1';
+CREATE VIEW suj AS SELECT id, salary FROM employee WHERE name = 'suj';
+USE userdb;
+CREATE VIEW user1 AS SELECT id, salary FROM default.employee WHERE name = 'user1';
+CREATE VIEW user2 AS SELECT id, salary FROM default.employee WHERE name = 'user2';
+USE default;
+CREATE GLOBAL TEMP VIEW temp1 AS SELECT 1 AS col1;
+CREATE TEMP VIEW temp2 AS SELECT 1 AS col1;
+
+-- List all views in default database
+SHOW VIEWS;
++-------------+------------+--------------+
+| namespace   | viewName   | isTemporary  |
++-------------+------------+--------------+
+| default     | sam        | false        |
+| default     | sam1       | false        |
+| default     | suj        | false        |
+|             | temp2      | true         |
++-------------+------------+--------------+
+
+-- List all views from userdb database 
+SHOW VIEWS FROM userdb;
++-------------+------------+--------------+
+| namespace   | viewName   | isTemporary  |
++-------------+------------+--------------+
+| userdb      | user1      | false        |
+| userdb      | user2      | false        |
+|             | temp2      | true         |
++-------------+------------+--------------+
+  
+-- List all views in global temp view database 
+SHOW VIEWS IN global_temp;
++-------------+------------+--------------+
+| namespace   | viewName   | isTemporary  |
++-------------+------------+--------------+
+| global_temp | temp1      | true         |
+|             | temp2      | true         |
++-------------+------------+--------------+
+
+-- List all views from default database matching the pattern `sam*`
+SHOW VIEWS FROM default LIKE 'sam*';
++-----------+------------+--------------+
+| namespace | viewName   | isTemporary  |
++-----------+------------+--------------+
+| default   | sam        | false        |
+| default   | sam1       | false        |
++-----------+------------+--------------+
+
+-- List all views from the current database matching the pattern `sam|suj｜temp*`
+SHOW VIEWS LIKE 'sam|suj|temp*';
++-------------+------------+--------------+
+| namespace   | viewName   | isTemporary  |
++-------------+------------+--------------+
+| default     | sam        | false        |
+| default     | suj        | false        |
+|             | temp2      | true         |
++-------------+------------+--------------+
+```
+
+### Related statements
+* [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+* [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-aux-show.md b/docs/sql-ref-syntax-aux-show.md
index ad5c3954c23fa..9f64ea2d50ae1 100644
--- a/docs/sql-ref-syntax-aux-show.md
+++ b/docs/sql-ref-syntax-aux-show.md
@@ -18,11 +18,13 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-* [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
-* [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
-* [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
-* [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
-* [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
-* [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
-* [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
-* [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
\ No newline at end of file
+
+ * [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
+ * [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
+ * [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
+ * [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
+ * [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
+ * [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
+ * [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
+ * [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
+ * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-aux.md b/docs/sql-ref-syntax-aux.md
index ba09d70b437a9..3cd758f076e99 100644
--- a/docs/sql-ref-syntax-aux.md
+++ b/docs/sql-ref-syntax-aux.md
@@ -21,9 +21,9 @@ license: |
 
 Besides the major SQL statements such as Data Definition Statements, Data Manipulation Statements and Data Retrieval Statements, Spark SQL also supports the following Auxiliary Statements:
 
-- [ANALYZE](sql-ref-syntax-aux-analyze.html)
-- [CACHE](sql-ref-syntax-aux-cache.html)
-- [DESCRIBE](sql-ref-syntax-aux-describe.html)
-- [SHOW](sql-ref-syntax-aux-show.html)
-- [CONFIGURATION MANAGEMENT](sql-ref-syntax-aux-conf-mgmt.html)
-- [RESOURCE MANAGEMENT](sql-ref-syntax-aux-resource-mgmt.html)
+ * [ANALYZE](sql-ref-syntax-aux-analyze.html)
+ * [CACHE](sql-ref-syntax-aux-cache.html)
+ * [DESCRIBE](sql-ref-syntax-aux-describe.html)
+ * [SHOW](sql-ref-syntax-aux-show.html)
+ * [CONFIGURATION MANAGEMENT](sql-ref-syntax-aux-conf-mgmt.html)
+ * [RESOURCE MANAGEMENT](sql-ref-syntax-aux-resource-mgmt.html)
diff --git a/docs/sql-ref-syntax-ddl-alter-database.md b/docs/sql-ref-syntax-ddl-alter-database.md
index a32343674feb0..fbc454e25fb0c 100644
--- a/docs/sql-ref-syntax-ddl-alter-database.md
+++ b/docs/sql-ref-syntax-ddl-alter-database.md
@@ -18,7 +18,9 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
+
 You can alter metadata associated with a database by setting `DBPROPERTIES`.  The specified property
 values override any existing value with the same property name. Please note that the usage of 
 `SCHEMA` and `DATABASE` are interchangeable and one can be used in place of the other. An error message
@@ -26,19 +28,21 @@ is issued if the database is not found in the system. This command is mostly use
 for a database and may be used for auditing purposes.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 ALTER { DATABASE | SCHEMA } database_name
-    SET DBPROPERTIES ( property_name = property_value, ... )
-{% endhighlight %}
+    SET DBPROPERTIES ( property_name = property_value [ , ... ] )
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>database_name</em></code></dt>
-  <dd>Specifies the name of the database to be altered.</dd>
-</dl>
+
+* **database_name**
+
+    Specifies the name of the database to be altered.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Creates a database named `inventory`.
 CREATE DATABASE inventory;
 
@@ -47,16 +51,16 @@ ALTER DATABASE inventory SET DBPROPERTIES ('Edited-by' = 'John', 'Edit-date' = '
 
 -- Verify that properties are set.
 DESCRIBE DATABASE EXTENDED inventory;
-
-   +-------------------------+--------------------------------------------+
-   |database_description_item|database_description_value                  |
-   +-------------------------+--------------------------------------------+
-   |Database Name            |inventory                                   |
-   |Description              |                                            |
-   |Location                 |file:/temp/spark-warehouse/inventory.db     |
-   |Properties               |((Edit-date,01/01/2001), (Edited-by,John))  |
-   +-------------------------+--------------------------------------------+
-{% endhighlight %}
++-------------------------+------------------------------------------+
+|database_description_item|                database_description_value|
++-------------------------+------------------------------------------+
+|            Database Name|                                 inventory|
+|              Description|                                          |
+|                 Location|   file:/temp/spark-warehouse/inventory.db|
+|               Properties|((Edit-date,01/01/2001), (Edited-by,John))|
++-------------------------+------------------------------------------+
+```
 
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
diff --git a/docs/sql-ref-syntax-ddl-alter-table.md b/docs/sql-ref-syntax-ddl-alter-table.md
index a921478daa470..eb0e9a9f9cf73 100644
--- a/docs/sql-ref-syntax-ddl-alter-table.md
+++ b/docs/sql-ref-syntax-ddl-alter-table.md
@@ -20,259 +20,414 @@ license: |
 ---
 
 ### Description
+
 `ALTER TABLE` statement changes the schema or properties of a table.
 
 ### RENAME 
-`ALTER TABLE RENAME` statement changes the table name of an existing table in the database.
+
+`ALTER TABLE RENAME TO` statement changes the table name of an existing table in the database.
 
 #### Syntax
-{% highlight sql %}
+
+```sql
 ALTER TABLE table_identifier RENAME TO table_identifier
 
 ALTER TABLE table_identifier partition_spec RENAME TO partition_spec
-
-{% endhighlight %}
+```
 
 #### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
-    Partition to be renamed. <br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>
 
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
+    Partition to be renamed.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
 
 ### ADD COLUMNS
+
 `ALTER TABLE ADD COLUMNS` statement adds mentioned columns to an existing table.
 
 #### Syntax
-{% highlight sql %}
-ALTER TABLE table_identifier ADD COLUMNS ( col_spec [ , col_spec ... ] )
-{% endhighlight %}
+
+```sql
+ALTER TABLE table_identifier ADD COLUMNS ( col_spec [ , ... ] )
+```
 
 #### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>COLUMNS ( col_spec )</em></code></dt>
-  <dd>Specifies the columns to be added to be renamed.</dd>
-</dl>
 
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **COLUMNS ( col_spec )**
+
+    Specifies the columns to be added.
+
+### ALTER OR CHANGE COLUMN
+
+`ALTER TABLE ALTER COLUMN` or `ALTER TABLE CHANGE COLUMN` statement changes column's definition.
+
+#### Syntax
+
+```sql
+ALTER TABLE table_identifier { ALTER | CHANGE } [ COLUMN ] col_spec alterColumnAction
+```
+
+#### Parameters
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **COLUMNS ( col_spec )**
+
+    Specifies the column to be altered or be changed.
+
+* **alterColumnAction**
+
+    Change column's definition.
+
+### ADD AND DROP PARTITION
+
+#### ADD PARTITION
+
+`ALTER TABLE ADD` statement adds partition to the partitioned table.
+
+##### Syntax
+
+```sql
+ALTER TABLE table_identifier ADD [IF NOT EXISTS] 
+    ( partition_spec [ partition_spec ... ] )
+```
+     
+##### Parameters
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
 
+    Partition to be added.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
+
+#### DROP PARTITION
+
+`ALTER TABLE DROP` statement drops the partition of the table.
+
+##### Syntax
+
+```sql
+ALTER TABLE table_identifier DROP [ IF EXISTS ] partition_spec [PURGE]
+```
+     
+##### Parameters
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
+    Partition to be dropped.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
+     
 ### SET AND UNSET
 
 #### SET TABLE PROPERTIES
+
 `ALTER TABLE SET` command is used for setting the table properties. If a particular property was already set, 
 this overrides the old value with the new one.
 
 `ALTER TABLE UNSET` is used to drop the table property. 
 
 ##### Syntax
-{% highlight sql %}
 
---Set Table Properties 
+```sql
+-- Set Table Properties 
 ALTER TABLE table_identifier SET TBLPROPERTIES ( key1 = val1, key2 = val2, ... )
 
---Unset Table Properties
+-- Unset Table Properties
 ALTER TABLE table_identifier UNSET TBLPROPERTIES [ IF EXISTS ] ( key1, key2, ... )
-  
-{% endhighlight %}
+```
 
 #### SET SERDE
-`ALTER TABLE SET` command is used for setting the SERDE or SERDE properties in Hive tables. If a particular property was already set,  
-this overrides the old value with the new one.
+
+`ALTER TABLE SET` command is used for setting the SERDE or SERDE properties in Hive tables. If a particular property was already set, this overrides the old value with the new one.
 
 ##### Syntax
-{% highlight sql %}
 
---Set SERDE Properties
+```sql
+-- Set SERDE Properties
 ALTER TABLE table_identifier [ partition_spec ]
     SET SERDEPROPERTIES ( key1 = val1, key2 = val2, ... )
 
 ALTER TABLE table_identifier [ partition_spec ] SET SERDE serde_class_name
     [ WITH SERDEPROPERTIES ( key1 = val1, key2 = val2, ... ) ]
-
-{% endhighlight %}
+```
 
 #### SET LOCATION And SET FILE FORMAT
+
 `ALTER TABLE SET` command can also be used for changing the file location and file format for 
 existing tables. 
 
 ##### Syntax
-{% highlight sql %}
 
---Changing File Format
+```sql
+-- Changing File Format
 ALTER TABLE table_identifier [ partition_spec ] SET FILEFORMAT file_format
 
---Changing File Location
+-- Changing File Location
 ALTER TABLE table_identifier [ partition_spec ] SET LOCATION 'new_location'
-
-{% endhighlight %}
+```
 
 #### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
-    Specifies the partition on which the property has to be set.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>SERDEPROPERTIES ( key1 = val1, key2 = val2, ... )</em></code></dt>
-  <dd>Specifies the SERDE properties to be set.</dd>
-</dl>
 
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
+    Specifies the partition on which the property has to be set.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
+
+* **SERDEPROPERTIES ( key1 = val1, key2 = val2, ... )**
+
+    Specifies the SERDE properties to be set.
 
 ### Examples
-{% highlight sql %}
 
---RENAME table 
+```sql
+-- RENAME table 
 DESC student;
-+--------------------------+------------+----------+--+
-|         col_name         | data_type  | comment  |
-+--------------------------+------------+----------+--+
-| name                     | string     | NULL     |
-| rollno                   | int        | NULL     |
-| age                      | int        | NULL     |
-| # Partition Information  |            |          |
-| # col_name               | data_type  | comment  |
-| age                      | int        | NULL     |
-+--------------------------+------------+----------+--+
++-----------------------+---------+-------+
+|               col_name|data_type|comment|
++-----------------------+---------+-------+
+|                   name|   string|   NULL|
+|                 rollno|      int|   NULL|
+|                    age|      int|   NULL|
+|# Partition Information|         |       |
+|             # col_name|data_type|comment|
+|                    age|      int|   NULL|
++-----------------------+---------+-------+
 
 ALTER TABLE Student RENAME TO StudentInfo;
 
---After Renaming the table
-
+-- After Renaming the table
 DESC StudentInfo;
-+--------------------------+------------+----------+--+
-|         col_name         | data_type  | comment  |
-+--------------------------+------------+----------+--+
-| name                     | string     | NULL     |
-| rollno                   | int        | NULL     |
-| age                      | int        | NULL     |
-| # Partition Information  |            |          |
-| # col_name               | data_type  | comment  |
-| age                      | int        | NULL     |
-+--------------------------+------------+----------+--+
-
---RENAME partition
++-----------------------+---------+-------+
+|               col_name|data_type|comment|
++-----------------------+---------+-------+
+|                   name|   string|   NULL|
+|                 rollno|      int|   NULL|
+|                    age|      int|   NULL|
+|# Partition Information|         |       |
+|             # col_name|data_type|comment|
+|                    age|      int|   NULL|
++-----------------------+---------+-------+
+
+-- RENAME partition
 
 SHOW PARTITIONS StudentInfo;
-+------------+--+
-| partition  |
-+------------+--+
-| age=10     |
-| age=11     |
-| age=12     |
-+------------+--+
++---------+
+|partition|
++---------+
+|   age=10|
+|   age=11|
+|   age=12|
++---------+
 
 ALTER TABLE default.StudentInfo PARTITION (age='10') RENAME TO PARTITION (age='15');
 
---After renaming Partition
+-- After renaming Partition
 SHOW PARTITIONS StudentInfo;
-+------------+--+
-| partition  |
-+------------+--+
-| age=11     |
-| age=12     |
-| age=15     |
-+------------+--+
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
++---------+
+
+-- Add new columns to a table
+DESC StudentInfo;
++-----------------------+---------+-------+
+|               col_name|data_type|comment|
++-----------------------+---------+-------+
+|                   name|   string|   NULL|
+|                 rollno|      int|   NULL|
+|                    age|      int|   NULL|
+|# Partition Information|         |       |
+|             # col_name|data_type|comment|
+|                    age|      int|   NULL|
++-----------------------+---------+-------+
 
--- Add new column to a table
+ALTER TABLE StudentInfo ADD columns (LastName string, DOB timestamp);
 
+-- After Adding New columns to the table
 DESC StudentInfo;
-+--------------------------+------------+----------+--+
-|         col_name         | data_type  | comment  |
-+--------------------------+------------+----------+--+
-| name                     | string     | NULL     |
-| rollno                   | int        | NULL     |
-| age                      | int        | NULL     |
-| # Partition Information  |            |          |
-| # col_name               | data_type  | comment  |
-| age                      | int        | NULL     |
-+--------------------------+------------+----------+
++-----------------------+---------+-------+
+|               col_name|data_type|comment|
++-----------------------+---------+-------+
+|                   name|   string|   NULL|
+|                 rollno|      int|   NULL|
+|               LastName|   string|   NULL|
+|                    DOB|timestamp|   NULL|
+|                    age|      int|   NULL|
+|# Partition Information|         |       |
+|             # col_name|data_type|comment|
+|                    age|      int|   NULL|
++-----------------------+---------+-------+
+
+-- Add a new partition to a table 
+SHOW PARTITIONS StudentInfo;
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
++---------+
 
-ALTER TABLE StudentInfo ADD columns (LastName string, DOB timestamp);
+ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18);
 
---After Adding New columns to the table
+-- After adding a new partition to the table
+SHOW PARTITIONS StudentInfo;
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
+|   age=18|
++---------+
+
+-- Drop a partition from the table 
+SHOW PARTITIONS StudentInfo;
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
+|   age=18|
++---------+
+
+ALTER TABLE StudentInfo DROP IF EXISTS PARTITION (age=18);
+
+-- After dropping the partition of the table
+SHOW PARTITIONS StudentInfo;
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
++---------+
+
+-- Adding multiple partitions to the table
+SHOW PARTITIONS StudentInfo;
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
++---------+
+
+ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18) PARTITION (age=20);
+
+-- After adding multiple partitions to the table
+SHOW PARTITIONS StudentInfo;
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
+|   age=18|
+|   age=20|
++---------+
+
+-- ALTER OR CHANGE COLUMNS
 DESC StudentInfo;
-+--------------------------+------------+----------+--+
-|         col_name         | data_type  | comment  |
-+--------------------------+------------+----------+--+
-| name                     | string     | NULL     |
-| rollno                   | int        | NULL     |
-| LastName                 | string     | NULL     |
-| DOB                      | timestamp  | NULL     |
-| age                      | int        | NULL     |
-| # Partition Information  |            |          |
-| # col_name               | data_type  | comment  |
-| age                      | int        | NULL     |
-+--------------------------+------------+----------+--+
-
-
---Change the fileformat
++-----------------------+---------+-------+
+|               col_name|data_type|comment|
++-----------------------+---------+-------+
+|                   name|   string|   NULL|
+|                 rollno|      int|   NULL|
+|               LastName|   string|   NULL|
+|                    DOB|timestamp|   NULL|
+|                    age|      int|   NULL|
+|# Partition Information|         |       |
+|             # col_name|data_type|comment|
+|                    age|      int|   NULL|
++-----------------------+---------+-------+
+
+ALTER TABLE StudentInfo ALTER COLUMN name COMMENT "new comment";
+
+--After ALTER or CHANGE COLUMNS
+DESC StudentInfo;
++-----------------------+---------+-----------+
+|               col_name|data_type|    comment|
++-----------------------+---------+-----------+
+|                   name|   string|new comment|
+|                 rollno|      int|       NULL|
+|               LastName|   string|       NULL|
+|                    DOB|timestamp|       NULL|
+|                    age|      int|       NULL|
+|# Partition Information|         |           |
+|             # col_name|data_type|    comment|
+|                    age|      int|       NULL|
++-----------------------+---------+-----------+
+
+-- Change the fileformat
 ALTER TABLE loc_orc SET fileformat orc;
 
 ALTER TABLE p1 partition (month=2, day=2) SET fileformat parquet;
 
---Change the file Location
+-- Change the file Location
 ALTER TABLE dbx.tab1 PARTITION (a='1', b='2') SET LOCATION '/path/to/part/ways'
 
 -- SET SERDE/ SERDE Properties
 ALTER TABLE test_tab SET SERDE 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe';
 
-ALTER TABLE dbx.tab1 SET SERDE 'org.apache.madoop' WITH SERDEPROPERTIES ('k' = 'v', 'kay' = 'vee')
+ALTER TABLE dbx.tab1 SET SERDE 'org.apache.hadoop' WITH SERDEPROPERTIES ('k' = 'v', 'kay' = 'vee')
 
---SET TABLE PROPERTIES
-ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('winner' = 'loser')
+-- SET TABLE PROPERTIES
+ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('winner' = 'loser');
 
---DROP TABLE PROPERTIES
-ALTER TABLE dbx.tab1 UNSET TBLPROPERTIES ('winner')
+-- SET TABLE COMMENT Using SET PROPERTIES
+ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('comment' = 'A table comment.');
 
-{% endhighlight %}
+-- Alter TABLE COMMENT Using SET PROPERTIES
+ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('comment' = 'This is a new comment.');
 
+-- DROP TABLE PROPERTIES
+ALTER TABLE dbx.tab1 UNSET TBLPROPERTIES ('winner');
+```
 
 ### Related Statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
-
 
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
diff --git a/docs/sql-ref-syntax-ddl-alter-view.md b/docs/sql-ref-syntax-ddl-alter-view.md
index d42d040522cae..a34e77decf593 100644
--- a/docs/sql-ref-syntax-ddl-alter-view.md
+++ b/docs/sql-ref-syntax-ddl-alter-view.md
@@ -29,21 +29,16 @@ Renames the existing view. If the new view name already exists in the source dat
 does not support moving the views across databases.
 
 #### Syntax
-{% highlight sql %}
+```sql
 ALTER VIEW view_identifier RENAME TO view_identifier
-{% endhighlight %}
+```
 
 #### Parameters
-<dl>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies a view name, which may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-</dl>
+* **view_identifier**
+
+    Specifies a view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
 
 #### SET View Properties
 Set one or more properties of an existing view. The properties are the key value pairs. If the properties' keys exist, 
@@ -51,119 +46,100 @@ the values are replaced with the new values. If the properties' keys do not exis
 the properties.
 
 #### Syntax
-{% highlight sql %}
+```sql
 ALTER VIEW view_identifier SET TBLPROPERTIES ( property_key = property_val [ , ... ] )
-{% endhighlight %}
+```
 
 #### Parameters
-<dl>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies a view name, which may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-  <dt><code><em>property_key</em></code></dt>
-  <dd>
-    Specifies the property key. The key may consists of multiple parts separated by dot.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ key_part1 ] [ .key_part2 ] [ ... ]
-      </code>
-  </dd>
-</dl>
+* **view_identifier**
+
+    Specifies a view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
+
+* **property_key**
+
+    Specifies the property key. The key may consists of multiple parts separated by dot.
+
+    **Syntax:** `[ key_part1 ] [ .key_part2 ] [ ... ]`
 
 #### UNSET View Properties
 Drop one or more properties of an existing view. If the specified keys do not exist, an exception is thrown. Use 
 `IF EXISTS` to avoid the exception. 
 
 #### Syntax
-{% highlight sql %}
+```sql
 ALTER VIEW view_identifier UNSET TBLPROPERTIES [ IF EXISTS ]  ( property_key [ , ... ] )
-{% endhighlight %}
+```
 
 #### Parameters
-<dl>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies a view name, which may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-  <dt><code><em>property_key</em></code></dt>
-  <dd>
-    Specifies the property key. The key may consists of multiple parts separated by dot.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ key_part1 ] [ .key_part2 ] [ ... ]
-      </code>
-  </dd>
-</dl>
+* **view_identifier**
+
+    Specifies a view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
+
+* **property_key**
+
+    Specifies the property key. The key may consists of multiple parts separated by dot.
+
+    **Syntax:** `[ key_part1 ] [ .key_part2 ] [ ... ]`
 
 #### ALTER View AS SELECT
-`ALTER VIEW view_identifier AS SELECT` statement changes the definition of a view, the `SELECT` statement must be valid,
+`ALTER VIEW view_identifier AS SELECT` statement changes the definition of a view. The `SELECT` statement must be valid,
 and the `view_identifier` must exist.
 
 #### Syntax
-{% highlight sql %}
+```sql
 ALTER VIEW view_identifier AS select_statement
-{% endhighlight %}
+```
+
+Note that `ALTER VIEW` statement does not support `SET SERDE` or `SET SERDEPROPERTIES` properties.
 
 #### Parameters
-<dl>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies a view name, which may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-  <dt><code><em>select_statement</em></code></dt>
-  <dd>
-    Specifies the definition of the view, detail check <a href="sql-ref-syntax-qry-select.html">select_statement</a>
-  </dd>
-</dl>
+* **view_identifier**
+
+    Specifies a view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
+
+* **select_statement**
+
+    Specifies the definition of the view. Check [select_statement](sql-ref-syntax-qry-select.html) for details.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Rename only changes the view name.
 -- The source and target databases of the view have to be the same.
--- Use qualified or unqualified name for the source and target view  
+-- Use qualified or unqualified name for the source and target view.
 ALTER VIEW tempdb1.v1 RENAME TO tempdb1.v2;
 
 -- Verify that the new view is created.
 DESCRIBE TABLE EXTENDED tempdb1.v2;
-
 +----------------------------+----------+-------+
-|col_name                    |data_type |comment|
+|                    col_name|data_type |comment|
 +----------------------------+----------+-------+
-|c1                          |int       |null   |
-|c2                          |string    |null   |
+|                          c1|       int|   null|
+|                          c2|    string|   null|
 |                            |          |       |
 |# Detailed Table Information|          |       |
-|Database                    |tempdb1   |       |
-|Table                       |v2        |       |
+|                    Database|   tempdb1|       |
+|                       Table|        v2|       |
 +----------------------------+----------+-------+
 
 -- Before ALTER VIEW SET TBLPROPERTIES
 DESC TABLE EXTENDED tempdb1.v2;
-
 +----------------------------+----------+-------+
-|col_name                    |data_type |comment|
+|                    col_name| data_type|comment|
 +----------------------------+----------+-------+
-|c1                          |int       |null   |
-|c2                          |string    |null   |
+|                          c1|       int|   null|
+|                          c2|    string|   null|
 |                            |          |       |
 |# Detailed Table Information|          |       |
-|Database                    |tempdb1   |       |
-|Table                       |v2        |       |
-|Table Properties            |[....]    |       |
+|                    Database|   tempdb1|       |
+|                       Table|        v2|       |
+|            Table Properties|    [....]|       |
 +----------------------------+----------+-------+
 
 -- Set properties in TBLPROPERTIES
@@ -171,17 +147,16 @@ ALTER VIEW tempdb1.v2 SET TBLPROPERTIES ('created.by.user' = "John", 'created.da
 
 -- Use `DESCRIBE TABLE EXTENDED tempdb1.v2` to verify
 DESC TABLE EXTENDED tempdb1.v2;
-
 +----------------------------+-----------------------------------------------------+-------+
-|col_name                    |data_type                                            |comment|
+|                    col_name|                                            data_type|comment|
 +----------------------------+-----------------------------------------------------+-------+
-|c1                          |int                                                  |null   |
-|c2                          |string                                               |null   |
+|                          c1|                                                  int|   null|
+|                          c2|                                               string|   null|
 |                            |                                                     |       |
 |# Detailed Table Information|                                                     |       |
-|Database                    |tempdb1                                              |       |
-|Table                       |v2                                                   |       |
-|Table Properties            |[created.by.user=John, created.date=01-01-2001, ....]|       |
+|                    Database|                                              tempdb1|       |
+|                       Table|                                                   v2|       |
+|            Table Properties|[created.by.user=John, created.date=01-01-2001, ....]|       |
 +----------------------------+-----------------------------------------------------+-------+
 
 -- Remove the key `created.by.user` and `created.date` from `TBLPROPERTIES`
@@ -189,17 +164,16 @@ ALTER VIEW tempdb1.v2 UNSET TBLPROPERTIES ('created.by.user', 'created.date');
 
 --Use `DESC TABLE EXTENDED tempdb1.v2` to verify the changes
 DESC TABLE EXTENDED tempdb1.v2;
-
 +----------------------------+----------+-------+
-|col_name                    |data_type |comment|
+|                    col_name| data_type|comment|
 +----------------------------+----------+-------+
-|c1                          |int       |null   |
-|c2                          |string    |null   |
+|                          c1|       int|   null|
+|                          c2|    string|   null|
 |                            |          |       |
 |# Detailed Table Information|          |       |
-|Database                    |tempdb1   |       |
-|Table                       |v2        |       |
-|Table Properties            |[....]    |       |
+|                    Database|   tempdb1|       |
+|                       Table|        v2|       |
+|            Table Properties|    [....]|       |
 +----------------------------+----------+-------+
 
 -- Change the view definition
@@ -207,29 +181,24 @@ ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;
 
 -- Use `DESC TABLE EXTENDED` to verify
 DESC TABLE EXTENDED tempdb1.v2;
-
 +----------------------------+---------------------------+-------+
-|col_name                    |data_type                  |comment|
+|                    col_name|                  data_type|comment|
 +----------------------------+---------------------------+-------+
-|c1                          |int                        |null   |
-|c2                          |string                     |null   |
+|                          c1|                        int|   null|
+|                          c2|                     string|   null|
 |                            |                           |       |
 |# Detailed Table Information|                           |       |
-|Database                    |tempdb1                    |       |
-|Table                       |v2                         |       |
-|Type                        |VIEW                       |       |
-|View Text                   |select * from tempdb1.v1   |       |
-|View Original Text          |select * from tempdb1.v1   |       |
+|                    Database|                    tempdb1|       |
+|                       Table|                         v2|       |
+|                        Type|                       VIEW|       |
+|                   View Text|   select * from tempdb1.v1|       |
+|          View Original Text|   select * from tempdb1.v1|       |
 +----------------------------+---------------------------+-------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
-- [describe-table](sql-ref-syntax-aux-describe-table.html)
-- [create-view](sql-ref-syntax-ddl-create-view.html)
-- [drop-view](sql-ref-syntax-ddl-drop-view.html)
-
-#### Note:
-
-`ALTER VIEW` statement does not support `SET SERDE` or `SET SERDEPROPERTIES` properties
-
+* [describe-table](sql-ref-syntax-aux-describe-table.html)
+* [create-view](sql-ref-syntax-ddl-create-view.html)
+* [drop-view](sql-ref-syntax-ddl-drop-view.html)
+* [show-views](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-ddl-create-database.md b/docs/sql-ref-syntax-ddl-create-database.md
index 4d2211c650953..9d8bf47844724 100644
--- a/docs/sql-ref-syntax-ddl-create-database.md
+++ b/docs/sql-ref-syntax-ddl-create-database.md
@@ -20,36 +20,43 @@ license: |
 ---
 
 ### Description
+
 Creates a database with the specified name. If database with the same name already exists, an exception will be thrown.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 CREATE { DATABASE | SCHEMA } [ IF NOT EXISTS ] database_name
-  [ COMMENT database_comment ]
-  [ LOCATION database_directory ]
-  [ WITH DBPROPERTIES ( property_name = property_value [ , ... ] ) ]
-{% endhighlight %}
+    [ COMMENT database_comment ]
+    [ LOCATION database_directory ]
+    [ WITH DBPROPERTIES ( property_name = property_value [ , ... ] ) ]
+```
 
 ### Parameters
-<dl>
-    <dt><code><em>database_name</em></code></dt>
-    <dd>Specifies the name of the database to be created.</dd>
 
-    <dt><code><em>IF NOT EXISTS</em></code></dt>
-    <dd>Creates a database with the given name if it doesn't exists. If a database with the same name already exists, nothing will happen.</dd>
+* **database_name**
+
+    Specifies the name of the database to be created.
+
+* **IF NOT EXISTS**
+
+    Creates a database with the given name if it does not exist. If a database with the same name already exists, nothing will happen.
 
-    <dt><code><em>database_directory</em></code></dt>
-    <dd>Path of the file system in which the specified database is to be created. If the specified path does not exist in the underlying file system, this command creates a directory with the path. If the location is not specified, the database will be created in the default warehouse directory, whose path is configured by the static configuration spark.sql.warehouse.dir.</dd>
+* **database_directory**
 
-    <dt><code><em>database_comment</em></code></dt>
-    <dd>Specifies the description for the database.</dd>
+    Path of the file system in which the specified database is to be created. If the specified path does not exist in the underlying file system, this command creates a directory with the path. If the location is not specified, the database will be created in the default warehouse directory, whose path is configured by the static configuration spark.sql.warehouse.dir.
 
-    <dt><code><em>WITH DBPROPERTIES ( property_name=property_value [ , ... ] )</em></code></dt>
-    <dd>Specifies the properties for the database in key-value pairs.</dd>
-</dl>
+* **database_comment**
+
+    Specifies the description for the database.
+
+* **WITH DBPROPERTIES ( property_name=property_value [ , ... ] )**
+
+    Specifies the properties for the database in key-value pairs.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Create database `customer_db`. This throws exception if database with name customer_db
 -- already exists.
 CREATE DATABASE customer_db;
@@ -60,20 +67,21 @@ CREATE DATABASE IF NOT EXISTS customer_db;
 -- Create database `customer_db` only if database with same name doesn't exist with 
 -- `Comments`,`Specific Location` and `Database properties`.
 CREATE DATABASE IF NOT EXISTS customer_db COMMENT 'This is customer database' LOCATION '/user'
- WITH DBPROPERTIES (ID=001, Name='John');
+    WITH DBPROPERTIES (ID=001, Name='John');
 
 -- Verify that properties are set.
 DESCRIBE DATABASE EXTENDED customer_db;
-   +----------------------------+-----------------------------+
-   | database_description_item  | database_description_value  |
-   +----------------------------+-----------------------------+
-   | Database Name              | customer_db                 |
-   | Description                | This is customer database   |
-   | Location                   | hdfs://hacluster/user       |
-   | Properties                 | ((ID,001), (Name,John))     |
-   +----------------------------+-----------------------------+
-{% endhighlight %}
++-------------------------+--------------------------+
+|database_description_item|database_description_value|
++-------------------------+--------------------------+
+|            Database Name|               customer_db|
+|              Description| This is customer database|
+|                 Location|     hdfs://hacluster/user|
+|               Properties|   ((ID,001), (Name,John))|
++-------------------------+--------------------------+
+```
 
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-ddl-create-function.md b/docs/sql-ref-syntax-ddl-create-function.md
index 1f94bf6d25aa5..dfa4f4f8123d8 100644
--- a/docs/sql-ref-syntax-ddl-create-function.md
+++ b/docs/sql-ref-syntax-ddl-create-function.md
@@ -20,6 +20,7 @@ license: |
 ---
 
 ### Description
+
 The `CREATE FUNCTION` statement is used to create a temporary or permanent function
 in Spark. Temporary functions are scoped at a session level where as permanent
 functions are created in the persistent catalog and are made available to
@@ -27,70 +28,64 @@ all sessions. The resources specified in the `USING` clause are made available
 to all executors when they are executed for the first time. In addition to the
 SQL interface, spark allows users to create custom user defined scalar and
 aggregate functions using Scala, Python and Java APIs. Please refer to 
-[scalar_functions](sql-getting-started.html#scalar-functions) and 
-[aggregate functions](sql-getting-started#aggregations) for more information.
+[Scalar UDFs](sql-ref-functions-udf-scalar.html) and
+[UDAFs](sql-ref-functions-udf-aggregate.html) for more information.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 CREATE [ OR REPLACE ] [ TEMPORARY ] FUNCTION [ IF NOT EXISTS ]
     function_name AS class_name [ resource_locations ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>OR REPLACE</em></code></dt>
-  <dd>
+
+* **OR REPLACE**
+
     If specified, the resources for the function are reloaded. This is mainly useful
     to pick up any changes made to the implementation of the function. This
-    parameter is mutually exclusive to <code>IF NOT EXISTS</code> and can not
+    parameter is mutually exclusive to `IF NOT EXISTS` and can not
     be specified together.
-  </dd>
-  <dt><code><em>TEMPORARY</em></code></dt>
-  <dd>
-    Indicates the scope of function being created. When <code>TEMPORARY</code> is specified, the
+
+* **TEMPORARY**
+
+    Indicates the scope of function being created. When `TEMPORARY` is specified, the
     created function is valid and visible in the current session. No persistent
     entry is made in the catalog for these kind of functions.
-  </dd>
-  <dt><code><em>IF NOT EXISTS</em></code></dt>
-  <dd>
+
+* **IF NOT EXISTS**
+
     If specified, creates the function only when it does not exist. The creation
     of function succeeds (no error is thrown) if the specified function already
-    exists in the system. This parameter is mutually exclusive to <code> OR REPLACE</code> 
+    exists in the system. This parameter is mutually exclusive to `OR REPLACE`
     and can not be specified together.
-  </dd>
-  <dt><code><em>function_name</em></code></dt>
-  <dd>
-    Specifies a name of function to be created. The function name may be
-    optionally qualified with a database name. <br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] function_name
-      </code>
-  </dd>
-  <dt><code><em>class_name</em></code></dt>
-  <dd>
+
+* **function_name**
+
+    Specifies a name of function to be created. The function name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] function_name`
+
+* **class_name**
+
     Specifies the name of the class that provides the implementation for function to be created.
     The implementing class should extend one of the base classes as follows:
-    <ul>
-      <li>Should extend <code>UDF</code> or <code>UDAF</code> in <code>org.apache.hadoop.hive.ql.exec</code> package.</li>
-      <li>Should extend <code>AbstractGenericUDAFResolver</code>, <code>GenericUDF</code>, or
-          <code>GenericUDTF</code> in <code>org.apache.hadoop.hive.ql.udf.generic</code> package.</li>
-      <li>Should extend <code>UserDefinedAggregateFunction</code> in <code>org.apache.spark.sql.expressions</code> package.</li>
-    </ul>
-  </dd>
-  <dt><code><em>resource_locations</em></code></dt>
-  <dd>
+
+    * Should extend `UDF` or `UDAF` in `org.apache.hadoop.hive.ql.exec` package.
+    * Should extend `AbstractGenericUDAFResolver`, `GenericUDF`, or
+      `GenericUDTF` in `org.apache.hadoop.hive.ql.udf.generic` package.
+    * Should extend `UserDefinedAggregateFunction` in `org.apache.spark.sql.expressions` package.
+
+* **resource_locations**
+
     Specifies the list of resources that contain the implementation of the function
-    along with its dependencies. <br><br>
-    <b>Syntax:</b>
-      <code>
-        USING { { (JAR | FILE ) resource_uri } , ... }
-      </code>
-  </dd>
-</dl>
+    along with its dependencies.
+
+    **Syntax:** `USING { { (JAR | FILE ) resource_uri } , ... }`
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- 1. Create a simple UDF `SimpleUdf` that increments the supplied integral value by 10.
 --    import org.apache.hadoop.hive.ql.exec.UDF;
 --    public class SimpleUdf extends UDF {
@@ -106,39 +101,39 @@ INSERT INTO test VALUES (1), (2);
 
 -- Create a permanent function called `simple_udf`. 
 CREATE FUNCTION simple_udf AS 'SimpleUdf'
-  USING JAR '/tmp/SimpleUdf.jar';
+    USING JAR '/tmp/SimpleUdf.jar';
 
 -- Verify that the function is in the registry.
 SHOW USER FUNCTIONS;
-  +------------------+
-  |          function|
-  +------------------+
-  |default.simple_udf|
-  +------------------+
++------------------+
+|          function|
++------------------+
+|default.simple_udf|
++------------------+
 
 -- Invoke the function. Every selected value should be incremented by 10.
-SELECT simple_udf(c1) AS function_return_value FROM t1;
-  +---------------------+                                                         
-  |function_return_value|
-  +---------------------+
-  |                   11|
-  |                   12|
-  +---------------------+
+SELECT simple_udf(c1) AS function_return_value FROM test;
++---------------------+
+|function_return_value|
++---------------------+
+|                   11|
+|                   12|
++---------------------+
 
 -- Created a temporary function.
 CREATE TEMPORARY FUNCTION simple_temp_udf AS 'SimpleUdf' 
-  USING JAR '/tmp/SimpleUdf.jar';
+    USING JAR '/tmp/SimpleUdf.jar';
 
 -- Verify that the newly created temporary function is in the registry.
 -- Please note that the temporary function does not have a qualified
 -- database associated with it.
 SHOW USER FUNCTIONS;
-  +------------------+
-  |          function|
-  +------------------+
-  |default.simple_udf|
-  |   simple_temp_udf|
-  +------------------+
++------------------+
+|          function|
++------------------+
+|default.simple_udf|
+|   simple_temp_udf|
++------------------+
 
 -- 1. Modify `SimpleUdf`'s implementation to add supplied integral value by 20.
 --    import org.apache.hadoop.hive.ql.exec.UDF;
@@ -152,20 +147,20 @@ SHOW USER FUNCTIONS;
 
 -- Replace the implementation of `simple_udf`
 CREATE OR REPLACE FUNCTION simple_udf AS 'SimpleUdfR'
-  USING JAR '/tmp/SimpleUdfR.jar';
+    USING JAR '/tmp/SimpleUdfR.jar';
 
 -- Invoke the function. Every selected value should be incremented by 20.
-SELECT simple_udf(c1) AS function_return_value FROM t1;
-+---------------------+                                                         
+SELECT simple_udf(c1) AS function_return_value FROM test;
++---------------------+
 |function_return_value|
 +---------------------+
 |                   21|
 |                   22|
 +---------------------+
+```
 
-{% endhighlight %}
+### Related Statements
 
-### Related statements
-- [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
-- [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
+* [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+* [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
diff --git a/docs/sql-ref-syntax-ddl-create-table-datasource.md b/docs/sql-ref-syntax-ddl-create-table-datasource.md
index 9b15c0865b38a..ba0516afbbfad 100644
--- a/docs/sql-ref-syntax-ddl-create-table-datasource.md
+++ b/docs/sql-ref-syntax-ddl-create-table-datasource.md
@@ -24,92 +24,117 @@ license: |
 The `CREATE TABLE` statement defines a new table using a Data Source. 
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 CREATE TABLE [ IF NOT EXISTS ] table_identifier
-  [ ( col_name1 col_type1 [ COMMENT col_comment1 ], ... ) ]
-  USING data_source
-  [ OPTIONS ( key1=val1, key2=val2, ... ) ]
-  [ PARTITIONED BY ( col_name1, col_name2, ... ) ]
-  [ CLUSTERED BY ( col_name3, col_name4, ... ) 
-    [ SORTED BY ( col_name [ ASC | DESC ], ... ) ] 
-    INTO num_buckets BUCKETS ]
-  [ LOCATION path ]
-  [ COMMENT table_comment ]
-  [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
-  [ AS select_statement ]
-{% endhighlight %}
+    [ ( col_name1 col_type1 [ COMMENT col_comment1 ], ... ) ]
+    USING data_source
+    [ OPTIONS ( key1=val1, key2=val2, ... ) ]
+    [ PARTITIONED BY ( col_name1, col_name2, ... ) ]
+    [ CLUSTERED BY ( col_name3, col_name4, ... ) 
+        [ SORTED BY ( col_name [ ASC | DESC ], ... ) ] 
+        INTO num_buckets BUCKETS ]
+    [ LOCATION path ]
+    [ COMMENT table_comment ]
+    [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
+    [ AS select_statement ]
+```
+
+Note that, the clauses between the USING clause and the AS SELECT clause can come in
+as any order. For example, you can write COMMENT table_comment after TBLPROPERTIES.
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>USING data_source</em></code></dt>
-  <dd>Data Source is the input format used to create the table. Data source can be CSV, TXT, ORC, JDBC, PARQUET, etc.</dd>
-</dl> 
-
-<dl>
-  <dt><code><em>PARTITIONED BY</em></code></dt>
-  <dd>Partitions are created on the table, based on the columns specified.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>CLUSTERED BY</em></code></dt>
-  <dd>
-	Partitions created on the table will be bucketed into fixed buckets based on the column specified for bucketing.<br><br>
-   	<b>NOTE:</b>Bucketing is an optimization technique that uses buckets (and bucketing columns) to determine data partitioning and avoid data shuffle.<br>
-	<dt><code><em>SORTED BY</em></code></dt>
-	<dd>Determines the order in which the data is stored in buckets. Default is Ascending order.</dd>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>LOCATION</em></code></dt>
-  <dd>Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>COMMENT</em></code></dt>
-  <dd>Table comments are added.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>TBLPROPERTIES</em></code></dt>
-  <dd>Table properties that have to be set are specified, such as `created.by.user`, `owner`, etc.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>AS select_statement</em></code></dt>
-  <dd>The table is populated using the data from the select statement.</dd>
-</dl>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **USING data_source**
+
+    Data Source is the input format used to create the table. Data source can be CSV, TXT, ORC, JDBC, PARQUET, etc.
+
+* **PARTITIONED BY**
+
+    Partitions are created on the table, based on the columns specified.
+
+* **CLUSTERED BY**
+
+    Partitions created on the table will be bucketed into fixed buckets based on the column specified for bucketing.
+
+    **NOTE:** Bucketing is an optimization technique that uses buckets (and bucketing columns) to determine data partitioning and avoid data shuffle.
+
+* **SORTED BY**
+
+    Specifies an ordering of bucket columns. Optionally, one can use ASC for an ascending order or DESC for a descending order after any column names in the SORTED BY clause.
+    If not specified, ASC is assumed by default.
+   
+* **INTO num_buckets BUCKETS**
+
+    Specifies buckets numbers, which is used in `CLUSTERED BY` clause.
+
+* **LOCATION**
+
+    Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc.
+
+* **COMMENT**
+
+    A string literal to describe the table.
+
+* **TBLPROPERTIES**
+
+    A list of key-value pairs that is used to tag the table definition.
+
+* **AS select_statement**
+
+    The table is populated using the data from the select statement.
+
+### Data Source Interaction
+
+A Data Source table acts like a pointer to the underlying data source. For example, you can create
+a table "foo" in Spark which points to a table "bar" in MySQL using JDBC Data Source. When you
+read/write table "foo", you actually read/write table "bar".
+ 
+In general CREATE TABLE is creating a "pointer", and you need to make sure it points to something
+existing. An exception is file source such as parquet, json. If you don't specify the LOCATION,
+Spark will create a default table location for you.
+
+For CREATE TABLE AS SELECT, Spark will overwrite the underlying data source with the data of the
+input query, to make sure the table gets created contains exactly the same data as the input query.
 
 ### Examples
-{% highlight sql %}
 
---Using data source
-CREATE TABLE Student (Id INT,name STRING ,age INT) USING CSV;
+```sql
 
---Using data from another table
-CREATE TABLE StudentInfo
-  AS SELECT * FROM Student;
+--Use data source
+CREATE TABLE student (id INT, name STRING, age INT) USING CSV;
 
---Partitioned and bucketed
-CREATE TABLE Student (Id INT,name STRING ,age INT)
-  USING CSV
-  PARTITIONED BY (age)
-  CLUSTERED BY (Id) INTO 4 buckets;
+--Use data from another table
+CREATE TABLE student_copy USING CSV
+    AS SELECT * FROM student;
+  
+--Omit the USING clause, which uses the default data source (parquet by default)
+CREATE TABLE student (id INT, name STRING, age INT);
 
-{% endhighlight %}
+--Specify table comment and properties
+CREATE TABLE student (id INT, name STRING, age INT) USING CSV
+    COMMENT 'this is a comment'
+    TBLPROPERTIES ('foo'='bar');
+
+--Specify table comment and properties with different clauses order
+CREATE TABLE student (id INT, name STRING, age INT) USING CSV
+    TBLPROPERTIES ('foo'='bar')
+    COMMENT 'this is a comment';
+
+--Create partitioned and bucketed table
+CREATE TABLE student (id INT, name STRING, age INT)
+    USING CSV
+    PARTITIONED BY (age)
+    CLUSTERED BY (Id) INTO 4 buckets;
+```
 
 ### Related Statements
+
 * [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
 * [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
diff --git a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
index 78b7d0581e985..3a8c8d5b1160a 100644
--- a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
+++ b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
@@ -18,105 +18,226 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 The `CREATE TABLE` statement defines a new table using Hive format.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 CREATE [ EXTERNAL ] TABLE [ IF NOT EXISTS ] table_identifier
-  [ ( col_name1[:] col_type1 [ COMMENT col_comment1 ], ... ) ]
-  [ COMMENT table_comment ]
-  [ PARTITIONED BY ( col_name2[:] col_type2 [ COMMENT col_comment2 ], ... ) 
-      | ( col_name1, col_name2, ... ) ]
-  [ ROW FORMAT row_format ]
-  [ STORED AS file_format ]
-  [ LOCATION path ]
-  [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
-  [ AS select_statement ]
-
-{% endhighlight %}
+    [ ( col_name1[:] col_type1 [ COMMENT col_comment1 ], ... ) ]
+    [ COMMENT table_comment ]
+    [ PARTITIONED BY ( col_name2[:] col_type2 [ COMMENT col_comment2 ], ... ) 
+        | ( col_name1, col_name2, ... ) ]
+    [ CLUSTERED BY ( col_name1, col_name2, ...) 
+        [ SORTED BY ( col_name1 [ ASC | DESC ], col_name2 [ ASC | DESC ], ... ) ] 
+        INTO num_buckets BUCKETS ]
+    [ ROW FORMAT row_format ]
+    [ STORED AS file_format ]
+    [ LOCATION path ]
+    [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
+    [ AS select_statement ]
+
+row_format:    
+    : SERDE serde_class [ WITH SERDEPROPERTIES (k1=v1, k2=v2, ... ) ]
+    | DELIMITED [ FIELDS TERMINATED BY fields_termiated_char [ ESCAPED BY escaped_char ] ] 
+        [ COLLECTION ITEMS TERMINATED BY collection_items_termiated_char ] 
+        [ MAP KEYS TERMINATED BY map_key_termiated_char ]
+        [ LINES TERMINATED BY row_termiated_char ]
+        [ NULL DEFINED AS null_char ]
+```
+
+Note that, the clauses between the columns definition clause and the AS SELECT clause can come in
+as any order. For example, you can write COMMENT table_comment after TBLPROPERTIES.
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>EXTERNAL</em></code></dt>
-  <dd>Table is defined using the path provided as LOCATION, does not use default location for this table.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>PARTITIONED BY</em></code></dt>
-  <dd>Partitions are created on the table, based on the columns specified.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>ROW FORMAT</em></code></dt>
-  <dd>SERDE is used to specify a custom SerDe or the DELIMITED clause in order to use the native SerDe.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>STORED AS</em></code></dt>
-    <dd>File format for table storage, could be TEXTFILE, ORC, PARQUET,etc.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>LOCATION</em></code></dt>
-  <dd>Path to the directory where table data is stored, Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>COMMENT</em></code></dt>
-  <dd>Table comments are added.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>TBLPROPERTIES</em></code></dt>
-  <dd>
-	Table properties that have to be set are specified, such as `created.by.user`, `owner`, etc.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>AS select_statement</em></code></dt>
-  <dd>The table is populated using the data from the select statement.</dd>
-</dl>
+* **table_identifier**
 
+    Specifies a table name, which may be optionally qualified with a database name.
 
-### Examples
-{% highlight sql %}
+    **Syntax:** `[ database_name. ] table_name`
+
+* **EXTERNAL**
+
+    Table is defined using the path provided as `LOCATION`, does not use default location for this table.
+
+* **PARTITIONED BY**
+
+    Partitions are created on the table, based on the columns specified.
+    
+* **CLUSTERED BY**
+
+    Partitions created on the table will be bucketed into fixed buckets based on the column specified for bucketing.
+
+    **NOTE:** Bucketing is an optimization technique that uses buckets (and bucketing columns) to determine data partitioning and avoid data shuffle.
+
+* **SORTED BY**
+
+    Specifies an ordering of bucket columns. Optionally, one can use ASC for an ascending order or DESC for a descending order after any column names in the SORTED BY clause.
+    If not specified, ASC is assumed by default.
+
+* **INTO num_buckets BUCKETS**
+
+    Specifies buckets numbers, which is used in `CLUSTERED BY` clause.
+    
+* **row_format**    
+
+    Use the `SERDE` clause to specify a custom SerDe for one table. Otherwise, use the `DELIMITED` clause to use the native SerDe and specify the delimiter, escape character, null character and so on.
+    
+* **SERDE**
+
+    Specifies a custom SerDe for one table.
+    
+* **serde_class**
+
+    Specifies a fully-qualified class name of a custom SerDe.
+
+* **SERDEPROPERTIES**
 
---Using Comment and loading data from another table into the created table
-CREATE TABLE StudentInfo
-  COMMENT 'Table is created using existing data'
-  AS SELECT * FROM Student;
+    A list of key-value pairs that is used to tag the SerDe definition.
+    
+* **DELIMITED**
 
---Partitioned table
-CREATE TABLE Student (Id INT,name STRING)
-  PARTITIONED BY (age INT)
-  TBLPROPERTIES ('owner'='xxxx');
+    The `DELIMITED` clause can be used to specify the native SerDe and state the delimiter, escape character, null character and so on.
+    
+* **FIELDS TERMINATED BY**
 
-CREATE TABLE Student (Id INT,name STRING,age INT)
-  PARTITIONED BY (name,age);
+    Used to define a column separator.
+    
+* **COLLECTION ITEMS TERMINATED BY**
 
---Using Row Format and file format
-CREATE TABLE Student (Id INT,name STRING)
-  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-  STORED AS TEXTFILE;
+    Used to define a collection item separator.
+   
+* **MAP KEYS TERMINATED BY**
 
-{% endhighlight %}
+    Used to define a map key separator.
+    
+* **LINES TERMINATED BY**
 
+    Used to define a row separator.
+    
+* **NULL DEFINED AS**
+
+    Used to define the specific value for NULL.
+    
+* **ESCAPED BY**
+
+    Used for escape mechanism.
+
+* **STORED AS**
+
+    File format for table storage, could be TEXTFILE, ORC, PARQUET, etc.
+
+* **LOCATION**
+
+    Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc.
+
+* **COMMENT**
+
+    A string literal to describe the table.
+
+* **TBLPROPERTIES**
+
+    A list of key-value pairs that is used to tag the table definition.
+
+* **AS select_statement**
+
+    The table is populated using the data from the select statement.
+
+### Examples
+
+```sql
+--Use hive format
+CREATE TABLE student (id INT, name STRING, age INT) STORED AS ORC;
+
+--Use data from another table
+CREATE TABLE student_copy STORED AS ORC
+    AS SELECT * FROM student;
+
+--Specify table comment and properties
+CREATE TABLE student (id INT, name STRING, age INT)
+    COMMENT 'this is a comment'
+    STORED AS ORC
+    TBLPROPERTIES ('foo'='bar');  
+
+--Specify table comment and properties with different clauses order
+CREATE TABLE student (id INT, name STRING, age INT)
+    STORED AS ORC
+    TBLPROPERTIES ('foo'='bar')
+    COMMENT 'this is a comment';
+
+--Create partitioned table
+CREATE TABLE student (id INT, name STRING)
+    PARTITIONED BY (age INT)
+    STORED AS ORC;
+
+--Create partitioned table with different clauses order
+CREATE TABLE student (id INT, name STRING)
+    STORED AS ORC
+    PARTITIONED BY (age INT);
+
+--Use Row Format and file format
+CREATE TABLE student (id INT, name STRING)
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+    STORED AS TEXTFILE;
+
+--Use complex datatype
+CREATE EXTERNAL TABLE family(
+        name STRING,
+        friends ARRAY<STRING>,
+        children MAP<STRING, INT>,
+        address STRUCT<street: STRING, city: STRING>
+    )
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' ESCAPED BY '\\'
+    COLLECTION ITEMS TERMINATED BY '_'
+    MAP KEYS TERMINATED BY ':'
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'foonull'
+    STORED AS TEXTFILE
+    LOCATION '/tmp/family/';
+
+--Use predefined custom SerDe
+CREATE TABLE avroExample
+    ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+    STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+        OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+    TBLPROPERTIES ('avro.schema.literal'='{ "namespace": "org.apache.hive",
+        "name": "first_schema",
+        "type": "record",
+        "fields": [
+                { "name":"string1", "type":"string" },
+                { "name":"string2", "type":"string" }
+            ] }');
+
+--Use personalized custom SerDe(we may need to `ADD JAR xxx.jar` first to ensure we can find the serde_class,
+--or you may run into `CLASSNOTFOUND` exception)
+ADD JAR /tmp/hive_serde_example.jar;
+
+CREATE EXTERNAL TABLE family (id INT, name STRING)
+    ROW FORMAT SERDE 'com.ly.spark.serde.SerDeExample'
+    STORED AS INPUTFORMAT 'com.ly.spark.example.serde.io.SerDeExampleInputFormat'
+        OUTPUTFORMAT 'com.ly.spark.example.serde.io.SerDeExampleOutputFormat'
+    LOCATION '/tmp/family/';
+
+--Use `CLUSTERED BY` clause to create bucket table without `SORTED BY`
+CREATE TABLE clustered_by_test1 (ID INT, AGE STRING)
+    CLUSTERED BY (ID)
+    INTO 4 BUCKETS
+    STORED AS ORC
+
+--Use `CLUSTERED BY` clause to create bucket table with `SORTED BY`
+CREATE TABLE clustered_by_test2 (ID INT, NAME STRING)
+    PARTITIONED BY (YEAR STRING)
+    CLUSTERED BY (ID, NAME)
+    SORTED BY (ID ASC)
+    INTO 3 BUCKETS
+    STORED AS PARQUET
+```
 
 ### Related Statements
+
 * [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
 * [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
diff --git a/docs/sql-ref-syntax-ddl-create-table-like.md b/docs/sql-ref-syntax-ddl-create-table-like.md
index b7d7bdd1d9805..cfb959ca6b23d 100644
--- a/docs/sql-ref-syntax-ddl-create-table-like.md
+++ b/docs/sql-ref-syntax-ddl-create-table-like.md
@@ -18,80 +18,71 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 The `CREATE TABLE` statement defines a new table using the definition/metadata of an existing table or view.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 CREATE TABLE [IF NOT EXISTS] table_identifier LIKE source_table_identifier
-USING data_source
-[ ROW FORMAT row_format ]
-[ STORED AS file_format ]
-[ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
-[ LOCATION path ]
-{% endhighlight %}
+    USING data_source
+    [ ROW FORMAT row_format ]
+    [ STORED AS file_format ]
+    [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
+    [ LOCATION path ]
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>  [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>USING data_source</em></code></dt>
-  <dd>Data Source is the input format used to create the table. Data source can be CSV, TXT, ORC, JDBC, PARQUET, etc.</dd>
-</dl> 
-
-<dl>
-  <dt><code><em>ROW FORMAT</em></code></dt>
-  <dd>SERDE is used to specify a custom SerDe or the DELIMITED clause in order to use the native SerDe.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>STORED AS</em></code></dt>
-  <dd>File format for table storage, could be TEXTFILE, ORC, PARQUET,etc.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>TBLPROPERTIES</em></code></dt>
-  <dd>Table properties that have to be set are specified, such as `created.by.user`, `owner`, etc.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>LOCATION</em></code></dt>
-  <dd>Path to the directory where table data is stored,Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc. Location to create an external table.</dd>
-</dl>
 
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **USING data_source**
+
+    Data Source is the input format used to create the table. Data source can be CSV, TXT, ORC, JDBC, PARQUET, etc.
+
+* **ROW FORMAT**
+
+    SERDE is used to specify a custom SerDe or the DELIMITED clause in order to use the native SerDe.
+
+* **STORED AS**
+
+    File format for table storage, could be TEXTFILE, ORC, PARQUET, etc.
+
+* **TBLPROPERTIES**
+
+    Table properties that have to be set are specified, such as `created.by.user`, `owner`, etc.
+
+* **LOCATION**
+
+    Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc. Location to create an external table.
 
 ### Examples
-{% highlight sql %}
 
---Create table using an exsisting table
+```sql
+-- Create table using an existing table
 CREATE TABLE Student_Dupli like Student;
 
---Create table like using a data source
+-- Create table like using a data source
 CREATE TABLE Student_Dupli like Student USING CSV;
 
---Table is created as external table at the location specified
+-- Table is created as external table at the location specified
 CREATE TABLE Student_Dupli like Student location  '/root1/home';
 
---Create table like using a rowformat
+-- Create table like using a rowformat
 CREATE TABLE Student_Dupli like Student
-  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-  STORED AS TEXTFILE
-  TBLPROPERTIES ('owner'='xxxx');
-
-{% endhighlight %}
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+    STORED AS TEXTFILE
+    TBLPROPERTIES ('owner'='xxxx');
+```
 
 ### Related Statements
+
 * [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
 * [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
 
diff --git a/docs/sql-ref-syntax-ddl-create-table.md b/docs/sql-ref-syntax-ddl-create-table.md
index c0e58a41cf5cc..85dc2020e6585 100644
--- a/docs/sql-ref-syntax-ddl-create-table.md
+++ b/docs/sql-ref-syntax-ddl-create-table.md
@@ -20,13 +20,16 @@ license: |
 ---
 
 ### Description
-`CREATE TABLE` statement is used to define a table in an exsisting database. 
+
+`CREATE TABLE` statement is used to define a table in an existing database. 
 
 The CREATE statements:
-* [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
+
+* [CREATE TABLE USING DATA_SOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
 * [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
 * [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
 
 ### Related Statements
-- [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+
+* [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+* [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
diff --git a/docs/sql-ref-syntax-ddl-create-view.md b/docs/sql-ref-syntax-ddl-create-view.md
index 67060d70f0790..1a9c1f62728e7 100644
--- a/docs/sql-ref-syntax-ddl-create-view.md
+++ b/docs/sql-ref-syntax-ddl-create-view.md
@@ -20,58 +20,54 @@ license: |
 ---
 
 ### Description
+
 Views are based on the result-set of an `SQL` query. `CREATE VIEW` constructs
 a virtual table that has no physical data therefore other operations like
 `ALTER VIEW` and `DROP VIEW` only change metadata. 
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 CREATE [ OR REPLACE ] [ [ GLOBAL ] TEMPORARY ] VIEW [ IF NOT EXISTS ] view_identifier
     create_view_clauses AS query
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>OR REPLACE</em></code></dt>
-  <dd>If a view of same name already exists, it will be replaced.</dd>
-</dl>
-<dl>
-  <dt><code><em>[ GLOBAL ] TEMPORARY</em></code></dt>
-  <dd>TEMPORARY views are session-scoped and will be dropped when session ends 
-      because it skips persisting the definition in the underlying metastore, if any.
-      GLOBAL TEMPORARY views are tied to a system preserved temporary database `global_temp`.</dd>
-</dl>
-<dl>
-  <dt><code><em>IF NOT EXISTS</em></code></dt>
-  <dd>Creates a view if it does not exists.</dd>
-</dl>
-<dl>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies a view name, which may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>create_view_clauses</em></code></dt>
-  <dd>These clauses are optional and order insensitive. It can be of following formats.
-    <ul>
-      <li><code>[ ( column_name [ COMMENT column_comment ], ... ) ]</code> to specify column-level comments.</li>
-      <li><code>[ COMMENT view_comment ]</code> to specify view-level comments.</li>
-      <li><code>[ TBLPROPERTIES ( property_name = property_value, ... ) ]</code> to add metadata key-value pairs.</li>
-    </ul>  
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A <a href="sql-ref-syntax-qry-select.html">SELECT</a> statement that constructs the view from base tables or other views.</dd>
-</dl>
+
+* **OR REPLACE**
+
+    If a view of same name already exists, it will be replaced.
+
+* **[ GLOBAL ] TEMPORARY**
+
+    TEMPORARY views are session-scoped and will be dropped when session ends
+    because it skips persisting the definition in the underlying metastore, if any.
+    GLOBAL TEMPORARY views are tied to a system preserved temporary database `global_temp`.
+
+* **IF NOT EXISTS**
+
+    Creates a view if it does not exist.
+
+* **view_identifier**
+
+    Specifies a view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
+
+* **create_view_clauses**
+
+    These clauses are optional and order insensitive. It can be of following formats.
+
+    * `[ ( column_name [ COMMENT column_comment ], ... ) ]` to specify column-level comments.
+    * `[ COMMENT view_comment ]` to specify view-level comments.
+    * `[ TBLPROPERTIES ( property_name = property_value [ , ... ] ) ]` to add metadata key-value pairs.
+
+* **query**
+  A [SELECT](sql-ref-syntax-qry-select.html) statement that constructs the view from base tables or other views.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 -- Create or replace view for `experienced_employee` with comments.
 CREATE OR REPLACE VIEW experienced_employee
     (ID COMMENT 'Unique identification number', Name) 
@@ -84,8 +80,10 @@ CREATE GLOBAL TEMPORARY VIEW IF NOT EXISTS subscribed_movies
     AS SELECT mo.member_id, mb.full_name, mo.movie_title
         FROM movies AS mo INNER JOIN members AS mb 
         ON mo.member_id = mb.id;
-{% endhighlight %}
+```
 
 ### Related Statements
-- [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
-- [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+
+* [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+* [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+* [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-database.md b/docs/sql-ref-syntax-ddl-drop-database.md
index 68244648c9a5b..4a3bc0c68b6d4 100644
--- a/docs/sql-ref-syntax-ddl-drop-database.md
+++ b/docs/sql-ref-syntax-ddl-drop-database.md
@@ -26,55 +26,43 @@ exception will be thrown if the database does not exist in the system.
 
 ### Syntax
 
-{% highlight sql %}
-DROP ( DATABASE | SCHEMA ) [ IF EXISTS ] dbname [ RESTRICT | CASCADE ]
-{% endhighlight %}
-
+```sql
+DROP { DATABASE | SCHEMA } [ IF EXISTS ] dbname [ RESTRICT | CASCADE ]
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>DATABASE | SCHEMA</em></code></dt>
-  <dd>`DATABASE` and `SCHEMA` mean the same thing, either of them can be used.</dd>
-</dl>
+* **DATABASE `|` SCHEMA**
+
+    `DATABASE` and `SCHEMA` mean the same thing, either of them can be used.
+
+* **IF EXISTS**
+
+    If specified, no exception is thrown when the database does not exist.
+
+* **RESTRICT**
+
+    If specified, will restrict dropping a non-empty database and is enabled by default.
 
-<dl>
-  <dt><code><em>IF EXISTS</em></code></dt>
-  <dd>If specified, no exception is thrown when the database does not exist.</dd>
-</dl>
+* **CASCADE**
 
-<dl>
-  <dt><code><em>RESTRICT</em></code></dt>
-  <dd>If specified, will restrict dropping a non-empty database and is enabled by default.</dd>
-</dl>
+    If specified, will drop all the associated tables and functions.
 
-<dl>
-  <dt><code><em>CASCADE</em></code></dt>
-  <dd>If specified, will drop all the associated tables and functions.</dd>
-</dl>
+### Examples
 
-### Example
-{% highlight sql %}
+```sql
 -- Create `inventory_db` Database
 CREATE DATABASE inventory_db COMMENT 'This database is used to maintain Inventory';
 
 -- Drop the database and it's tables
 DROP DATABASE inventory_db CASCADE;
-+---------+
-| Result  |
-+---------+
-+---------+
 
 -- Drop the database using IF EXISTS
 DROP DATABASE IF EXISTS inventory_db CASCADE;
-+---------+
-| Result  |
-+---------+
-+---------+
+```
 
-{% endhighlight %}
+### Related Statements
 
-### Related statements
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
\ No newline at end of file
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-function.md b/docs/sql-ref-syntax-ddl-drop-function.md
index a059960920bef..bef31d74afcff 100644
--- a/docs/sql-ref-syntax-ddl-drop-function.md
+++ b/docs/sql-ref-syntax-ddl-drop-function.md
@@ -20,86 +20,83 @@ license: |
 ---
 
 ### Description
+
 The `DROP FUNCTION` statement drops a temporary or user defined function (UDF). An exception will
- be thrown if the function does not exist. 
+be thrown if the function does not exist. 
 
 ### Syntax
-{% highlight sql %}
-DROP [ TEMPORARY ] FUNCTION [ IF EXISTS ] [ db_name. ] function_name
-{% endhighlight %}
 
+```sql
+DROP [ TEMPORARY ] FUNCTION [ IF EXISTS ] function_name
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>function_name</em></code></dt>
-  <dd>The name of an existing function.</dd>
-</dl>
+* **function_name**
+
+    Specifies the name of an existing function. The function name may be
+    optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] function_name`
+
+* **TEMPORARY**
 
-<dl>
-  <dt><code><em>TEMPORARY</em></code></dt>
-  <dd>Should be used to delete the `temporary` function.</dd>
-</dl>
+    Should be used to delete the `TEMPORARY` function.
 
-<dl>
-  <dt><code><em>IF EXISTS</em></code></dt>
-  <dd>If specified, no exception is thrown when the function does not exist.</dd>
-</dl>
+* **IF EXISTS**
 
-### Example
-{% highlight sql %}
+    If specified, no exception is thrown when the function does not exist.
+
+### Examples
+
+```sql
 -- Create a permanent function `test_avg`
-CREATE FUNCTION test_avg as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage';
+CREATE FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage';
 
 -- List user functions
 SHOW USER FUNCTIONS;
-  +-------------------+
-  |     function      |
-  +-------------------+
-  | default.test_avg  |
-  +-------------------+
++----------------+
+|        function|
++----------------+
+|default.test_avg|
++----------------+
 
 -- Create Temporary function `test_avg`
-CREATE TEMPORARY FUNCTION test_avg as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage';
+CREATE TEMPORARY FUNCTION test_avg AS
+    'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage';
 
 -- List user functions
 SHOW USER FUNCTIONS;
-  +-------------------+
-  |     function      |
-  +-------------------+
-  | default.test_avg  |
-  | test_avg          |
-  +-------------------+
++----------------+
+|        function|
++----------------+
+|default.test_avg|
+|        test_avg|
++----------------+
 
 -- Drop Permanent function
 DROP FUNCTION test_avg;
-  +---------+
-  | Result  |
-  +---------+
-  +---------+
 
 -- Try to drop Permanent function which is not present
 DROP FUNCTION test_avg;
-  Error: Error running query:
-  org.apache.spark.sql.catalyst.analysis.NoSuchPermanentFunctionException:
-  Function 'default.test_avg' not found in database 'default'; (state=,code=0)
+Error: Error running query:
+org.apache.spark.sql.catalyst.analysis.NoSuchPermanentFunctionException:
+Function 'default.test_avg' not found in database 'default'; (state=,code=0)
 
 -- List the functions after dropping, it should list only temporary function
 SHOW USER FUNCTIONS;
-  +-----------+
-  | function  |
-  +-----------+
-  | test_avg  |
-  +-----------+
++--------+
+|function|
++--------+
+|test_avg|
++--------+
   
 -- Drop Temporary function
 DROP TEMPORARY FUNCTION IF EXISTS test_avg;
-  +---------+
-  | Result  |
-  +---------+
-  +---------+
-{% endhighlight %}
-### Related statements
-- [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
-- [SHOW FUNCTION](sql-ref-syntax-aux-show-functions.html)
\ No newline at end of file
+```
+
+### Related Statements
+
+* [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+* [SHOW FUNCTION](sql-ref-syntax-aux-show-functions.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-table.md b/docs/sql-ref-syntax-ddl-drop-table.md
index d1d8534efe7a2..a15a9928f437d 100644
--- a/docs/sql-ref-syntax-ddl-drop-table.md
+++ b/docs/sql-ref-syntax-ddl-drop-table.md
@@ -27,61 +27,45 @@ if the table is not `EXTERNAL` table. If the table is not present it throws an e
 In case of an external table, only the associated metadata information is removed from the metastore database.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 DROP TABLE [ IF EXISTS ] table_identifier
-{% endhighlight %}
+```
 
 ### Parameter
-<dl>
-  <dt><code><em>IF EXISTS</em></code></dt>
-  <dd>
-     If specified, no exception is thrown when the table does not exists.
-  </dd>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies the table name to be dropped. The table name may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-### Example
-{% highlight sql %}
+
+* **IF EXISTS**
+
+    If specified, no exception is thrown when the table does not exist.
+
+* **table_identifier**
+
+    Specifies the table name to be dropped. The table name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+### Examples
+
+```sql
 -- Assumes a table named `employeetable` exists.
 DROP TABLE employeetable;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
 
 -- Assumes a table named `employeetable` exists in the `userdb` database
 DROP TABLE userdb.employeetable;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
 
--- Assumes a table named `employeetable` does not exists.
+-- Assumes a table named `employeetable` does not exist.
 -- Throws exception
 DROP TABLE employeetable;
 Error: org.apache.spark.sql.AnalysisException: Table or view not found: employeetable;
 (state=,code=0)
 
--- Assumes a table named `employeetable` does not exists,Try with IF EXISTS
+-- Assumes a table named `employeetable` does not exist,Try with IF EXISTS
 -- this time it will not throw exception
 DROP TABLE IF EXISTS employeetable;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
-
-{% endhighlight %}
+```
 
 ### Related Statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
-
 
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-view.md b/docs/sql-ref-syntax-ddl-drop-view.md
index bf9e497b3a418..5b680d7f907e0 100644
--- a/docs/sql-ref-syntax-ddl-drop-view.md
+++ b/docs/sql-ref-syntax-ddl-drop-view.md
@@ -20,63 +20,51 @@ license: |
 ---
 
 ### Description
+
 `DROP VIEW` removes the metadata associated with a specified view from the catalog.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 DROP VIEW [ IF EXISTS ] view_identifier
-{% endhighlight %}
+```
 
 ### Parameter
-<dl>
-  <dt><code><em>IF EXISTS</em></code></dt>
-  <dd>
-     If specified, no exception is thrown when the view does not exists.
-  </dd>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies the view name to be dropped. The view name may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-</dl>
-
-### Example
-{% highlight sql %}
+
+* **IF EXISTS**
+
+    If specified, no exception is thrown when the view does not exist.
+
+* **view_identifier**
+
+    Specifies the view name to be dropped. The view name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
+
+### Examples
+
+```sql
 -- Assumes a view named `employeeView` exists.
 DROP VIEW employeeView;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
 
 -- Assumes a view named `employeeView` exists in the `userdb` database
 DROP VIEW userdb.employeeView;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
 
--- Assumes a view named `employeeView` does not exists.
+-- Assumes a view named `employeeView` does not exist.
 -- Throws exception
 DROP VIEW employeeView;
 Error: org.apache.spark.sql.AnalysisException: Table or view not found: employeeView;
 (state=,code=0)
 
--- Assumes a view named `employeeView` does not exists,Try with IF EXISTS
+-- Assumes a view named `employeeView` does not exist,Try with IF EXISTS
 -- this time it will not throw exception
 DROP VIEW IF EXISTS employeeView;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
-
-{% endhighlight %}
+```
 
 ### Related Statements
-- [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
-- [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+
+* [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+* [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+* [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-ddl-repair-table.md b/docs/sql-ref-syntax-ddl-repair-table.md
index daa6a46fca58f..c2ef0a7b7fbe9 100644
--- a/docs/sql-ref-syntax-ddl-repair-table.md
+++ b/docs/sql-ref-syntax-ddl-repair-table.md
@@ -20,50 +20,49 @@ license: |
 ---
 
 ### Description
+
 `MSCK REPAIR TABLE` recovers all the partitions in the directory of a table and updates the Hive metastore. When creating a table using `PARTITIONED BY` clause, partitions are generated and registered in the Hive metastore. However, if the partitioned table is created from existing data, partitions are not registered automatically in the Hive metastore. User needs to run `MSCK REPAIR TABLE` to register the partitions. `MSCK REPAIR TABLE` on a non-existent table or a table without partitions throws an exception. Another way to recover partitions is to use `ALTER TABLE RECOVER PARTITIONS`.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 MSCK REPAIR TABLE table_identifier
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies the name of the table to be repaired. The table name may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
+
+* **table_identifier**
+
+    Specifies the name of the table to be repaired. The table name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
 
 ### Examples
-{% highlight sql %}
- -- create a partitioned table from existing data /tmp/namesAndAges.parquet
- CREATE TABLE t1 (name STRING, age INT) USING parquet PARTITIONED BY (age)
-     location "/tmp/namesAndAges.parquet";
 
- -- SELECT * FROM t1 does not return results
- SELECT * FROM t1;
+```sql
+-- create a partitioned table from existing data /tmp/namesAndAges.parquet
+CREATE TABLE t1 (name STRING, age INT) USING parquet PARTITIONED BY (age)
+    LOCATION "/tmp/namesAndAges.parquet";
 
- -- run MSCK REPAIR TABLE to recovers all the partitions
- MSCK REPAIR TABLE t1;
+-- SELECT * FROM t1 does not return results
+SELECT * FROM t1;
 
- -- SELECT * FROM t1 returns results
- SELECT * FROM t1;
+-- run MSCK REPAIR TABLE to recovers all the partitions
+MSCK REPAIR TABLE t1;
 
-     + -------------- + ------+
-     | name           | age   |
-     + -------------- + ------+
-     | Michael        | 20    |
-     + -------------- + ------+
-     | Justin         | 19    |
-     + -------------- + ----- +
-     | Andy           | 30    |
-     + -------------- + ----- +
+-- SELECT * FROM t1 returns results
+SELECT * FROM t1;
++-------+---+
+|   name|age|
++-------+---+
+|Michael| 20|
++-------+---+
+| Justin| 19|
++-------+---+
+|   Andy| 30|
++-------+---+
+```
 
-{% endhighlight %}
 ### Related Statements
- * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+
+* [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
diff --git a/docs/sql-ref-syntax-ddl-truncate-table.md b/docs/sql-ref-syntax-ddl-truncate-table.md
index 3a0569e809d84..6139814a3259a 100644
--- a/docs/sql-ref-syntax-ddl-truncate-table.md
+++ b/docs/sql-ref-syntax-ddl-truncate-table.md
@@ -20,79 +20,69 @@ license: |
 ---
 
 ### Description
+
 The `TRUNCATE TABLE` statement removes all the rows from a table or partition(s). The table must not be a view 
 or an external/temporary table. In order to truncate multiple partitions at once, the user can specify the partitions 
 in `partition_spec`. If no `partition_spec` is specified it will remove all partitions in the table.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 TRUNCATE TABLE table_identifier [ partition_spec ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>
+    for partitions.
 
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
 
 ### Examples
-{% highlight sql %}
 
---Create table Student with partition
-CREATE TABLE Student ( name String, rollno INT) PARTITIONED BY (age int);
+```sql
+-- Create table Student with partition
+CREATE TABLE Student (name STRING, rollno INT) PARTITIONED BY (age INT);
 
-SELECT * from Student;
-+-------+---------+------+--+
-| name  | rollno  | age  |
-+-------+---------+------+--+
-| ABC   | 1       | 10   |
-| DEF   | 2       | 10   |
-| XYZ   | 3       | 12   |
-+-------+---------+------+--+
+SELECT * FROM Student;
++----+------+---+
+|name|rollno|age|
++----+------+---+
+| ABC|     1| 10|
+| DEF|     2| 10|
+| XYZ|     3| 12|
++----+------+---+
 
 -- Removes all rows from the table in the partition specified
 TRUNCATE TABLE Student partition(age=10);
 
---After truncate execution, records belonging to partition age=10 are removed
-SELECT * from Student;
-+-------+---------+------+--+
-| name  | rollno  | age  |
-+-------+---------+------+--+
-| XYZ   | 3       | 12   |
-+-------+---------+------+--+
+-- After truncate execution, records belonging to partition age=10 are removed
+SELECT * FROM Student;
++----+------+---+
+|name|rollno|age|
++----+------+---+
+| XYZ|     3| 12|
++----+------+---+
 
 -- Removes all rows from the table from all partitions
 TRUNCATE TABLE Student;
 
-SELECT * from Student;
-+-------+---------+------+--+
-| name  | rollno  | age  |
-+-------+---------+------+--+
-+-------+---------+------+--+
-No rows selected 
-
-{% endhighlight %}
-
+SELECT * FROM Student;
++----+------+---+
+|name|rollno|age|
++----+------+---+
++----+------+---+
+```
 
 ### Related Statements
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
-- [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
 
+* [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+* [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
diff --git a/docs/sql-ref-syntax-qry-select-usedb.md b/docs/sql-ref-syntax-ddl-usedb.md
similarity index 70%
rename from docs/sql-ref-syntax-qry-select-usedb.md
rename to docs/sql-ref-syntax-ddl-usedb.md
index 92ac91ac51769..90076e0306125 100644
--- a/docs/sql-ref-syntax-qry-select-usedb.md
+++ b/docs/sql-ref-syntax-ddl-usedb.md
@@ -20,41 +20,38 @@ license: |
 ---
 
 ### Description
+
 `USE` statement is used to set the current database. After the current database is set,
 the unqualified database artifacts such as tables, functions and views that are 
 referenced by SQLs are resolved from the current database. 
 The default database name is 'default'.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 USE database_name
-{% endhighlight %}
+```
 
 ### Parameter
 
-<dl>
-  <dt><code><em>database_name</em></code></dt>
-  <dd>
-     Name of the database will be used. If the database does not exist, an exception will be thrown.
-  </dd>
-</dl>
+* **database_name**
+
+    Name of the database will be used. If the database does not exist, an exception will be thrown.
+
+### Examples
 
-### Example
-{% highlight sql %}
+```sql
 -- Use the 'userdb' which exists.
 USE userdb;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
 
 -- Use the 'userdb1' which doesn't exist
 USE userdb1;
-Error: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: Database 'userdb1' not found;(state=,code=0)
-{% endhighlight %}
+Error: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: Database 'userdb1' not found;
+(state=,code=0)
+```
 
-### Related statements.
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
-- [CREATE TABLE ](sql-ref-syntax-ddl-create-table.html)
+### Related Statements
 
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+* [CREATE TABLE ](sql-ref-syntax-ddl-create-table.html)
diff --git a/docs/sql-ref-syntax-ddl.md b/docs/sql-ref-syntax-ddl.md
index 954020a874455..cb3e04c0ec910 100644
--- a/docs/sql-ref-syntax-ddl.md
+++ b/docs/sql-ref-syntax-ddl.md
@@ -19,20 +19,19 @@ license: |
   limitations under the License.
 ---
 
-
 Data Definition Statements are used to create or modify the structure of database objects in a database. Spark SQL supports the following Data Definition Statements:
 
-
-- [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
-- [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
-- [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
-- [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
-- [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
-- [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
-- [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
+ * [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
+ * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+ * [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+ * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+ * [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
+ * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+ * [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+ * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+ * [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
+ * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+ * [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+ * [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
+ * [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
+ * [USE DATABASE](sql-ref-syntax-ddl-usedb.html)
diff --git a/docs/sql-ref-syntax-dml-insert-into.md b/docs/sql-ref-syntax-dml-insert-into.md
index 715f43c9b80ea..ed5da2b2d28df 100644
--- a/docs/sql-ref-syntax-dml-insert-into.md
+++ b/docs/sql-ref-syntax-dml-insert-into.md
@@ -24,195 +24,182 @@ license: |
 The `INSERT INTO` statement inserts new rows into a table. The inserted rows can be specified by value expressions or result from a query.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 INSERT INTO [ TABLE ] table_identifier [ partition_spec ]
-    { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
-{% endhighlight %}
+    { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION (partition_col_name  = partition_col_val [ , ... ])
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ]</em></code></dt>
-  <dd>Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted. A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A query that produces the rows to be inserted. It can be in one of following formats:
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-   </dd>
-</dl>
+    for partitions.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
+
+* **VALUES ( { value `|` NULL } [ , ... ] ) [ , ( ... ) ]**
+
+    Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
+    A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
+
+* **query**
+
+    A query that produces the rows to be inserted. It can be in one of following formats:
+    * a `SELECT` statement
+    * a `TABLE` statement
+    * a `FROM` statement
 
 ### Examples
+
 #### Single Row Insert Using a VALUES Clause
-{% highlight sql %}
- CREATE TABLE students (name VARCHAR(64), address VARCHAR(64), student_id INT)
-     USING PARQUET PARTITIONED BY (student_id);
 
- INSERT INTO students
-     VALUES ('Amy Smith', '123 Park Ave, San Jose', 111111);
+```sql
+CREATE TABLE students (name VARCHAR(64), address VARCHAR(64), student_id INT)
+    USING PARQUET PARTITIONED BY (student_id);
 
- SELECT * FROM students;
+INSERT INTO students VALUES
+    ('Amy Smith', '123 Park Ave, San Jose', 111111);
 
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-{% endhighlight %}
+SELECT * FROM students;
++---------+---------------------+----------+
+|     name|              address|student_id|
++---------+---------------------+----------+
+|Amy Smith|123 Park Ave,San Jose|    111111|
++---------+---------------------+----------+
+```
 
 #### Multi-Row Insert Using a VALUES Clause
-{% highlight sql %}
- INSERT INTO students
-     VALUES ('Bob Brown', '456 Taylor St, Cupertino', 222222),
-            ('Cathy Johnson', '789 Race Ave, Palo Alto', 333333);
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Bob Brown      | 456 Taylor St, Cupertino       | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-     | Cathy Johnson  | 789 Race Ave, Palo Alto        | 333333         |
-     + -------------- + ------------------------------ + -------------- +
-{% endhighlight %}
+
+```sql
+INSERT INTO students VALUES
+    ('Bob Brown', '456 Taylor St, Cupertino', 222222),
+    ('Cathy Johnson', '789 Race Ave, Palo Alto', 333333);
+
+SELECT * FROM students;
++-------------+------------------------+----------+
+|         name|                 address|student_id|
++-------------+------------------------+----------+
+|    Amy Smith|  123 Park Ave, San Jose|    111111|
++-------------+------------------------+----------+
+|    Bob Brown|456 Taylor St, Cupertino|    222222|
++-------------+------------------------+----------+
+|Cathy Johnson| 789 Race Ave, Palo Alto|    333333|
++--------------+-----------------------+----------+
+```
 
 #### Insert Using a SELECT Statement
-{% highlight sql %}
- -- Assuming the persons table has already been created and populated.
- SELECT * FROM persons;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | ssn            |
-     + -------------- + ------------------------------ + -------------- +
-     | Dora Williams  | 134 Forest Ave, Melo Park      | 123456789      |
-     + -------------- + ------------------------------ + -------------- +
-     | Eddie Davis    | 245 Market St, Milpitas        | 345678901      |
-     + -------------- + ------------------------------ + ---------------+
-
- INSERT INTO students PARTITION (student_id = 444444)
-     SELECT name, address FROM persons WHERE name = "Dora Williams";
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Bob Brown      | 456 Taylor St, Cupertino       | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-     | Cathy Johnson  | 789 Race Ave, Palo Alto        | 333333         |
-     + -------------- + ------------------------------ + -------------- +
-     | Dora Williams  | 134 Forest Ave, Melo Park      | 444444         |
-     + -------------- + ------------------------------ + -------------- +
-{% endhighlight %}
+
+```sql
+-- Assuming the persons table has already been created and populated.
+SELECT * FROM persons;
++-------------+-------------------------+---------+
+|         name|                  address|      ssn|
++-------------+-------------------------+---------+
+|Dora Williams|134 Forest Ave, Melo Park|123456789|
++-------------+-------------------------+---------+
+|  Eddie Davis|  245 Market St, Milpitas|345678901|
++-------------+-------------------------+---------+
+
+INSERT INTO students PARTITION (student_id = 444444)
+    SELECT name, address FROM persons WHERE name = "Dora Williams";
+
+SELECT * FROM students;
++-------------+-------------------------+----------+
+|         name|                  address|student_id|
++-------------+-------------------------+----------+
+|    Amy Smith|   123 Park Ave, San Jose|    111111|
++-------------+-------------------------+----------+
+|    Bob Brown| 456 Taylor St, Cupertino|    222222|
++-------------+-------------------------+----------+
+|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
++-------------+-------------------------+----------+
+|Dora Williams|134 Forest Ave, Melo Park|    444444|
++-------------+-------------------------+----------+
+```
 
 #### Insert Using a TABLE Statement
-{% highlight sql %}
- -- Assuming the visiting_students table has already been created and populated.
- SELECT * FROM visiting_students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Fleur Laurent  | 345 Copper St, London          | 777777         |
-     + -------------- + ------------------------------ + -------------- +
-     | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-     + -------------- + ------------------------------ + -------------- +
-
- INSERT INTO students TABLE visiting_students;
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Bob Brown      | 456 Taylor St, Cupertino       | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-     | Cathy Johnson  | 789 Race Ave, Palo Alto        | 333333         |
-     + -------------- + ------------------------------ + -------------- +
-     | Dora Williams  | 134 Forest Ave, Melo Park      | 444444         |
-     + -------------- + ------------------------------ + -------------- +
-     | Fleur Laurent  | 345 Copper St, London          | 777777         |
-     + -------------- + ------------------------------ + -------------- +
-     | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-     + -------------- + ------------------------------ + -------------- +
-{% endhighlight %}
+
+```sql
+-- Assuming the visiting_students table has already been created and populated.
+SELECT * FROM visiting_students;
++-------------+---------------------+----------+
+|         name|              address|student_id|
++-------------+---------------------+----------+
+|Fleur Laurent|345 Copper St, London|    777777|
++-------------+---------------------+----------+
+|Gordon Martin| 779 Lake Ave, Oxford|    888888|
++-------------+---------------------+----------+
+
+INSERT INTO students TABLE visiting_students;
+
+SELECT * FROM students;
++-------------+-------------------------+----------+
+|         name|                  address|student_id|
++-------------+-------------------------+----------+
+|    Amy Smith|    123 Park Ave,San Jose|    111111|
++-------------+-------------------------+----------+
+|    Bob Brown| 456 Taylor St, Cupertino|    222222|
++-------------+-------------------------+----------+
+|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
++-------------+-------------------------+----------+
+|Dora Williams|134 Forest Ave, Melo Park|    444444|
++-------------+-------------------------+----------+
+|Fleur Laurent|    345 Copper St, London|    777777|
++-------------+-------------------------+----------+
+|Gordon Martin|     779 Lake Ave, Oxford|    888888|
++-------------+-------------------------+----------+
+```
 
 #### Insert Using a FROM Statement
-{% highlight sql %}
- -- Assuming the applicants table has already been created and populated.
- SELECT * FROM applicants;
-
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | name           | address                        | student_id     | qualified      |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Helen Davis    | 469 Mission St, San Diego      | 999999         | true           |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Ivy King       | 367 Leigh Ave, Santa Clara     | 101010         | false          |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Jason Wang     | 908 Bird St, Saratoga          | 121212         | true           |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-
- INSERT INTO students
-      FROM applicants SELECT name, address, id applicants WHERE qualified = true;
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Bob Brown      | 456 Taylor St, Cupertino       | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-     | Cathy Johnson  | 789 Race Ave, Palo Alto        | 333333         |
-     + -------------- + ------------------------------ + -------------- +
-     | Dora Williams  | 134 Forest Ave, Melo Park      | 444444         |
-     + -------------- + ------------------------------ + -------------- +
-     | Fleur Laurent  | 345 Copper St, London          | 777777         |
-     + -------------- + ------------------------------ + -------------- +
-     | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-     + -------------- + ------------------------------ + -------------- +
-     | Helen Davis    | 469 Mission St, San Diego      | 999999         |
-     + -------------- + ------------------------------ + -------------- +
-     | Jason Wang     | 908 Bird St, Saratoga          | 121212         |
-     + -------------- + ------------------------------ + -------------- +
-{% endhighlight %}
+
+```sql
+-- Assuming the applicants table has already been created and populated.
+SELECT * FROM applicants;
++-----------+--------------------------+----------+---------+
+|       name|                   address|student_id|qualified|
++-----------+--------------------------+----------+---------+
+|Helen Davis| 469 Mission St, San Diego|    999999|     true|
++-----------+--------------------------+----------+---------+
+|   Ivy King|367 Leigh Ave, Santa Clara|    101010|    false|
++-----------+--------------------------+----------+---------+
+| Jason Wang|     908 Bird St, Saratoga|    121212|     true|
++-----------+--------------------------+----------+---------+
+
+INSERT INTO students
+     FROM applicants SELECT name, address, id applicants WHERE qualified = true;
+
+SELECT * FROM students;
++-------------+-------------------------+----------+
+|         name|                  address|student_id|
++-------------+-------------------------+----------+
+|    Amy Smith|   123 Park Ave, San Jose|    111111|
++-------------+-------------------------+----------+
+|    Bob Brown| 456 Taylor St, Cupertino|    222222|
++-------------+-------------------------+----------+
+|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
++-------------+-------------------------+----------+
+|Dora Williams|134 Forest Ave, Melo Park|    444444|
++-------------+-------------------------+----------+
+|Fleur Laurent|    345 Copper St, London|    777777|
++-------------+-------------------------+----------+
+|Gordon Martin|     779 Lake Ave, Oxford|    888888|
++-------------+-------------------------+----------+
+|  Helen Davis|469 Mission St, San Diego|    999999|
++-------------+-------------------------+----------+
+|   Jason Wang|    908 Bird St, Saratoga|    121212|
++-------------+-------------------------+----------+
+```
 
 ### Related Statements
-  * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
-  * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
-  * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+
+* [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+* [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
+* [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
index 6d56cdeb051c6..8ed6a3cd1be09 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
@@ -20,68 +20,58 @@ license: |
 ---
 
 ### Description
+
 The `INSERT OVERWRITE DIRECTORY` with Hive format overwrites the existing data in the directory with the new values using Hive `SerDe`.
 Hive support must be enabled to use this command. The inserted rows can be specified by value expressions or result from a query.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 INSERT OVERWRITE [ LOCAL ] DIRECTORY directory_path
-   [ ROW FORMAT row_format ] [ STORED AS file_format ]
-   { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
-{% endhighlight %}
+    [ ROW FORMAT row_format ] [ STORED AS file_format ]
+    { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>directory_path</em></code></dt>
-  <dd>
-  Specifies the destination directory. The <code>LOCAL</code> keyword is used to specify that the directory is on the local file system.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>row_format</em></code></dt>
-  <dd>
-  Specifies the row format for this insert. Valid options are <code>SERDE</code> clause and <code>DELIMITED</code> clause. <code>SERDE</code> clause can be used to specify a custom <code>SerDe</code> for this insert. Alternatively, <code>DELIMITED</code> clause can be used to specify the native <code>SerDe</code> and state the delimiter, escape character, null character, and so on.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>file_format</em></code></dt>
-  <dd>
-  Specifies the file format for this insert. Valid options are <code>TEXTFILE</code>, <code>SEQUENCEFILE</code>, <code>RCFILE</code>, <code>ORC</code>, <code>PARQUET</code>, and <code>AVRO</code>. You can also specify your own input and output format using <code>INPUTFORMAT</code> and <code>OUTPUTFORMAT</code>. <code>ROW FORMAT SERDE</code> can only be used with <code>TEXTFILE</code>, <code>SEQUENCEFILE</code>, or <code>RCFILE</code>, while <code>ROW FORMAT DELIMITED</code> can only be used with <code>TEXTFILE</code>.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ]</em></code></dt>
-  <dd>
-  Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted. A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A query that produces the rows to be inserted. It can be in one of following formats:
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-   </dd>
-</dl>
+
+* **directory_path**
+
+    Specifies the destination directory. The `LOCAL` keyword is used to specify that the directory is on the local file system.
+
+* **row_format**
+
+    Specifies the row format for this insert. Valid options are `SERDE` clause and `DELIMITED` clause. `SERDE` clause can be used to specify a custom `SerDe` for this insert. Alternatively, `DELIMITED` clause can be used to specify the native `SerDe` and state the delimiter, escape character, null character, and so on.
+
+* **file_format**
+
+    Specifies the file format for this insert. Valid options are `TEXTFILE`, `SEQUENCEFILE`, `RCFILE`, `ORC`, `PARQUET`, and `AVRO`. You can also specify your own input and output format using `INPUTFORMAT` and `OUTPUTFORMAT`. `ROW FORMAT SERDE` can only be used with `TEXTFILE`, `SEQUENCEFILE`, or `RCFILE`, while `ROW FORMAT DELIMITED` can only be used with `TEXTFILE`.
+
+* **VALUES ( { value `|` NULL } [ , ... ] ) [ , ( ... ) ]**
+
+    Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
+    A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
+
+* **query**
+
+    A query that produces the rows to be inserted. It can be in one of following formats:
+    * a `SELECT` statement
+    * a `TABLE` statement
+    * a `FROM` statement
 
 ### Examples
-{% highlight sql %}
- INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
-     STORED AS orc
-     SELECT * FROM test_table;
 
- INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
-     ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-     SELECT * FROM test_table;
-{% endhighlight %}
+```sql
+INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
+    STORED AS orc
+    SELECT * FROM test_table;
+
+INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+    SELECT * FROM test_table;
+```
 
 ### Related Statements
-  * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
-  * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
-  * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
\ No newline at end of file
+
+* [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+* [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+* [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
index 06000b593bda4..fd7437d37c909 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
@@ -9,65 +9,58 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
-The `INSERT OVERWRITE DIRECTORY` statement overwrites the existing data in the directory with the new values using Spark native format. The inserted rows can be specified by value expressions or result from a query.
+
+The `INSERT OVERWRITE DIRECTORY` statement overwrites the existing data in the directory with the new values using a given Spark file format. The inserted rows can be specified by value expressions or result from a query.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 INSERT OVERWRITE [ LOCAL ] DIRECTORY [ directory_path ]
     USING file_format [ OPTIONS ( key = val [ , ... ] ) ]
-    { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
-{% endhighlight %}
+    { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>directory_path</em></code></dt>
-  <dd>
-  Specifies the destination directory. It can also be specified in <code>OPTIONS</code> using <code>path</code>. The <code>LOCAL</code> keyword is used to specify that the directory is on the local file system.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>file_format</em></code></dt>
-  <dd>
-  Specifies the file format to use for the insert. Valid options are <code>TEXT</code>, <code>CSV</code>, <code>JSON</code>, <code>JDBC</code>, <code>PARQUET</code>, <code>ORC</code>, <code>HIVE</code>, <code>DELTA</code>, <code>LIBSVM</code>, or a fully qualified class name of a custom implementation of <code>org.apache.spark.sql.sources.DataSourceRegister</code>.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>OPTIONS ( key = val [ , ... ] )</em></code></dt>
-  <dd>Specifies one or more table property key and value pairs.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ]</em></code></dt>
-  <dd>
-  Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted. A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A query that produces the rows to be inserted. It can be in one of following formats:
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-   </dd>
-</dl>
+
+* **directory_path**
+
+    Specifies the destination directory. It can also be specified in `OPTIONS` using `path`.
+    The `LOCAL` keyword is used to specify that the directory is on the local file system.
+
+* **file_format**
+
+    Specifies the file format to use for the insert. Valid options are `TEXT`, `CSV`, `JSON`, `JDBC`, `PARQUET`, `ORC`, `HIVE`, `LIBSVM`, or a fully qualified class name of a custom implementation of `org.apache.spark.sql.execution.datasources.FileFormat`.
+
+* **OPTIONS ( key = val [ , ... ] )**
+
+    Specifies one or more options for the writing of the file format.
+
+* **VALUES ( { value `|` NULL } [ , ... ] ) [ , ( ... ) ]**
+
+    Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
+    A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
+
+* **query**
+
+    A query that produces the rows to be inserted. It can be in one of following formats:
+    * a `SELECT` statement
+    * a `TABLE` statement
+    * a `FROM` statement
 
 ### Examples
-{% highlight sql %}
+
+```sql
 INSERT OVERWRITE DIRECTORY '/tmp/destination'
     USING parquet
     OPTIONS (col1 1, col2 2, col3 'test')
@@ -77,9 +70,10 @@ INSERT OVERWRITE DIRECTORY
     USING parquet
     OPTIONS ('path' '/tmp/destination', col1 1, col2 2, col3 'test')
     SELECT * FROM test_table;
-{% endhighlight %}
+```
 
 ### Related Statements
-  * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
-  * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
-  * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
\ No newline at end of file
+
+* [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+* [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+* [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-table.md b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
index 1f32342642b2a..ecfd060dfd5ee 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-table.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -24,175 +24,153 @@ license: |
 The `INSERT OVERWRITE` statement overwrites the existing data in the table using the new values. The inserted rows can be specified by value expressions or result from a query.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ]
-    { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
-{% endhighlight %}
+    { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ]</em></code></dt>
-  <dd>Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted. A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A query that produces the rows to be inserted. It can be in one of following formats:
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-   </dd>
-</dl>
+    for partitions.
+
+    **Syntax:** `PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )`
+
+* **VALUES ( { value `|` NULL } [ , ... ] ) [ , ( ... ) ]**
+
+    Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
+    A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
+
+* **query**
+
+    A query that produces the rows to be inserted. It can be in one of following formats:
+    * a `SELECT` statement
+    * a `TABLE` statement
+    * a `FROM` statement
 
 ### Examples
+
 #### Insert Using a VALUES Clause
-{% highlight sql %}
- -- Assuming the students table has already been created and populated.
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Bob Brown      | 456 Taylor St, Cupertino       | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-     | Cathy Johnson  | 789 Race Ave, Palo Alto        | 333333         |
-     + -------------- + ------------------------------ + -------------- +
-     | Dora Williams  | 134 Forest Ave, Melo Park      | 444444         |
-     + -------------- + ------------------------------ + -------------- +
-     | Fleur Laurent  | 345 Copper St, London          | 777777         |
-     + -------------- + ------------------------------ + -------------- +
-     | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-     + -------------- + ------------------------------ + -------------- +
-     | Helen Davis    | 469 Mission St, San Diego      | 999999         |
-     + -------------- + ------------------------------ + -------------- +
-     | Jason Wang     | 908 Bird St, Saratoga          | 121212         |
-     + -------------- + ------------------------------ + -------------- +
-
- INSERT OVERWRITE students
-     VALUES ('Ashua Hill', '456 Erica Ct, Cupertino', 111111),
-            ('Brian Reed', '723 Kern Ave, Palo Alto', 222222);
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Ashua Hill     | 456 Erica Ct, Cupertino        | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Brian Reed     | 723 Kern Ave, Palo Alto        | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-{% endhighlight %}
+
+```sql
+-- Assuming the students table has already been created and populated.
+SELECT * FROM students;
++-------------+-------------------------+----------+
+|         name|                  address|student_id|
++-------------+-------------------------+----------+
+|    Amy Smith|   123 Park Ave, San Jose|    111111|
+|    Bob Brown| 456 Taylor St, Cupertino|    222222|
+|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
+|Dora Williams|134 Forest Ave, Melo Park|    444444|
+|Fleur Laurent|    345 Copper St, London|    777777|
+|Gordon Martin|     779 Lake Ave, Oxford|    888888|
+|  Helen Davis|469 Mission St, San Diego|    999999|
+|   Jason Wang|    908 Bird St, Saratoga|    121212|
++-------------+-------------------------+----------+
+
+INSERT OVERWRITE students VALUES
+    ('Ashua Hill', '456 Erica Ct, Cupertino', 111111),
+    ('Brian Reed', '723 Kern Ave, Palo Alto', 222222);
+
+SELECT * FROM students;
++----------+-----------------------+----------+
+|      name|                address|student_id|
++----------+-----------------------+----------+
+|Ashua Hill|456 Erica Ct, Cupertino|    111111|
+|Brian Reed|723 Kern Ave, Palo Alto|    222222|
++----------+-----------------------+----------+
+```
 
 #### Insert Using a SELECT Statement
-{% highlight sql %}
- -- Assuming the persons table has already been created and populated.
- SELECT * FROM persons;
-
-      + -------------- + ------------------------------ + -------------- +
-      | name           | address                        | ssn            |
-      + -------------- + ------------------------------ + -------------- +
-      | Dora Williams  | 134 Forest Ave, Melo Park      | 123456789      |
-      + -------------- + ------------------------------ + -------------- +
-      | Eddie Davis    | 245 Market St, Milpitas        | 345678901      |
-      + -------------- + ------------------------------ + ---------------+
-
- INSERT OVERWRITE students PARTITION (student_id = 222222)
-     SELECT name, address FROM persons WHERE name = "Dora Williams";
-
- SELECT * FROM students;
-
-      + -------------- + ------------------------------ + -------------- +
-      | name           | address                        | student_id     |
-      + -------------- + ------------------------------ + -------------- +
-      | Ashua Hill     | 456 Erica Ct, Cupertino        | 111111         |
-      + -------------- + ------------------------------ + -------------- +
-      | Dora Williams  | 134 Forest Ave, Melo Park      | 222222         |
-      + -------------- + ------------------------------ + -------------- +
-{% endhighlight %}
+
+```sql
+-- Assuming the persons table has already been created and populated.
+SELECT * FROM persons;
++-------------+-------------------------+---------+
+|         name|                  address|      ssn|
++-------------+-------------------------+---------+
+|Dora Williams|134 Forest Ave, Melo Park|123456789|
++-------------+-------------------------+---------+
+|  Eddie Davis|   245 Market St,Milpitas|345678901|
++-------------+-------------------------+---------+
+
+INSERT OVERWRITE students PARTITION (student_id = 222222)
+    SELECT name, address FROM persons WHERE name = "Dora Williams";
+
+SELECT * FROM students;
++-------------+-------------------------+----------+
+|         name|                  address|student_id|
++-------------+-------------------------+----------+
+|   Ashua Hill|  456 Erica Ct, Cupertino|    111111|
++-------------+-------------------------+----------+
+|Dora Williams|134 Forest Ave, Melo Park|    222222|
++-------------+-------------------------+----------+
+```
 
 #### Insert Using a TABLE Statement
-{% highlight sql %}
- -- Assuming the visiting_students table has already been created and populated.
- SELECT * FROM visiting_students;
-
-      + -------------- + ------------------------------ + -------------- +
-      | name           | address                        | student_id     |
-      + -------------- + ------------------------------ + -------------- +
-      | Fleur Laurent  | 345 Copper St, London          | 777777         |
-      + -------------- + ------------------------------ + -------------- +
-      | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-      + -------------- + ------------------------------ + -------------- +
-
- INSERT OVERWRITE students TABLE visiting_students;
-
- SELECT * FROM students;
-
-      + -------------- + ------------------------------ + -------------- +
-      | name           | address                        | student_id     |
-      + -------------- + ------------------------------ + -------------- +
-      | Fleur Laurent  | 345 Copper St, London          | 777777         |
-      + -------------- + ------------------------------ + -------------- +
-      | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-      + -------------- + ------------------------------ + -------------- +
-{% endhighlight %}
+
+```sql
+-- Assuming the visiting_students table has already been created and populated.
+SELECT * FROM visiting_students;
++-------------+---------------------+----------+
+|         name|              address|student_id|
++-------------+---------------------+----------+
+|Fleur Laurent|345 Copper St, London|    777777|
++-------------+---------------------+----------+
+|Gordon Martin| 779 Lake Ave, Oxford|    888888|
++-------------+---------------------+----------+
+
+INSERT OVERWRITE students TABLE visiting_students;
+
+SELECT * FROM students;
++-------------+---------------------+----------+
+|         name|              address|student_id|
++-------------+---------------------+----------+
+|Fleur Laurent|345 Copper St, London|    777777|
++-------------+---------------------+----------+
+|Gordon Martin| 779 Lake Ave, Oxford|    888888|
++-------------+---------------------+----------+
+```
 
 #### Insert Using a FROM Statement
-{% highlight sql %}
- -- Assuming the applicants table has already been created and populated.
- SELECT * FROM applicants;
-
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | name           | address                        | student_id     | qualified      |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Helen Davis    | 469 Mission St, San Diego      | 999999         | true           |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Ivy King       | 367 Leigh Ave, Santa Clara     | 101010         | false          |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Jason Wang     | 908 Bird St, Saratoga          | 121212         | true           |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-
- INSERT OVERWRITE students;
-      FROM applicants SELECT name, address, id applicants WHERE qualified = true;
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Helen Davis    | 469 Mission St, San Diego      | 999999         |
-     + -------------- + ------------------------------ + -------------- +
-     | Jason Wang     | 908 Bird St, Saratoga          | 121212         |
-     + -------------- + ------------------------------ + -------------- +
-{% endhighlight %}
+
+```sql
+-- Assuming the applicants table has already been created and populated.
+SELECT * FROM applicants;
++-----------+--------------------------+----------+---------+
+|       name|                   address|student_id|qualified|
++-----------+--------------------------+----------+---------+
+|Helen Davis| 469 Mission St, San Diego|    999999|     true|
++-----------+--------------------------+----------+---------+
+|   Ivy King|367 Leigh Ave, Santa Clara|    101010|    false|
++-----------+--------------------------+----------+---------+
+| Jason Wang|     908 Bird St, Saratoga|    121212|     true|
++-----------+--------------------------+----------+---------+
+
+INSERT OVERWRITE students
+    FROM applicants SELECT name, address, id applicants WHERE qualified = true;
+
+SELECT * FROM students;
++-----------+-------------------------+----------+
+|       name|                  address|student_id|
++-----------+-------------------------+----------+
+|Helen Davis|469 Mission St, San Diego|    999999|
++-----------+-------------------------+----------+
+| Jason Wang|    908 Bird St, Saratoga|    121212|
++-----------+-------------------------+----------+
+```
 
 ### Related Statements
-  * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
-  * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
-  * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+
+* [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+* [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
+* [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-insert.md b/docs/sql-ref-syntax-dml-insert.md
index 15a2e28896943..62f6dee876450 100644
--- a/docs/sql-ref-syntax-dml-insert.md
+++ b/docs/sql-ref-syntax-dml-insert.md
@@ -20,6 +20,7 @@ license: |
 ---
 
 The INSERT statements:
+
 * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
 * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
 * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
diff --git a/docs/sql-ref-syntax-dml-load.md b/docs/sql-ref-syntax-dml-load.md
index aadfd708e274e..9381b4267fb24 100644
--- a/docs/sql-ref-syntax-dml-load.md
+++ b/docs/sql-ref-syntax-dml-load.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,112 +20,94 @@ license: |
 ---
 
 ### Description
-`LOAD DATA` statement loads the data into a table from the user specified directory or file. If a directory is specified then all the files from the directory are loaded. If a file is specified then only the single file is loaded. Additionally the `LOAD DATA` statement takes an optional partition specification. When a partition is specified, the data files (when input source is a directory) or the single file (when input source is a file) are loaded into the partition of the target table.
+
+`LOAD DATA` statement loads the data into a Hive serde table from the user specified directory or file. If a directory is specified then all the files from the directory are loaded. If a file is specified then only the single file is loaded. Additionally the `LOAD DATA` statement takes an optional partition specification. When a partition is specified, the data files (when input source is a directory) or the single file (when input source is a file) are loaded into the partition of the target table.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 LOAD DATA [ LOCAL ] INPATH path [ OVERWRITE ] INTO TABLE table_identifier [ partition_spec ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>path</em></code></dt>
-  <dd>Path of the file system. It can be either an absolute or a relative path.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
-    An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>LOCAL</em></code></dt>
-  <dd>If specified, it causes the <code>INPATH</code> to be resolved against the local file system, instead of the default file system, which is typically a distributed storage.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>OVERWRITE</em></code></dt>
-  <dd>By default, new data is appended to the table. If <code>OVERWRITE</code> is used, the table is instead overwritten with new data.</dd>
-</dl>
-
-### Examples
-{% highlight sql %}
- -- Example without partition specification.
- -- Assuming the students table has already been created and populated.
- SELECT * FROM students;
 
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
+* **path**
 
- CREATE TABLE test_load (name VARCHAR(64), address VARCHAR(64), student_id INT);
+    Path of the file system. It can be either an absolute or a relative path.
 
- -- Assuming the students table is in '/user/hive/warehouse/'
- LOAD DATA LOCAL INPATH '/user/hive/warehouse/students' OVERWRITE INTO TABLE test_load;
+* **table_identifier**
 
- SELECT * FROM test_load;
+    Specifies a table name, which may be optionally qualified with a database name.
 
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
+    **Syntax:** `[ database_name. ] table_name`
 
- -- Example with partition specification.
- CREATE TABLE test_partition (c1 INT, c2 INT, c3 INT) USING HIVE PARTITIONED BY (c2, c3);
+* **partition_spec**
 
- INSERT INTO test_partition PARTITION (c2 = 2, c3 = 3) VALUES (1);
-
- INSERT INTO test_partition PARTITION (c2 = 5, c3 = 6) VALUES (4);
-
- INSERT INTO test_partition PARTITION (c2 = 8, c3 = 9) VALUES (7);
-
- SELECT * FROM test_partition;
-
-      + ------- + ------- + ----- +
-      | c1      | c2      | c3    |
-      + ------- + --------------- +
-      | 1       | 2       | 3     |
-      + ------- + ------- + ----- +
-      | 4       | 5       | 6     |
-      + ------- + ------- + ----- +
-      | 7       | 8       | 9     |
-      + ------- + ------- + ----- +
+    An optional parameter that specifies a comma separated list of key and value pairs
+    for partitions.
 
- CREATE TABLE test_load_partition (c1 INT, c2 INT, c3 INT) USING HIVE PARTITIONED BY (c2, c3);
+    **Syntax:** `PARTITION ( partition_col_name = partition_col_val [ , ... ] )`
 
- -- Assuming the test_partition table is in '/user/hive/warehouse/'
- LOAD DATA LOCAL INPATH '/user/hive/warehouse/test_partition/c2=2/c3=3'
-     OVERWRITE INTO TABLE test_load_partition PARTITION (c2=2, c3=3);
+* **LOCAL**
 
- SELECT * FROM test_load_partition;
+    If specified, it causes the `INPATH` to be resolved against the local file system, instead of the default file system, which is typically a distributed storage.
 
-      + ------- + ------- + ----- +
-      | c1      | c2      | c3    |
-      + ------- + --------------- +
-      | 1       | 2       | 3     |
-      + ------- + ------- + ----- +
+* **OVERWRITE**
 
+    By default, new data is appended to the table. If `OVERWRITE` is used, the table is instead overwritten with new data.
 
-{% endhighlight %}
+### Examples
 
+```sql
+-- Example without partition specification.
+-- Assuming the students table has already been created and populated.
+SELECT * FROM students;
++---------+----------------------+----------+
+|     name|               address|student_id|
++---------+----------------------+----------+
+|Amy Smith|123 Park Ave, San Jose|    111111|
++---------+----------------------+----------+
+
+CREATE TABLE test_load (name VARCHAR(64), address VARCHAR(64), student_id INT) USING HIVE;
+
+-- Assuming the students table is in '/user/hive/warehouse/'
+LOAD DATA LOCAL INPATH '/user/hive/warehouse/students' OVERWRITE INTO TABLE test_load;
+
+SELECT * FROM test_load;
++---------+----------------------+----------+
+|     name|               address|student_id|
++---------+----------------------+----------+
+|Amy Smith|123 Park Ave, San Jose|    111111|
++---------+----------------------+----------+
+
+-- Example with partition specification.
+CREATE TABLE test_partition (c1 INT, c2 INT, c3 INT) PARTITIONED BY (c2, c3);
+
+INSERT INTO test_partition PARTITION (c2 = 2, c3 = 3) VALUES (1);
+
+INSERT INTO test_partition PARTITION (c2 = 5, c3 = 6) VALUES (4);
+
+INSERT INTO test_partition PARTITION (c2 = 8, c3 = 9) VALUES (7);
+
+SELECT * FROM test_partition;
++---+---+---+
+| c1| c2| c3|
++---+---+---+
+|  1|  2|  3|
+|  4|  5|  6|
+|  7|  8|  9|
++---+---+---+
+
+CREATE TABLE test_load_partition (c1 INT, c2 INT, c3 INT) USING HIVE PARTITIONED BY (c2, c3);
+
+-- Assuming the test_partition table is in '/user/hive/warehouse/'
+LOAD DATA LOCAL INPATH '/user/hive/warehouse/test_partition/c2=2/c3=3'
+    OVERWRITE INTO TABLE test_load_partition PARTITION (c2=2, c3=3);
+
+SELECT * FROM test_load_partition;
++---+---+---+
+| c1| c2| c3|
++---+---+---+
+|  1|  2|  3|
++---+---+---+
+```
diff --git a/docs/sql-ref-syntax-dml.md b/docs/sql-ref-syntax-dml.md
index af09e2054562e..fc408e1d38d26 100644
--- a/docs/sql-ref-syntax-dml.md
+++ b/docs/sql-ref-syntax-dml.md
@@ -21,5 +21,5 @@ license: |
 
 Data Manipulation Statements are used to add, change, or delete data. Spark SQL supports the following Data Manipulation Statements:
 
-- [INSERT](sql-ref-syntax-dml-insert.html)
-- [LOAD](sql-ref-syntax-dml-load.html)
\ No newline at end of file
+* [INSERT](sql-ref-syntax-dml-insert.html)
+* [LOAD](sql-ref-syntax-dml-load.html)
diff --git a/docs/sql-ref-syntax-qry-aggregation.md b/docs/sql-ref-syntax-qry-aggregation.md
deleted file mode 100644
index fbe6e61c84fd1..0000000000000
--- a/docs/sql-ref-syntax-qry-aggregation.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Aggregation (CUBE/ROLLUP/GROUPING)
-displayTitle: Aggregation (CUBE/ROLLUP/GROUPING) 
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-**This page is under construction**
diff --git a/docs/sql-ref-syntax-qry-explain.md b/docs/sql-ref-syntax-qry-explain.md
index 7e18e16bc8ea6..7b84264a28cca 100644
--- a/docs/sql-ref-syntax-qry-explain.md
+++ b/docs/sql-ref-syntax-qry-explain.md
@@ -24,116 +24,104 @@ license: |
 The `EXPLAIN` statement is used to provide logical/physical plans for an input statement. 
 By default, this clause provides information about a physical plan only.
 
-
 ### Syntax
-{% highlight sql %}
-EXPLAIN [EXTENDED | CODEGEN | COST | FORMATTED] statement
-{% endhighlight %}
+
+```sql
+EXPLAIN [ EXTENDED | CODEGEN | COST | FORMATTED ] statement
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>EXTENDED</em></code></dt>
-  <dd>Generates parsed logical plan, analyzed logical plan, optimized logical plan and physical plan.
-   Parsed Logical plan is a unresolved plan that extracted from the query.
-   Analyzed logical plans transforms which translates unresolvedAttribute and unresolvedRelation into fully typed objects.
-   The optimized logical plan transforms through a set of optimization rules, resulting in the physical plan.
-  </dd>
-</dl> 
-
-<dl>
-  <dt><code><em>CODEGEN</em></code></dt>
-  <dd>Generates code for the statement, if any and a physical plan.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>COST</em></code></dt>
-  <dd>If plan node statistics are available, generates a logical plan and the statistics.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>FORMATTED</em></code></dt>
-  <dd>Generates two sections: a physical plan outline and node details.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>statement</em></code></dt>
-  <dd>
+* **EXTENDED**
+
+    Generates parsed logical plan, analyzed logical plan, optimized logical plan and physical plan.
+    Parsed Logical plan is a unresolved plan that extracted from the query.
+    Analyzed logical plans transforms which translates unresolvedAttribute and unresolvedRelation into fully typed objects.
+    The optimized logical plan transforms through a set of optimization rules, resulting in the physical plan.
+
+* **CODEGEN**
+
+    Generates code for the statement, if any and a physical plan.
+
+* **COST**
+
+    If plan node statistics are available, generates a logical plan and the statistics.
+
+* **FORMATTED**
+
+    Generates two sections: a physical plan outline and node details.
+
+* **statement**
+
     Specifies a SQL statement to be explained.
-  </dd>
- </dl>
 
 ### Examples
-{% highlight sql %}
-
---Default Output
 
+```sql
+-- Default Output
 EXPLAIN select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;
 +----------------------------------------------------+
-|                        plan                        |
+|                                                plan|
 +----------------------------------------------------+
 | == Physical Plan ==
-*(2) HashAggregate(keys=[k#33], functions=[sum(cast(v#34 as bigint))])
-+- Exchange hashpartitioning(k#33, 200), true, [id=#59]
-   +- *(1) HashAggregate(keys=[k#33], functions=[partial_sum(cast(v#34 as bigint))])
-      +- *(1) LocalTableScan [k#33, v#34]
+ *(2) HashAggregate(keys=[k#33], functions=[sum(cast(v#34 as bigint))])
+ +- Exchange hashpartitioning(k#33, 200), true, [id=#59]
+    +- *(1) HashAggregate(keys=[k#33], functions=[partial_sum(cast(v#34 as bigint))])
+       +- *(1) LocalTableScan [k#33, v#34]
 |
- +----------------------------------------------------
++----------------------------------------------------
 
 -- Using Extended
-
 EXPLAIN EXTENDED select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;
 +----------------------------------------------------+
-|                        plan                        |
+|                                                plan|
 +----------------------------------------------------+
 | == Parsed Logical Plan ==
-'Aggregate ['k], ['k, unresolvedalias('sum('v), None)]
-+- 'SubqueryAlias `t`
-   +- 'UnresolvedInlineTable [k, v], [List(1, 2), List(1, 3)]
-
-== Analyzed Logical Plan ==
-k: int, sum(v): bigint
-Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
-+- SubqueryAlias `t`
-   +- LocalRelation [k#47, v#48]
-
-== Optimized Logical Plan ==
-Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
-+- LocalRelation [k#47, v#48]
-
-== Physical Plan ==
-*(2) HashAggregate(keys=[k#47], functions=[sum(cast(v#48 as bigint))], output=[k#47, sum(v)#50L])
+ 'Aggregate ['k], ['k, unresolvedalias('sum('v), None)]
+ +- 'SubqueryAlias `t`
+    +- 'UnresolvedInlineTable [k, v], [List(1, 2), List(1, 3)]
+   
+ == Analyzed Logical Plan ==
+ k: int, sum(v): bigint
+ Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
+ +- SubqueryAlias `t`
+    +- LocalRelation [k#47, v#48]
+   
+ == Optimized Logical Plan ==
+ Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
+ +- LocalRelation [k#47, v#48]
+   
+ == Physical Plan ==
+ *(2) HashAggregate(keys=[k#47], functions=[sum(cast(v#48 as bigint))], output=[k#47, sum(v)#50L])
 +- Exchange hashpartitioning(k#47, 200), true, [id=#79]
    +- *(1) HashAggregate(keys=[k#47], functions=[partial_sum(cast(v#48 as bigint))], output=[k#47, sum#52L])
-      +- *(1) LocalTableScan [k#47, v#48]
- |
+    +- *(1) LocalTableScan [k#47, v#48]
+|
 +----------------------------------------------------+
 
---Using Formatted
-
+-- Using Formatted
 EXPLAIN FORMATTED select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;
 +----------------------------------------------------+
-|                        plan                        |
+|                                                plan|
 +----------------------------------------------------+
 | == Physical Plan ==
-* HashAggregate (4)
-+- Exchange (3)
-   +- * HashAggregate (2)
-      +- * LocalTableScan (1)
-
-
-(1) LocalTableScan [codegen id : 1]
-Output: [k#19, v#20]
-     
-(2) HashAggregate [codegen id : 1]
-Input: [k#19, v#20]
-     
-(3) Exchange 
-Input: [k#19, sum#24L]
-     
-(4) HashAggregate [codegen id : 2]
-Input: [k#19, sum#24L] 
- |
+ * HashAggregate (4)
+ +- Exchange (3)
+    +- * HashAggregate (2)
+       +- * LocalTableScan (1)
+   
+   
+ (1) LocalTableScan [codegen id : 1]
+ Output: [k#19, v#20]
+        
+ (2) HashAggregate [codegen id : 1]
+ Input: [k#19, v#20]
+        
+ (3) Exchange
+ Input: [k#19, sum#24L]
+        
+ (4) HashAggregate [codegen id : 2]
+ Input: [k#19, sum#24L]
+|
 +----------------------------------------------------+
-
-{% endhighlight %}
+```
diff --git a/docs/sql-ref-syntax-qry-sampling.md b/docs/sql-ref-syntax-qry-sampling.md
deleted file mode 100644
index a5efb361ee847..0000000000000
--- a/docs/sql-ref-syntax-qry-sampling.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Sampling Queries
-displayTitle: Sampling Queries
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-**This page is under construction**
diff --git a/docs/sql-ref-syntax-qry-select-case.md b/docs/sql-ref-syntax-qry-select-case.md
new file mode 100644
index 0000000000000..6136b161d234b
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-case.md
@@ -0,0 +1,109 @@
+---
+layout: global
+title: CASE Clause
+displayTitle: CASE Clause
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+`CASE` clause uses a rule to return a specific result based on the specified condition, similar to if/else statements in other programming languages.
+
+### Syntax
+
+```sql
+CASE [ expression ] { WHEN boolean_expression THEN then_expression } [ ... ]
+    [ ELSE else_expression ]
+END
+```
+
+### Parameters
+    
+* **boolean_expression**
+
+    Specifies any expression that evaluates to a result type `boolean`. Two or
+    more expressions may be combined together using the logical
+    operators ( `AND`, `OR` ).
+
+* **then_expression**
+
+    Specifies the then expression based on the `boolean_expression` condition; `then_expression` and `else_expression` should all be same type or coercible to a common type.
+    
+* **else_expression**
+
+    Specifies the default expression; `then_expression` and `else_expression` should all be same type or coercible to a common type.
+    
+### Examples
+
+```sql
+CREATE TABLE person (id INT, name STRING, age INT);
+INSERT INTO person VALUES
+    (100, 'John', 30),
+    (200, 'Mary', NULL),
+    (300, 'Mike', 80),
+    (400, 'Dan', 50);
+
+SELECT id, CASE WHEN id > 200 THEN 'bigger' ELSE 'small' END FROM person;
++------+--------------------------------------------------+
+|  id  | CASE WHEN (id > 200) THEN bigger ELSE small END  |
++------+--------------------------------------------------+
+| 100  | small                                            |
+| 200  | small                                            |
+| 300  | bigger                                           |
+| 400  | bigger                                           |
++------+--------------------------------------------------+
+
+SELECT id, CASE id WHEN 100 then 'bigger' WHEN  id > 300 THEN '300' ELSE 'small' END FROM person;
++------+-----------------------------------------------------------------------------------------------+
+|  id  | CASE WHEN (id = 100) THEN bigger WHEN (id = CAST((id > 300) AS INT)) THEN 300 ELSE small END  |
++------+-----------------------------------------------------------------------------------------------+
+| 100  | bigger                                                                                        |
+| 200  | small                                                                                         |
+| 300  | small                                                                                         |
+| 400  | small                                                                                         |
++------+-----------------------------------------------------------------------------------------------+
+
+SELECT * FROM person
+    WHERE 
+        CASE 1 = 1 
+            WHEN 100 THEN 'big' 
+            WHEN 200 THEN 'bigger'
+            WHEN 300 THEN 'biggest' 
+            ELSE 'small'
+        END = 'small';
++------+-------+-------+
+|  id  | name  |  age  |
++------+-------+-------+
+| 100  | John  | 30    |
+| 200  | Mary  | NULL  |
+| 300  | Mike  | 80    |
+| 400  | Dan   | 50    |
++------+-------+-------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-clusterby.md b/docs/sql-ref-syntax-qry-select-clusterby.md
index c96c4419212bb..9bcfac5b3b5c6 100644
--- a/docs/sql-ref-syntax-qry-select-clusterby.md
+++ b/docs/sql-ref-syntax-qry-select-clusterby.md
@@ -9,16 +9,19 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-The <code>CLUSTER BY</code> clause is used to first repartition the data based
+
+### Description
+
+The `CLUSTER BY` clause is used to first repartition the data based
 on the input expressions and then sort the data within each partition. This is
 semantically equivalent to performing a
 [DISTRIBUTE BY](sql-ref-syntax-qry-select-distribute-by.html) followed by a
@@ -26,74 +29,76 @@ semantically equivalent to performing a
 resultant rows are sorted within each partition and does not guarantee a total order of output.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 CLUSTER BY { expression [ , ... ] }
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>expression</em></code></dt>
-  <dd>
+
+* **expression**
+
     Specifies combination of one or more values, operators and SQL functions that results in a value.
-  </dd>
-</dl>
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CREATE TABLE person (name STRING, age INT);
-INSERT INTO person VALUES 
-    ('Zen Hui', 25), 
-    ('Anil B', 18), 
-    ('Shone S', 16), 
+INSERT INTO person VALUES
+    ('Zen Hui', 25),
+    ('Anil B', 18),
+    ('Shone S', 16),
     ('Mike A', 25),
-    ('John A', 18), 
+    ('John A', 18),
     ('Jack N', 16);
 
 -- Reduce the number of shuffle partitions to 2 to illustrate the behavior of `CLUSTER BY`.
 -- It's easier to see the clustering and sorting behavior with less number of partitions.
 SET spark.sql.shuffle.partitions = 2;
-                        
+
 -- Select the rows with no ordering. Please note that without any sort directive, the results
--- of the query is not deterministic. It's included here to show the difference in behavior 
+-- of the query is not deterministic. It's included here to show the difference in behavior
 -- of a query when `CLUSTER BY` is not used vs when it's used. The query below produces rows
 -- where age column is not sorted.
 SELECT age, name FROM person;
-
-  +---+-------+
-  |age|name   |
-  +---+-------+
-  |16 |Shone S|
-  |25 |Zen Hui|
-  |16 |Jack N |
-  |25 |Mike A |
-  |18 |John A |
-  |18 |Anil B |
-  +---+-------+
++---+-------+
+|age|   name|
++---+-------+
+| 16|Shone S|
+| 25|Zen Hui|
+| 16| Jack N|
+| 25| Mike A|
+| 18| John A|
+| 18| Anil B|
++---+-------+
 
 -- Produces rows clustered by age. Persons with same age are clustered together.
 -- In the query below, persons with age 18 and 25 are in first partition and the
 -- persons with age 16 are in the second partition. The rows are sorted based
 -- on age within each partition.
 SELECT age, name FROM person CLUSTER BY age;
++---+-------+
+|age|   name|
++---+-------+
+| 18| John A|
+| 18| Anil B|
+| 25|Zen Hui|
+| 25| Mike A|
+| 16|Shone S|
+| 16| Jack N|
++---+-------+
+```
+
+### Related Statements
 
-  +---+-------+
-  |age|name   |
-  +---+-------+
-  |18 |John A |
-  |18 |Anil B |
-  |25 |Zen Hui|
-  |25 |Mike A |
-  |16 |Shone S|
-  |16 |Jack N |
-  +---+-------+
-{% endhighlight %}
-
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
\ No newline at end of file
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-cte.md b/docs/sql-ref-syntax-qry-select-cte.md
index 2bd7748291a9f..351de64a2d026 100644
--- a/docs/sql-ref-syntax-qry-select-cte.md
+++ b/docs/sql-ref-syntax-qry-select-cte.md
@@ -19,4 +19,106 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+A common table expression (CTE) defines a temporary result set that a user can reference possibly multiple times within the scope of a SQL statement. A CTE is used mainly in a SELECT statement.
+
+### Syntax
+
+```sql
+WITH common_table_expression [ , ... ]
+```
+
+While `common_table_expression` is defined as
+```sql
+expression_name [ ( column_name [ , ... ] ) ] [ AS ] ( query )
+```
+
+### Parameters
+
+* **expression_name**
+
+    Specifies a name for the common table expression.
+
+* **query**
+
+    A [SELECT statement](sql-ref-syntax-qry-select.html).
+
+### Examples
+
+```sql
+-- CTE with multiple column aliases
+WITH t(x, y) AS (SELECT 1, 2)
+SELECT * FROM t WHERE x = 1 AND y = 2;
++---+---+
+|  x|  y|
++---+---+
+|  1|  2|
++---+---+
+
+-- CTE in CTE definition
+WITH t AS (
+    WITH t2 AS (SELECT 1)
+    SELECT * FROM t2
+)
+SELECT * FROM t;
++---+
+|  1|
++---+
+|  1|
++---+
+
+-- CTE in subquery
+SELECT max(c) FROM (
+    WITH t(c) AS (SELECT 1)
+    SELECT * FROM t
+);
++------+
+|max(c)|
++------+
+|     1|
++------+
+
+-- CTE in subquery expression
+SELECT (
+    WITH t AS (SELECT 1)
+    SELECT * FROM t
+);
++----------------+
+|scalarsubquery()|
++----------------+
+|               1|
++----------------+
+
+-- CTE in CREATE VIEW statement
+CREATE VIEW v AS
+    WITH t(a, b, c, d) AS (SELECT 1, 2, 3, 4)
+    SELECT * FROM t;
+SELECT * FROM v;
++---+---+---+---+
+|  a|  b|  c|  d|
++---+---+---+---+
+|  1|  2|  3|  4|
++---+---+---+---+
+
+-- If name conflict is detected in nested CTE, then AnalysisException is thrown by default.
+-- SET spark.sql.legacy.ctePrecedencePolicy = CORRECTED (which is recommended),
+-- inner CTE definitions take precedence over outer definitions.
+SET spark.sql.legacy.ctePrecedencePolicy = CORRECTED;
+WITH
+    t AS (SELECT 1),
+    t2 AS (
+        WITH t AS (SELECT 2)
+        SELECT * FROM t
+    )
+SELECT * FROM t2;
++---+
+|  2|
++---+
+|  2|
++---+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-distinct.md b/docs/sql-ref-syntax-qry-select-distinct.md
deleted file mode 100644
index 2ed7931c202f3..0000000000000
--- a/docs/sql-ref-syntax-qry-select-distinct.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Distinct operator
-displayTitle: Distinct operator
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-**This page is under construction**
diff --git a/docs/sql-ref-syntax-qry-select-distribute-by.md b/docs/sql-ref-syntax-qry-select-distribute-by.md
index e706ccf03917b..fbf662d3359f4 100644
--- a/docs/sql-ref-syntax-qry-select-distribute-by.md
+++ b/docs/sql-ref-syntax-qry-select-distribute-by.md
@@ -9,86 +9,91 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-The <code>DISTRIBUTE BY</code> clause is used to repartition the data based
+
+### Description
+
+The `DISTRIBUTE BY` clause is used to repartition the data based
 on the input expressions. Unlike the [CLUSTER BY](sql-ref-syntax-qry-select-clusterby.html)
-clause, this does not sort the data within each partition. 
+clause, this does not sort the data within each partition.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 DISTRIBUTE BY { expression [ , ... ] }
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>expression</em></code></dt>
-  <dd>
+
+* **expression**
+
     Specifies combination of one or more values, operators and SQL functions that results in a value.
-  </dd>
-</dl>
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CREATE TABLE person (name STRING, age INT);
 INSERT INTO person VALUES
-    ('Zen Hui', 25), 
-    ('Anil B', 18), 
-    ('Shone S', 16), 
+    ('Zen Hui', 25),
+    ('Anil B', 18),
+    ('Shone S', 16),
     ('Mike A', 25),
-    ('John A', 18), 
+    ('John A', 18),
     ('Jack N', 16);
 
 -- Reduce the number of shuffle partitions to 2 to illustrate the behavior of `DISTRIBUTE BY`.
 -- It's easier to see the clustering and sorting behavior with less number of partitions.
 SET spark.sql.shuffle.partitions = 2;
-                        
+
 -- Select the rows with no ordering. Please note that without any sort directive, the result
--- of the query is not deterministic. It's included here to just contrast it with the 
+-- of the query is not deterministic. It's included here to just contrast it with the
 -- behavior of `DISTRIBUTE BY`. The query below produces rows where age columns are not
 -- clustered together.
 SELECT age, name FROM person;
-
-  +---+-------+
-  |age|name   |
-  +---+-------+
-  |16 |Shone S|
-  |25 |Zen Hui|
-  |16 |Jack N |
-  |25 |Mike A |
-  |18 |John A |
-  |18 |Anil B |
-  +---+-------+
++---+-------+
+|age|   name|
++---+-------+
+| 16|Shone S|
+| 25|Zen Hui|
+| 16| Jack N|
+| 25| Mike A|
+| 18| John A|
+| 18| Anil B|
++---+-------+
 
 -- Produces rows clustered by age. Persons with same age are clustered together.
 -- Unlike `CLUSTER BY` clause, the rows are not sorted within a partition.
 SELECT age, name FROM person DISTRIBUTE BY age;
++---+-------+
+|age|   name|
++---+-------+
+| 25|Zen Hui|
+| 25| Mike A|
+| 18| John A|
+| 18| Anil B|
+| 16|Shone S|
+| 16| Jack N|
++---+-------+
+```
+
+### Related Statements
 
-  +---+-------+
-  |age|name   |
-  +---+-------+
-  |25 |Zen Hui|
-  |25 |Mike A |
-  |18 |John A |
-  |18 |Anil B |
-  |16 |Shone S|
-  |16 |Jack N |
-  +---+-------+
-{% endhighlight %}
-
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
\ No newline at end of file
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-qry-select-file.md b/docs/sql-ref-syntax-qry-select-file.md
new file mode 100644
index 0000000000000..c3dc406ee79e6
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-file.md
@@ -0,0 +1,76 @@
+---
+layout: global
+title: File
+displayTitle: File
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+You can query a file with a specified format directly with SQL.
+
+### Syntax
+
+```sql
+file_format.`file_path`
+```
+
+### Parameters
+
+* **file_format**
+
+    Specifies a file format for a given file path, could be TEXTFILE, ORC, PARQUET, etc.
+
+* **file_path**
+
+    Specifies a file path with a given format.
+
+### Examples
+
+```sql
+-- PARQUET file
+SELECT * FROM parquet.`examples/src/main/resources/users.parquet`;
++------+--------------+----------------+
+|  name|favorite_color|favorite_numbers|
++------+--------------+----------------+
+|Alyssa|          null|  [3, 9, 15, 20]|
+|   Ben|           red|              []|
++------+--------------+----------------+
+
+-- ORC file
+SELECT * FROM orc.`examples/src/main/resources/users.orc`;
++------+--------------+----------------+
+|  name|favorite_color|favorite_numbers|
++------+--------------+----------------+
+|Alyssa|          null|  [3, 9, 15, 20]|
+|   Ben|           red|              []|
++------+--------------+----------------+
+
+-- JSON file
+SELECT * FROM json.`examples/src/main/resources/people.json`;
++----+-------+
+| age|   name|
++----+-------+
+|null|Michael|
+|  30|   Andy|
+|  19| Justin|
++----+-------+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-groupby.md b/docs/sql-ref-syntax-qry-select-groupby.md
index ab1c5d6eb5ee5..934e5f70d4b08 100644
--- a/docs/sql-ref-syntax-qry-select-groupby.md
+++ b/docs/sql-ref-syntax-qry-select-groupby.md
@@ -9,71 +9,93 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-The <code>GROUP BY</code> clause is used to group the rows based on a set of specified grouping expressions and compute aggregations on 
-the group of rows based on one or more specified aggregate functions. Spark also supports advanced aggregations to do multiple 
+
+### Description
+
+The `GROUP BY` clause is used to group the rows based on a set of specified grouping expressions and compute aggregations on
+the group of rows based on one or more specified aggregate functions. Spark also supports advanced aggregations to do multiple
 aggregations for the same input record set via `GROUPING SETS`, `CUBE`, `ROLLUP` clauses.
+When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
 
 ### Syntax
-{% highlight sql %}
-GROUP BY [ GROUPING SETS grouping_sets ] group_expression [ , group_expression [ , ... ] ]
-    [ ( WITH ROLLUP | WITH CUBE | GROUPING SETS grouping_sets ) ) ]
-{% endhighlight %}
+
+```sql
+GROUP BY group_expression [ , group_expression [ , ... ] ]
+    [ { WITH ROLLUP | WITH CUBE | GROUPING SETS (grouping_set [ , ...]) } ]
+
+GROUP BY GROUPING SETS (grouping_set [ , ...])
+```
+
+While aggregate functions are defined as
+```sql
+aggregate_name ( [ DISTINCT ] expression [ , ... ] ) [ FILTER ( WHERE boolean_expression ) ]
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>GROUPING SETS</em></code></dt>
-  <dd>
-    Groups the rows for each subset of the expressions specified in the grouping sets. For example, 
-    <code>GROUP BY GROUPING SETS (warehouse, product)</code> is semantically equivalent
-    to union of results of <code>GROUP BY warehouse</code> and <code>GROUP BY product</code>. This clause
-    is shorthand for a <code>UNION ALL</code> where each leg of the <code>UNION ALL</code> 
-    operator performs aggregation of subset of the columns specified in the <code>GROUPING SETS</code> clause.
-  </dd>
-  <dt><code><em>grouping_sets</em></code></dt>
-  <dd>
-    Specifies one of more groupings based on which the <code>GROUP BY</code> clause performs aggregations. A grouping
-    set is specified by a list of comma-separated expressions in parentheses.<br><br>
-    <b>Syntax:</b>
-      <code>
-        ( () | ( expression [ , ... ] ) )
-      </code>
-  </dd>
-  <dt><code><em>grouping_expression</em></code></dt>
-  <dd>
-    Specifies the critieria based on which the rows are grouped together. The grouping of rows is performed based on
+
+* **GROUPING SETS**
+
+    Groups the rows for each subset of the expressions specified in the grouping sets. For example,
+    `GROUP BY GROUPING SETS (warehouse, product)` is semantically equivalent
+    to union of results of `GROUP BY warehouse` and `GROUP BY product`. This clause
+    is a shorthand for a `UNION ALL` where each leg of the `UNION ALL`
+    operator performs aggregation of subset of the columns specified in the `GROUPING SETS` clause.
+
+* **grouping_set**
+
+    A grouping set is specified by zero or more comma-separated expressions in parentheses.
+
+    **Syntax:** `( [ expression [ , ... ] ] )`
+
+* **grouping_expression**
+
+    Specifies the criteria based on which the rows are grouped together. The grouping of rows is performed based on
     result values of the grouping expressions. A grouping expression may be a column alias, a column position
     or an expression.
-  </dd>
-  <dt><code><em>ROLLUP</em></code></dt>
-  <dd>
-    Specifies multiple levels of aggregations in a single statement. This clause is used to compute aggregations 
-    based on multiple grouping sets. <code>ROLLUP</code> is shorthand for <code>GROUPING SETS</code>. For example,
-    GROUP BY warehouse, product  WITH ROLLUP is equivalent to GROUP BY <code>warehouse, product</code> GROUPING SETS
-    <code> ((warehouse, product), (warehouse), ())</code>.
-    The N elements of a <code>ROLLUP</code> specification results in N+1 <code>GROUPING SETS</code>.
-  </dd>
-  <dt><code><em>CUBE</em></code></dt>
-  <dd>
-    <code>CUBE</code> clause is used to perform aggregations based on combination of grouping columns specified in the 
-    <code>GROUP BY</code> clause. For example, <code>GROUP BY warehouse, product  WITH CUBE</code> is equivalent 
-    to GROUP BY <code>warehouse, product</code> GROUPING SETS <code>((warehouse, product), (warehouse), (product), ())</code>.
-    The N elements of a <code>CUBE</code> specification results in 2^N <code>GROUPING SETS</code>.
-  </dd>
-</dl>
+
+* **ROLLUP**
+
+    Specifies multiple levels of aggregations in a single statement. This clause is used to compute aggregations
+    based on multiple grouping sets. `ROLLUP` is a shorthand for `GROUPING SETS`. For example,
+    `GROUP BY warehouse, product WITH ROLLUP` is equivalent to `GROUP BY GROUPING SETS
+    ((warehouse, product), (warehouse), ())`.
+    The N elements of a `ROLLUP` specification results in N+1 `GROUPING SETS`.
+
+* **CUBE**
+
+    `CUBE` clause is used to perform aggregations based on combination of grouping columns specified in the
+    `GROUP BY` clause. `CUBE` is a shorthand for `GROUPING SETS`. For example,
+    `GROUP BY warehouse, product WITH CUBE` is equivalent to `GROUP BY GROUPING SETS
+    ((warehouse, product), (warehouse), (product), ())`.
+    The N elements of a `CUBE` specification results in 2^N `GROUPING SETS`.
+
+* **aggregate_name**
+
+    Specifies an aggregate function name (MIN, MAX, COUNT, SUM, AVG, etc.).
+
+* **DISTINCT**
+
+    Removes duplicates in input rows before they are passed to aggregate functions.
+
+* **FILTER**
+
+    Filters the input rows for which the `boolean_expression` in the `WHERE` clause evaluates
+    to true are passed to the aggregate function; other rows are discarded.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CREATE TABLE dealer (id INT, city STRING, car_model STRING, quantity INT);
-INSERT INTO dealer VALUES 
+INSERT INTO dealer VALUES
     (100, 'Fremont', 'Honda Civic', 10),
     (100, 'Fremont', 'Honda Accord', 15),
     (100, 'Fremont', 'Honda CRV', 7),
@@ -85,38 +107,58 @@ INSERT INTO dealer VALUES
 
 -- Sum of quantity per dealership. Group by `id`.
 SELECT id, sum(quantity) FROM dealer GROUP BY id ORDER BY id;
-
-  +---+-------------+
-  |id |sum(quantity)|
-  +---+-------------+
-  |100|32           |
-  |200|33           |
-  |300|13           |
-  +---+-------------+
++---+-------------+
+| id|sum(quantity)|
++---+-------------+
+|100|           32|
+|200|           33|
+|300|           13|
++---+-------------+
 
 -- Use column position in GROUP by clause.
 SELECT id, sum(quantity) FROM dealer GROUP BY 1 ORDER BY 1;
-
-  +---+-------------+
-  |id |sum(quantity)|
-  +---+-------------+
-  |100|32           |
-  |200|33           |
-  |300|13           |
-  +---+-------------+
++---+-------------+
+| id|sum(quantity)|
++---+-------------+
+|100|           32|
+|200|           33|
+|300|           13|
++---+-------------+
 
 -- Multiple aggregations.
 -- 1. Sum of quantity per dealership.
--- 2. Max quantity per dealership. 
+-- 2. Max quantity per dealership.
 SELECT id, sum(quantity) AS sum, max(quantity) AS max FROM dealer GROUP BY id ORDER BY id;
++---+---+---+
+| id|sum|max|
++---+---+---+
+|100| 32| 15|
+|200| 33| 20|
+|300| 13|  8|
++---+---+---+
+
+-- Count the number of distinct dealer cities per car_model.
+SELECT car_model, count(DISTINCT city) AS count FROM dealer GROUP BY car_model;
++------------+-----+
+|   car_model|count|
++------------+-----+
+| Honda Civic|    3|
+|   Honda CRV|    2|
+|Honda Accord|    3|
++------------+-----+
 
-  +---+---+---+
-  |id |sum|max|
-  +---+---+---+
-  |100|32 |15 |
-  |200|33 |20 |
-  |300|13 |8  |
-  +---+---+---+
+-- Sum of only 'Honda Civic' and 'Honda CRV' quantities per dealership.
+SELECT id, sum(quantity) FILTER (
+            WHERE car_model IN ('Honda Civic', 'Honda CRV')
+        ) AS `sum(quantity)` FROM dealer
+    GROUP BY id ORDER BY id;
++---+-------------+
+| id|sum(quantity)|
++---+-------------+
+|100|           17|
+|200|           23|
+|300|            5|
++---+-------------+
 
 -- Aggregations using multiple sets of grouping columns in a single statement.
 -- Following performs aggregations based on four sets of grouping columns.
@@ -125,112 +167,135 @@ SELECT id, sum(quantity) AS sum, max(quantity) AS max FROM dealer GROUP BY id OR
 -- 3. car_model
 -- 4. Empty grouping set. Returns quantities for all city and car models.
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
-   GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
-   ORDER BY city;
-
-  +--------+------------+---+
-  |city    |car_model   |sum|
-  +--------+------------+---+
-  |null    |null        |78 |
-  |null    |Honda Accord|33 |
-  |null    |Honda CRV   |10 |
-  |null    |Honda Civic |35 |
-  |Dublin  |null        |33 |
-  |Dublin  |Honda Accord|10 |
-  |Dublin  |Honda CRV   |3  |
-  |Dublin  |Honda Civic |20 |
-  |Fremont |null        |32 |
-  |Fremont |Honda Accord|15 |
-  |Fremont |Honda CRV   |7  |
-  |Fremont |Honda Civic |10 |
-  |San Jose|null        |13 |
-  |San Jose|Honda Accord|8  |
-  |San Jose|Honda Civic |5  |
-  +--------+------------+---+
-
--- Alternate syntax for `GROUPING SETS` in which both `GROUP BY` and `GROUPING SETS` 
+    GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
+    ORDER BY city;
++---------+------------+---+
+|     city|   car_model|sum|
++---------+------------+---+
+|     null|        null| 78|
+|     null| HondaAccord| 33|
+|     null|    HondaCRV| 10|
+|     null|  HondaCivic| 35|
+|   Dublin|        null| 33|
+|   Dublin| HondaAccord| 10|
+|   Dublin|    HondaCRV|  3|
+|   Dublin|  HondaCivic| 20|
+|  Fremont|        null| 32|
+|  Fremont| HondaAccord| 15|
+|  Fremont|    HondaCRV|  7|
+|  Fremont|  HondaCivic| 10|
+| San Jose|        null| 13|
+| San Jose| HondaAccord|  8|
+| San Jose|  HondaCivic|  5|
++---------+------------+---+
+
+-- Alternate syntax for `GROUPING SETS` in which both `GROUP BY` and `GROUPING SETS`
 -- specifications are present.
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
-   GROUP BY city, car_model GROUPING SETS ((city, car_model), (city), (car_model), ())
-   ORDER BY city, car_model;
-
-  +--------+------------+---+
-  |city    |car_model   |sum|
-  +--------+------------+---+
-  |null    |null        |78 |
-  |null    |Honda Accord|33 |
-  |null    |Honda CRV   |10 |
-  |null    |Honda Civic |35 |
-  |Dublin  |null        |33 |
-  |Dublin  |Honda Accord|10 |
-  |Dublin  |Honda CRV   |3  |
-  |Dublin  |Honda Civic |20 |
-  |Fremont |null        |32 |
-  |Fremont |Honda Accord|15 |
-  |Fremont |Honda CRV   |7  |
-  |Fremont |Honda Civic |10 |
-  |San Jose|null        |13 |
-  |San Jose|Honda Accord|8  |
-  |San Jose|Honda Civic |5  |
-  +--------+------------+---+
+    GROUP BY city, car_model GROUPING SETS ((city, car_model), (city), (car_model), ())
+    ORDER BY city, car_model;
++---------+------------+---+
+|     city|   car_model|sum|
++---------+------------+---+
+|     null|        null| 78|
+|     null| HondaAccord| 33|
+|     null|    HondaCRV| 10|
+|     null|  HondaCivic| 35|
+|   Dublin|        null| 33|
+|   Dublin| HondaAccord| 10|
+|   Dublin|    HondaCRV|  3|
+|   Dublin|  HondaCivic| 20|
+|  Fremont|        null| 32|
+|  Fremont| HondaAccord| 15|
+|  Fremont|    HondaCRV|  7|
+|  Fremont|  HondaCivic| 10|
+| San Jose|        null| 13|
+| San Jose| HondaAccord|  8|
+| San Jose|  HondaCivic|  5|
++---------+------------+---+
 
 -- Group by processing with `ROLLUP` clause.
 -- Equivalent GROUP BY GROUPING SETS ((city, car_model), (city), ())
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
-   GROUP BY city, car_model WITH ROLLUP
-   ORDER BY city, car_model;
-
-  +--------+------------+---+
-  |city    |car_model   |sum|
-  +--------+------------+---+
-  |null    |null        |78 |
-  |Dublin  |null        |33 |
-  |Dublin  |Honda Accord|10 |
-  |Dublin  |Honda CRV   |3  |
-  |Dublin  |Honda Civic |20 |
-  |Fremont |null        |32 |
-  |Fremont |Honda Accord|15 |
-  |Fremont |Honda CRV   |7  |
-  |Fremont |Honda Civic |10 |
-  |San Jose|null        |13 |
-  |San Jose|Honda Accord|8  |
-  |San Jose|Honda Civic |5  |
-  +--------+------------+---+
+    GROUP BY city, car_model WITH ROLLUP
+    ORDER BY city, car_model;
++---------+------------+---+
+|     city|   car_model|sum|
++---------+------------+---+
+|     null|        null| 78|
+|   Dublin|        null| 33|
+|   Dublin| HondaAccord| 10|
+|   Dublin|    HondaCRV|  3|
+|   Dublin|  HondaCivic| 20|
+|  Fremont|        null| 32|
+|  Fremont| HondaAccord| 15|
+|  Fremont|    HondaCRV|  7|
+|  Fremont|  HondaCivic| 10|
+| San Jose|        null| 13|
+| San Jose| HondaAccord|  8|
+| San Jose|  HondaCivic|  5|
++---------+------------+---+
 
 -- Group by processing with `CUBE` clause.
 -- Equivalent GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
-   GROUP BY city, car_model WITH CUBE 
-   ORDER BY city, car_model;
-
-  +--------+------------+---+
-  |city    |car_model   |sum|
-  +--------+------------+---+
-  |null    |null        |78 |
-  |null    |Honda Accord|33 |
-  |null    |Honda CRV   |10 |
-  |null    |Honda Civic |35 |
-  |Dublin  |null        |33 |
-  |Dublin  |Honda Accord|10 |
-  |Dublin  |Honda CRV   |3  |
-  |Dublin  |Honda Civic |20 |
-  |Fremont |null        |32 |
-  |Fremont |Honda Accord|15 |
-  |Fremont |Honda CRV   |7  |
-  |Fremont |Honda Civic |10 |
-  |San Jose|null        |13 |
-  |San Jose|Honda Accord|8  |
-  |San Jose|Honda Civic |5  |
-  +--------+------------+---+
-
-{% endhighlight %}
-
-### Related clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+    GROUP BY city, car_model WITH CUBE
+    ORDER BY city, car_model;
++---------+------------+---+
+|     city|   car_model|sum|
++---------+------------+---+
+|     null|        null| 78|
+|     null| HondaAccord| 33|
+|     null|    HondaCRV| 10|
+|     null|  HondaCivic| 35|
+|   Dublin|        null| 33|
+|   Dublin| HondaAccord| 10|
+|   Dublin|    HondaCRV|  3|
+|   Dublin|  HondaCivic| 20|
+|  Fremont|        null| 32|
+|  Fremont| HondaAccord| 15|
+|  Fremont|    HondaCRV|  7|
+|  Fremont|  HondaCivic| 10|
+| San Jose|        null| 13|
+| San Jose| HondaAccord|  8|
+| San Jose|  HondaCivic|  5|
++---------+------------+---+
+
+--Prepare data for ignore nulls example
+CREATE TABLE person (id INT, name STRING, age INT);
+INSERT INTO person VALUES
+    (100, 'Mary', NULL),
+    (200, 'John', 30),
+    (300, 'Mike', 80),
+    (400, 'Dan', 50);
+
+--Select the first row in cloumn age
+SELECT FIRST(age) FROM person;
++--------------------+
+| first(age, false)  |
++--------------------+
+| NULL               |
++--------------------+
+
+--Get the first row in cloumn `age` ignore nulls,last row in column `id` and sum of cloumn `id`.
+SELECT FIRST(age IGNORE NULLS), LAST(id), SUM(id) FROM person;
++-------------------+------------------+----------+
+| first(age, true)  | last(id, false)  | sum(id)  |
++-------------------+------------------+----------+
+| 30                | 400              | 1000     |
++-------------------+------------------+----------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-having.md b/docs/sql-ref-syntax-qry-select-having.md
index 94d9be649f106..59a8c680945e6 100644
--- a/docs/sql-ref-syntax-qry-select-having.md
+++ b/docs/sql-ref-syntax-qry-select-having.md
@@ -9,45 +9,47 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-The <code>HAVING</code> clause is used to filter the results produced by
-<code>GROUP BY</code> based on the specified condition. It is often used
+
+### Description
+
+The `HAVING` clause is used to filter the results produced by
+`GROUP BY` based on the specified condition. It is often used
 in conjunction with a [GROUP BY](sql-ref-syntax-qry-select-groupby.html)
 clause.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 HAVING boolean_expression
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>boolean_expression</em></code></dt>
-  <dd>
-    Specifies any expression that evaluates to a result type <code>boolean</code>. Two or
-    more expressions may be combined together using the logical 
-    operators ( <code>AND</code>, <code>OR</code> ).<br><br>
-
-    <b>Note</b><br>
-    The expressions specified in the <code>HAVING</code> clause can only refer to:
-     <ol>
-      <li>Constants</li>
-      <li>Expressions that appear in GROUP BY</li>
-      <li>Aggregate functions</li>
-    </ol>
-  </dd>
-</dl>
+
+* **boolean_expression**
+
+    Specifies any expression that evaluates to a result type `boolean`. Two or
+    more expressions may be combined together using the logical
+    operators ( `AND`, `OR` ).
+
+    **Note**
+
+    The expressions specified in the `HAVING` clause can only refer to:
+    1. Constants
+    2. Expressions that appear in GROUP BY
+    3. Aggregate functions
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CREATE TABLE dealer (id INT, city STRING, car_model STRING, quantity INT);
 INSERT INTO dealer VALUES
     (100, 'Fremont', 'Honda Civic', 10),
@@ -61,70 +63,68 @@ INSERT INTO dealer VALUES
 
 -- `HAVING` clause referring to column in `GROUP BY`.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING city = 'Fremont';
-
-  +-------+---+
-  |city   |sum|
-  +-------+---+
-  |Fremont|32 |
-  +-------+---+
++-------+---+
+|   city|sum|
++-------+---+
+|Fremont| 32|
++-------+---+
 
 -- `HAVING` clause referring to aggregate function.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING sum(quantity) > 15;
- 
-  +-------+---+
-  |   city|sum|
-  +-------+---+
-  | Dublin| 33|
-  |Fremont| 32|
-  +-------+---+
++-------+---+
+|   city|sum|
++-------+---+
+| Dublin| 33|
+|Fremont| 32|
++-------+---+
 
 -- `HAVING` clause referring to aggregate function by its alias.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING sum > 15;
-
-  +-------+---+
-  |   city|sum|
-  +-------+---+
-  | Dublin| 33|
-  |Fremont| 32|
-  +-------+---+
++-------+---+
+|   city|sum|
++-------+---+
+| Dublin| 33|
+|Fremont| 32|
++-------+---+
 
 -- `HAVING` clause referring to a different aggregate function than what is present in
 -- `SELECT` list.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING max(quantity) > 15;
-
-  +------+---+
-  |city  |sum|
-  +------+---+
-  |Dublin|33 |
-  +------+---+
++------+---+
+|  city|sum|
++------+---+
+|Dublin| 33|
++------+---+
 
 -- `HAVING` clause referring to constant expression.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING 1 > 0 ORDER BY city;
-  
-  +--------+---+
-  |    city|sum|
-  +--------+---+
-  |  Dublin| 33|
-  | Fremont| 32|
-  |San Jose| 13|
-  +--------+---+
++--------+---+
+|    city|sum|
++--------+---+
+|  Dublin| 33|
+| Fremont| 32|
+|San Jose| 13|
++--------+---+
 
 -- `HAVING` clause without a `GROUP BY` clause.
 SELECT sum(quantity) AS sum FROM dealer HAVING sum(quantity) > 10;
-  +---+
-  |sum|
-  +---+
-  | 78|
-  +---+
- 
-{% endhighlight %}
-
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
++---+
+|sum|
++---+
+| 78|
++---+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-hints.md b/docs/sql-ref-syntax-qry-select-hints.md
index eb9655fc0d731..5f1cb4c5bed0b 100644
--- a/docs/sql-ref-syntax-qry-select-hints.md
+++ b/docs/sql-ref-syntax-qry-select-hints.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Query Hints
-displayTitle: Query Hints
+title: Hints
+displayTitle: Hints
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,4 +19,134 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+Hints give users a way to suggest how Spark SQL to use specific approaches to generate its execution plan.
+
+### Syntax
+
+```sql
+/*+ hint [ , ... ] */
+```
+
+### Partitioning Hints
+
+Partitioning hints allow users to suggest a partitioning strategy that Spark should follow. `COALESCE`, `REPARTITION`,
+and `REPARTITION_BY_RANGE` hints are supported and are equivalent to `coalesce`, `repartition`, and
+`repartitionByRange` [Dataset APIs](api/scala/org/apache/spark/sql/Dataset.html), respectively. These hints give users
+a way to tune performance and control the number of output files in Spark SQL. When multiple partitioning hints are
+specified, multiple nodes are inserted into the logical plan, but the leftmost hint is picked by the optimizer.
+
+#### Partitioning Hints Types
+
+* **COALESCE**
+
+  The `COALESCE` hint can be used to reduce the number of partitions to the specified number of partitions. It takes a partition number as a parameter.
+
+* **REPARTITION**
+
+  The `REPARTITION` hint can be used to repartition to the specified number of partitions using the specified partitioning expressions. It takes a partition number, column names, or both as parameters.
+
+* **REPARTITION_BY_RANGE**
+
+  The `REPARTITION_BY_RANGE` hint can be used to repartition to the specified number of partitions using the specified partitioning expressions. It takes column names and an optional partition number as parameters.
+
+#### Examples
+
+```sql
+SELECT /*+ COALESCE(3) */ * FROM t;
+
+SELECT /*+ REPARTITION(3) */ * FROM t;
+
+SELECT /*+ REPARTITION(c) */ * FROM t;
+
+SELECT /*+ REPARTITION(3, c) */ * FROM t;
+
+SELECT /*+ REPARTITION_BY_RANGE(c) */ * FROM t;
+
+SELECT /*+ REPARTITION_BY_RANGE(3, c) */ * FROM t;
+
+-- multiple partitioning hints
+EXPLAIN EXTENDED SELECT /*+ REPARTITION(100), COALESCE(500), REPARTITION_BY_RANGE(3, c) */ * FROM t;
+== Parsed Logical Plan ==
+'UnresolvedHint REPARTITION, [100]
++- 'UnresolvedHint COALESCE, [500]
+   +- 'UnresolvedHint REPARTITION_BY_RANGE, [3, 'c]
+      +- 'Project [*]
+         +- 'UnresolvedRelation [t]
+
+== Analyzed Logical Plan ==
+name: string, c: int
+Repartition 100, true
++- Repartition 500, false
+   +- RepartitionByExpression [c#30 ASC NULLS FIRST], 3
+      +- Project [name#29, c#30]
+         +- SubqueryAlias spark_catalog.default.t
+            +- Relation[name#29,c#30] parquet
+
+== Optimized Logical Plan ==
+Repartition 100, true
++- Relation[name#29,c#30] parquet
+
+== Physical Plan ==
+Exchange RoundRobinPartitioning(100), false, [id=#121]
++- *(1) ColumnarToRow
+   +- FileScan parquet default.t[name#29,c#30] Batched: true, DataFilters: [], Format: Parquet,
+      Location: CatalogFileIndex[file:/spark/spark-warehouse/t], PartitionFilters: [],
+      PushedFilters: [], ReadSchema: struct<name:string>
+```
+
+### Join Hints
+
+Join hints allow users to suggest the join strategy that Spark should use. Prior to Spark 3.0, only the `BROADCAST` Join Hint was supported. `MERGE`, `SHUFFLE_HASH` and `SHUFFLE_REPLICATE_NL` Joint Hints support was added in 3.0. When different join strategy hints are specified on both sides of a join, Spark prioritizes hints in the following order: `BROADCAST` over `MERGE` over `SHUFFLE_HASH` over `SHUFFLE_REPLICATE_NL`. When both sides are specified with the `BROADCAST` hint or the `SHUFFLE_HASH` hint, Spark will pick the build side based on the join type and the sizes of the relations. Since a given strategy may not support all join types, Spark is not guaranteed to use the join strategy suggested by the hint.
+
+#### Join Hints Types
+
+* **BROADCAST**
+
+    Suggests that Spark use broadcast join. The join side with the hint will be broadcast regardless of `autoBroadcastJoinThreshold`. If both sides of the join have the broadcast hints, the one with the smaller size (based on stats) will be broadcast. The aliases for `BROADCAST` are `BROADCASTJOIN` and `MAPJOIN`.
+
+* **MERGE**
+
+    Suggests that Spark use shuffle sort merge join. The aliases for `MERGE` are `SHUFFLE_MERGE` and `MERGEJOIN`.
+
+* **SHUFFLE_HASH**
+
+    Suggests that Spark use shuffle hash join. If both sides have the shuffle hash hints, Spark chooses the smaller side (based on stats) as the build side.
+
+* **SHUFFLE_REPLICATE_NL**
+
+    Suggests that Spark use shuffle-and-replicate nested loop join.
+
+#### Examples
+
+```sql
+-- Join Hints for broadcast join
+SELECT /*+ BROADCAST(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+SELECT /*+ BROADCASTJOIN (t1) */ * FROM t1 left JOIN t2 ON t1.key = t2.key;
+SELECT /*+ MAPJOIN(t2) */ * FROM t1 right JOIN t2 ON t1.key = t2.key;
+
+-- Join Hints for shuffle sort merge join
+SELECT /*+ SHUFFLE_MERGE(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+SELECT /*+ MERGEJOIN(t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+SELECT /*+ MERGE(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+
+-- Join Hints for shuffle hash join
+SELECT /*+ SHUFFLE_HASH(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+
+-- Join Hints for shuffle-and-replicate nested loop join
+SELECT /*+ SHUFFLE_REPLICATE_NL(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+
+-- When different join strategy hints are specified on both sides of a join, Spark
+-- prioritizes the BROADCAST hint over the MERGE hint over the SHUFFLE_HASH hint
+-- over the SHUFFLE_REPLICATE_NL hint.
+-- Spark will issue Warning in the following example
+-- org.apache.spark.sql.catalyst.analysis.HintErrorLogger: Hint (strategy=merge)
+-- is overridden by another hint and will not take effect.
+SELECT /*+ BROADCAST(t1), MERGE(t1, t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+```
+
+### Related Statements
+
+* [JOIN](sql-ref-syntax-qry-select-join.html)
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-inline-table.md b/docs/sql-ref-syntax-qry-select-inline-table.md
new file mode 100644
index 0000000000000..7f0372ea787e5
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-inline-table.md
@@ -0,0 +1,77 @@
+---
+layout: global
+title: Inline Table
+displayTitle: Inline Table
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+An inline table is a temporary table created using a VALUES clause.
+
+### Syntax
+
+```sql
+VALUES ( expression [ , ... ] ) [ table_alias ]
+```
+
+### Parameters
+
+* **expression**
+
+    Specifies a combination of one or more values, operators and SQL functions that results in a value.
+
+* **table_alias**
+
+    Specifies a temporary name with an optional column name list.
+
+    **Syntax:** `[ AS ] table_name [ ( column_name [ , ... ] ) ]`
+
+### Examples
+
+```sql
+-- single row, without a table alias
+SELECT * FROM VALUES ("one", 1);
++----+----+
+|col1|col2|
++----+----+
+| one|   1|
++----+----+
+
+-- three rows with a table alias
+SELECT * FROM VALUES ("one", 1), ("two", 2), ("three", null) AS data(a, b);
++-----+----+
+|    a|   b|
++-----+----+
+|  one|   1|
+|  two|   2|
+|three|null|
++-----+----+
+
+-- complex types with a table alias
+SELECT * FROM VALUES ("one", array(0, 1)), ("two", array(2, 3)) AS data(a, b);
++---+------+
+|  a|     b|
++---+------+
+|one|[0, 1]|
+|two|[2, 3]|
++---+------+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-join.md b/docs/sql-ref-syntax-qry-select-join.md
index ee66dbdb8c823..09b0efd7b5751 100644
--- a/docs/sql-ref-syntax-qry-select-join.md
+++ b/docs/sql-ref-syntax-qry-select-join.md
@@ -19,4 +19,220 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+A SQL join is used to combine rows from two relations based on join criteria. The following section describes the overall join syntax and the sub-sections cover different types of joins along with examples.
+
+### Syntax
+
+```sql
+relation { [ join_type ] JOIN relation [ join_criteria ] | NATURAL join_type JOIN relation }
+```
+
+### Parameters
+
+* **relation**
+
+    Specifies the relation to be joined.
+
+* **join_type**
+
+    Specifies the join type.
+
+    **Syntax:**
+
+    `[ INNER ] | CROSS | LEFT [ OUTER ] | [ LEFT ] SEMI | RIGHT [ OUTER ] | FULL [ OUTER ] | [ LEFT ] ANTI`
+
+* **join_criteria**
+
+    Specifies how the rows from one relation will be combined with the rows of another relation.
+
+    **Syntax:** `ON boolean_expression | USING ( column_name [ , ... ] )`
+
+    `boolean_expression`
+
+    Specifies an expression with a return type of boolean.
+
+### Join Types
+
+#### **Inner Join**
+
+The inner join is the default join in Spark SQL. It selects rows that have matching values in both relations.
+
+**Syntax:**
+
+`relation [ INNER ] JOIN relation [ join_criteria ]`
+
+#### **Left Join**
+
+A left join returns all values from the left relation and the matched values from the right relation, or appends NULL if there is no match. It is also referred to as a left outer join.
+
+**Syntax:**
+
+`relation LEFT [ OUTER ] JOIN relation [ join_criteria ]`
+
+#### **Right Join**
+
+A right join returns all values from the right relation and the matched values from the left relation, or appends NULL if there is no match. It is also referred to as a right outer join.
+
+**Syntax:**
+
+`relation RIGHT [ OUTER ] JOIN relation [ join_criteria ]`
+
+#### **Full Join**
+
+A full join returns all values from both relations, appending NULL values on the side that does not have a match. It is also referred to as a full outer join.
+
+**Syntax:**
+
+`relation FULL [ OUTER ] JOIN relation [ join_criteria ]`
+
+#### **Cross Join**
+
+A cross join returns the Cartesian product of two relations.
+
+**Syntax:**
+
+`relation CROSS JOIN relation [ join_criteria ]`
+
+#### **Semi Join**
+
+A semi join returns values from the left side of the relation that has a match with the right. It is also referred to as a left semi join.
+
+**Syntax:**
+
+`relation [ LEFT ] SEMI JOIN relation [ join_criteria ]`
+
+#### **Anti Join**
+
+An anti join returns values from the left relation that has no match with the right. It is also referred to as a left anti join.
+
+**Syntax:**
+
+`relation [ LEFT ] ANTI JOIN relation [ join_criteria ]`
+
+### Examples
+
+```sql
+-- Use employee and department tables to demonstrate different type of joins.
+SELECT * FROM employee;
++---+-----+------+
+| id| name|deptno|
++---+-----+------+
+|105|Chloe|     5|
+|103| Paul|     3|
+|101| John|     1|
+|102| Lisa|     2|
+|104| Evan|     4|
+|106|  Amy|     6|
++---+-----+------+
+
+SELECT * FROM department;
++------+-----------+
+|deptno|   deptname|
++------+-----------+
+|     3|Engineering|
+|     2|      Sales|
+|     1|  Marketing|
++------+-----------+
+
+-- Use employee and department tables to demonstrate inner join.
+SELECT id, name, employee.deptno, deptname
+    FROM employee INNER JOIN department ON employee.deptno = department.deptno;
++---+-----+------+-----------|
+| id| name|deptno|   deptname|
++---+-----+------+-----------|
+|103| Paul|     3|Engineering|
+|101| John|     1|  Marketing|
+|102| Lisa|     2|      Sales|
++---+-----+------+-----------|
+
+-- Use employee and department tables to demonstrate left join.
+SELECT id, name, employee.deptno, deptname
+    FROM employee LEFT JOIN department ON employee.deptno = department.deptno;
++---+-----+------+-----------|
+| id| name|deptno|   deptname|
++---+-----+------+-----------|
+|105|Chloe|     5|       NULL|
+|103| Paul|     3|Engineering|
+|101| John|     1|  Marketing|
+|102| Lisa|     2|      Sales|
+|104| Evan|     4|       NULL|
+|106|  Amy|     6|       NULL|
++---+-----+------+-----------|
+
+-- Use employee and department tables to demonstrate right join.
+SELECT id, name, employee.deptno, deptname
+    FROM employee RIGHT JOIN department ON employee.deptno = department.deptno;
++---+-----+------+-----------|
+| id| name|deptno|   deptname|
++---+-----+------+-----------|
+|103| Paul|     3|Engineering|
+|101| John|     1|  Marketing|
+|102| Lisa|     2|      Sales|
++---+-----+------+-----------|
+
+-- Use employee and department tables to demonstrate full join.
+SELECT id, name, employee.deptno, deptname
+    FROM employee FULL JOIN department ON employee.deptno = department.deptno;
++---+-----+------+-----------|
+| id| name|deptno|   deptname|
++---+-----+------+-----------|
+|101| John|     1|  Marketing|
+|106|  Amy|     6|       NULL|
+|103| Paul|     3|Engineering|
+|105|Chloe|     5|       NULL|
+|104| Evan|     4|       NULL|
+|102| Lisa|     2|      Sales|
++---+-----+------+-----------|
+
+-- Use employee and department tables to demonstrate cross join.
+SELECT id, name, employee.deptno, deptname FROM employee CROSS JOIN department;
++---+-----+------+-----------|
+| id| name|deptno|   deptname|
++---+-----+------+-----------|
+|105|Chloe|     5|Engineering|
+|105|Chloe|     5|  Marketing|
+|105|Chloe|     5|      Sales|
+|103| Paul|     3|Engineering|
+|103| Paul|     3|  Marketing|
+|103| Paul|     3|      Sales|
+|101| John|     1|Engineering|
+|101| John|     1|  Marketing|
+|101| John|     1|      Sales|
+|102| Lisa|     2|Engineering|
+|102| Lisa|     2|  Marketing|
+|102| Lisa|     2|      Sales|
+|104| Evan|     4|Engineering|
+|104| Evan|     4|  Marketing|
+|104| Evan|     4|      Sales|
+|106|  Amy|     4|Engineering|
+|106|  Amy|     4|  Marketing|
+|106|  Amy|     4|      Sales|
++---+-----+------+-----------|
+
+-- Use employee and department tables to demonstrate semi join.
+SELECT * FROM employee SEMI JOIN department ON employee.deptno = department.deptno;
++---+-----+------+
+| id| name|deptno|
++---+-----+------+
+|103| Paul|     3|
+|101| John|     1|
+|102| Lisa|     2|
++---+-----+------+
+
+-- Use employee and department tables to demonstrate anti join.
+SELECT * FROM employee ANTI JOIN department ON employee.deptno = department.deptno;
++---+-----+------+
+| id| name|deptno|
++---+-----+------+
+|105|Chloe|     5|
+|104| Evan|     4|
+|106|  Amy|     6|
++---+-----+------+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
+* [Hints](sql-ref-syntax-qry-select-hints.html)
diff --git a/docs/sql-ref-syntax-qry-select-lateral-view.md b/docs/sql-ref-syntax-qry-select-lateral-view.md
new file mode 100644
index 0000000000000..f742c8fa57043
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-lateral-view.md
@@ -0,0 +1,125 @@
+---
+layout: global
+title: LATERAL VIEW Clause
+displayTitle: LATERAL VIEW Clause
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `LATERAL VIEW` clause is used in conjunction with generator functions such as `EXPLODE`, which will generate a virtual table containing one or more rows. `LATERAL VIEW` will apply the rows to each original output row.
+
+### Syntax
+
+```sql
+LATERAL VIEW [ OUTER ] generator_function ( expression [ , ... ] ) [ table_alias ] AS column_alias [ , ... ]
+```
+
+### Parameters
+
+* **OUTER**
+
+    If `OUTER` specified, returns null if an input array/map is empty or null. 
+    
+* **generator_function**
+
+    Specifies a generator function (EXPLODE, INLINE, etc.).
+    
+* **table_alias**
+
+    The alias for `generator_function`, which is optional.
+     
+* **column_alias**
+
+    Lists the column aliases of `generator_function`, which may be used in output rows. We may have multiple aliases if `generator_function` have multiple output columns.
+         
+### Examples
+
+```sql
+CREATE TABLE person (id INT, name STRING, age INT, class INT, address STRING);
+INSERT INTO person VALUES
+    (100, 'John', 30, 1, 'Street 1'),
+    (200, 'Mary', NULL, 1, 'Street 2'),
+    (300, 'Mike', 80, 3, 'Street 3'),
+    (400, 'Dan', 50, 4, 'Street 4');
+
+SELECT * FROM person
+    LATERAL VIEW EXPLODE(ARRAY(30, 60)) tabelName AS c_age
+    LATERAL VIEW EXPLODE(ARRAY(40, 80)) AS d_age;
++------+-------+-------+--------+-----------+--------+--------+
+|  id  | name  |  age  | class  |  address  | c_age  | d_age  |
++------+-------+-------+--------+-----------+--------+--------+
+| 100  | John  | 30    | 1      | Street 1  | 30     | 40     |
+| 100  | John  | 30    | 1      | Street 1  | 30     | 80     |
+| 100  | John  | 30    | 1      | Street 1  | 60     | 40     |
+| 100  | John  | 30    | 1      | Street 1  | 60     | 80     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 30     | 40     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 30     | 80     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 60     | 40     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 60     | 80     |
+| 300  | Mike  | 80    | 3      | Street 3  | 30     | 40     |
+| 300  | Mike  | 80    | 3      | Street 3  | 30     | 80     |
+| 300  | Mike  | 80    | 3      | Street 3  | 60     | 40     |
+| 300  | Mike  | 80    | 3      | Street 3  | 60     | 80     |
+| 400  | Dan   | 50    | 4      | Street 4  | 30     | 40     |
+| 400  | Dan   | 50    | 4      | Street 4  | 30     | 80     |
+| 400  | Dan   | 50    | 4      | Street 4  | 60     | 40     |
+| 400  | Dan   | 50    | 4      | Street 4  | 60     | 80     |
++------+-------+-------+--------+-----------+--------+--------+
+
+SELECT c_age, COUNT(1) FROM person
+    LATERAL VIEW EXPLODE(ARRAY(30, 60)) AS c_age
+    LATERAL VIEW EXPLODE(ARRAY(40, 80)) AS d_age 
+GROUP BY c_age;
++--------+-----------+
+| c_age  | count(1)  |
++--------+-----------+
+| 60     | 8         |
+| 30     | 8         |
++--------+-----------+
+
+SELECT * FROM person
+    LATERAL VIEW EXPLODE(ARRAY()) tabelName AS c_age;
++-----+-------+------+--------+----------+--------+
+| id  | name  | age  | class  | address  | c_age  |
++-----+-------+------+--------+----------+--------+
++-----+-------+------+--------+----------+--------+
+
+SELECT * FROM person
+    LATERAL VIEW OUTER EXPLODE(ARRAY()) tabelName AS c_age;
++------+-------+-------+--------+-----------+--------+
+|  id  | name  |  age  | class  |  address  | c_age  |
++------+-------+-------+--------+-----------+--------+
+| 100  | John  | 30    | 1      | Street 1  | NULL   |
+| 200  | Mary  | NULL  | 1      | Street 2  | NULL   |
+| 300  | Mike  | 80    | 3      | Street 3  | NULL   |
+| 400  | Dan   | 50    | 4      | Street 4  | NULL   |
++------+-------+-------+--------+-----------+--------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md
new file mode 100644
index 0000000000000..6211faa8d529e
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-like.md
@@ -0,0 +1,119 @@
+---
+layout: global
+title: LIKE Predicate
+displayTitle: LIKE Predicate
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+A LIKE predicate is used to search for a specific pattern.
+
+### Syntax
+
+```sql
+[ NOT ] { LIKE search_pattern [ ESCAPE esc_char ] | [ RLIKE | REGEXP ] regex_pattern }
+```
+
+### Parameters
+
+* **search_pattern**
+
+    Specifies a string pattern to be searched by the `LIKE` clause. It can contain special pattern-matching characters:
+
+    * `%` matches zero or more characters.
+    * `_` matches exactly one character.
+
+* **esc_char**
+
+    Specifies the escape character. The default escape character is `\`.
+
+* **regex_pattern**
+
+    Specifies a regular expression search pattern to be searched by the `RLIKE` or `REGEXP` clause.
+
+### Examples
+
+```sql
+CREATE TABLE person (id INT, name STRING, age INT);
+INSERT INTO person VALUES
+    (100, 'John', 30),
+    (200, 'Mary', NULL),
+    (300, 'Mike', 80),
+    (400, 'Dan',  50),
+    (500, 'Evan_w', 16);
+
+SELECT * FROM person WHERE name LIKE 'M%';
++---+----+----+
+| id|name| age|
++---+----+----+
+|300|Mike|  80|
+|200|Mary|null|
++---+----+----+
+
+SELECT * FROM person WHERE name LIKE 'M_ry';
++---+----+----+
+| id|name| age|
++---+----+----+
+|200|Mary|null|
++---+----+----+
+
+SELECT * FROM person WHERE name NOT LIKE 'M_ry';
++---+------+---+
+| id|  name|age|
++---+------+---+
+|500|Evan_W| 16|
+|300|  Mike| 80|
+|100|  John| 30|
+|400|   Dan| 50|
++---+------+---+
+
+SELECT * FROM person WHERE name RLIKE 'M+';
++---+----+----+
+| id|name| age|
++---+----+----+
+|300|Mike|  80|
+|200|Mary|null|
++---+----+----+
+
+SELECT * FROM person WHERE name REGEXP 'M+';
++---+----+----+
+| id|name| age|
++---+----+----+
+|300|Mike|  80|
+|200|Mary|null|
++---+----+----+
+
+SELECT * FROM person WHERE name LIKE '%\_%';
++---+------+---+
+| id|  name|age|
++---+------+---+
+|500|Evan_W| 16|
++---+------+---+
+
+SELECT * FROM person WHERE name LIKE '%$_%' ESCAPE '$';
++---+------+---+
+| id|  name|age|
++---+------+---+
+|500|Evan_W| 16|
++---+------+---+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
diff --git a/docs/sql-ref-syntax-qry-select-limit.md b/docs/sql-ref-syntax-qry-select-limit.md
index 2b9999cc40783..bd64ba890901b 100644
--- a/docs/sql-ref-syntax-qry-select-limit.md
+++ b/docs/sql-ref-syntax-qry-select-limit.md
@@ -9,93 +9,101 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-The <code>LIMIT</code> clause is used to constrain the number of rows returned by
+
+### Description
+
+The `LIMIT` clause is used to constrain the number of rows returned by
 the [SELECT](sql-ref-syntax-qry-select.html) statement. In general, this clause
-is used in conjuction with [ORDER BY](sql-ref-syntax-qry-select-orderby.html) to
+is used in conjunction with [ORDER BY](sql-ref-syntax-qry-select-orderby.html) to
 ensure that the results are deterministic.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 LIMIT { ALL | integer_expression }
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>ALL</em></code></dt>
-  <dd>
+
+* **ALL**
+
     If specified, the query returns all the rows. In other words, no limit is applied if this
     option is specified.
-  </dd>
-  <dt><code><em>integer_expression</em></code></dt>
-  <dd>
-    Specifies an expression that returns an integer. 
-  </dd>
-</dl>
+
+* **integer_expression**
+
+    Specifies a foldable expression that returns an integer.
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CREATE TABLE person (name STRING, age INT);
 INSERT INTO person VALUES
-    ('Zen Hui', 25), 
-    ('Anil B', 18), 
-    ('Shone S', 16), 
+    ('Zen Hui', 25),
+    ('Anil B', 18),
+    ('Shone S', 16),
     ('Mike A', 25),
-    ('John A', 18), 
+    ('John A', 18),
     ('Jack N', 16);
-                        
+
 -- Select the first two rows.
 SELECT name, age FROM person ORDER BY name LIMIT 2;
-
-  +------+---+
-  |name  |age|
-  +------+---+
-  |Anil B|18 |
-  |Jack N|16 |
-  +------+---+
++------+---+
+|  name|age|
++------+---+
+|Anil B| 18|
+|Jack N| 16|
++------+---+
 
 -- Specifying ALL option on LIMIT returns all the rows.
 SELECT name, age FROM person ORDER BY name LIMIT ALL;
++-------+---+
+|   name|age|
++-------+---+
+| Anil B| 18|
+| Jack N| 16|
+| John A| 18|
+| Mike A| 25|
+|Shone S| 16|
+|Zen Hui| 25|
++-------+---+
+
+-- A function expression as an input to LIMIT.
+SELECT name, age FROM person ORDER BY name LIMIT length('SPARK');
++-------+---+
+|   name|age|
++-------+---+
+| Anil B| 18|
+| Jack N| 16|
+| John A| 18|
+| Mike A| 25|
+|Shone S| 16|
++-------+---+
+
+-- A non-foldable expression as an input to LIMIT is not allowed.
+SELECT name, age FROM person ORDER BY name LIMIT length(name);
+org.apache.spark.sql.AnalysisException: The limit expression must evaluate to a constant value ...
+```
+
+### Related Statements
 
-  +-------+---+
-  |name   |age|
-  +-------+---+
-  |Anil B |18 |
-  |Jack N |16 |
-  |John A |18 |
-  |Mike A |25 |
-  |Shone S|16 |
-  |Zen Hui|25 |
-  +-------+---+
-
--- A function expression as an input to limit.
-SELECT name, age FROM person ORDER BY name LIMIT length('SPARK')
-
-  +-------+---+
-  |   name|age|
-  +-------+---+
-  | Anil B| 18|
-  | Jack N| 16|
-  | John A| 18|
-  | Mike A| 25|
-  |Shone S| 16|
-  +-------+---+
-{% endhighlight %}
-
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-orderby.md b/docs/sql-ref-syntax-qry-select-orderby.md
index c4b4ced0b7e73..13f0ae40cb828 100644
--- a/docs/sql-ref-syntax-qry-select-orderby.md
+++ b/docs/sql-ref-syntax-qry-select-orderby.md
@@ -9,64 +9,60 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-The <code>ORDER BY</code> clause is used to return the result rows in a sorted manner
+
+### Description
+
+The `ORDER BY` clause is used to return the result rows in a sorted manner
 in the user specified order. Unlike the [SORT BY](sql-ref-syntax-qry-select-sortby.html)
-clause, this clause guarantees a total order in the output. 
+clause, this clause guarantees a total order in the output.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 ORDER BY { expression [ sort_direction | nulls_sort_oder ] [ , ... ] }
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>ORDER BY</em></code></dt>
-  <dd>
-    Specifies a comma-separated list of expressions along with optional parameters <code>sort_direction</code>
-    and <code>nulls_sort_order</code> which are used to sort the rows.
-  </dd>
-  <dt><code><em>sort_direction</em></code></dt>
-  <dd>
+
+* **ORDER BY**
+
+    Specifies a comma-separated list of expressions along with optional parameters `sort_direction`
+    and `nulls_sort_order` which are used to sort the rows.
+
+* **sort_direction**
+
     Optionally specifies whether to sort the rows in ascending or descending
-    order. The valid values for the sort direction are <code>ASC</code> for ascending
-    and <code>DESC</code> for descending. If sort direction is not explicitly specified, then by default
-    rows are sorted ascending. <br><br>
-    <b>Syntax:</b>
-    <code>
-       [ ASC | DESC ]
-    </code>
-  </dd>
-  <dt><code><em>nulls_sort_order</em></code></dt>
-  <dd>
-    Optionally specifies whether NULL values are returned before/after non-NULL values, based on the 
-    sort direction. In Spark, NULL values are considered to be lower than any non-NULL values by default.
-    Therefore the ordering of NULL values depend on the sort direction. If <code>null_sort_order</code> is
-    not specified, then NULLs sort first if sort order is <code>ASC</code> and NULLS sort last if 
-    sort order is <code>DESC</code>.<br><br>
-    <ol>
-      <li> If <code>NULLS FIRST</code> (the default) is specified, then NULL values are returned first 
-           regardless of the sort order.</li>
-      <li>If <code>NULLS LAST</code> is specified, then NULL values are returned last regardless of
-           the sort order. </li>
-    </ol><br>
-    <b>Syntax:</b>
-    <code>
-       [ NULLS { FIRST | LAST } ] 
-    </code>
-  </dd>
-</dl>
+    order. The valid values for the sort direction are `ASC` for ascending
+    and `DESC` for descending. If sort direction is not explicitly specified, then by default
+    rows are sorted ascending.
+
+    **Syntax:** [ ASC `|` DESC ]
+
+* **nulls_sort_order**
+
+    Optionally specifies whether NULL values are returned before/after non-NULL values. If
+    `null_sort_order` is not specified, then NULLs sort first if sort order is
+    `ASC` and NULLS sort last if sort order is `DESC`.
+
+    1. If `NULLS FIRST` is specified, then NULL values are returned first
+       regardless of the sort order.
+    2. If `NULLS LAST` is specified, then NULL values are returned last regardless of
+       the sort order.
+
+    **Syntax:** `[ NULLS { FIRST | LAST } ]`
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CREATE TABLE person (id INT, name STRING, age INT);
 INSERT INTO person VALUES
     (100, 'John', 30),
@@ -75,79 +71,78 @@ INSERT INTO person VALUES
     (400, 'Jerry', NULL),
     (500, 'Dan',  50);
 
--- Sort rows by age. By default rows are sorted in ascending manner.
+-- Sort rows by age. By default rows are sorted in ascending manner with NULL FIRST.
 SELECT name, age FROM person ORDER BY age;
-
-  +-----+----+
-  |name |age |
-  +-----+----+
-  |Jerry|null|
-  |Mary |null|
-  |John |30  |
-  |Dan  |50  |
-  |Mike |80  |
-  +-----+----+
++-----+----+
+| name| age|
++-----+----+
+|Jerry|null|
+| Mary|null|
+| John|  30|
+|  Dan|  50|
+| Mike|  80|
++-----+----+
 
 -- Sort rows in ascending manner keeping null values to be last.
 SELECT name, age FROM person ORDER BY age NULLS LAST;
++-----+----+
+| name| age|
++-----+----+
+| John|  30|
+|  Dan|  50|
+| Mike|  80|
+| Mary|null|
+|Jerry|null|
++-----+----+
 
-  +-----+----+
-  |name |age |
-  +-----+----+
-  |John |30  |
-  |Dan  |50  |
-  |Mike |80  |
-  |Mary |null|
-  |Jerry|null|
-  +-----+----+
-
--- Sort rows by age in descending manner.
+-- Sort rows by age in descending manner, which defaults to NULL LAST.
 SELECT name, age FROM person ORDER BY age DESC;
- 
-  +-----+----+
-  |name |age |
-  +-----+----+
-  |Mike |80  |
-  |Dan  |50  |
-  |John |30  |
-  |Jerry|null|
-  |Mary |null|
-  +-----+----+
++-----+----+
+| name| age|
++-----+----+
+| Mike|  80|
+|  Dan|  50|
+| John|  30|
+|Jerry|null|
+| Mary|null|
++-----+----+
 
 -- Sort rows in ascending manner keeping null values to be first.
 SELECT name, age FROM person ORDER BY age DESC NULLS FIRST;
-
-  +-----+----+
-  |name |age |
-  +-----+----+
-  |Jerry|null|
-  |Mary |null|
-  |Mike |80  |
-  |Dan  |50  |
-  |John |30  |
-  +-----+----+
++-----+----+
+| name| age|
++-----+----+
+|Jerry|null|
+| Mary|null|
+| Mike|  80|
+|  Dan|  50|
+| John|  30|
++-----+----+
 
 -- Sort rows based on more than one column with each column having different
 -- sort direction.
 SELECT * FROM person ORDER BY name ASC, age DESC;
++---+-----+----+
+| id| name| age|
++---+-----+----+
+|500|  Dan|  50|
+|400|Jerry|null|
+|100| John|  30|
+|200| Mary|null|
+|300| Mike|  80|
++---+-----+----+
+```
+
+### Related Statements
 
-  +---+-----+----+
-  |id |name |age |
-  +---+-----+----+
-  |500|Dan  |50  |
-  |400|Jerry|null|
-  |100|John |30  |
-  |200|Mary |null|
-  |300|Mike |80  |
-  +---+-----+----+
-{% endhighlight %}
-
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-pivot.md b/docs/sql-ref-syntax-qry-select-pivot.md
new file mode 100644
index 0000000000000..649c2518b28eb
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-pivot.md
@@ -0,0 +1,101 @@
+---
+layout: global
+title: PIVOT Clause
+displayTitle: PIVOT Clause
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `PIVOT` clause is used for data perspective. We can get the aggregated values based on specific column values, which will be turned to multiple columns used in `SELECT` clause. The `PIVOT` clause can be specified after the table name or subquery.
+
+### Syntax
+
+```sql
+PIVOT ( { aggregate_expression [ AS aggregate_expression_alias ] } [ , ... ]
+    FOR column_list IN ( expression_list ) )
+```
+
+### Parameters
+    
+* **aggregate_expression**
+
+    Specifies an aggregate expression (SUM(a), COUNT(DISTINCT b), etc.).
+    
+* **aggregate_expression_alias**
+
+    Specifies an alias for the aggregate expression.
+     
+* **column_list**
+
+    Contains columns in the `FROM` clause, which specifies the columns we want to replace with new columns. We can use brackets to surround the columns, such as `(c1, c2)`.
+      
+* **expression_list**
+
+    Specifies new columns, which are used to match values in `column_list` as the aggregating condition. We can also add aliases for them.
+    
+### Examples
+
+```sql
+CREATE TABLE person (id INT, name STRING, age INT, class INT, address STRING);
+INSERT INTO person VALUES
+    (100, 'John', 30, 1, 'Street 1'),
+    (200, 'Mary', NULL, 1, 'Street 2'),
+    (300, 'Mike', 80, 3, 'Street 3'),
+    (400, 'Dan', 50, 4, 'Street 4');
+
+SELECT * FROM person
+    PIVOT (
+        SUM(age) AS a, AVG(class) AS c
+        FOR name IN ('John' AS john, 'Mike' AS mike)
+    );
++------+-----------+---------+---------+---------+---------+
+|  id  |  address  | john_a  | john_c  | mike_a  | mike_c  |
++------+-----------+---------+---------+---------+---------+
+| 200  | Street 2  | NULL    | NULL    | NULL    | NULL    |
+| 100  | Street 1  | 30      | 1.0     | NULL    | NULL    |
+| 300  | Street 3  | NULL    | NULL    | 80      | 3.0     |
+| 400  | Street 4  | NULL    | NULL    | NULL    | NULL    |
++------+-----------+---------+---------+---------+---------+
+
+SELECT * FROM person
+    PIVOT (
+        SUM(age) AS a, AVG(class) AS c
+        FOR (name, age) IN (('John', 30) AS c1, ('Mike', 40) AS c2)
+    );
++------+-----------+-------+-------+-------+-------+
+|  id  |  address  | c1_a  | c1_c  | c2_a  | c2_c  |
++------+-----------+-------+-------+-------+-------+
+| 200  | Street 2  | NULL  | NULL  | NULL  | NULL  |
+| 100  | Street 1  | 30    | 1.0   | NULL  | NULL  |
+| 300  | Street 3  | NULL  | NULL  | NULL  | NULL  |
+| 400  | Street 4  | NULL  | NULL  | NULL  | NULL  |
++------+-----------+-------+-------+-------+-------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-sampling.md b/docs/sql-ref-syntax-qry-select-sampling.md
new file mode 100644
index 0000000000000..f9c7b760a239f
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-sampling.md
@@ -0,0 +1,94 @@
+---
+layout: global
+title: Sampling Queries
+displayTitle: Sampling Queries
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `TABLESAMPLE` statement is used to sample the table. It supports the following sampling methods:
+  * `TABLESAMPLE`(x `ROWS`): Sample the table down to the given number of rows.
+  * `TABLESAMPLE`(x `PERCENT`): Sample the table down to the given percentage. Note that percentages are defined as a number between 0 and 100.
+  * `TABLESAMPLE`(`BUCKET` x `OUT OF` y): Sample the table down to a `x` out of `y` fraction.
+
+**Note:** `TABLESAMPLE` returns the approximate number of rows or fraction requested.
+
+### Syntax
+
+```sql
+TABLESAMPLE ({ integer_expression | decimal_expression } PERCENT)
+    | TABLESAMPLE ( integer_expression ROWS )
+    | TABLESAMPLE ( BUCKET integer_expression OUT OF integer_expression )
+```
+
+### Examples
+
+```sql
+SELECT * FROM test;
++--+----+
+|id|name|
++--+----+
+| 5|Alex|
+| 8|Lucy|
+| 2|Mary|
+| 4|Fred|
+| 1|Lisa|
+| 9|Eric|
+|10|Adam|
+| 6|Mark|
+| 7|Lily|
+| 3|Evan|
++--+----+
+
+SELECT * FROM test TABLESAMPLE (50 PERCENT);
++--+----+
+|id|name|
++--+----+
+| 5|Alex|
+| 2|Mary|
+| 4|Fred|
+| 9|Eric|
+|10|Adam|
+| 3|Evan|
++--+----+
+
+SELECT * FROM test TABLESAMPLE (5 ROWS);
++--+----+
+|id|name|
++--+----+
+| 5|Alex|
+| 8|Lucy|
+| 2|Mary|
+| 4|Fred|
+| 1|Lisa|
++--+----+
+
+SELECT * FROM test TABLESAMPLE (BUCKET 4 OUT OF 10);
++--+----+
+|id|name|
++--+----+
+| 8|Lucy|
+| 2|Mary|
+| 9|Eric|
+| 6|Mark|
++--+----+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-qry-select-setops.md b/docs/sql-ref-syntax-qry-select-setops.md
index 526f857520aa2..8cd12c37fa603 100644
--- a/docs/sql-ref-syntax-qry-select-setops.md
+++ b/docs/sql-ref-syntax-qry-select-setops.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Set Operations
-displayTitle: Set Operations
+title: Set Operators
+displayTitle: Set Operators
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -19,4 +19,178 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+Set operators are used to combine two input relations into a single one. Spark SQL supports three types of set operators:
+
+ - `EXCEPT` or `MINUS`
+ - `INTERSECT`
+ - `UNION`
+
+Note that input relations must have the same number of columns and compatible data types for the respective columns.
+
+### EXCEPT
+
+`EXCEPT` and `EXCEPT ALL` return the rows that are found in one relation but not the other. `EXCEPT` (alternatively, `EXCEPT DISTINCT`) takes only distinct rows while `EXCEPT ALL` does not remove duplicates from the result rows. Note that `MINUS` is an alias for `EXCEPT`.
+
+#### Syntax
+
+```sql
+[ ( ] relation [ ) ] EXCEPT | MINUS [ ALL | DISTINCT ] [ ( ] relation [ ) ]
+```
+
+#### Examples
+
+```sql
+-- Use number1 and number2 tables to demonstrate set operators in this page.
+SELECT * FROM number1;
++---+
+|  c|
++---+
+|  3|
+|  1|
+|  2|
+|  2|
+|  3|
+|  4|
++---+
+  
+SELECT * FROM number2;
++---+
+|  c|
++---+
+|  5|
+|  1|
+|  2|
+|  2|
++---+
+
+SELECT c FROM number1 EXCEPT SELECT c FROM number2;
++---+
+|  c|
++---+
+|  3|
+|  4|
++---+
+
+SELECT c FROM number1 MINUS SELECT c FROM number2;
++---+
+|  c|
++---+
+|  3|
+|  4|
++---+
+
+SELECT c FROM number1 EXCEPT ALL (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  3|
+|  3|
+|  4|
++---+
+
+SELECT c FROM number1 MINUS ALL (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  3|
+|  3|
+|  4|
++---+
+```
+
+### INTERSECT
+
+`INTERSECT` and `INTERSECT ALL` return the rows that are found in both relations. `INTERSECT` (alternatively, `INTERSECT DISTINCT`) takes only distinct rows while `INTERSECT ALL` does not remove duplicates from the result rows.
+
+#### Syntax
+
+```sql
+[ ( ] relation [ ) ] INTERSECT [ ALL | DISTINCT ] [ ( ] relation [ ) ]
+```
+
+#### Examples
+
+```sql
+(SELECT c FROM number1) INTERSECT (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  1|
+|  2|
++---+
+
+(SELECT c FROM number1) INTERSECT DISTINCT (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  1|
+|  2|
++---+
+
+(SELECT c FROM number1) INTERSECT ALL (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  1|
+|  2|
+|  2|
++---+
+```
+
+### UNION
+
+`UNION` and `UNION ALL` return the rows that are found in either relation. `UNION` (alternatively, `UNION DISTINCT`) takes only distinct rows while `UNION ALL` does not remove duplicates from the result rows.
+
+#### Syntax
+
+```sql
+[ ( ] relation [ ) ] UNION [ ALL | DISTINCT ] [ ( ] relation [ ) ]
+```
+
+### Examples
+
+```sql
+(SELECT c FROM number1) UNION (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  1|
+|  3|
+|  5|
+|  4|
+|  2|
++---+
+
+(SELECT c FROM number1) UNION DISTINCT (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  1|
+|  3|
+|  5|
+|  4|
+|  2|
++---+
+
+SELECT c FROM number1 UNION ALL (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  3|
+|  1|
+|  2|
+|  2|
+|  3|
+|  4|
+|  5|
+|  1|
+|  2|
+|  2|
++---+
+```
+
+### Related Statements
+
+* [SELECT Statement](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-sortby.md b/docs/sql-ref-syntax-qry-select-sortby.md
index c0a491b78ee83..09e559adcd6ab 100644
--- a/docs/sql-ref-syntax-qry-select-sortby.md
+++ b/docs/sql-ref-syntax-qry-select-sortby.md
@@ -9,179 +9,173 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-The <code>SORT BY</code> clause is used to return the result rows sorted
+
+### Description
+
+The `SORT BY` clause is used to return the result rows sorted
 within each partition in the user specified order. When there is more than one partition
-<code>SORT BY</code> may return result that is partially ordered. This is different
+`SORT BY` may return result that is partially ordered. This is different
 than [ORDER BY](sql-ref-syntax-qry-select-orderby.html) clause which guarantees a
 total order of the output.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 SORT BY { expression [ sort_direction | nulls_sort_order ] [ , ... ] }
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>SORT BY</em></code></dt>
-  <dd>
-    Specifies a comma-separated list of expressions along with optional parameters <code>sort_direction</code>
-    and <code>nulls_sort_order</code> which are used to sort the rows within each partition.
-  </dd>
-  <dt><code><em>sort_direction</em></code></dt>
-  <dd>
+
+* **SORT BY**
+
+    Specifies a comma-separated list of expressions along with optional parameters `sort_direction`
+    and `nulls_sort_order` which are used to sort the rows within each partition.
+
+* **sort_direction**
+
     Optionally specifies whether to sort the rows in ascending or descending
-    order. The valid values for the sort direction are <code>ASC</code> for ascending
-    and <code>DESC</code> for descending. If sort direction is not explicitly specified, then by default
-    rows are sorted ascending. <br><br>
-    <b>Syntax:</b>
-    <code>
-       [ ASC | DESC ]
-    </code>
-  </dd>
-  <dt><code><em>nulls_sort_order</em></code></dt>
-  <dd>
-    Optionally specifies whether NULL values are returned before/after non-NULL values, based on the 
-    sort direction. In Spark, NULL values are considered to be lower than any non-NULL values by default.
-    Therefore the ordering of NULL values depend on the sort direction. If <code>null_sort_order</code> is
-    not specified, then NULLs sort first if sort order is <code>ASC</code> and NULLS sort last if 
-    sort order is <code>DESC</code>.<br><br>
-    <ol>
-      <li> If <code>NULLS FIRST</code> (the default) is specified, then NULL values are returned first 
-           regardless of the sort order.</li>
-      <li>If <code>NULLS LAST</code> is specified, then NULL values are returned last regardless of
-           the sort order. </li>
-    </ol><br>
-    <b>Syntax:</b>
-    <code>
-       [ NULLS { FIRST | LAST } ] 
-    </code>
-  </dd>
-</dl>
+    order. The valid values for the sort direction are `ASC` for ascending
+    and `DESC` for descending. If sort direction is not explicitly specified, then by default
+    rows are sorted ascending.
+
+    **Syntax:** `[ ASC | DESC ]`
+
+* **nulls_sort_order**
+
+    Optionally specifies whether NULL values are returned before/after non-NULL values. If
+    `null_sort_order` is not specified, then NULLs sort first if sort order is
+    `ASC` and NULLS sort last if sort order is `DESC`.
+
+    1. If `NULLS FIRST` is specified, then NULL values are returned first
+       regardless of the sort order.
+    2. If `NULLS LAST` is specified, then NULL values are returned last regardless of
+       the sort order.
+
+    **Syntax:** `[ NULLS { FIRST | LAST } ]`
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CREATE TABLE person (zip_code INT, name STRING, age INT);
 INSERT INTO person VALUES
-    (94588, 'Zen Hui', 50), 
-    (94588, 'Dan Li', 18), 
+    (94588, 'Zen Hui', 50),
+    (94588, 'Dan Li', 18),
     (94588, 'Anil K', 27),
     (94588, 'John V', NULL),
     (94511, 'David K', 42),
     (94511, 'Aryan B.', 18),
     (94511, 'Lalit B.', NULL);
 
--- Use `REPARTITION` hint to partition the data by `zip_code` to 
+-- Use `REPARTITION` hint to partition the data by `zip_code` to
 -- examine the `SORT BY` behavior. This is used in rest of the
 -- examples.
 
 -- Sort rows by `name` within each partition in ascending manner
 SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person SORT BY name;
-
-  +--------+----+--------+
-  |name    |age |zip_code|
-  +--------+----+--------+
-  |Anil K  |27  |94588   |
-  |Dan Li  |18  |94588   |
-  |John V  |null|94588   |
-  |Zen Hui |50  |94588   |
-  |Aryan B.|18  |94511   |
-  |David K |42  |94511   |
-  |Lalit B.|null|94511   |
-  +--------+----+--------+
++--------+----+--------+
+|    name| age|zip_code|
++--------+----+--------+
+|  Anil K|  27|   94588|
+|  Dan Li|  18|   94588|
+|  John V|null|   94588|
+| Zen Hui|  50|   94588|
+|Aryan B.|  18|   94511|
+| David K|  42|   94511|
+|Lalit B.|null|   94511|
++--------+----+--------+
 
 -- Sort rows within each partition using column position.
 SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person SORT BY 1;
-
-  +--------+----+--------+
-  |name    |age |zip_code|
-  +--------+----+--------+
-  |Anil K  |27  |94588   |
-  |Dan Li  |18  |94588   |
-  |John V  |null|94588   |
-  |Zen Hui |50  |94588   |
-  |Aryan B.|18  |94511   |
-  |David K |42  |94511   |
-  |Lalit B.|null|94511   |
-  +--------+----+--------+
++--------+----+--------+
+|    name| age|zip_code|
++--------+----+--------+
+|  Anil K|  27|   94588|
+|  Dan Li|  18|   94588|
+|  John V|null|   94588|
+| Zen Hui|  50|   94588|
+|Aryan B.|  18|   94511|
+| David K|  42|   94511|
+|Lalit B.|null|   94511|
++--------+----+--------+
 
 -- Sort rows within partition in ascending manner keeping null values to be last.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age NULLS LAST;
-
-  +----+--------+--------+
-  |age |name    |zip_code|
-  +----+--------+--------+
-  |18  |Dan Li  |94588   |
-  |27  |Anil K  |94588   |
-  |50  |Zen Hui |94588   |
-  |null|John V  |94588   |
-  |18  |Aryan B.|94511   |
-  |42  |David K |94511   |
-  |null|Lalit B.|94511   |
-  +----+--------+--------+
-
--- Sort rows by age within each partition in descending manner.
++----+--------+--------+
+| age|    name|zip_code|
++----+--------+--------+
+|  18|  Dan Li|   94588|
+|  27|  Anil K|   94588|
+|  50| Zen Hui|   94588|
+|null|  John V|   94588|
+|  18|Aryan B.|   94511|
+|  42| David K|   94511|
+|null|Lalit B.|   94511|
++----+--------+--------+
+
+-- Sort rows by age within each partition in descending manner, which defaults to NULL LAST.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age DESC;
- 
-  +----+--------+--------+
-  |age |name    |zip_code|
-  +----+--------+--------+
-  |50  |Zen Hui |94588   |
-  |27  |Anil K  |94588   |
-  |18  |Dan Li  |94588   |
-  |null|John V  |94588   |
-  |42  |David K |94511   |
-  |18  |Aryan B.|94511   |
-  |null|Lalit B.|94511   |
-  +----+--------+--------+
-
--- Sort rows by age within each partition in ascending manner keeping null values to be first.
++----+--------+--------+
+| age|    name|zip_code|
++----+--------+--------+
+|  50| Zen Hui|   94588|
+|  27|  Anil K|   94588|
+|  18|  Dan Li|   94588|
+|null|  John V|   94588|
+|  42| David K|   94511|
+|  18|Aryan B.|   94511|
+|null|Lalit B.|   94511|
++----+--------+--------+
+
+-- Sort rows by age within each partition in descending manner keeping null values to be first.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age DESC NULLS FIRST;
-
-  +----+--------+--------+
-  |age |name    |zip_code|
-  +----+--------+--------+
-  |null|John V  |94588   |
-  |50  |Zen Hui |94588   |
-  |27  |Anil K  |94588   |
-  |18  |Dan Li  |94588   |
-  |null|Lalit B.|94511   |
-  |42  |David K |94511   |
-  |18  |Aryan B.|94511   |
-  +----+--------+--------+
++----+--------+--------+
+| age|    name|zip_code|
++----+--------+--------+
+|null|  John V|   94588|
+|  50| Zen Hui|   94588|
+|  27|  Anil K|   94588|
+|  18|  Dan Li|   94588|
+|null|Lalit B.|   94511|
+|  42| David K|   94511|
+|  18|Aryan B.|   94511|
++----+--------+--------+
 
 -- Sort rows within each partition based on more than one column with each column having
 -- different sort direction.
 SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person
-   SORT BY name ASC, age DESC;
-
-  +--------+----+--------+
-  |name    |age |zip_code|
-  +--------+----+--------+
-  |Anil K  |27  |94588   |
-  |Dan Li  |18  |94588   |
-  |John V  |null|94588   |
-  |Zen Hui |50  |94588   |
-  |Aryan B.|18  |94511   |
-  |David K |42  |94511   |
-  |Lalit B.|null|94511   |
-  +--------+----+--------+
-{% endhighlight %}
-
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
\ No newline at end of file
+    SORT BY name ASC, age DESC;
++--------+----+--------+
+|    name| age|zip_code|
++--------+----+--------+
+|  Anil K|  27|   94588|
+|  Dan Li|  18|   94588|
+|  John V|null|   94588|
+| Zen Hui|  50|   94588|
+|Aryan B.|  18|   94511|
+| David K|  42|   94511|
+|Lalit B.|null|   94511|
++--------+----+--------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-tvf.md b/docs/sql-ref-syntax-qry-select-tvf.md
new file mode 100644
index 0000000000000..b04e2f5c3fbd7
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-tvf.md
@@ -0,0 +1,166 @@
+---
+layout: global
+title: Table-valued Functions (TVF)
+displayTitle: Table-valued Functions (TVF)
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+A table-valued function (TVF) is a function that returns a relation or a set of rows. There are two types of TVFs in Spark SQL:
+1. a TVF that can be specified in a FROM clause, e.g. range;
+2. a TVF that can be specified in SELECT/LATERAL VIEW clauses, e.g. explode.
+
+### Supported Table-valued Functions
+
+#### TVFs that can be specified in a FROM clause:
+
+|Function|Argument Type(s)|Description|
+|--------|----------------|-----------|
+|**range** ( *end* )|Long|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from 0 to *end* (exclusive) with step value 1.|
+|**range** ( *start, end* )|Long, Long|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from *start* to *end* (exclusive) with step value 1.|
+|**range** ( *start, end, step* )|Long, Long, Long|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from *start* to *end* (exclusive) with *step* value.|
+|**range** ( *start, end, step, numPartitions* )|Long, Long, Long, Int|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from *start* to *end* (exclusive) with *step* value, with partition number *numPartitions* specified.|
+
+#### TVFs that can be specified in SELECT/LATERAL VIEW clauses:
+
+|Function|Argument Type(s)|Description|
+|--------|----------------|-----------|
+|**explode** ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows, or the elements of map *expr* into multiple rows and columns. Unless specified otherwise, uses the default column name col for elements of the array or key and value for the elements of the map.|
+|**explode_outer** <br> ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows, or the elements of map *expr* into multiple rows and columns. Unless specified otherwise, uses the default column name col for elements of the array or key and value for the elements of the map.|
+|**inline** ( *expr* )|Expression|Explodes an array of structs into a table. Uses column names col1, col2, etc. by default unless specified otherwise.|
+|**inline_outer** <br> ( *expr* )|Expression|Explodes an array of structs into a table. Uses column names col1, col2, etc. by default unless specified otherwise.|
+|**posexplode** <br> ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows with positions, or the elements of map *expr* into multiple rows and columns with positions. Unless specified otherwise, uses the column name pos for position, col for elements of the array or key and value for elements of the map.|
+|**posexplode_outer** ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows with positions, or the elements of map *expr* into multiple rows and columns with positions. Unless specified otherwise, uses the column name pos for position, col for elements of the array or key and value for elements of the map.|
+|**stack** ( *n, expr1, ..., exprk* )|Seq[Expression]|Separates *expr1, ..., exprk* into n rows. Uses column names col0, col1, etc. by default unless specified otherwise.|
+|**json_tuple** <br> ( *jsonStr, p1, p2, ..., pn* )|Seq[Expression]|Returns a tuple like the function *get_json_object*, but it takes multiple names. All the input parameters and output column types are string.|
+|**parse_url** <br> ( *url, partToExtract[, key]* )|Seq[Expression]|Extracts a part from a URL.|
+
+### Examples
+
+```sql
+-- range call with end
+SELECT * FROM range(6 + cos(3));
++---+
+| id|
++---+
+|  0|
+|  1|
+|  2|
+|  3|
+|  4|
++---+
+
+-- range call with start and end
+SELECT * FROM range(5, 10);
++---+
+| id|
++---+
+|  5|
+|  6|
+|  7|
+|  8|
+|  9|
++---+
+
+-- range call with numPartitions
+SELECT * FROM range(0, 10, 2, 200);
++---+
+| id|
++---+
+|  0|
+|  2|
+|  4|
+|  6|
+|  8|
++---+
+
+-- range call with a table alias
+SELECT * FROM range(5, 8) AS test;
++---+
+| id|
++---+
+|  5|
+|  6|
+|  7|
++---+
+
+SELECT explode(array(10, 20));
++---+
+|col|
++---+
+| 10|
+| 20|
++---+
+
+SELECT inline(array(struct(1, 'a'), struct(2, 'b')));
++----+----+
+|col1|col2|
++----+----+
+|   1|   a|
+|   2|   b|
++----+----+
+
+SELECT posexplode(array(10,20));
++---+---+
+|pos|col|
++---+---+
+|  0| 10|
+|  1| 20|
++---+---+
+
+SELECT stack(2, 1, 2, 3);
++----+----+
+|col0|col1|
++----+----+
+|   1|   2|
+|   3|null|
++----+----+
+
+SELECT json_tuple('{"a":1, "b":2}', 'a', 'b');
++---+---+
+| c0| c1|
++---+---+
+|  1|  2|
++---+---+
+
+SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST');
++-----------------------------------------------------+
+|parse_url(http://spark.apache.org/path?query=1, HOST)|
++-----------------------------------------------------+
+|                                     spark.apache.org|
++-----------------------------------------------------+
+
+-- Use explode in a LATERAL VIEW clause
+CREATE TABLE test (c1 INT);
+INSERT INTO test VALUES (1);
+INSERT INTO test VALUES (2);
+SELECT * FROM test LATERAL VIEW explode (ARRAY(3,4)) AS c2;
++--+--+
+|c1|c2|
++--+--+
+| 1| 3|
+| 1| 4|
+| 2| 3|
+| 2| 4|
++--+--+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-where.md b/docs/sql-ref-syntax-qry-select-where.md
index a493623df480b..9ff7993d40c58 100644
--- a/docs/sql-ref-syntax-qry-select-where.md
+++ b/docs/sql-ref-syntax-qry-select-where.md
@@ -9,35 +9,38 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-The <code>WHERE</code> clause is used to limit the results of the <code>FROM</code>
+
+### Description
+
+The `WHERE` clause is used to limit the results of the `FROM`
 clause of a query or a subquery based on the specified condition.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 WHERE boolean_expression
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>boolean_expression</em></code></dt>
-  <dd>
-    Specifies any expression that evaluates to a result type <code>boolean</code>. Two or
-    more expressions may be combined together using the logical 
-    operators ( <code>AND</code>, <code>OR</code> ).
-  </dd>
-</dl>
+
+* **boolean_expression**
+
+    Specifies any expression that evaluates to a result type `boolean`. Two or
+    more expressions may be combined together using the logical
+    operators ( `AND`, `OR` ).
 
 ### Examples
-{% highlight sql %}
+
+```sql
 CREATE TABLE person (id INT, name STRING, age INT);
 INSERT INTO person VALUES
     (100, 'John', 30),
@@ -47,78 +50,81 @@ INSERT INTO person VALUES
 
 -- Comparison operator in `WHERE` clause.
 SELECT * FROM person WHERE id > 200 ORDER BY id;
-  +---+----+---+
-  |id |name|age|
-  +---+----+---+
-  |300|Mike|80 |
-  |400|Dan |50 |
-  +---+----+---+
++---+----+---+
+| id|name|age|
++---+----+---+
+|300|Mike| 80|
+|400| Dan| 50|
++---+----+---+
 
 -- Comparison and logical operators in `WHERE` clause.
 SELECT * FROM person WHERE id = 200 OR id = 300 ORDER BY id;
-  +---+----+----+
-  |id |name|age |
-  +---+----+----+
-  |200|Mary|null|
-  |300|Mike|80  |
-  +---+----+----+
++---+----+----+
+| id|name| age|
++---+----+----+
+|200|Mary|null|
+|300|Mike|  80|
++---+----+----+
 
 -- IS NULL expression in `WHERE` clause.
 SELECT * FROM person WHERE id > 300 OR age IS NULL ORDER BY id;
-  +---+----+----+
-  |id |name|age |
-  +---+----+----+
-  |200|Mary|null|
-  |400|Dan |50  |
-  +---+----+----+
++---+----+----+
+| id|name| age|
++---+----+----+
+|200|Mary|null|
+|400| Dan|  50|
++---+----+----+
 
 -- Function expression in `WHERE` clause.
 SELECT * FROM person WHERE length(name) > 3 ORDER BY id;
-  +---+----+----+
-  |id |name|age |
-  +---+----+----+
-  |100|John|30  |
-  |200|Mary|null|
-  |300|Mike|80  |
-  +---+----+----+
++---+----+----+
+| id|name| age|
++---+----+----+
+|100|John|  30|
+|200|Mary|null|
+|300|Mike|  80|
++---+----+----+
 
 -- `BETWEEN` expression in `WHERE` clause.
 SELECT * FROM person WHERE id BETWEEN 200 AND 300 ORDER BY id;
-  +---+----+----+
-  | id|name| age|
-  +---+----+----+
-  |200|Mary|null|
-  |300|Mike|  80|
-  +---+----+----+
++---+----+----+
+| id|name| age|
++---+----+----+
+|200|Mary|null|
+|300|Mike|  80|
++---+----+----+
 
 -- Scalar Subquery in `WHERE` clause.
 SELECT * FROM person WHERE age > (SELECT avg(age) FROM person);
-  +---+----+---+
-  |id |name|age|
-  +---+----+---+
-  |300|Mike|80 |
-  +---+----+---+
-
--- Correlated column reference in `WHERE` clause of subquery.
-SELECT * FROM person AS parent 
-WHERE EXISTS (
-              SELECT 1 FROM person AS child
-              WHERE parent.id = child.id AND child.age IS NULL
-             );
-  +---+----+----+
-  |id |name|age |
-  +---+----+----+
-  |200|Mary|null|
-  +---+----+----+
-
-{% endhighlight %}
-
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
++---+----+---+
+| id|name|age|
++---+----+---+
+|300|Mike| 80|
++---+----+---+
+
+-- Correlated Subquery in `WHERE` clause.
+SELECT * FROM person AS parent
+    WHERE EXISTS (
+        SELECT 1 FROM person AS child
+        WHERE parent.id = child.id AND child.age IS NULL
+    );
++---+----+----+
+|id |name|age |
++---+----+----+
+|200|Mary|null|
++---+----+----+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-qry-select-window.md b/docs/sql-ref-syntax-qry-select-window.md
new file mode 100644
index 0000000000000..a1c2b18b04fce
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-window.md
@@ -0,0 +1,191 @@
+---
+layout: global
+title: Window Functions
+displayTitle: Window Functions
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+Window functions operate on a group of rows, referred to as a window, and calculate a return value for each row based on the group of rows. Window functions are useful for processing tasks such as calculating a moving average, computing a cumulative statistic, or accessing the value of rows given the relative position of the current row.
+
+### Syntax
+
+```sql
+window_function OVER
+( [  { PARTITION | DISTRIBUTE } BY partition_col_name = partition_col_val ( [ , ... ] ) ]
+  { ORDER | SORT } BY expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ... ]
+  [ window_frame ] )
+```
+
+### Parameters
+
+* **window_function**
+
+    * Ranking Functions
+
+      **Syntax:** `RANK | DENSE_RANK | PERCENT_RANK | NTILE | ROW_NUMBER`
+
+    * Analytic Functions
+
+      **Syntax:** `CUME_DIST | LAG | LEAD`
+
+    * Aggregate Functions
+
+      **Syntax:** `MAX | MIN | COUNT | SUM | AVG | ...`
+
+      Please refer to the [Built-in Aggregation Functions](sql-ref-functions-builtin.html#aggregate-functions) document for a complete list of Spark aggregate functions.
+
+* **window_frame**
+
+    Specifies which row to start the window on and where to end it.
+
+    **Syntax:**
+
+    `{ RANGE | ROWS } { frame_start | BETWEEN frame_start AND frame_end }`
+
+    * `frame_start` and `frame_end` have the following syntax:
+
+      **Syntax:**
+
+      `UNBOUNDED PRECEDING | offset PRECEDING | CURRENT ROW | offset FOLLOWING | UNBOUNDED FOLLOWING`
+
+      `offset:` specifies the `offset` from the position of the current row.
+
+    **Note:** If `frame_end` is omitted it defaults to `CURRENT ROW`.
+
+### Examples
+
+```sql
+CREATE TABLE employees (name STRING, dept STRING, salary INT, age INT);
+
+INSERT INTO employees VALUES ("Lisa", "Sales", 10000, 35);
+INSERT INTO employees VALUES ("Evan", "Sales", 32000, 38);
+INSERT INTO employees VALUES ("Fred", "Engineering", 21000, 28);
+INSERT INTO employees VALUES ("Alex", "Sales", 30000, 33);
+INSERT INTO employees VALUES ("Tom", "Engineering", 23000, 33);
+INSERT INTO employees VALUES ("Jane", "Marketing", 29000, 28);
+INSERT INTO employees VALUES ("Jeff", "Marketing", 35000, 38);
+INSERT INTO employees VALUES ("Paul", "Engineering", 29000, 23);
+INSERT INTO employees VALUES ("Chloe", "Engineering", 23000, 25);
+
+SELECT * FROM employees;
++-----+-----------+------+-----+
+| name|       dept|salary|  age|
++-----+-----------+------+-----+
+|Chloe|Engineering| 23000|   25|
+| Fred|Engineering| 21000|   28|
+| Paul|Engineering| 29000|   23|
+|Helen|  Marketing| 29000|   40|
+|  Tom|Engineering| 23000|   33|
+| Jane|  Marketing| 29000|   28|
+| Jeff|  Marketing| 35000|   38|
+| Evan|      Sales| 32000|   38|
+| Lisa|      Sales| 10000|   35|
+| Alex|      Sales| 30000|   33|
++-----+-----------+------+-----+
+
+SELECT name, dept, RANK() OVER (PARTITION BY dept ORDER BY salary) AS rank FROM employees;
++-----+-----------+------+----+
+| name|       dept|salary|rank|
++-----+-----------+------+----+
+| Lisa|      Sales| 10000|   1|
+| Alex|      Sales| 30000|   2|
+| Evan|      Sales| 32000|   3|
+| Fred|Engineering| 21000|   1|
+|  Tom|Engineering| 23000|   2|
+|Chloe|Engineering| 23000|   2|
+| Paul|Engineering| 29000|   4|
+|Helen|  Marketing| 29000|   1|
+| Jane|  Marketing| 29000|   1|
+| Jeff|  Marketing| 35000|   3|
++-----+-----------+------+----+
+
+SELECT name, dept, DENSE_RANK() OVER (PARTITION BY dept ORDER BY salary ROWS BETWEEN
+    UNBOUNDED PRECEDING AND CURRENT ROW) AS dense_rank FROM employees;
++-----+-----------+------+----------+
+| name|       dept|salary|dense_rank|
++-----+-----------+------+----------+
+| Lisa|      Sales| 10000|         1|
+| Alex|      Sales| 30000|         2|
+| Evan|      Sales| 32000|         3|
+| Fred|Engineering| 21000|         1|
+|  Tom|Engineering| 23000|         2|
+|Chloe|Engineering| 23000|         2|
+| Paul|Engineering| 29000|         3|
+|Helen|  Marketing| 29000|         1|
+| Jane|  Marketing| 29000|         1|
+| Jeff|  Marketing| 35000|         2|
++-----+-----------+------+----------+
+
+SELECT name, dept, age, CUME_DIST() OVER (PARTITION BY dept ORDER BY age
+    RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cume_dist FROM employees;
++-----+-----------+------+------------------+
+| name|       dept|age   |         cume_dist|
++-----+-----------+------+------------------+
+| Alex|      Sales|    33|0.3333333333333333|
+| Lisa|      Sales|    35|0.6666666666666666|
+| Evan|      Sales|    38|               1.0|
+| Paul|Engineering|    23|              0.25|
+|Chloe|Engineering|    25|              0.75|
+| Fred|Engineering|    28|              0.25|
+|  Tom|Engineering|    33|               1.0|
+| Jane|  Marketing|    28|0.3333333333333333|
+| Jeff|  Marketing|    38|0.6666666666666666|
+|Helen|  Marketing|    40|               1.0|
++-----+-----------+------+------------------+
+
+SELECT name, dept, salary, MIN(salary) OVER (PARTITION BY dept ORDER BY salary) AS min
+    FROM employees;
++-----+-----------+------+-----+
+| name|       dept|salary|  min|
++-----+-----------+------+-----+
+| Lisa|      Sales| 10000|10000|
+| Alex|      Sales| 30000|10000|
+| Evan|      Sales| 32000|10000|
+|Helen|  Marketing| 29000|29000|
+| Jane|  Marketing| 29000|29000|
+| Jeff|  Marketing| 35000|29000|
+| Fred|Engineering| 21000|21000|
+|  Tom|Engineering| 23000|21000|
+|Chloe|Engineering| 23000|21000|
+| Paul|Engineering| 29000|21000|
++-----+-----------+------+-----+
+
+SELECT name, salary,
+    LAG(salary) OVER (PARTITION BY dept ORDER BY salary) AS lag,
+    LEAD(salary, 1, 0) OVER (PARTITION BY dept ORDER BY salary) AS lead
+    FROM employees;
++-----+-----------+------+-----+-----+
+| name|       dept|salary|  lag| lead|
++-----+-----------+------+-----+-----+
+| Lisa|      Sales| 10000|NULL |30000|
+| Alex|      Sales| 30000|10000|32000|
+| Evan|      Sales| 32000|30000|    0|
+| Fred|Engineering| 21000| NULL|23000|
+|Chloe|Engineering| 23000|21000|23000|
+|  Tom|Engineering| 23000|23000|29000|
+| Paul|Engineering| 29000|23000|    0|
+|Helen|  Marketing| 29000| NULL|29000|
+| Jane|  Marketing| 29000|29000|35000|
+| Jeff|  Marketing| 35000|29000|    0|
++-----+-----------+------+-----+-----+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 00bd719004fb8..bac7c2bc6a06d 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -1,7 +1,7 @@
 ---
 layout: global
 title: SELECT
-displayTitle: SELECT 
+displayTitle: SELECT
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,138 +9,169 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
+### Description
+
 Spark supports a `SELECT` statement and conforms to the ANSI SQL standard. Queries are
-used to retrieve result sets from one or more tables. The following section 
+used to retrieve result sets from one or more tables. The following section
 describes the overall query syntax and the sub-sections cover different constructs
-of a query along with examples. 
+of a query along with examples.
 
 ### Syntax
-{% highlight sql %}
+
+```sql
 [ WITH with_query [ , ... ] ]
+select_statement [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select_statement, ... ]
+    [ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ... ] } ]
+    [ SORT BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ... ] } ]
+    [ CLUSTER BY { expression [ , ... ] } ]
+    [ DISTRIBUTE BY { expression [, ... ] } ]
+    [ WINDOW { named_window [ , WINDOW named_window, ... ] } ]
+    [ LIMIT { ALL | expression } ]
+```
+
+While `select_statement` is defined as
+```sql
 SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
-  FROM { from_item [ , ...] }
-  [ WHERE boolean_expression ]
-  [ GROUP BY expression [ , ...] ]
-  [ HAVING boolean_expression ]
-  [ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
-  [ SORT  BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
-  [ CLUSTER BY { expression [ , ...] } ]
-  [ DISTRIBUTE BY { expression [, ...] } ]
-  { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select ]
-  [ WINDOW { named_window [ , WINDOW named_window, ... ] } ]
-  [ LIMIT { ALL | expression } ]
-{% endhighlight %}
+    FROM { from_item [ , ... ] }
+    [ PIVOT clause ]
+    [ LATERAL VIEW clause ] [ ... ] 
+    [ WHERE boolean_expression ]
+    [ GROUP BY expression [ , ... ] ]
+    [ HAVING boolean_expression ]
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>with_query</em></code></dt>
-  <dd>
-    Specifies the common table expressions (CTEs) before the main <code>SELECT</code> query block.
-    These table expressions are allowed to be referenced later in the main query. This is useful to abstract
-    out repeated subquery blocks in the main query and improves readability of the query.
-  </dd>
-  <dt><code><em>hints</em></code></dt>
-  <dd>
+
+* **with_query**
+
+    Specifies the [common table expressions (CTEs)](sql-ref-syntax-qry-select-cte.html) before the main query block.
+    These table expressions are allowed to be referenced later in the FROM clause. This is useful to abstract
+    out repeated subquery blocks in the FROM clause and improves readability of the query.
+
+* **hints**
+
     Hints can be specified to help spark optimizer make better planning decisions. Currently spark supports hints
-    that influence selection of join strategies and repartitioning of the data. 
-  </dd>
-  <dt><code><em>ALL</em></code></dt>
-  <dd>
+    that influence selection of join strategies and repartitioning of the data.
+
+* **ALL**
+
     Select all matching rows from the relation and is enabled by default.
-  </dd>
-  <dt><code><em>DISTINCT</em></code></dt>
-  <dd>
+
+* **DISTINCT**
+
     Select all matching rows from the relation after removing duplicates in results.
-  </dd>
-  <dt><code><em>named_expression</em></code></dt>
-  <dd>
-    An expression with an assigned name. In general, it denotes a column expression.<br><br>
-    <b>Syntax:</b>
-      <code>
-        expression [AS] [alias]
-      </code>
-  </dd>
-  <dt><code><em>from_item</em></code></dt>
-  <dd>
-    Specifies a source of input for the query. It can be one of the following:
-    <ol>
-      <li>Table relation</li>
-      <li>Join relation</li>
-      <li>Table valued function</li>
-      <li>Inlined table</li>
-      <li>Subquery</li>    
-    </ol>
-  </dd>
-  <dt><code><em>WHERE</em></code></dt>
-  <dd>
-    Filters the result of the FROM clause based on the supplied predicates.
-  </dd>
-  <dt><code><em>GROUP BY</em></code></dt>
-  <dd>
-    Specifies the expressions that are used to group the rows. This is used in conjunction with aggregate functions
-    (MIN, MAX, COUNT, SUM, AVG) to group rows based on the grouping expressions.
-  </dd>
-  <dt><code><em>HAVING</em></code></dt>
-  <dd>
-    Specifies the predicates by which the rows produced by GROUP BY are filtered. The HAVING clause is used to
-    filter rows after the grouping is performed.
-  </dd>
-  <dt><code><em>ORDER BY</em></code></dt>
-  <dd>
-    Specifies an ordering of the rows of the complete result set of the query. The output rows are ordered
-    across the partitions. This parameter is mutually exclusive with <code>SORT BY</code>,
-    <code>CLUSTER BY</code> and <code>DISTRIBUTE BY</code> and can not be specified together.
-  </dd>
-  <dt><code><em>SORT BY</em></code></dt>
-  <dd>
-    Specifies an ordering by which the rows are ordered within each partition. This parameter is mutually
-    exclusive with <code>ORDER BY</code> and <code>CLUSTER BY</code> and can not be specified together.
-  </dd>
-  <dt><code><em>CLUSTER BY</em></code></dt>
-  <dd>
-    Specifies a set of expressions that is used to repartition and sort the rows. Using this clause has
-    the same effect of using <code>DISTRIBUTE BY</code> and <code>SORT BY</code> together. 
-  </dd>
-  <dt><code><em>DISTRIBUTE BY</em></code></dt>
-  <dd>
-    Specifies a set of expressions by which the result rows are repartitioned. This parameter is mutually 
-    exclusive with <code>ORDER BY</code> and <code>CLUSTER BY</code> and can not be specified together. 
-  </dd>
-  <dt><code><em>LIMIT</em></code></dt>
-  <dd>
-    Specifies the maximum number of rows that can be returned by a statement or subquery. This clause 
-    is mostly used in the conjunction with <code>ORDER BY</code> to produce a deterministic result. 
-  </dd>
-  <dt><code><em>boolean_expression</em></code></dt>
-  <dd>
-    Specifies an expression with a return type of boolean.
-  </dd>
-  <dt><code><em>expression</em></code></dt>
-  <dd>
-    Specifies a combination of one or more values, operators, and SQL functions that evaluates to a value.
-  </dd>
-  <dt><code><em>named_window</em></code></dt>
-  <dd>
-    Specifies aliases for one or more source window specifications. The source window specifications can 
-    be referenced in the widow definitions in the query.
-  </dd>
-</dl>
-
-### Related Clauses
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
\ No newline at end of file
+
+* **named_expression**
+
+    An expression with an assigned name. In general, it denotes a column expression.
+
+    **Syntax:** `expression [AS] [alias]`
+
+* **from_item**
+
+     Specifies a source of input for the query. It can be one of the following:
+     * Table relation
+     * [Join relation](sql-ref-syntax-qry-select-join.html)
+     * [Table-value function](sql-ref-syntax-qry-select-tvf.html)
+     * [Inline table](sql-ref-syntax-qry-select-inline-table.html)
+     * Subquery
+     * [File](sql-ref-syntax-qry-select-file.html)
+     
+* **PIVOT**
+
+     The `PIVOT` clause is used for data perspective; We can get the aggregated values based on specific column value.
+
+* **LATERAL VIEW**
+     
+     The `LATERAL VIEW` clause is used in conjunction with generator functions such as `EXPLODE`, which will generate a virtual table containing one or more rows. `LATERAL VIEW` will apply the rows to each original output row.
+ 
+* **WHERE**
+
+     Filters the result of the FROM clause based on the supplied predicates.
+
+* **GROUP BY**
+
+     Specifies the expressions that are used to group the rows. This is used in conjunction with aggregate functions
+     (MIN, MAX, COUNT, SUM, AVG, etc.) to group rows based on the grouping expressions and aggregate values in each group.
+     When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
+
+* **HAVING**
+
+     Specifies the predicates by which the rows produced by GROUP BY are filtered. The HAVING clause is used to
+     filter rows after the grouping is performed. If HAVING is specified without GROUP BY, it indicates a GROUP BY
+     without grouping expressions (global aggregate).
+
+* **ORDER BY**
+
+     Specifies an ordering of the rows of the complete result set of the query. The output rows are ordered
+     across the partitions. This parameter is mutually exclusive with `SORT BY`,
+     `CLUSTER BY` and `DISTRIBUTE BY` and can not be specified together.
+
+* **SORT BY**
+
+     Specifies an ordering by which the rows are ordered within each partition. This parameter is mutually
+     exclusive with `ORDER BY` and `CLUSTER BY` and can not be specified together.
+
+* **CLUSTER BY**
+
+     Specifies a set of expressions that is used to repartition and sort the rows. Using this clause has
+     the same effect of using `DISTRIBUTE BY` and `SORT BY` together.
+
+* **DISTRIBUTE BY**
+
+     Specifies a set of expressions by which the result rows are repartitioned. This parameter is mutually
+     exclusive with `ORDER BY` and `CLUSTER BY` and can not be specified together.
+
+* **LIMIT**
+
+     Specifies the maximum number of rows that can be returned by a statement or subquery. This clause
+     is mostly used in the conjunction with `ORDER BY` to produce a deterministic result.
+
+* **boolean_expression**
+
+     Specifies any expression that evaluates to a result type `boolean`. Two or
+     more expressions may be combined together using the logical
+     operators ( `AND`, `OR` ).
+
+* **expression**
+
+     Specifies a combination of one or more values, operators, and SQL functions that evaluates to a value.
+
+* **named_window**
+
+     Specifies aliases for one or more source window specifications. The source window specifications can
+     be referenced in the widow definitions in the query.
+
+### Related Statements
+
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+* [Hints](sql-ref-syntax-qry-select-hints.html)
+* [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+* [File](sql-ref-syntax-qry-select-file.html)
+* [JOIN](sql-ref-syntax-qry-select-join.html)
+* [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
+* [Set Operators](sql-ref-syntax-qry-select-setops.html)
+* [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
+* [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+* [Window Function](sql-ref-syntax-qry-select-window.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-window.md b/docs/sql-ref-syntax-qry-window.md
deleted file mode 100644
index 767f47715e89c..0000000000000
--- a/docs/sql-ref-syntax-qry-window.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Windowing Analytic Functions
-displayTitle: Windowing Analytic Functions
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-**This page is under construction**
diff --git a/docs/sql-ref-syntax-qry.md b/docs/sql-ref-syntax-qry.md
index cd7c0ffccdc0f..6751b90e12443 100644
--- a/docs/sql-ref-syntax-qry.md
+++ b/docs/sql-ref-syntax-qry.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,20 +19,34 @@ license: |
   limitations under the License.
 ---
 
-Spark supports <code>SELECT</code> statement that is  used to retrieve rows
+Spark supports <code>SELECT</code> statement that is used to retrieve rows
 from one or more tables according to the specified clauses. The full syntax
 and brief description of supported clauses are explained in
-[SELECT](sql-ref-syntax-qry-select.html) section. Spark also provides the
+[SELECT](sql-ref-syntax-qry-select.html) section. The SQL statements related
+to SELECT are also included in this section. Spark also provides the
 ability to generate logical and physical plan for a given query using
 [EXPLAIN](sql-ref-syntax-qry-explain.html) statement.
 
- 
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
-- [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)
+* [SELECT Statement](sql-ref-syntax-qry-select.html)
+  * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+  * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+  * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+  * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+  * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+  * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+  * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+  * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+  * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+  * [Hints](sql-ref-syntax-qry-select-hints.html)
+  * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+  * [File](sql-ref-syntax-qry-select-file.html)
+  * [JOIN](sql-ref-syntax-qry-select-join.html)
+  * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
+  * [Set Operators](sql-ref-syntax-qry-select-setops.html)
+  * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
+  * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+  * [Window Function](sql-ref-syntax-qry-select-window.html)
+  * [CASE Clause](sql-ref-syntax-qry-select-case.html)
+  * [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+  * [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
+* [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)
diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md
index 2510278ab252c..148ce107c94bd 100644
--- a/docs/sql-ref-syntax.md
+++ b/docs/sql-ref-syntax.md
@@ -19,4 +19,83 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL is Apache Spark's module for working with structured data. The SQL Syntax section describes the SQL syntax in detail along with usage examples when applicable.
+Spark SQL is Apache Spark's module for working with structured data. The SQL Syntax section describes the SQL syntax in detail along with usage examples when applicable. This document provides a list of Data Definition and Data Manipulation Statements, as well as Data Retrieval and Auxiliary Statements.
+
+### DDL Statements
+
+ * [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
+ * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+ * [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+ * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+ * [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
+ * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+ * [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+ * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+ * [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
+ * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+ * [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+ * [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
+ * [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
+ * [USE DATABASE](sql-ref-syntax-ddl-usedb.html)
+
+### DML Statements
+
+ * [INSERT INTO](sql-ref-syntax-dml-insert-into.html)
+ * [INSERT OVERWRITE](sql-ref-syntax-dml-insert-overwrite-table.html)
+ * [INSERT OVERWRITE DIRECTORY](sql-ref-syntax-dml-insert-overwrite-directory.html)
+ * [INSERT OVERWRITE DIRECTORY with Hive format](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+ * [LOAD](sql-ref-syntax-dml-load.html)
+
+### Data Retrieval Statements
+
+ * [SELECT Statement](sql-ref-syntax-qry-select.html)
+   * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+   * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+   * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+   * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+   * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+   * [Hints](sql-ref-syntax-qry-select-hints.html)
+   * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+   * [File](sql-ref-syntax-qry-select-file.html)
+   * [JOIN](sql-ref-syntax-qry-select-join.html)
+   * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
+   * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+   * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+   * [Set Operators](sql-ref-syntax-qry-select-setops.html)
+   * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+   * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
+   * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+   * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+   * [Window Function](sql-ref-syntax-qry-select-window.html)
+   * [CASE Clause](sql-ref-syntax-qry-select-case.html)
+   * [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+   * [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
+ * [EXPLAIN](sql-ref-syntax-qry-explain.html)
+
+### Auxiliary Statements
+
+ * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+ * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+ * [ANALYZE TABLE](sql-ref-syntax-aux-analyze-table.html)
+ * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+ * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+ * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+ * [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+ * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+ * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+ * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+ * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+ * [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
+ * [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
+ * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
+ * [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
+ * [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
+ * [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
+ * [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
+ * [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
+ * [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
+ * [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
+ * [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
+ * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
+ * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
diff --git a/docs/sql-ref.md b/docs/sql-ref.md
index 6c57b0d6fdf6f..6a87166f7133d 100644
--- a/docs/sql-ref.md
+++ b/docs/sql-ref.md
@@ -1,7 +1,8 @@
 ---
 layout: global
-title: Reference
-displayTitle: Reference
+title: SQL Reference
+redirect_from: /sql-reference
+displayTitle: SQL Reference
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -19,7 +20,21 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL is Apache Spark's module for working with structured data.
-This guide is a reference for Structured Query Language (SQL) for Apache 
-Spark. This document describes the SQL constructs supported by Spark in detail
-along with usage examples when applicable.
+Spark SQL is Apache Spark's module for working with structured data. This guide is a reference for Structured Query Language (SQL) and includes syntax, semantics, keywords, and examples for common SQL usage. It contains information for the following topics:
+
+ * [ANSI Compliance](sql-ref-ansi-compliance.html)
+ * [Data Types](sql-ref-datatypes.html)
+ * [Datetime Pattern](sql-ref-datetime-pattern.html)
+ * [Functions](sql-ref-functions.html)
+   * [Built-in Functions](sql-ref-functions-builtin.html)
+   * [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+   * [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+   * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
+ * [Identifiers](sql-ref-identifier.html)
+ * [Literals](sql-ref-literals.html)
+ * [Null Semantics](sql-ref-null-semantics.html)
+ * [SQL Syntax](sql-ref-syntax.html)
+   * [DDL Statements](sql-ref-syntax-ddl.html)
+   * [DML Statements](sql-ref-syntax-dml.html)
+   * [Data Retrieval Statements](sql-ref-syntax-qry.html)
+   * [Auxiliary Statements](sql-ref-syntax-aux.html)
diff --git a/docs/ss-migration-guide.md b/docs/ss-migration-guide.md
index db8fdff8b2ac4..002058b69bf30 100644
--- a/docs/ss-migration-guide.md
+++ b/docs/ss-migration-guide.md
@@ -30,4 +30,6 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
 
 - In Spark 3.0, Structured Streaming forces the source schema into nullable when file-based datasources such as text, json, csv, parquet and orc are used via `spark.readStream(...)`. Previously, it respected the nullability in source schema; however, it caused issues tricky to debug with NPE. To restore the previous behavior, set `spark.sql.streaming.fileSource.schema.forceNullable` to `false`.
 
-- Spark 3.0 fixes the correctness issue on Stream-stream outer join, which changes the schema of state. (SPARK-26154 for more details) Spark 3.0 will fail the query if you start your query from checkpoint constructed from Spark 2.x which uses stream-stream outer join. Please discard the checkpoint and replay previous inputs to recalculate outputs.
\ No newline at end of file
+- Spark 3.0 fixes the correctness issue on Stream-stream outer join, which changes the schema of state. (See [SPARK-26154](https://issues.apache.org/jira/browse/SPARK-26154) for more details). If you start your query from checkpoint constructed from Spark 2.x which uses stream-stream outer join, Spark 3.0 fails the query. To recalculate outputs, discard the checkpoint and replay previous inputs.
+
+- In Spark 3.0, the deprecated class `org.apache.spark.sql.streaming.ProcessingTime` has been removed. Use `org.apache.spark.sql.streaming.Trigger.ProcessingTime` instead. Likewise, `org.apache.spark.sql.execution.streaming.continuous.ContinuousTrigger` has been removed in favor of `Trigger.Continuous`, and `org.apache.spark.sql.execution.streaming.OneTimeTrigger` has been hidden in favor of `Trigger.Once`.
\ No newline at end of file
diff --git a/docs/streaming-custom-receivers.md b/docs/streaming-custom-receivers.md
index a37b961243f47..6ce73b27d11ba 100644
--- a/docs/streaming-custom-receivers.md
+++ b/docs/streaming-custom-receivers.md
@@ -28,7 +28,7 @@ in Scala or Java.
 ## Implementing a Custom Receiver
 
 This starts with implementing a **Receiver**
-([Scala doc](api/scala/index.html#org.apache.spark.streaming.receiver.Receiver),
+([Scala doc](api/scala/org/apache/spark/streaming/receiver/Receiver.html),
 [Java doc](api/java/org/apache/spark/streaming/receiver/Receiver.html)).
 A custom receiver must extend this abstract class by implementing two methods
 
diff --git a/docs/streaming-kafka-integration.md b/docs/streaming-kafka-integration.md
index 6fa363285f385..3282124b0be5d 100644
--- a/docs/streaming-kafka-integration.md
+++ b/docs/streaming-kafka-integration.md
@@ -23,4 +23,4 @@ replicated commit log service.  Please read the [Kafka documentation](https://ka
 thoroughly before starting an integration using Spark.
 
 At the moment, Spark requires Kafka 0.10 and higher. See 
-<a href="streaming-kafka-0-10-integration.html">Kafka 0.10 integration documentation</a> for details.
\ No newline at end of file
+<a href="streaming-kafka-0-10-integration.html">Kafka 0.10 integration documentation</a> for details.
diff --git a/docs/streaming-kinesis-integration.md b/docs/streaming-kinesis-integration.md
index 3389d453c2cbd..c7959d4201151 100644
--- a/docs/streaming-kinesis-integration.md
+++ b/docs/streaming-kinesis-integration.md
@@ -46,20 +46,20 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
             import org.apache.spark.storage.StorageLevel
             import org.apache.spark.streaming.kinesis.KinesisInputDStream
             import org.apache.spark.streaming.{Seconds, StreamingContext}
-            import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream
+            import org.apache.spark.streaming.kinesis.KinesisInitialPositions
 
             val kinesisStream = KinesisInputDStream.builder
                 .streamingContext(streamingContext)
                 .endpointUrl([endpoint URL])
                 .regionName([region name])
                 .streamName([streamName])
-                .initialPositionInStream([initial position])
+                .initialPosition([initial position])
                 .checkpointAppName([Kinesis app name])
                 .checkpointInterval([checkpoint interval])
                 .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
                 .build()
 
-	See the [API docs](api/scala/index.html#org.apache.spark.streaming.kinesis.KinesisInputDStream)
+	See the [API docs](api/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.html)
 	and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala). Refer to the [Running the Example](#running-the-example) subsection for instructions on how to run the example.
 
 	</div>
@@ -68,14 +68,14 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
             import org.apache.spark.streaming.kinesis.KinesisInputDStream;
             import org.apache.spark.streaming.Seconds;
             import org.apache.spark.streaming.StreamingContext;
-            import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream;
+            import org.apache.spark.streaming.kinesis.KinesisInitialPositions;
 
             KinesisInputDStream<byte[]> kinesisStream = KinesisInputDStream.builder()
                 .streamingContext(streamingContext)
                 .endpointUrl([endpoint URL])
                 .regionName([region name])
                 .streamName([streamName])
-                .initialPositionInStream([initial position])
+                .initialPosition([initial position])
                 .checkpointAppName([Kinesis app name])
                 .checkpointInterval([checkpoint interval])
                 .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
@@ -110,7 +110,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
                 import org.apache.spark.storage.StorageLevel
                 import org.apache.spark.streaming.kinesis.KinesisInputDStream
                 import org.apache.spark.streaming.{Seconds, StreamingContext}
-                import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream
+                import org.apache.spark.streaming.kinesis.KinesisInitialPositions
                 import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration
                 import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel
 
@@ -119,7 +119,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
                     .endpointUrl([endpoint URL])
                     .regionName([region name])
                     .streamName([streamName])
-                    .initialPositionInStream([initial position])
+                    .initialPosition([initial position])
                     .checkpointAppName([Kinesis app name])
                     .checkpointInterval([checkpoint interval])
                     .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
@@ -133,7 +133,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
                 import org.apache.spark.streaming.kinesis.KinesisInputDStream;
                 import org.apache.spark.streaming.Seconds;
                 import org.apache.spark.streaming.StreamingContext;
-                import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream;
+                import org.apache.spark.streaming.kinesis.KinesisInitialPositions;
                 import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration;
                 import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel;
                 import scala.collection.JavaConverters;
@@ -143,7 +143,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
                     .endpointUrl([endpoint URL])
                     .regionName([region name])
                     .streamName([streamName])
-                    .initialPositionInStream([initial position])
+                    .initialPosition([initial position])
                     .checkpointAppName([Kinesis app name])
                     .checkpointInterval([checkpoint interval])
                     .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
@@ -170,7 +170,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
 
 	- `[checkpoint interval]`: The interval (e.g., Duration(2000) = 2 seconds) at which the Kinesis Client Library saves its position in the stream.  For starters, set it to the same as the batch interval of the streaming application.
 
-	- `[initial position]`: Can be either `InitialPositionInStream.TRIM_HORIZON` or `InitialPositionInStream.LATEST` (see [`Kinesis Checkpointing`](#kinesis-checkpointing) section and [`Amazon Kinesis API documentation`](http://docs.aws.amazon.com/streams/latest/dev/developing-consumers-with-sdk.html) for more details).
+	- `[initial position]`: Can be either `KinesisInitialPositions.TrimHorizon` or `KinesisInitialPositions.Latest` or `KinesisInitialPositions.AtTimestamp` (see [`Kinesis Checkpointing`](#kinesis-checkpointing) section and [`Amazon Kinesis API documentation`](http://docs.aws.amazon.com/streams/latest/dev/developing-consumers-with-sdk.html) for more details).
 
 	- `[message handler]`: A function that takes a Kinesis `Record` and outputs generic `T`.
 
@@ -229,7 +229,7 @@ To run the example,
 
 - Set up Kinesis stream (see earlier section) within AWS. Note the name of the Kinesis stream and the endpoint URL corresponding to the region where the stream was created.
 
-- Set up the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY` with your AWS credentials.
+- Set up the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` with your AWS credentials.
 
 - In the Spark root directory, run the example as
 
@@ -246,8 +246,7 @@ To run the example,
 	</div>
 	<div data-lang="python" markdown="1">
 
-        ./bin/spark-submit --jars external/kinesis-asl/target/scala-*/\
-            spark-streaming-kinesis-asl-assembly_*.jar \
+        ./bin/spark-submit --jars 'external/kinesis-asl-assembly/target/spark-streaming-kinesis-asl-assembly_*.jar' \
             external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \
             [Kinesis app name] [Kinesis stream name] [endpoint URL] [region name]
 
@@ -272,9 +271,9 @@ de-aggregate records during consumption.
 
 - Checkpointing too frequently will cause excess load on the AWS checkpoint storage layer and may lead to AWS throttling.  The provided example handles this throttling with a random-backoff-retry strategy.
 
-- If no Kinesis checkpoint info exists when the input DStream starts, it will start either from the oldest record available (`InitialPositionInStream.TRIM_HORIZON`) or from the latest tip (`InitialPositionInStream.LATEST`).  This is configurable.
-  - `InitialPositionInStream.LATEST` could lead to missed records if data is added to the stream while no input DStreams are running (and no checkpoint info is being stored).
-  - `InitialPositionInStream.TRIM_HORIZON` may lead to duplicate processing of records where the impact is dependent on checkpoint frequency and processing idempotency.
+- If no Kinesis checkpoint info exists when the input DStream starts, it will start either from the oldest record available (`KinesisInitialPositions.TrimHorizon`), or from the latest tip (`KinesisInitialPositions.Latest`), or (except Python) from the position denoted by the provided UTC timestamp (`KinesisInitialPositions.AtTimestamp(Date timestamp)`).  This is configurable.
+  - `KinesisInitialPositions.Latest` could lead to missed records if data is added to the stream while no input DStreams are running (and no checkpoint info is being stored).
+  - `KinesisInitialPositions.TrimHorizon` may lead to duplicate processing of records where the impact is dependent on checkpoint frequency and processing idempotency.
 
 #### Kinesis retry configuration
  - `spark.streaming.kinesis.retry.waitTime` : Wait time between Kinesis retries as a duration string. When reading from Amazon Kinesis, users may hit `ProvisionedThroughputExceededException`'s, when consuming faster than 5 transactions/second or, exceeding the maximum read rate of 2 MiB/second. This configuration can be tweaked to increase the sleep between fetches when a fetch fails to reduce these exceptions. Default is "100ms".
diff --git a/docs/streaming-programming-guide.md b/docs/streaming-programming-guide.md
index f6b579fbf74d1..ac4aa9255ae68 100644
--- a/docs/streaming-programming-guide.md
+++ b/docs/streaming-programming-guide.md
@@ -57,7 +57,7 @@ Spark Streaming provides a high-level abstraction called *discretized stream* or
 which represents a continuous stream of data. DStreams can be created either from input data
 streams from sources such as Kafka, and Kinesis, or by applying high-level
 operations on other DStreams. Internally, a DStream is represented as a sequence of
-[RDDs](api/scala/index.html#org.apache.spark.rdd.RDD).
+[RDDs](api/scala/org/apache/spark/rdd/RDD.html).
 
 This guide shows you how to start writing Spark Streaming programs with DStreams. You can
 write Spark Streaming programs in Scala, Java or Python (introduced in Spark 1.2),
@@ -80,7 +80,7 @@ do is as follows.
 <div data-lang="scala"  markdown="1" >
 First, we import the names of the Spark Streaming classes and some implicit
 conversions from StreamingContext into our environment in order to add useful methods to
-other classes we need (like DStream). [StreamingContext](api/scala/index.html#org.apache.spark.streaming.StreamingContext) is the
+other classes we need (like DStream). [StreamingContext](api/scala/org/apache/spark/streaming/StreamingContext.html) is the
 main entry point for all streaming functionality. We create a local StreamingContext with two execution threads,  and a batch interval of 1 second.
 
 {% highlight scala %}
@@ -185,7 +185,7 @@ JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(x.split(" ")).itera
 generating multiple new records from each record in the source DStream. In this case,
 each line will be split into multiple words and the stream of words is represented as the
 `words` DStream. Note that we defined the transformation using a
-[FlatMapFunction](api/scala/index.html#org.apache.spark.api.java.function.FlatMapFunction) object.
+[FlatMapFunction](api/scala/org/apache/spark/api/java/function/FlatMapFunction.html) object.
 As we will discover along the way, there are a number of such convenience classes in the Java API
 that help defines DStream transformations.
 
@@ -201,9 +201,9 @@ wordCounts.print();
 {% endhighlight %}
 
 The `words` DStream is further mapped (one-to-one transformation) to a DStream of `(word,
-1)` pairs, using a [PairFunction](api/scala/index.html#org.apache.spark.api.java.function.PairFunction)
+1)` pairs, using a [PairFunction](api/scala/org/apache/spark/api/java/function/PairFunction.html)
 object. Then, it is reduced to get the frequency of words in each batch of data,
-using a [Function2](api/scala/index.html#org.apache.spark.api.java.function.Function2) object.
+using a [Function2](api/scala/org/apache/spark/api/java/function/Function2.html) object.
 Finally, `wordCounts.print()` will print a few of the counts generated every second.
 
 Note that when these lines are executed, Spark Streaming only sets up the computation it
@@ -435,7 +435,7 @@ To initialize a Spark Streaming program, a **StreamingContext** object has to be
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-A [StreamingContext](api/scala/index.html#org.apache.spark.streaming.StreamingContext) object can be created from a [SparkConf](api/scala/index.html#org.apache.spark.SparkConf) object.
+A [StreamingContext](api/scala/org/apache/spark/streaming/StreamingContext.html) object can be created from a [SparkConf](api/scala/org/apache/spark/SparkConf.html) object.
 
 {% highlight scala %}
 import org.apache.spark._
@@ -451,7 +451,7 @@ or a special __"local[\*]"__ string to run in local mode. In practice, when runn
 you will not want to hardcode `master` in the program,
 but rather [launch the application with `spark-submit`](submitting-applications.html) and
 receive it there. However, for local testing and unit tests, you can pass "local[\*]" to run Spark Streaming
-in-process (detects the number of cores in the local system). Note that this internally creates a [SparkContext](api/scala/index.html#org.apache.spark.SparkContext) (starting point of all Spark functionality) which can be accessed as `ssc.sparkContext`.
+in-process (detects the number of cores in the local system). Note that this internally creates a [SparkContext](api/scala/org/apache/spark/SparkContext.html) (starting point of all Spark functionality) which can be accessed as `ssc.sparkContext`.
 
 The batch interval must be set based on the latency requirements of your application
 and available cluster resources. See the [Performance Tuning](#setting-the-right-batch-interval)
@@ -584,7 +584,7 @@ Input DStreams are DStreams representing the stream of input data received from
 sources. In the [quick example](#a-quick-example), `lines` was an input DStream as it represented
 the stream of data received from the netcat server. Every input DStream
 (except file stream, discussed later in this section) is associated with a **Receiver**
-([Scala doc](api/scala/index.html#org.apache.spark.streaming.receiver.Receiver),
+([Scala doc](api/scala/org/apache/spark/streaming/receiver/Receiver.html),
 [Java doc](api/java/org/apache/spark/streaming/receiver/Receiver.html)) object which receives the
 data from a source and stores it in Spark's memory for processing.
 
@@ -739,7 +739,7 @@ DStreams can be created with data streams received through custom receivers. See
 For testing a Spark Streaming application with test data, one can also create a DStream based on a queue of RDDs, using `streamingContext.queueStream(queueOfRDDs)`. Each RDD pushed into the queue will be treated as a batch of data in the DStream, and processed like a stream.
 
 For more details on streams from sockets and files, see the API documentations of the relevant functions in
-[StreamingContext](api/scala/index.html#org.apache.spark.streaming.StreamingContext) for
+[StreamingContext](api/scala/org/apache/spark/streaming/StreamingContext.html) for
 Scala, [JavaStreamingContext](api/java/index.html?org/apache/spark/streaming/api/java/JavaStreamingContext.html)
 for Java, and [StreamingContext](api/python/pyspark.streaming.html#pyspark.streaming.StreamingContext) for Python.
 
@@ -1219,8 +1219,8 @@ joinedStream = windowedStream.transform(lambda rdd: rdd.join(dataset))
 In fact, you can also dynamically change the dataset you want to join against. The function provided to `transform` is evaluated every batch interval and therefore will use the current dataset that `dataset` reference points to.
 
 The complete list of DStream transformations is available in the API documentation. For the Scala API,
-see [DStream](api/scala/index.html#org.apache.spark.streaming.dstream.DStream)
-and [PairDStreamFunctions](api/scala/index.html#org.apache.spark.streaming.dstream.PairDStreamFunctions).
+see [DStream](api/scala/org/apache/spark/streaming/dstream/DStream.html)
+and [PairDStreamFunctions](api/scala/org/apache/spark/streaming/dstream/PairDStreamFunctions.html).
 For the Java API, see [JavaDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaDStream.html)
 and [JavaPairDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaPairDStream.html).
 For the Python API, see [DStream](api/python/pyspark.streaming.html#pyspark.streaming.DStream).
@@ -2067,7 +2067,7 @@ for prime time, the old one be can be brought down. Note that this can be done f
 sending the data to two destinations (i.e., the earlier and upgraded applications).
 
 - The existing application is shutdown gracefully (see
-[`StreamingContext.stop(...)`](api/scala/index.html#org.apache.spark.streaming.StreamingContext)
+[`StreamingContext.stop(...)`](api/scala/org/apache/spark/streaming/StreamingContext.html)
 or [`JavaStreamingContext.stop(...)`](api/java/index.html?org/apache/spark/streaming/api/java/JavaStreamingContext.html)
 for graceful shutdown options) which ensure data that has been received is completely
 processed before shutdown. Then the
@@ -2104,7 +2104,7 @@ In that case, consider
 [reducing](#reducing-the-batch-processing-times) the batch processing time.
 
 The progress of a Spark Streaming program can also be monitored using the
-[StreamingListener](api/scala/index.html#org.apache.spark.streaming.scheduler.StreamingListener) interface,
+[StreamingListener](api/scala/org/apache/spark/streaming/scheduler/StreamingListener.html) interface,
 which allows you to get receiver status and processing times. Note that this is a developer API
 and it is likely to be improved upon (i.e., more information reported) in the future.
 
@@ -2197,7 +2197,7 @@ computation is not high enough. For example, for distributed reduce operations l
 and `reduceByKeyAndWindow`, the default number of parallel tasks is controlled by
 the `spark.default.parallelism` [configuration property](configuration.html#spark-properties). You
 can pass the level of parallelism as an argument (see
-[`PairDStreamFunctions`](api/scala/index.html#org.apache.spark.streaming.dstream.PairDStreamFunctions)
+[`PairDStreamFunctions`](api/scala/org/apache/spark/streaming/dstream/PairDStreamFunctions.html)
 documentation), or set the `spark.default.parallelism`
 [configuration property](configuration.html#spark-properties) to change the default.
 
@@ -2205,9 +2205,9 @@ documentation), or set the `spark.default.parallelism`
 {:.no_toc}
 The overheads of data serialization can be reduced by tuning the serialization formats. In the case of streaming, there are two types of data that are being serialized.
 
-* **Input data**: By default, the input data received through Receivers is stored in the executors' memory with [StorageLevel.MEMORY_AND_DISK_SER_2](api/scala/index.html#org.apache.spark.storage.StorageLevel$). That is, the data is serialized into bytes to reduce GC overheads, and replicated for tolerating executor failures. Also, the data is kept first in memory, and spilled over to disk only if the memory is insufficient to hold all of the input data necessary for the streaming computation. This serialization obviously has overheads -- the receiver must deserialize the received data and re-serialize it using Spark's serialization format. 
+* **Input data**: By default, the input data received through Receivers is stored in the executors' memory with [StorageLevel.MEMORY_AND_DISK_SER_2](api/scala/org/apache/spark/storage/StorageLevel$.html). That is, the data is serialized into bytes to reduce GC overheads, and replicated for tolerating executor failures. Also, the data is kept first in memory, and spilled over to disk only if the memory is insufficient to hold all of the input data necessary for the streaming computation. This serialization obviously has overheads -- the receiver must deserialize the received data and re-serialize it using Spark's serialization format.
 
-* **Persisted RDDs generated by Streaming Operations**: RDDs generated by streaming computations may be persisted in memory. For example, window operations persist data in memory as they would be processed multiple times. However, unlike the Spark Core default of [StorageLevel.MEMORY_ONLY](api/scala/index.html#org.apache.spark.storage.StorageLevel$), persisted RDDs generated by streaming computations are persisted with [StorageLevel.MEMORY_ONLY_SER](api/scala/index.html#org.apache.spark.storage.StorageLevel$) (i.e. serialized) by default to minimize GC overheads.
+* **Persisted RDDs generated by Streaming Operations**: RDDs generated by streaming computations may be persisted in memory. For example, window operations persist data in memory as they would be processed multiple times. However, unlike the Spark Core default of [StorageLevel.MEMORY_ONLY](api/scala/org/apache/spark/storage/StorageLevel$.html), persisted RDDs generated by streaming computations are persisted with [StorageLevel.MEMORY_ONLY_SER](api/scala/org/apache/spark/storage/StorageLevel.html$) (i.e. serialized) by default to minimize GC overheads.
 
 In both cases, using Kryo serialization can reduce both CPU and memory overheads. See the [Spark Tuning Guide](tuning.html#data-serialization) for more details. For Kryo, consider registering custom classes, and disabling object reference tracking (see Kryo-related configurations in the [Configuration Guide](configuration.html#compression-and-serialization)).
 
@@ -2247,7 +2247,7 @@ A good approach to figure out the right batch size for your application is to te
 conservative batch interval (say, 5-10 seconds) and a low data rate. To verify whether the system
 is able to keep up with the data rate, you can check the value of the end-to-end delay experienced
 by each processed batch (either look for "Total delay" in Spark driver log4j logs, or use the
-[StreamingListener](api/scala/index.html#org.apache.spark.streaming.scheduler.StreamingListener)
+[StreamingListener](api/scala/org/apache/spark/streaming/scheduler/StreamingListener.html)
 interface).
 If the delay is maintained to be comparable to the batch size, then system is stable. Otherwise,
 if the delay is continuously increasing, it means that the system is unable to keep up and it
@@ -2485,10 +2485,10 @@ additional effort may be necessary to achieve exactly-once semantics. There are
 * Third-party DStream data sources can be found in [Third Party Projects](https://spark.apache.org/third-party-projects.html)
 * API documentation
   - Scala docs
-    * [StreamingContext](api/scala/index.html#org.apache.spark.streaming.StreamingContext) and
-  [DStream](api/scala/index.html#org.apache.spark.streaming.dstream.DStream)
-    * [KafkaUtils](api/scala/index.html#org.apache.spark.streaming.kafka.KafkaUtils$),
-    [KinesisUtils](api/scala/index.html#org.apache.spark.streaming.kinesis.KinesisInputDStream),
+    * [StreamingContext](api/scala/org/apache/spark/streaming/StreamingContext.html) and
+  [DStream](api/scala/org/apache/spark/streaming/dstream/DStream.html)
+    * [KafkaUtils](api/scala/org/apache/spark/streaming/kafka/KafkaUtils$.html),
+    [KinesisUtils](api/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.html),
   - Java docs
     * [JavaStreamingContext](api/java/index.html?org/apache/spark/streaming/api/java/JavaStreamingContext.html),
     [JavaDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaDStream.html) and
diff --git a/docs/structured-streaming-kafka-integration.md b/docs/structured-streaming-kafka-integration.md
index a1eeee54987db..600cf58a9fd8e 100644
--- a/docs/structured-streaming-kafka-integration.md
+++ b/docs/structured-streaming-kafka-integration.md
@@ -61,7 +61,7 @@ val df = spark
   .option("includeHeaders", "true")
   .load()
 df.selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)", "headers")
-  .as[(String, String, Map)]
+  .as[(String, String, Array[(String, Array[Byte])])]
 
 // Subscribe to multiple topics
 val df = spark
@@ -525,28 +525,32 @@ The caching key is built up from the following information:
 The following properties are available to configure the consumer pool:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.consumer.cache.capacity</td>
-  <td>The maximum number of consumers cached. Please note that it's a soft limit.</td>
   <td>64</td>
+  <td>The maximum number of consumers cached. Please note that it's a soft limit.</td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.timeout</td>
-  <td>The minimum amount of time a consumer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>5m (5 minutes)</td>
+  <td>The minimum amount of time a consumer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.evictorThreadRunInterval</td>
-  <td>The interval of time between runs of the idle evictor thread for consumer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>The interval of time between runs of the idle evictor thread for consumer pool. When non-positive, no idle evictor thread will be run.</td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.jmx.enable</td>
+  <td>false</td>
   <td>Enable or disable JMX for pools created with this configuration instance. Statistics of the pool are available via JMX instance.
   The prefix of JMX name is set to "kafka010-cached-simple-kafka-consumer-pool".
   </td>
-  <td>false</td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -571,16 +575,18 @@ Note that it doesn't leverage Apache Commons Pool due to the difference of chara
 The following properties are available to configure the fetched data pool:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.consumer.fetchedData.cache.timeout</td>
-  <td>The minimum amount of time a fetched data may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>5m (5 minutes)</td>
+  <td>The minimum amount of time a fetched data may sit idle in the pool before it is eligible for eviction by the evictor.</td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.fetchedData.cache.evictorThreadRunInterval</td>
-  <td>The interval of time between runs of the idle evictor thread for fetched data pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>The interval of time between runs of the idle evictor thread for fetched data pool. When non-positive, no idle evictor thread will be run.</td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -816,16 +822,18 @@ It will use different Kafka producer when delegation token is renewed; Kafka pro
 The following properties are available to configure the producer pool:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.producer.cache.timeout</td>
-  <td>The minimum amount of time a producer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>10m (10 minutes)</td>
+  <td>The minimum amount of time a producer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
+  <td>2.2.1</td>
 </tr>
 <tr>
   <td>spark.kafka.producer.cache.evictorThreadRunInterval</td>
-  <td>The interval of time between runs of the idle evictor thread for producer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>The interval of time between runs of the idle evictor thread for producer pool. When non-positive, no idle evictor thread will be run.</td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -935,7 +943,7 @@ When none of the above applies then unsecure connection assumed.
 Delegation tokens can be obtained from multiple clusters and <code>${cluster}</code> is an arbitrary unique identifier which helps to group different configurations.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.auth.bootstrap.servers</code></td>
     <td>None</td>
@@ -943,6 +951,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       A list of coma separated host/port pairs to use for establishing the initial connection
       to the Kafka cluster. For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.target.bootstrap.servers.regex</code></td>
@@ -953,6 +962,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       If multiple clusters match the address, an exception will be thrown and the query won't be started.
       Kafka's secure and unsecure listeners are bound to different ports. When both used the secure listener port has to be part of the regular expression.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.security.protocol</code></td>
@@ -962,6 +972,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       <code>bootstrap.servers</code> config matches (for further details please see <code>spark.kafka.clusters.${cluster}.target.bootstrap.servers.regex</code>),
       and can be overridden by setting <code>kafka.security.protocol</code> on the source or sink.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.sasl.kerberos.service.name</code></td>
@@ -970,6 +981,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       The Kerberos principal name that Kafka runs as. This can be defined either in Kafka's JAAS config or in Kafka's config.
       For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.truststore.location</code></td>
@@ -977,6 +989,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
     <td>
       The location of the trust store file. For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.truststore.password</code></td>
@@ -985,6 +998,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       The store password for the trust store file. This is optional and only needed if <code>spark.kafka.clusters.${cluster}.ssl.truststore.location</code> is configured.
       For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.keystore.location</code></td>
@@ -993,6 +1007,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       The location of the key store file. This is optional for client and can be used for two-way authentication for client.
       For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.keystore.password</code></td>
@@ -1001,6 +1016,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       The store password for the key store file. This is optional and only needed if <code>spark.kafka.clusters.${cluster}.ssl.keystore.location</code> is configured.
       For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.key.password</code></td>
@@ -1009,6 +1025,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       The password of the private key in the key store file. This is optional for client.
       For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.sasl.token.mechanism</code></td>
@@ -1017,6 +1034,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       SASL mechanism used for client connections with delegation token. Because SCRAM login module used for authentication a compatible mechanism has to be set here.
       For further details please see Kafka documentation (<code>sasl.mechanism</code>). Only used to authenticate against Kafka broker with delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
 </table>
 
diff --git a/docs/structured-streaming-programming-guide.md b/docs/structured-streaming-programming-guide.md
index 727bda18a14fb..31b1ca9d47c0e 100644
--- a/docs/structured-streaming-programming-guide.md
+++ b/docs/structured-streaming-programming-guide.md
@@ -498,13 +498,13 @@ to track the read position in the stream. The engine uses checkpointing and writ
 
 # API using Datasets and DataFrames
 Since Spark 2.0, DataFrames and Datasets can represent static, bounded data, as well as streaming, unbounded data. Similar to static Datasets/DataFrames, you can use the common entry point `SparkSession`
-([Scala](api/scala/index.html#org.apache.spark.sql.SparkSession)/[Java](api/java/org/apache/spark/sql/SparkSession.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.SparkSession)/[R](api/R/sparkR.session.html) docs)
+([Scala](api/scala/org/apache/spark/sql/SparkSession.html)/[Java](api/java/org/apache/spark/sql/SparkSession.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.SparkSession)/[R](api/R/sparkR.session.html) docs)
 to create streaming DataFrames/Datasets from streaming sources, and apply the same operations on them as static DataFrames/Datasets. If you are not familiar with Datasets/DataFrames, you are strongly advised to familiarize yourself with them using the
 [DataFrame/Dataset Programming Guide](sql-programming-guide.html).
 
 ## Creating streaming DataFrames and streaming Datasets
 Streaming DataFrames can be created through the `DataStreamReader` interface
-([Scala](api/scala/index.html#org.apache.spark.sql.streaming.DataStreamReader)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamReader.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamReader) docs)
+([Scala](api/scala/org/apache/spark/sql/streaming/DataStreamReader.html)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamReader.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamReader) docs)
 returned by `SparkSession.readStream()`. In [R](api/R/read.stream.html), with the `read.stream()` method. Similar to the read interface for creating static DataFrame, you can specify the details of the source – data format, schema, options, etc.
 
 #### Input Sources
@@ -540,12 +540,13 @@ Here are the details of all the sources in Spark.
         <br/>
         <code>fileNameOnly</code>: whether to check new files based on only the filename instead of on the full path (default: false). With this set to `true`, the following files would be considered as the same file, because their filenames, "dataset.txt", are the same:
         <br/>
-        <code>maxFileAge</code>: Maximum age of a file that can be found in this directory, before it is ignored. For the first batch all files will be considered valid. If <code>latestFirst</code> is set to `true` and <code>maxFilesPerTrigger</code> is set, then this parameter will be ignored, because old files that are valid, and should be processed, may be ignored. The max age is specified with respect to the timestamp of the latest file, and not the timestamp of the current system.(default: 1 week)
-        <br/>
         "file:///dataset.txt"<br/>
         "s3://a/dataset.txt"<br/>
         "s3n://a/b/dataset.txt"<br/>
-        "s3a://a/b/c/dataset.txt"<br/>
+        "s3a://a/b/c/dataset.txt"
+        <br/>
+        <code>maxFileAge</code>: Maximum age of a file that can be found in this directory, before it is ignored. For the first batch all files will be considered valid. If <code>latestFirst</code> is set to `true` and <code>maxFilesPerTrigger</code> is set, then this parameter will be ignored, because old files that are valid, and should be processed, may be ignored. The max age is specified with respect to the timestamp of the latest file, and not the timestamp of the current system.(default: 1 week)
+        <br/>
         <code>cleanSource</code>: option to clean up completed files after processing.<br/>
         Available options are "archive", "delete", "off". If the option is not provided, the default value is "off".<br/>
         When "archive" is provided, additional option <code>sourceArchiveDir</code> must be provided as well. The value of "sourceArchiveDir" must not match with source pattern in depth (the number of directories from the root directory), where the depth is minimum of depth on both paths. This will ensure archived files are never included as new source files.<br/>
@@ -557,7 +558,7 @@ Here are the details of all the sources in Spark.
         NOTE 3: Both delete and move actions are best effort. Failing to delete or move files will not fail the streaming query. Spark may not clean up some source files in some circumstances - e.g. the application doesn't shut down gracefully, too many files are queued to clean up.
         <br/><br/>
         For file-format-specific options, see the related methods in <code>DataStreamReader</code>
-        (<a href="api/scala/index.html#org.apache.spark.sql.streaming.DataStreamReader">Scala</a>/<a href="api/java/org/apache/spark/sql/streaming/DataStreamReader.html">Java</a>/<a href="api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamReader">Python</a>/<a
+        (<a href="api/scala/org/apache/spark/sql/streaming/DataStreamReader.html">Scala</a>/<a href="api/java/org/apache/spark/sql/streaming/DataStreamReader.html">Java</a>/<a href="api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamReader">Python</a>/<a
         href="api/R/read.stream.html">R</a>).
         E.g. for "parquet" format options see <code>DataStreamReader.parquet()</code>.
         <br/><br/>
@@ -1114,7 +1115,7 @@ val staticDf = spark.read. ...
 val streamingDf = spark.readStream. ...
 
 streamingDf.join(staticDf, "type")          // inner equi-join with a static DF
-streamingDf.join(staticDf, "type", "right_join")  // right outer join with a static DF  
+streamingDf.join(staticDf, "type", "left_outer")  // left outer join with a static DF
 
 {% endhighlight %}
 
@@ -1125,7 +1126,7 @@ streamingDf.join(staticDf, "type", "right_join")  // right outer join with a sta
 Dataset<Row> staticDf = spark.read(). ...;
 Dataset<Row> streamingDf = spark.readStream(). ...;
 streamingDf.join(staticDf, "type");         // inner equi-join with a static DF
-streamingDf.join(staticDf, "type", "right_join");  // right outer join with a static DF
+streamingDf.join(staticDf, "type", "left_outer");  // left outer join with a static DF
 {% endhighlight %}
 
 
@@ -1136,7 +1137,7 @@ streamingDf.join(staticDf, "type", "right_join");  // right outer join with a st
 staticDf = spark.read. ...
 streamingDf = spark.readStream. ...
 streamingDf.join(staticDf, "type")  # inner equi-join with a static DF
-streamingDf.join(staticDf, "type", "right_join")  # right outer join with a static DF
+streamingDf.join(staticDf, "type", "left_outer")  # left outer join with a static DF
 {% endhighlight %}
 
 </div>
@@ -1148,10 +1149,10 @@ staticDf <- read.df(...)
 streamingDf <- read.stream(...)
 joined <- merge(streamingDf, staticDf, sort = FALSE)  # inner equi-join with a static DF
 joined <- join(
+            streamingDf,
             staticDf,
-            streamingDf, 
             streamingDf$value == staticDf$value,
-            "right_outer")  # right outer join with a static DF
+            "left_outer")  # left outer join with a static DF
 {% endhighlight %}
 
 </div>
@@ -1622,7 +1623,7 @@ However, as a side effect, data from the slower streams will be aggressively dro
 this configuration judiciously.
 
 ### Arbitrary Stateful Operations
-Many usecases require more advanced stateful operations than aggregations. For example, in many usecases, you have to track sessions from data streams of events. For doing such sessionization, you will have to save arbitrary types of data as state, and perform arbitrary operations on the state using the data stream events in every trigger. Since Spark 2.2, this can be done using the operation `mapGroupsWithState` and the more powerful operation `flatMapGroupsWithState`. Both operations allow you to apply user-defined code on grouped Datasets to update user-defined state. For more concrete details, take a look at the API documentation ([Scala](api/scala/index.html#org.apache.spark.sql.streaming.GroupState)/[Java](api/java/org/apache/spark/sql/streaming/GroupState.html)) and the examples ([Scala]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_SHORT}}/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredSessionization.scala)/[Java]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_SHORT}}/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredSessionization.java)).
+Many usecases require more advanced stateful operations than aggregations. For example, in many usecases, you have to track sessions from data streams of events. For doing such sessionization, you will have to save arbitrary types of data as state, and perform arbitrary operations on the state using the data stream events in every trigger. Since Spark 2.2, this can be done using the operation `mapGroupsWithState` and the more powerful operation `flatMapGroupsWithState`. Both operations allow you to apply user-defined code on grouped Datasets to update user-defined state. For more concrete details, take a look at the API documentation ([Scala](api/scala/org/apache/spark/sql/streaming/GroupState.html)/[Java](api/java/org/apache/spark/sql/streaming/GroupState.html)) and the examples ([Scala]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_SHORT}}/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredSessionization.scala)/[Java]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_SHORT}}/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredSessionization.java)).
 
 Though Spark cannot check and force it, the state function should be implemented with respect to the semantics of the output mode. For example, in Update mode Spark doesn't expect that the state function will emit rows which are older than current watermark plus allowed late record delay, whereas in Append mode the state function can emit these rows.
 
@@ -1679,7 +1680,7 @@ end-to-end exactly once per query. Ensuring end-to-end exactly once for the last
 
 ## Starting Streaming Queries
 Once you have defined the final result DataFrame/Dataset, all that is left is for you to start the streaming computation. To do that, you have to use the `DataStreamWriter`
-([Scala](api/scala/index.html#org.apache.spark.sql.streaming.DataStreamWriter)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamWriter.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamWriter) docs)
+([Scala](api/scala/org/apache/spark/sql/streaming/DataStreamWriter.html)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamWriter.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamWriter) docs)
 returned through `Dataset.writeStream()`. You will have to specify one or more of the following in this interface.
 
 - *Details of the output sink:* Data format, location, etc.
@@ -1863,7 +1864,7 @@ Here are the details of all the sinks in Spark.
         <code>path</code>: path to the output directory, must be specified.
         <br/><br/>
         For file-format-specific options, see the related methods in DataFrameWriter
-        (<a href="api/scala/index.html#org.apache.spark.sql.DataFrameWriter">Scala</a>/<a href="api/java/org/apache/spark/sql/DataFrameWriter.html">Java</a>/<a href="api/python/pyspark.sql.html#pyspark.sql.DataFrameWriter">Python</a>/<a
+        (<a href="api/scala/org/apache/spark/sql/DataFrameWriter.html">Scala</a>/<a href="api/java/org/apache/spark/sql/DataFrameWriter.html">Java</a>/<a href="api/python/pyspark.sql.html#pyspark.sql.DataFrameWriter">Python</a>/<a
         href="api/R/write.stream.html">R</a>).
         E.g. for "parquet" format options see <code>DataFrameWriter.parquet()</code>
     </td>
@@ -2105,7 +2106,7 @@ streamingDF.writeStream.foreachBatch { (batchDF: DataFrame, batchId: Long) =>
 
 {% highlight java %}
 streamingDatasetOfString.writeStream().foreachBatch(
-  new VoidFunction2<Dataset<String>, Long> {
+  new VoidFunction2<Dataset<String>, Long>() {
     public void call(Dataset<String> dataset, Long batchId) {
       // Transform and write batchDF
     }    
@@ -2175,7 +2176,7 @@ Since Spark 2.4, `foreach` is available in Scala, Java and Python.
 <div class="codetabs">
 <div data-lang="scala"  markdown="1">
 
-In Scala, you have to extend the class `ForeachWriter` ([docs](api/scala/index.html#org.apache.spark.sql.ForeachWriter)).
+In Scala, you have to extend the class `ForeachWriter` ([docs](api/scala/org/apache/spark/sql/ForeachWriter.html)).
 
 {% highlight scala %}
 streamingDatasetOfString.writeStream.foreach(
@@ -2202,7 +2203,7 @@ streamingDatasetOfString.writeStream.foreach(
 In Java, you have to extend the class `ForeachWriter` ([docs](api/java/org/apache/spark/sql/ForeachWriter.html)).
 {% highlight java %}
 streamingDatasetOfString.writeStream().foreach(
-  new ForeachWriter[String] {
+  new ForeachWriter<String>() {
 
     @Override public boolean open(long partitionId, long version) {
       // Open connection
@@ -2564,7 +2565,7 @@ lastProgress(query)       # the most recent progress update of this streaming qu
 </div>
 
 You can start any number of queries in a single SparkSession. They will all be running concurrently sharing the cluster resources. You can use `sparkSession.streams()` to get the `StreamingQueryManager`
-([Scala](api/scala/index.html#org.apache.spark.sql.streaming.StreamingQueryManager)/[Java](api/java/org/apache/spark/sql/streaming/StreamingQueryManager.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.StreamingQueryManager) docs)
+([Scala](api/scala/org/apache/spark/sql/streaming/StreamingQueryManager.html)/[Java](api/java/org/apache/spark/sql/streaming/StreamingQueryManager.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.StreamingQueryManager) docs)
 that can be used to manage the currently active queries.
 
 <div class="codetabs">
@@ -2624,7 +2625,7 @@ There are multiple ways to monitor active streaming queries. You can either push
 You can directly get the current status and metrics of an active query using 
 `streamingQuery.lastProgress()` and `streamingQuery.status()`. 
 `lastProgress()` returns a `StreamingQueryProgress` object 
-in [Scala](api/scala/index.html#org.apache.spark.sql.streaming.StreamingQueryProgress) 
+in [Scala](api/scala/org/apache/spark/sql/streaming/StreamingQueryProgress.html) 
 and [Java](api/java/org/apache/spark/sql/streaming/StreamingQueryProgress.html)
 and a dictionary with the same fields in Python. It has all the information about
 the progress made in the last trigger of the stream - what data was processed, 
@@ -2632,7 +2633,7 @@ what were the processing rates, latencies, etc. There is also
 `streamingQuery.recentProgress` which returns an array of last few progresses.  
 
 In addition, `streamingQuery.status()` returns a `StreamingQueryStatus` object 
-in [Scala](api/scala/index.html#org.apache.spark.sql.streaming.StreamingQueryStatus) 
+in [Scala](api/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.html) 
 and [Java](api/java/org/apache/spark/sql/streaming/StreamingQueryStatus.html)
 and a dictionary with the same fields in Python. It gives information about
 what the query is immediately doing - is a trigger active, is data being processed, etc.
@@ -2853,7 +2854,7 @@ Will print something like the following.
 
 You can also asynchronously monitor all queries associated with a
 `SparkSession` by attaching a `StreamingQueryListener`
-([Scala](api/scala/index.html#org.apache.spark.sql.streaming.StreamingQueryListener)/[Java](api/java/org/apache/spark/sql/streaming/StreamingQueryListener.html) docs).
+([Scala](api/scala/org/apache/spark/sql/streaming/StreamingQueryListener.html)/[Java](api/java/org/apache/spark/sql/streaming/StreamingQueryListener.html) docs).
 Once you attach your custom `StreamingQueryListener` object with
 `sparkSession.streams.attachListener()`, you will get callbacks when a query is started and
 stopped and when there is progress made in an active query. Here is an example,
diff --git a/docs/tuning.md b/docs/tuning.md
index 1faf7cfe0d68e..18d4a6205f4ff 100644
--- a/docs/tuning.md
+++ b/docs/tuning.md
@@ -260,10 +260,21 @@ enough. Spark automatically sets the number of "map" tasks to run on each file a
 (though you can control it through optional parameters to `SparkContext.textFile`, etc), and for
 distributed "reduce" operations, such as `groupByKey` and `reduceByKey`, it uses the largest
 parent RDD's number of partitions. You can pass the level of parallelism as a second argument
-(see the [`spark.PairRDDFunctions`](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions) documentation),
+(see the [`spark.PairRDDFunctions`](api/scala/org/apache/spark/rdd/PairRDDFunctions.html) documentation),
 or set the config property `spark.default.parallelism` to change the default.
 In general, we recommend 2-3 tasks per CPU core in your cluster.
 
+## Parallel Listing on Input Paths
+
+Sometimes you may also need to increase directory listing parallelism when job input has large number of directories,
+otherwise the process could take a very long time, especially when against object store like S3.
+If your job works on RDD with Hadoop input formats (e.g., via `SparkContext.sequenceFile`), the parallelism is
+controlled via [`spark.hadoop.mapreduce.input.fileinputformat.list-status.num-threads`](https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml) (currently default is 1).
+
+For Spark SQL with file-based data sources, you can tune `spark.sql.sources.parallelPartitionDiscovery.threshold` and
+`spark.sql.sources.parallelPartitionDiscovery.parallelism` to improve listing parallelism. Please
+refer to [Spark SQL performance tuning guide](sql-performance-tuning.html) for more details.
+
 ## Memory Usage of Reduce Tasks
 
 Sometimes, you will get an OutOfMemoryError not because your RDDs don't fit in memory, but because the
diff --git a/docs/web-ui.md b/docs/web-ui.md
index e28a689c8de50..fe2604364a12b 100644
--- a/docs/web-ui.md
+++ b/docs/web-ui.md
@@ -99,7 +99,7 @@ This page displays the details of a specific job identified by its job ID.
 The Stages tab displays a summary page that shows the current state of all stages of all jobs in
 the Spark application.
 
-At the beginning of the page is the summary with the count of all stages by status (active, pending, completed, sikipped, and failed)
+At the beginning of the page is the summary with the count of all stages by status (active, pending, completed, skipped, and failed)
 
 <p style="text-align: center;">
   <img src="img/AllStagesPageDetail1.png" title="Stages header" alt="Stages header" width="30%">
@@ -136,7 +136,7 @@ Summary metrics for all task are represented in a table and in a timeline.
 * **[Tasks deserialization time](configuration.html#compression-and-serialization)**
 * **Duration of tasks**.
 * **GC time** is the total JVM garbage collection time.
-* **Result serialization time** is the time spent serializing the task result on a executor before sending it back to the driver.
+* **Result serialization time** is the time spent serializing the task result on an executor before sending it back to the driver.
 * **Getting result time** is the time that the driver spends fetching task results from workers.
 * **Scheduler delay** is the time the task waits to be scheduled for execution.
 * **Peak execution memory** is the maximum memory used by the internal data structures created during shuffles, aggregations and joins.
@@ -406,10 +406,38 @@ Here is the list of SQL metrics:
 
 </table>
 
-## Streaming Tab
-The web UI includes a Streaming tab if the application uses Spark streaming. This tab displays
-scheduling delay and processing time for each micro-batch in the data stream, which can be useful
-for troubleshooting the streaming application.
+## Structured Streaming Tab
+When running Structured Streaming jobs in micro-batch mode, a Structured Streaming tab will be 
+available on the Web UI. The overview page displays some brief statistics for running and completed
+queries. Also, you can check the latest exception of a failed query. For detailed statistics, please
+click a "run id" in the tables.
+
+<p style="text-align: center;">
+  <img src="img/webui-structured-streaming-detail.png" title="Structured Streaming Query Statistics" alt="Structured Streaming Query Statistics">
+</p>
+
+The statistics page displays some useful metrics for insight into the status of your streaming 
+queries. Currently, it contains the following metrics.
+
+* **Input Rate.** The aggregate (across all sources) rate of data arriving.
+* **Process Rate.** The aggregate (across all sources) rate at which Spark is processing data.
+* **Input Rows.** The aggregate (across all sources) number of records processed in a trigger.
+* **Batch Duration.** The process duration of each batch. 
+* **Operation Duration.** The amount of time taken to perform various operations in milliseconds.
+The tracked operations are listed as follows.
+    * addBatch: Time taken to read the micro-batch's input data from the sources, process it, and write the batch's output to the sink. This should take the bulk of the micro-batch's time.
+    * getBatch: Time taken to prepare the logical query to read the input of the current micro-batch from the sources.
+    * latestOffset & getOffset: Time taken to query the maximum available offset for this source.
+    * queryPlanning: Time taken to generates the execution plan.
+    * walCommit: Time taken to write the offsets to the metadata log.
+    
+As an early-release version, the statistics page is still under development and will be improved in
+future releases.
+
+## Streaming (DStreams) Tab
+The web UI includes a Streaming tab if the application uses Spark Streaming with DStream API.
+This tab displays scheduling delay and processing time for each micro-batch in the data stream,
+which can be useful for troubleshooting the streaming application.
 
 ## JDBC/ODBC Server Tab
 We can see this tab when Spark is running as a [distributed SQL engine](sql-distributed-sql-engine.html). It shows information about sessions and submitted SQL operations.
diff --git a/examples/pom.xml b/examples/pom.xml
index a099f1e042e99..2082ece41569a 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
index b2ce0bc08642a..2295225387a33 100644
--- a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
@@ -98,6 +98,7 @@ public static void main(String[] args) {
       .getOrCreate();
 
     runBasicDataSourceExample(spark);
+    runGenericFileSourceOptionsExample(spark);
     runBasicParquetExample(spark);
     runParquetSchemaMergingExample(spark);
     runJsonDatasetExample(spark);
@@ -106,6 +107,48 @@ public static void main(String[] args) {
     spark.stop();
   }
 
+  private static void runGenericFileSourceOptionsExample(SparkSession spark) {
+    // $example on:ignore_corrupt_files$
+    // enable ignore corrupt files
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=true");
+    // dir1/file3.json is corrupt from parquet's view
+    Dataset<Row> testCorruptDF = spark.read().parquet(
+            "examples/src/main/resources/dir1/",
+            "examples/src/main/resources/dir1/dir2/");
+    testCorruptDF.show();
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // |file2.parquet|
+    // +-------------+
+    // $example off:ignore_corrupt_files$
+    // $example on:recursive_file_lookup$
+    Dataset<Row> recursiveLoadedDF = spark.read().format("parquet")
+            .option("recursiveFileLookup", "true")
+            .load("examples/src/main/resources/dir1");
+    recursiveLoadedDF.show();
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // |file2.parquet|
+    // +-------------+
+    // $example off:recursive_file_lookup$
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=false");
+    // $example on:load_with_path_glob_filter$
+    Dataset<Row> testGlobFilterDF = spark.read().format("parquet")
+            .option("pathGlobFilter", "*.parquet") // json file should be filtered out
+            .load("examples/src/main/resources/dir1");
+    testGlobFilterDF.show();
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // +-------------+
+    // $example off:load_with_path_glob_filter$
+  }
+
   private static void runBasicDataSourceExample(SparkSession spark) {
     // $example on:generic_load_save_functions$
     Dataset<Row> usersDF = spark.read().load("examples/src/main/resources/users.parquet");
@@ -123,11 +166,6 @@ private static void runBasicDataSourceExample(SparkSession spark) {
       .option("header", "true")
       .load("examples/src/main/resources/people.csv");
     // $example off:manual_load_options_csv$
-    // $example on:load_with_path_glob_filter$
-    Dataset<Row> partitionedUsersDF = spark.read().format("orc")
-      .option("pathGlobFilter", "*.orc")
-      .load("examples/src/main/resources/partitioned_users.orc");
-    // $example off:load_with_path_glob_filter$
     // $example on:manual_save_options_orc$
     usersDF.write().format("orc")
       .option("orc.bloom.filter.columns", "favorite_color")
diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java
new file mode 100644
index 0000000000000..e5e698809b215
--- /dev/null
+++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.examples.sql;
+
+// $example on:udf_scalar$
+import org.apache.spark.sql.*;
+import org.apache.spark.sql.api.java.UDF1;
+import org.apache.spark.sql.expressions.UserDefinedFunction;
+import static org.apache.spark.sql.functions.udf;
+import org.apache.spark.sql.types.DataTypes;
+// $example off:udf_scalar$
+
+public class JavaUserDefinedScalar {
+
+  public static void main(String[] args) {
+
+    // $example on:udf_scalar$
+    SparkSession spark = SparkSession
+      .builder()
+      .appName("Java Spark SQL UDF scalar example")
+      .getOrCreate();
+
+    // Define and register a zero-argument non-deterministic UDF
+    // UDF is deterministic by default, i.e. produces the same result for the same input.
+    UserDefinedFunction random = udf(
+      () -> Math.random(), DataTypes.DoubleType
+    );
+    random.asNondeterministic();
+    spark.udf().register("random", random);
+    spark.sql("SELECT random()").show();
+    // +-------+
+    // |UDF()  |
+    // +-------+
+    // |xxxxxxx|
+    // +-------+
+
+    // Define and register a one-argument UDF
+    spark.udf().register("plusOne", new UDF1<Integer, Integer>() {
+      @Override
+      public Integer call(Integer x) {
+        return x + 1;
+      }
+    }, DataTypes.IntegerType);
+    spark.sql("SELECT plusOne(5)").show();
+    // +----------+
+    // |plusOne(5)|
+    // +----------+
+    // |         6|
+    // +----------+
+
+    // Define and register a two-argument UDF
+    UserDefinedFunction strLen = udf(
+      (String s, Integer x) -> s.length() + x, DataTypes.IntegerType
+    );
+    spark.udf().register("strLen", strLen);
+    spark.sql("SELECT strLen('test', 1)").show();
+    // +------------+
+    // |UDF(test, 1)|
+    // +------------+
+    // |           5|
+    // +------------+
+
+    // UDF in a WHERE clause
+    spark.udf().register("oneArgFilter", new UDF1<Long, Boolean>() {
+      @Override
+      public Boolean call(Long x) {
+        return  x > 5;
+      }
+    }, DataTypes.BooleanType);
+    spark.range(1, 10).createOrReplaceTempView("test");
+    spark.sql("SELECT * FROM test WHERE oneArgFilter(id)").show();
+    // +---+
+    // | id|
+    // +---+
+    // |  6|
+    // |  7|
+    // |  8|
+    // |  9|
+    // +---+
+
+    // $example off:udf_scalar$
+    spark.stop();
+  }
+}
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java
index 47692ec982890..f84a1978de1ad 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java
@@ -67,7 +67,7 @@ public static void main(String[] args) throws Exception {
     JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, new Duration(1000));
 
     // Create an input stream with the custom receiver on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     JavaReceiverInputDStream<String> lines = ssc.receiverStream(
       new JavaCustomReceiver(args[0], Integer.parseInt(args[1])));
     JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(SPACE.split(x)).iterator());
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java
index b217672def88e..d56134bd99e36 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java
@@ -57,7 +57,7 @@ public static void main(String[] args) throws Exception {
     JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, Durations.seconds(1));
 
     // Create a JavaReceiverInputDStream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     JavaReceiverInputDStream<String> lines = ssc.socketTextStream(
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
index 45a876decff8b..c3543237fc7a8 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
@@ -126,7 +126,7 @@ private static JavaStreamingContext createContext(String ip,
     ssc.checkpoint(checkpointDirectory);
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     JavaReceiverInputDStream<String> lines = ssc.socketTextStream(ip, port);
     JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(SPACE.split(x)).iterator());
     JavaPairDStream<String, Integer> wordCounts = words.mapToPair(s -> new Tuple2<>(s, 1))
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
index 948d1a2111780..5d30698c93372 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
@@ -59,7 +59,7 @@ public static void main(String[] args) throws Exception {
     JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, Durations.seconds(1));
 
     // Create a JavaReceiverInputDStream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     JavaReceiverInputDStream<String> lines = ssc.socketTextStream(
diff --git a/examples/src/main/python/ml/train_validation_split.py b/examples/src/main/python/ml/train_validation_split.py
index d4f9184bf576e..5e3dc7b3ec2fa 100644
--- a/examples/src/main/python/ml/train_validation_split.py
+++ b/examples/src/main/python/ml/train_validation_split.py
@@ -17,7 +17,7 @@
 
 """
 This example demonstrates applying TrainValidationSplit to split data
-and preform model selection.
+and perform model selection.
 Run with:
 
   bin/spark-submit examples/src/main/python/ml/train_validation_split.py
diff --git a/examples/src/main/python/sql/arrow.py b/examples/src/main/python/sql/arrow.py
index 1c983172d36ef..b9fb29f466a40 100644
--- a/examples/src/main/python/sql/arrow.py
+++ b/examples/src/main/python/sql/arrow.py
@@ -23,12 +23,19 @@
 
 from __future__ import print_function
 
+import sys
+
 from pyspark.sql import SparkSession
 from pyspark.sql.pandas.utils import require_minimum_pandas_version, require_minimum_pyarrow_version
 
 require_minimum_pandas_version()
 require_minimum_pyarrow_version()
 
+if sys.version_info < (3, 6):
+    raise Exception(
+        "Running this example file requires Python 3.6+; however, "
+        "your Python version was:\n %s" % sys.version)
+
 
 def dataframe_with_arrow_example(spark):
     # $example on:dataframe_with_arrow$
@@ -50,15 +57,45 @@ def dataframe_with_arrow_example(spark):
     print("Pandas DataFrame result statistics:\n%s\n" % str(result_pdf.describe()))
 
 
-def scalar_pandas_udf_example(spark):
-    # $example on:scalar_pandas_udf$
+def ser_to_frame_pandas_udf_example(spark):
+    # $example on:ser_to_frame_pandas_udf$
+    import pandas as pd
+
+    from pyspark.sql.functions import pandas_udf
+
+    @pandas_udf("col1 string, col2 long")
+    def func(s1: pd.Series, s2: pd.Series, s3: pd.DataFrame) -> pd.DataFrame:
+        s3['col2'] = s1 + s2.str.len()
+        return s3
+
+    # Create a Spark DataFrame that has three columns including a struct column.
+    df = spark.createDataFrame(
+        [[1, "a string", ("a nested string",)]],
+        "long_col long, string_col string, struct_col struct<col1:string>")
+
+    df.printSchema()
+    # root
+    # |-- long_column: long (nullable = true)
+    # |-- string_column: string (nullable = true)
+    # |-- struct_column: struct (nullable = true)
+    # |    |-- col1: string (nullable = true)
+
+    df.select(func("long_col", "string_col", "struct_col")).printSchema()
+    # |-- func(long_col, string_col, struct_col): struct (nullable = true)
+    # |    |-- col1: string (nullable = true)
+    # |    |-- col2: long (nullable = true)
+    # $example off:ser_to_frame_pandas_udf$$
+
+
+def ser_to_ser_pandas_udf_example(spark):
+    # $example on:ser_to_ser_pandas_udf$
     import pandas as pd
 
     from pyspark.sql.functions import col, pandas_udf
     from pyspark.sql.types import LongType
 
     # Declare the function and create the UDF
-    def multiply_func(a, b):
+    def multiply_func(a: pd.Series, b: pd.Series) -> pd.Series:
         return a * b
 
     multiply = pandas_udf(multiply_func, returnType=LongType())
@@ -83,26 +120,27 @@ def multiply_func(a, b):
     # |                  4|
     # |                  9|
     # +-------------------+
-    # $example off:scalar_pandas_udf$
+    # $example off:ser_to_ser_pandas_udf$
 
 
-def scalar_iter_pandas_udf_example(spark):
-    # $example on:scalar_iter_pandas_udf$
+def iter_ser_to_iter_ser_pandas_udf_example(spark):
+    # $example on:iter_ser_to_iter_ser_pandas_udf$
+    from typing import Iterator
+
     import pandas as pd
 
-    from pyspark.sql.functions import col, pandas_udf, struct, PandasUDFType
+    from pyspark.sql.functions import pandas_udf
 
     pdf = pd.DataFrame([1, 2, 3], columns=["x"])
     df = spark.createDataFrame(pdf)
 
-    # When the UDF is called with a single column that is not StructType,
-    # the input to the underlying function is an iterator of pd.Series.
-    @pandas_udf("long", PandasUDFType.SCALAR_ITER)
-    def plus_one(batch_iter):
-        for x in batch_iter:
+    # Declare the function and create the UDF
+    @pandas_udf("long")
+    def plus_one(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
+        for x in iterator:
             yield x + 1
 
-    df.select(plus_one(col("x"))).show()
+    df.select(plus_one("x")).show()
     # +-----------+
     # |plus_one(x)|
     # +-----------+
@@ -110,15 +148,28 @@ def plus_one(batch_iter):
     # |          3|
     # |          4|
     # +-----------+
+    # $example off:iter_ser_to_iter_ser_pandas_udf$
+
+
+def iter_sers_to_iter_ser_pandas_udf_example(spark):
+    # $example on:iter_sers_to_iter_ser_pandas_udf$
+    from typing import Iterator, Tuple
+
+    import pandas as pd
 
-    # When the UDF is called with more than one columns,
-    # the input to the underlying function is an iterator of pd.Series tuple.
-    @pandas_udf("long", PandasUDFType.SCALAR_ITER)
-    def multiply_two_cols(batch_iter):
-        for a, b in batch_iter:
+    from pyspark.sql.functions import pandas_udf
+
+    pdf = pd.DataFrame([1, 2, 3], columns=["x"])
+    df = spark.createDataFrame(pdf)
+
+    # Declare the function and create the UDF
+    @pandas_udf("long")
+    def multiply_two_cols(
+            iterator: Iterator[Tuple[pd.Series, pd.Series]]) -> Iterator[pd.Series]:
+        for a, b in iterator:
             yield a * b
 
-    df.select(multiply_two_cols(col("x"), col("x"))).show()
+    df.select(multiply_two_cols("x", "x")).show()
     # +-----------------------+
     # |multiply_two_cols(x, x)|
     # +-----------------------+
@@ -126,92 +177,32 @@ def multiply_two_cols(batch_iter):
     # |                      4|
     # |                      9|
     # +-----------------------+
+    # $example off:iter_sers_to_iter_ser_pandas_udf$
 
-    # When the UDF is called with a single column that is StructType,
-    # the input to the underlying function is an iterator of pd.DataFrame.
-    @pandas_udf("long", PandasUDFType.SCALAR_ITER)
-    def multiply_two_nested_cols(pdf_iter):
-        for pdf in pdf_iter:
-            yield pdf["a"] * pdf["b"]
-
-    df.select(
-        multiply_two_nested_cols(
-            struct(col("x").alias("a"), col("x").alias("b"))
-        ).alias("y")
-    ).show()
-    # +---+
-    # |  y|
-    # +---+
-    # |  1|
-    # |  4|
-    # |  9|
-    # +---+
-
-    # In the UDF, you can initialize some states before processing batches.
-    # Wrap your code with try/finally or use context managers to ensure
-    # the release of resources at the end.
-    y_bc = spark.sparkContext.broadcast(1)
-
-    @pandas_udf("long", PandasUDFType.SCALAR_ITER)
-    def plus_y(batch_iter):
-        y = y_bc.value  # initialize states
-        try:
-            for x in batch_iter:
-                yield x + y
-        finally:
-            pass  # release resources here, if any
-
-    df.select(plus_y(col("x"))).show()
-    # +---------+
-    # |plus_y(x)|
-    # +---------+
-    # |        2|
-    # |        3|
-    # |        4|
-    # +---------+
-    # $example off:scalar_iter_pandas_udf$
-
-
-def grouped_map_pandas_udf_example(spark):
-    # $example on:grouped_map_pandas_udf$
-    from pyspark.sql.functions import pandas_udf, PandasUDFType
-
-    df = spark.createDataFrame(
-        [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-        ("id", "v"))
-
-    @pandas_udf("id long, v double", PandasUDFType.GROUPED_MAP)
-    def subtract_mean(pdf):
-        # pdf is a pandas.DataFrame
-        v = pdf.v
-        return pdf.assign(v=v - v.mean())
-
-    df.groupby("id").apply(subtract_mean).show()
-    # +---+----+
-    # | id|   v|
-    # +---+----+
-    # |  1|-0.5|
-    # |  1| 0.5|
-    # |  2|-3.0|
-    # |  2|-1.0|
-    # |  2| 4.0|
-    # +---+----+
-    # $example off:grouped_map_pandas_udf$
 
+def ser_to_scalar_pandas_udf_example(spark):
+    # $example on:ser_to_scalar_pandas_udf$
+    import pandas as pd
 
-def grouped_agg_pandas_udf_example(spark):
-    # $example on:grouped_agg_pandas_udf$
-    from pyspark.sql.functions import pandas_udf, PandasUDFType
+    from pyspark.sql.functions import pandas_udf
     from pyspark.sql import Window
 
     df = spark.createDataFrame(
         [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
         ("id", "v"))
 
-    @pandas_udf("double", PandasUDFType.GROUPED_AGG)
-    def mean_udf(v):
+    # Declare the function and create the UDF
+    @pandas_udf("double")
+    def mean_udf(v: pd.Series) -> float:
         return v.mean()
 
+    df.select(mean_udf(df['v'])).show()
+    # +-----------+
+    # |mean_udf(v)|
+    # +-----------+
+    # |        4.2|
+    # +-----------+
+
     df.groupby("id").agg(mean_udf(df['v'])).show()
     # +---+-----------+
     # | id|mean_udf(v)|
@@ -233,37 +224,54 @@ def mean_udf(v):
     # |  2| 5.0|   6.0|
     # |  2|10.0|   6.0|
     # +---+----+------+
-    # $example off:grouped_agg_pandas_udf$
+    # $example off:ser_to_scalar_pandas_udf$
 
 
-def map_iter_pandas_udf_example(spark):
-    # $example on:map_iter_pandas_udf$
-    import pandas as pd
+def grouped_apply_in_pandas_example(spark):
+    # $example on:grouped_apply_in_pandas$
+    df = spark.createDataFrame(
+        [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
+        ("id", "v"))
 
-    from pyspark.sql.functions import pandas_udf, PandasUDFType
+    def subtract_mean(pdf):
+        # pdf is a pandas.DataFrame
+        v = pdf.v
+        return pdf.assign(v=v - v.mean())
+
+    df.groupby("id").applyInPandas(subtract_mean, schema="id long, v double").show()
+    # +---+----+
+    # | id|   v|
+    # +---+----+
+    # |  1|-0.5|
+    # |  1| 0.5|
+    # |  2|-3.0|
+    # |  2|-1.0|
+    # |  2| 4.0|
+    # +---+----+
+    # $example off:grouped_apply_in_pandas$
 
+
+def map_in_pandas_example(spark):
+    # $example on:map_in_pandas$
     df = spark.createDataFrame([(1, 21), (2, 30)], ("id", "age"))
 
-    @pandas_udf(df.schema, PandasUDFType.MAP_ITER)
-    def filter_func(batch_iter):
-        for pdf in batch_iter:
+    def filter_func(iterator):
+        for pdf in iterator:
             yield pdf[pdf.id == 1]
 
-    df.mapInPandas(filter_func).show()
+    df.mapInPandas(filter_func, schema=df.schema).show()
     # +---+---+
     # | id|age|
     # +---+---+
     # |  1| 21|
     # +---+---+
-    # $example off:map_iter_pandas_udf$
+    # $example off:map_in_pandas$
 
 
-def cogrouped_map_pandas_udf_example(spark):
-    # $example on:cogrouped_map_pandas_udf$
+def cogrouped_apply_in_pandas_example(spark):
+    # $example on:cogrouped_apply_in_pandas$
     import pandas as pd
 
-    from pyspark.sql.functions import pandas_udf, PandasUDFType
-
     df1 = spark.createDataFrame(
         [(20000101, 1, 1.0), (20000101, 2, 2.0), (20000102, 1, 3.0), (20000102, 2, 4.0)],
         ("time", "id", "v1"))
@@ -272,11 +280,11 @@ def cogrouped_map_pandas_udf_example(spark):
         [(20000101, 1, "x"), (20000101, 2, "y")],
         ("time", "id", "v2"))
 
-    @pandas_udf("time int, id int, v1 double, v2 string", PandasUDFType.COGROUPED_MAP)
     def asof_join(l, r):
         return pd.merge_asof(l, r, on="time", by="id")
 
-    df1.groupby("id").cogroup(df2.groupby("id")).apply(asof_join).show()
+    df1.groupby("id").cogroup(df2.groupby("id")).applyInPandas(
+        asof_join, schema="time int, id int, v1 double, v2 string").show()
     # +--------+---+---+---+
     # |    time| id| v1| v2|
     # +--------+---+---+---+
@@ -285,7 +293,7 @@ def asof_join(l, r):
     # |20000101|  2|2.0|  y|
     # |20000102|  2|4.0|  y|
     # +--------+---+---+---+
-    # $example off:cogrouped_map_pandas_udf$
+    # $example off:cogrouped_apply_in_pandas$
 
 
 if __name__ == "__main__":
@@ -296,17 +304,21 @@ def asof_join(l, r):
 
     print("Running Pandas to/from conversion example")
     dataframe_with_arrow_example(spark)
-    print("Running pandas_udf scalar example")
-    scalar_pandas_udf_example(spark)
-    print("Running pandas_udf scalar iterator example")
-    scalar_iter_pandas_udf_example(spark)
-    print("Running pandas_udf grouped map example")
-    grouped_map_pandas_udf_example(spark)
-    print("Running pandas_udf grouped agg example")
-    grouped_agg_pandas_udf_example(spark)
-    print("Running pandas_udf map iterator example")
-    map_iter_pandas_udf_example(spark)
-    print("Running pandas_udf cogrouped map example")
-    cogrouped_map_pandas_udf_example(spark)
+    print("Running pandas_udf example: Series to Frame")
+    ser_to_frame_pandas_udf_example(spark)
+    print("Running pandas_udf example: Series to Series")
+    ser_to_ser_pandas_udf_example(spark)
+    print("Running pandas_udf example: Iterator of Series to Iterator of Series")
+    iter_ser_to_iter_ser_pandas_udf_example(spark)
+    print("Running pandas_udf example: Iterator of Multiple Series to Iterator of Series")
+    iter_sers_to_iter_ser_pandas_udf_example(spark)
+    print("Running pandas_udf example: Series to Scalar")
+    ser_to_scalar_pandas_udf_example(spark)
+    print("Running pandas function example: Grouped Map")
+    grouped_apply_in_pandas_example(spark)
+    print("Running pandas function example: Map")
+    map_in_pandas_example(spark)
+    print("Running pandas function example: Co-grouped Map")
+    cogrouped_apply_in_pandas_example(spark)
 
     spark.stop()
diff --git a/examples/src/main/python/sql/datasource.py b/examples/src/main/python/sql/datasource.py
index 0d78097ea975e..9f8fdd74e0242 100644
--- a/examples/src/main/python/sql/datasource.py
+++ b/examples/src/main/python/sql/datasource.py
@@ -28,6 +28,48 @@
 # $example off:schema_merging$
 
 
+def generic_file_source_options_example(spark):
+    # $example on:ignore_corrupt_files$
+    # enable ignore corrupt files
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=true")
+    # dir1/file3.json is corrupt from parquet's view
+    test_corrupt_df = spark.read.parquet("examples/src/main/resources/dir1/",
+                                         "examples/src/main/resources/dir1/dir2/")
+    test_corrupt_df.show()
+    # +-------------+
+    # |         file|
+    # +-------------+
+    # |file1.parquet|
+    # |file2.parquet|
+    # +-------------+
+    # $example off:ignore_corrupt_files$
+
+    # $example on:recursive_file_lookup$
+    recursive_loaded_df = spark.read.format("parquet")\
+        .option("recursiveFileLookup", "true")\
+        .load("examples/src/main/resources/dir1")
+    recursive_loaded_df.show()
+    # +-------------+
+    # |         file|
+    # +-------------+
+    # |file1.parquet|
+    # |file2.parquet|
+    # +-------------+
+    # $example off:recursive_file_lookup$
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=false")
+
+    # $example on:load_with_path_glob_filter$
+    df = spark.read.load("examples/src/main/resources/dir1",
+                         format="parquet", pathGlobFilter="*.parquet")
+    df.show()
+    # +-------------+
+    # |         file|
+    # +-------------+
+    # |file1.parquet|
+    # +-------------+
+    # $example off:load_with_path_glob_filter$
+
+
 def basic_datasource_example(spark):
     # $example on:generic_load_save_functions$
     df = spark.read.load("examples/src/main/resources/users.parquet")
@@ -54,14 +96,9 @@ def basic_datasource_example(spark):
 
     # $example on:manual_load_options_csv$
     df = spark.read.load("examples/src/main/resources/people.csv",
-                         format="csv", sep=":", inferSchema="true", header="true")
+                         format="csv", sep=";", inferSchema="true", header="true")
     # $example off:manual_load_options_csv$
 
-    # $example on:load_with_path_glob_filter$
-    df = spark.read.load("examples/src/main/resources/partitioned_users.orc",
-                         format="orc", pathGlobFilter="*.orc")
-    # $example off:load_with_path_glob_filter$
-
     # $example on:manual_save_options_orc$
     df = spark.read.orc("examples/src/main/resources/users.orc")
     (df.write.format("orc")
@@ -233,6 +270,7 @@ def jdbc_dataset_example(spark):
         .getOrCreate()
 
     basic_datasource_example(spark)
+    generic_file_source_options_example(spark)
     parquet_example(spark)
     parquet_schema_merging_example(spark)
     json_dataset_example(spark)
diff --git a/examples/src/main/python/streaming/recoverable_network_wordcount.py b/examples/src/main/python/streaming/recoverable_network_wordcount.py
index a39c4d0b5b8cd..e9f2db97d87e6 100644
--- a/examples/src/main/python/streaming/recoverable_network_wordcount.py
+++ b/examples/src/main/python/streaming/recoverable_network_wordcount.py
@@ -68,7 +68,7 @@ def createContext(host, port, outputPath):
     ssc = StreamingContext(sc, 1)
 
     # Create a socket stream on target ip:port and count the
-    # words in input stream of \n delimited text (eg. generated by 'nc')
+    # words in input stream of \n delimited text (e.g. generated by 'nc')
     lines = ssc.socketTextStream(host, port)
     words = lines.flatMap(lambda line: line.split(" "))
     wordCounts = words.map(lambda x: (x, 1)).reduceByKey(lambda x, y: x + y)
diff --git a/examples/src/main/python/streaming/sql_network_wordcount.py b/examples/src/main/python/streaming/sql_network_wordcount.py
index ab3cfc067994d..22fe26073a690 100644
--- a/examples/src/main/python/streaming/sql_network_wordcount.py
+++ b/examples/src/main/python/streaming/sql_network_wordcount.py
@@ -54,7 +54,7 @@ def getSparkSessionInstance(sparkConf):
     ssc = StreamingContext(sc, 1)
 
     # Create a socket stream on target ip:port and count the
-    # words in input stream of \n delimited text (eg. generated by 'nc')
+    # words in input stream of \n delimited text (e.g. generated by 'nc')
     lines = ssc.socketTextStream(host, int(port))
     words = lines.flatMap(lambda line: line.split(" "))
 
diff --git a/examples/src/main/r/RSparkSQLExample.R b/examples/src/main/r/RSparkSQLExample.R
index fa083d5542fae..8685cfb5c05f2 100644
--- a/examples/src/main/r/RSparkSQLExample.R
+++ b/examples/src/main/r/RSparkSQLExample.R
@@ -99,6 +99,26 @@ createOrReplaceTempView(df, "table")
 df <- sql("SELECT * FROM table")
 # $example off:run_sql$
 
+# Ignore corrupt files
+# $example on:ignore_corrupt_files$
+# enable ignore corrupt files
+sql("set spark.sql.files.ignoreCorruptFiles=true")
+# dir1/file3.json is corrupt from parquet's view
+testCorruptDF <- read.parquet(c("examples/src/main/resources/dir1/", "examples/src/main/resources/dir1/dir2/"))
+head(testCorruptDF)
+#            file
+# 1 file1.parquet
+# 2 file2.parquet
+# $example off:ignore_corrupt_files$
+
+# $example on:recursive_file_lookup$
+recursiveLoadedDF <- read.df("examples/src/main/resources/dir1", "parquet", recursiveFileLookup = "true")
+head(recursiveLoadedDF)
+#            file
+# 1 file1.parquet
+# 2 file2.parquet
+# $example off:recursive_file_lookup$
+sql("set spark.sql.files.ignoreCorruptFiles=false")
 
 # $example on:generic_load_save_functions$
 df <- read.df("examples/src/main/resources/users.parquet")
@@ -119,7 +139,9 @@ namesAndAges <- select(df, "name", "age")
 # $example off:manual_load_options_csv$
 
 # $example on:load_with_path_glob_filter$
-df <- read.df("examples/src/main/resources/partitioned_users.orc", "orc", pathGlobFilter = "*.orc")
+df <- read.df("examples/src/main/resources/dir1", "parquet", pathGlobFilter = "*.parquet")
+#            file
+# 1 file1.parquet
 # $example off:load_with_path_glob_filter$
 
 # $example on:manual_save_options_orc$
diff --git a/examples/src/main/resources/dir1/dir2/file2.parquet b/examples/src/main/resources/dir1/dir2/file2.parquet
new file mode 100644
index 0000000000000..d1895bf29b75c
Binary files /dev/null and b/examples/src/main/resources/dir1/dir2/file2.parquet differ
diff --git a/examples/src/main/resources/dir1/file1.parquet b/examples/src/main/resources/dir1/file1.parquet
new file mode 100644
index 0000000000000..ad360b16fd898
Binary files /dev/null and b/examples/src/main/resources/dir1/file1.parquet differ
diff --git a/examples/src/main/resources/dir1/file3.json b/examples/src/main/resources/dir1/file3.json
new file mode 100644
index 0000000000000..0490f92d7f317
--- /dev/null
+++ b/examples/src/main/resources/dir1/file3.json
@@ -0,0 +1 @@
+{"file":"corrupt.json"}
diff --git a/examples/src/main/resources/partitioned_users.orc/do_not_read_this.txt b/examples/src/main/resources/partitioned_users.orc/do_not_read_this.txt
deleted file mode 100644
index 9c19f2a0449eb..0000000000000
--- a/examples/src/main/resources/partitioned_users.orc/do_not_read_this.txt
+++ /dev/null
@@ -1 +0,0 @@
-do not read this
diff --git a/examples/src/main/resources/partitioned_users.orc/favorite_color=__HIVE_DEFAULT_PARTITION__/users.orc b/examples/src/main/resources/partitioned_users.orc/favorite_color=__HIVE_DEFAULT_PARTITION__/users.orc
deleted file mode 100644
index 890395a9281ab..0000000000000
Binary files a/examples/src/main/resources/partitioned_users.orc/favorite_color=__HIVE_DEFAULT_PARTITION__/users.orc and /dev/null differ
diff --git a/examples/src/main/resources/partitioned_users.orc/favorite_color=red/users.orc b/examples/src/main/resources/partitioned_users.orc/favorite_color=red/users.orc
deleted file mode 100644
index 150615a6f3b24..0000000000000
Binary files a/examples/src/main/resources/partitioned_users.orc/favorite_color=red/users.orc and /dev/null differ
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
index c6312d71cc912..a606cc4218778 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
@@ -59,13 +59,13 @@ object BinaryClassificationMetricsExample {
 
     // Precision by threshold
     val precision = metrics.precisionByThreshold
-    precision.foreach { case (t, p) =>
+    precision.collect.foreach { case (t, p) =>
       println(s"Threshold: $t, Precision: $p")
     }
 
     // Recall by threshold
     val recall = metrics.recallByThreshold
-    recall.foreach { case (t, r) =>
+    recall.collect.foreach { case (t, r) =>
       println(s"Threshold: $t, Recall: $r")
     }
 
@@ -74,13 +74,13 @@ object BinaryClassificationMetricsExample {
 
     // F-measure
     val f1Score = metrics.fMeasureByThreshold
-    f1Score.foreach { case (t, f) =>
+    f1Score.collect.foreach { case (t, f) =>
       println(s"Threshold: $t, F-score: $f, Beta = 1")
     }
 
     val beta = 0.5
     val fScore = metrics.fMeasureByThreshold(beta)
-    f1Score.foreach { case (t, f) =>
+    fScore.collect.foreach { case (t, f) =>
       println(s"Threshold: $t, F-score: $f, Beta = 0.5")
     }
 
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala
index 5e400b7d715b4..6ed59a3366d93 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala
@@ -53,7 +53,7 @@ object ChiSqSelectorExample {
     // $example off$
 
     println("filtered data: ")
-    filteredData.foreach(x => println(x))
+    filteredData.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala
index 1855058bb4f3c..d6ec678b422f9 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala
@@ -45,10 +45,10 @@ object ElementwiseProductExample {
     // $example off$
 
     println("transformedData: ")
-    transformedData.foreach(x => println(x))
+    transformedData.collect.foreach(x => println(x))
 
     println("transformedData2: ")
-    transformedData2.foreach(x => println(x))
+    transformedData2.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala
index b3a9604c2be3e..b1cad7bf8125f 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala
@@ -46,10 +46,10 @@ object NormalizerExample {
     // $example off$
 
     println("data1: ")
-    data1.foreach(x => println(x))
+    data1.collect.foreach(x => println(x))
 
     println("data2: ")
-    data2.foreach(x => println(x))
+    data2.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala
index 769fc17b3dc65..66a608ced0f83 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala
@@ -49,10 +49,10 @@ object StandardScalerExample {
     // $example off$
 
     println("data1: ")
-    data1.foreach(x => println(x))
+    data1.collect.foreach(x => println(x))
 
     println("data2: ")
-    data2.foreach(x => println(x))
+    data2.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala
index a5bdcd8f2ed32..14b2a2015c241 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala
@@ -55,10 +55,10 @@ object TFIDFExample {
     // $example off$
 
     println("tfidf: ")
-    tfidf.foreach(x => println(x))
+    tfidf.collect.foreach(x => println(x))
 
     println("tfidfIgnore: ")
-    tfidfIgnore.foreach(x => println(x))
+    tfidfIgnore.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala b/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
index d4c05e5ad9944..2c7abfcd335d1 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
@@ -32,6 +32,7 @@ object SQLDataSourceExample {
       .getOrCreate()
 
     runBasicDataSourceExample(spark)
+    runGenericFileSourceOptionsExample(spark)
     runBasicParquetExample(spark)
     runParquetSchemaMergingExample(spark)
     runJsonDatasetExample(spark)
@@ -40,6 +41,48 @@ object SQLDataSourceExample {
     spark.stop()
   }
 
+  private def runGenericFileSourceOptionsExample(spark: SparkSession): Unit = {
+    // $example on:ignore_corrupt_files$
+    // enable ignore corrupt files
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=true")
+    // dir1/file3.json is corrupt from parquet's view
+    val testCorruptDF = spark.read.parquet(
+      "examples/src/main/resources/dir1/",
+      "examples/src/main/resources/dir1/dir2/")
+    testCorruptDF.show()
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // |file2.parquet|
+    // +-------------+
+    // $example off:ignore_corrupt_files$
+    // $example on:recursive_file_lookup$
+    val recursiveLoadedDF = spark.read.format("parquet")
+      .option("recursiveFileLookup", "true")
+      .load("examples/src/main/resources/dir1")
+    recursiveLoadedDF.show()
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // |file2.parquet|
+    // +-------------+
+    // $example off:recursive_file_lookup$
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=false")
+    // $example on:load_with_path_glob_filter$
+    val testGlobFilterDF = spark.read.format("parquet")
+      .option("pathGlobFilter", "*.parquet") // json file should be filtered out
+      .load("examples/src/main/resources/dir1")
+    testGlobFilterDF.show()
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // +-------------+
+    // $example off:load_with_path_glob_filter$
+  }
+
   private def runBasicDataSourceExample(spark: SparkSession): Unit = {
     // $example on:generic_load_save_functions$
     val usersDF = spark.read.load("examples/src/main/resources/users.parquet")
@@ -56,11 +99,6 @@ object SQLDataSourceExample {
       .option("header", "true")
       .load("examples/src/main/resources/people.csv")
     // $example off:manual_load_options_csv$
-    // $example on:load_with_path_glob_filter$
-    val partitionedUsersDF = spark.read.format("orc")
-      .option("pathGlobFilter", "*.orc")
-      .load("examples/src/main/resources/partitioned_users.orc")
-    // $example off:load_with_path_glob_filter$
     // $example on:manual_save_options_orc$
     usersDF.write.format("orc")
       .option("orc.bloom.filter.columns", "favorite_color")
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedScalar.scala b/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedScalar.scala
new file mode 100644
index 0000000000000..25218a6dcc5af
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedScalar.scala
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.examples.sql
+
+// $example on:udf_scalar$
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.functions.udf
+// $example off:udf_scalar$
+
+object UserDefinedScalar {
+
+  def main(args: Array[String]): Unit = {
+    // $example on:udf_scalar$
+    val spark = SparkSession
+      .builder()
+      .appName("Spark SQL UDF scalar example")
+      .getOrCreate()
+
+    // Define and register a zero-argument non-deterministic UDF
+    // UDF is deterministic by default, i.e. produces the same result for the same input.
+    val random = udf(() => Math.random())
+    spark.udf.register("random", random.asNondeterministic())
+    spark.sql("SELECT random()").show()
+    // +-------+
+    // |UDF()  |
+    // +-------+
+    // |xxxxxxx|
+    // +-------+
+
+    // Define and register a one-argument UDF
+    val plusOne = udf((x: Int) => x + 1)
+    spark.udf.register("plusOne", plusOne)
+    spark.sql("SELECT plusOne(5)").show()
+    // +------+
+    // |UDF(5)|
+    // +------+
+    // |     6|
+    // +------+
+
+    // Define a two-argument UDF and register it with Spark in one step
+    spark.udf.register("strLenScala", (_: String).length + (_: Int))
+    spark.sql("SELECT strLenScala('test', 1)").show()
+    // +--------------------+
+    // |strLenScala(test, 1)|
+    // +--------------------+
+    // |                   5|
+    // +--------------------+
+
+    // UDF in a WHERE clause
+    spark.udf.register("oneArgFilter", (n: Int) => { n > 5 })
+    spark.range(1, 10).createOrReplaceTempView("test")
+    spark.sql("SELECT * FROM test WHERE oneArgFilter(id)").show()
+    // +---+
+    // | id|
+    // +---+
+    // |  6|
+    // |  7|
+    // |  8|
+    // |  9|
+    // +---+
+
+    // $example off:udf_scalar$
+
+    spark.stop()
+  }
+}
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
index 0f47deaf1021b..626f4b4d3ccdf 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
@@ -50,7 +50,7 @@ object CustomReceiver {
     val ssc = new StreamingContext(sparkConf, Seconds(1))
 
     // Create an input stream with the custom receiver on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     val lines = ssc.receiverStream(new CustomReceiver(args(0), args(1).toInt))
     val words = lines.flatMap(_.split(" "))
     val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala
index 26bb51dde3a1d..7d981dfb949ea 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala
@@ -47,7 +47,7 @@ object NetworkWordCount {
     val ssc = new StreamingContext(sparkConf, Seconds(1))
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK_SER)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
index 243c22e71275c..98c1807b1ed62 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
@@ -112,7 +112,7 @@ object RecoverableNetworkWordCount {
     ssc.checkpoint(checkpointDirectory)
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     val lines = ssc.socketTextStream(ip, port)
     val words = lines.flatMap(_.split(" "))
     val wordCounts = words.map((_, 1)).reduceByKey(_ + _)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala
index 778be7baaeeac..7daa0014e0f1c 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala
@@ -51,7 +51,7 @@ object SqlNetworkWordCount {
     val ssc = new StreamingContext(sparkConf, Seconds(2))
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK_SER)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala
index 46f01edf7deec..8a5fcda9cd990 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala
@@ -52,7 +52,7 @@ object StatefulNetworkWordCount {
     val initialRDD = ssc.sparkContext.parallelize(List(("hello", 1), ("world", 1)))
 
     // Create a ReceiverInputDStream on target ip:port and count the
-    // words in input stream of \n delimited test (eg. generated by 'nc')
+    // words in input stream of \n delimited test (e.g. generated by 'nc')
     val lines = ssc.socketTextStream(args(0), args(1).toInt)
     val words = lines.flatMap(_.split(" "))
     val wordDstream = words.map(x => (x, 1))
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index ba6f20bfdbf58..923b3bd6981dc 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java b/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java
new file mode 100644
index 0000000000000..a4555844b5117
--- /dev/null
+++ b/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.avro;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+import org.apache.avro.file.CodecFactory;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.mapred.AvroKey;
+import org.apache.avro.mapreduce.AvroKeyOutputFormat;
+import org.apache.avro.mapreduce.Syncable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+// A variant of `AvroKeyOutputFormat`, which is used to inject the custom `RecordWriterFactory` so
+// that we can set avro file metadata.
+class SparkAvroKeyOutputFormat extends AvroKeyOutputFormat<GenericRecord> {
+  SparkAvroKeyOutputFormat(Map<String, String> metadata) {
+    super(new SparkRecordWriterFactory(metadata));
+  }
+
+  static class SparkRecordWriterFactory extends RecordWriterFactory<GenericRecord> {
+    private final Map<String, String> metadata;
+    SparkRecordWriterFactory(Map<String, String> metadata) {
+      this.metadata = metadata;
+    }
+
+    protected RecordWriter<AvroKey<GenericRecord>, NullWritable> create(
+        Schema writerSchema,
+        GenericData dataModel,
+        CodecFactory compressionCodec,
+        OutputStream outputStream,
+        int syncInterval) throws IOException {
+      return new SparkAvroKeyRecordWriter(
+        writerSchema, dataModel, compressionCodec, outputStream, syncInterval, metadata);
+    }
+  }
+}
+
+// This a fork of org.apache.avro.mapreduce.AvroKeyRecordWriter, in order to set file metadata.
+class SparkAvroKeyRecordWriter<T> extends RecordWriter<AvroKey<T>, NullWritable>
+    implements Syncable {
+
+  private final DataFileWriter<T> mAvroFileWriter;
+
+  SparkAvroKeyRecordWriter(
+      Schema writerSchema,
+      GenericData dataModel,
+      CodecFactory compressionCodec,
+      OutputStream outputStream,
+      int syncInterval,
+      Map<String, String> metadata) throws IOException {
+    this.mAvroFileWriter = new DataFileWriter(dataModel.createDatumWriter(writerSchema));
+    for (Map.Entry<String, String> entry : metadata.entrySet()) {
+      this.mAvroFileWriter.setMeta(entry.getKey(), entry.getValue());
+    }
+    this.mAvroFileWriter.setCodec(compressionCodec);
+    this.mAvroFileWriter.setSyncInterval(syncInterval);
+    this.mAvroFileWriter.create(writerSchema, outputStream);
+  }
+
+  public void write(AvroKey<T> record, NullWritable ignore) throws IOException {
+    this.mAvroFileWriter.append(record.datum());
+  }
+
+  public void close(TaskAttemptContext context) throws IOException {
+    this.mAvroFileWriter.close();
+  }
+
+  public long sync() throws IOException {
+    return this.mAvroFileWriter.sync();
+  }
+}
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
index 79c72057c5823..0fdf5c6af13b9 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGe
 import org.apache.spark.sql.catalyst.util.{FailFastMode, ParseMode, PermissiveMode}
 import org.apache.spark.sql.types._
 
-case class AvroDataToCatalyst(
+private[avro] case class AvroDataToCatalyst(
     child: Expression,
     jsonFormatSchema: String,
     options: Map[String, String])
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 2c17c16f06da7..7580957a0de36 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -32,16 +32,35 @@ import org.apache.avro.util.Utf8
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
+import org.apache.spark.sql.execution.datasources.DataSourceUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 /**
  * A deserializer to deserialize data in avro format to data in catalyst format.
  */
-class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
+private[sql] class AvroDeserializer(
+    rootAvroType: Schema,
+    rootCatalystType: DataType,
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value) {
+
+  def this(rootAvroType: Schema, rootCatalystType: DataType) {
+    this(rootAvroType, rootCatalystType,
+      LegacyBehaviorPolicy.withName(
+        SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ)))
+  }
+
   private lazy val decimalConversions = new DecimalConversion()
 
+  private val dateRebaseFunc = DataSourceUtils.creteDateRebaseFuncInRead(
+    datetimeRebaseMode, "Avro")
+
+  private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInRead(
+    datetimeRebaseMode, "Avro")
+
   private val converter: Any => Any = rootCatalystType match {
     // A shortcut for empty schema.
     case st: StructType if st.isEmpty =>
@@ -89,20 +108,21 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
         updater.setInt(ordinal, value.asInstanceOf[Int])
 
       case (INT, DateType) => (updater, ordinal, value) =>
-        updater.setInt(ordinal, value.asInstanceOf[Int])
+        updater.setInt(ordinal, dateRebaseFunc(value.asInstanceOf[Int]))
 
       case (LONG, LongType) => (updater, ordinal, value) =>
         updater.setLong(ordinal, value.asInstanceOf[Long])
 
       case (LONG, TimestampType) => avroType.getLogicalType match {
-        case _: TimestampMillis => (updater, ordinal, value) =>
-          updater.setLong(ordinal, value.asInstanceOf[Long] * 1000)
+        // For backward compatibility, if the Avro type is Long and it is not logical type
+        // (the `null` case), the value is processed as timestamp type with millisecond precision.
+        case null | _: TimestampMillis => (updater, ordinal, value) =>
+          val millis = value.asInstanceOf[Long]
+          val micros = DateTimeUtils.fromMillis(millis)
+          updater.setLong(ordinal, timestampRebaseFunc(micros))
         case _: TimestampMicros => (updater, ordinal, value) =>
-          updater.setLong(ordinal, value.asInstanceOf[Long])
-        case null => (updater, ordinal, value) =>
-          // For backward compatibility, if the Avro type is Long and it is not logical type,
-          // the value is processed as timestamp type with millisecond precision.
-          updater.setLong(ordinal, value.asInstanceOf[Long] * 1000)
+          val micros = value.asInstanceOf[Long]
+          updater.setLong(ordinal, timestampRebaseFunc(micros))
         case other => throw new IncompatibleSchemaException(
           s"Cannot convert Avro logical type ${other} to Catalyst Timestamp type.")
       }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
index 123669ba1376a..59d54bc433f8b 100755
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
@@ -34,7 +34,8 @@ import org.apache.spark.TaskContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.datasources.{FileFormat, OutputWriterFactory, PartitionedFile}
+import org.apache.spark.sql.execution.datasources.{DataSourceUtils, FileFormat, OutputWriterFactory, PartitionedFile}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.{DataSourceRegister, Filter}
 import org.apache.spark.sql.types._
 import org.apache.spark.util.SerializableConfiguration
@@ -123,8 +124,12 @@ private[sql] class AvroFileFormat extends FileFormat
         reader.sync(file.start)
         val stop = file.start + file.length
 
-        val deserializer =
-          new AvroDeserializer(userProvidedSchema.getOrElse(reader.getSchema), requiredSchema)
+        val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
+          reader.asInstanceOf[DataFileReader[_]].getMetaString,
+          SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ))
+
+        val deserializer = new AvroDeserializer(
+          userProvidedSchema.getOrElse(reader.getSchema), requiredSchema, datetimeRebaseMode)
 
         new Iterator[InternalRow] {
           private[this] var completed = false
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
index f3ea78583fbc0..8972b0553c522 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.internal.SQLConf
 /**
  * Options for Avro Reader and Writer stored in case insensitive manner.
  */
-class AvroOptions(
+private[sql] class AvroOptions(
     @transient val parameters: CaseInsensitiveMap[String],
     @transient val conf: Configuration) extends Logging with Serializable {
 
@@ -95,7 +95,7 @@ class AvroOptions(
     parameters.get("mode").map(ParseMode.fromString).getOrElse(FailFastMode)
 }
 
-object AvroOptions {
+private[sql] object AvroOptions {
   def apply(parameters: Map[String, String]): AvroOptions = {
     val hadoopConf = SparkSession
       .getActiveSession
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
index 06507115f5ed8..ac9608c867937 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
@@ -19,16 +19,21 @@ package org.apache.spark.sql.avro
 
 import java.io.{IOException, OutputStream}
 
+import scala.collection.JavaConverters._
+
 import org.apache.avro.Schema
 import org.apache.avro.generic.GenericRecord
 import org.apache.avro.mapred.AvroKey
-import org.apache.avro.mapreduce.AvroKeyOutputFormat
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.io.NullWritable
 import org.apache.hadoop.mapreduce.{RecordWriter, TaskAttemptContext}
 
+import org.apache.spark.SPARK_VERSION_SHORT
+import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.datasources.OutputWriter
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 
 // NOTE: This class is instantiated and used on executor side only, no need to be serializable.
@@ -38,15 +43,28 @@ private[avro] class AvroOutputWriter(
     schema: StructType,
     avroSchema: Schema) extends OutputWriter {
 
+  // Whether to rebase datetimes from Gregorian to Julian calendar in write
+  private val datetimeRebaseMode = LegacyBehaviorPolicy.withName(
+    SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE))
+
   // The input rows will never be null.
-  private lazy val serializer = new AvroSerializer(schema, avroSchema, nullable = false)
+  private lazy val serializer =
+    new AvroSerializer(schema, avroSchema, nullable = false, datetimeRebaseMode)
 
   /**
    * Overrides the couple of methods responsible for generating the output streams / files so
    * that the data can be correctly partitioned
    */
-  private val recordWriter: RecordWriter[AvroKey[GenericRecord], NullWritable] =
-    new AvroKeyOutputFormat[GenericRecord]() {
+  private val recordWriter: RecordWriter[AvroKey[GenericRecord], NullWritable] = {
+    val fileMeta = Map(SPARK_VERSION_METADATA_KEY -> SPARK_VERSION_SHORT) ++ {
+      if (datetimeRebaseMode == LegacyBehaviorPolicy.LEGACY) {
+        Some(SPARK_LEGACY_DATETIME -> "")
+      } else {
+        None
+      }
+    }
+
+    new SparkAvroKeyOutputFormat(fileMeta.asJava) {
 
       override def getDefaultWorkFile(context: TaskAttemptContext, extension: String): Path = {
         new Path(path)
@@ -57,8 +75,8 @@ private[avro] class AvroOutputWriter(
         val path = getDefaultWorkFile(context, ".avro")
         path.getFileSystem(context.getConfiguration).create(path)
       }
-
     }.getRecordWriter(context)
+  }
 
   override def write(row: InternalRow): Unit = {
     val key = new AvroKey(serializer.serialize(row).asInstanceOf[GenericRecord])
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index b7bf7e5543033..2c0bffa509ec8 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -34,18 +34,37 @@ import org.apache.avro.util.Utf8
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.execution.datasources.DataSourceUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 
 /**
  * A serializer to serialize data in catalyst format to data in avro format.
  */
-class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean)
-  extends Logging {
+private[sql] class AvroSerializer(
+    rootCatalystType: DataType,
+    rootAvroType: Schema,
+    nullable: Boolean,
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value) extends Logging {
+
+  def this(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean) {
+    this(rootCatalystType, rootAvroType, nullable,
+      LegacyBehaviorPolicy.withName(SQLConf.get.getConf(
+        SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE)))
+  }
 
   def serialize(catalystData: Any): Any = {
     converter.apply(catalystData)
   }
 
+  private val dateRebaseFunc = DataSourceUtils.creteDateRebaseFuncInWrite(
+    datetimeRebaseMode, "Avro")
+
+  private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInWrite(
+    datetimeRebaseMode, "Avro")
+
   private val converter: Any => Any = {
     val actualAvroType = resolveNullableType(rootAvroType, nullable)
     val baseConverter = rootCatalystType match {
@@ -136,14 +155,15 @@ class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable:
         (getter, ordinal) => ByteBuffer.wrap(getter.getBinary(ordinal))
 
       case (DateType, INT) =>
-        (getter, ordinal) => getter.getInt(ordinal)
+        (getter, ordinal) => dateRebaseFunc(getter.getInt(ordinal))
 
       case (TimestampType, LONG) => avroType.getLogicalType match {
-          case _: TimestampMillis => (getter, ordinal) => getter.getLong(ordinal) / 1000
-          case _: TimestampMicros => (getter, ordinal) => getter.getLong(ordinal)
-          // For backward compatibility, if the Avro type is Long and it is not logical type,
-          // output the timestamp value as with millisecond precision.
-          case null => (getter, ordinal) => getter.getLong(ordinal) / 1000
+          // For backward compatibility, if the Avro type is Long and it is not logical type
+          // (the `null` case), output the timestamp value as with millisecond precision.
+          case null | _: TimestampMillis => (getter, ordinal) =>
+            DateTimeUtils.toMillis(timestampRebaseFunc(getter.getLong(ordinal)))
+          case _: TimestampMicros => (getter, ordinal) =>
+            timestampRebaseFunc(getter.getLong(ordinal))
           case other => throw new IncompatibleSchemaException(
             s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
         }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
index 70dcd58a600fc..a9f34bbea4c87 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
@@ -37,12 +37,12 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
-object AvroUtils extends Logging {
+private[sql] object AvroUtils extends Logging {
   def inferSchema(
       spark: SparkSession,
       options: Map[String, String],
       files: Seq[FileStatus]): Option[StructType] = {
-    val conf = spark.sessionState.newHadoopConf()
+    val conf = spark.sessionState.newHadoopConfWithOptions(options)
     val parsedOptions = new AvroOptions(options, conf)
 
     if (parsedOptions.parameters.contains(ignoreExtensionKey)) {
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
index 7732c83aaf7c4..53910b752fdd6 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.types.{BinaryType, DataType}
 
-case class CatalystDataToAvro(
+private[avro] case class CatalystDataToAvro(
     child: Expression,
     jsonFormatSchema: Option[String]) extends UnaryExpression {
 
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
index 3947d327dfac6..0e01fbf6fe2e8 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
@@ -24,6 +24,7 @@ import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
 import org.apache.avro.LogicalTypes.{Date, Decimal, TimestampMicros, TimestampMillis}
 import org.apache.avro.Schema.Type._
 
+import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.sql.catalyst.util.RandomUUIDGenerator
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.types.Decimal.{maxPrecisionForBytes, minBytesForPrecision}
@@ -32,21 +33,29 @@ import org.apache.spark.sql.types.Decimal.{maxPrecisionForBytes, minBytesForPrec
  * This object contains method that are used to convert sparkSQL schemas to avro schemas and vice
  * versa.
  */
+@DeveloperApi
 object SchemaConverters {
   private lazy val uuidGenerator = RandomUUIDGenerator(new Random().nextLong())
 
   private lazy val nullSchema = Schema.create(Schema.Type.NULL)
 
+  /**
+   * Internal wrapper for SQL data type and nullability.
+   *
+   * @since 2.4.0
+   */
   case class SchemaType(dataType: DataType, nullable: Boolean)
 
   /**
-   * This function takes an avro schema and returns a sql schema.
+   * Converts an Avro schema to a corresponding Spark SQL schema.
+   *
+   * @since 2.4.0
    */
   def toSqlType(avroSchema: Schema): SchemaType = {
     toSqlTypeHelper(avroSchema, Set.empty)
   }
 
-  def toSqlTypeHelper(avroSchema: Schema, existingRecordNames: Set[String]): SchemaType = {
+  private def toSqlTypeHelper(avroSchema: Schema, existingRecordNames: Set[String]): SchemaType = {
     avroSchema.getType match {
       case INT => avroSchema.getLogicalType match {
         case _: Date => SchemaType(DateType, nullable = false)
@@ -133,6 +142,11 @@ object SchemaConverters {
     }
   }
 
+  /**
+   * Converts a Spark SQL schema to a corresponding Avro schema.
+   *
+   * @since 2.4.0
+   */
   def toAvroType(
       catalystType: DataType,
       nullable: Boolean = false,
@@ -192,4 +206,5 @@ object SchemaConverters {
   }
 }
 
-class IncompatibleSchemaException(msg: String, ex: Throwable = null) extends Exception(msg, ex)
+private[avro] class IncompatibleSchemaException(
+  msg: String, ex: Throwable = null) extends Exception(msg, ex)
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
index c6f52d676422c..2d21c493d5711 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
@@ -31,13 +31,15 @@ class AvroDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    AvroTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    AvroTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    AvroTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    AvroTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
index 8230dbaf8ea6c..15918f46a83bb 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
@@ -32,7 +32,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.avro.{AvroDeserializer, AvroOptions}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.read.PartitionReader
-import org.apache.spark.sql.execution.datasources.PartitionedFile
+import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile}
 import org.apache.spark.sql.execution.datasources.v2.{EmptyPartitionReader, FilePartitionReaderFactory, PartitionReaderWithPartitionValues}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
@@ -88,8 +88,11 @@ case class AvroPartitionReaderFactory(
       reader.sync(partitionedFile.start)
       val stop = partitionedFile.start + partitionedFile.length
 
-      val deserializer =
-        new AvroDeserializer(userProvidedSchema.getOrElse(reader.getSchema), readDataSchema)
+      val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
+        reader.asInstanceOf[DataFileReader[_]].getMetaString,
+        SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ))
+      val deserializer = new AvroDeserializer(
+        userProvidedSchema.getOrElse(reader.getSchema), readDataSchema, datetimeRebaseMode)
 
       val fileReader = new PartitionReader[InternalRow] {
         private[this] var completed = false
diff --git a/external/avro/src/test/resources/before_1582_date_v2_4_5.avro b/external/avro/src/test/resources/before_1582_date_v2_4_5.avro
new file mode 100644
index 0000000000000..5c15601f7ee4b
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_date_v2_4_5.avro differ
diff --git a/external/avro/src/test/resources/before_1582_date_v2_4_6.avro b/external/avro/src/test/resources/before_1582_date_v2_4_6.avro
new file mode 100644
index 0000000000000..212ea1d5efa5c
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_date_v2_4_6.avro differ
diff --git a/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_5.avro b/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_5.avro
new file mode 100644
index 0000000000000..c3445e3999bc1
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_5.avro differ
diff --git a/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_6.avro b/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_6.avro
new file mode 100644
index 0000000000000..96008d2378b1f
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_6.avro differ
diff --git a/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_5.avro b/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_5.avro
new file mode 100644
index 0000000000000..be12a0782073c
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_5.avro differ
diff --git a/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_6.avro b/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_6.avro
new file mode 100644
index 0000000000000..262f5dd6e77a4
Binary files /dev/null and b/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_6.avro differ
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
index 82569653c1f23..12ebddf72b03d 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
@@ -246,7 +246,7 @@ abstract class AvroLogicalTypeSuite extends QueryTest with SharedSparkSession {
     dataFileWriter.create(schema, new File(avroFile))
     val logicalType = LogicalTypes.decimal(precision, scale)
 
-    decimalInputData.map { x =>
+    decimalInputData.foreach { x =>
       val avroRec = new GenericData.Record(schema)
       val decimal = new java.math.BigDecimal(x).setScale(scale)
       val bytes =
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 360160c9c9398..5d7d2e486c524 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -19,9 +19,9 @@ package org.apache.spark.sql.avro
 
 import java.io._
 import java.net.URL
-import java.nio.file.{Files, Paths}
+import java.nio.file.{Files, Paths, StandardCopyOption}
 import java.sql.{Date, Timestamp}
-import java.util.{Locale, TimeZone, UUID}
+import java.util.{Locale, UUID}
 
 import scala.collection.JavaConverters._
 
@@ -33,15 +33,17 @@ import org.apache.avro.generic.{GenericData, GenericDatumReader, GenericDatumWri
 import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed}
 import org.apache.commons.io.FileUtils
 
-import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf, SparkException, SparkUpgradeException}
 import org.apache.spark.sql._
-import org.apache.spark.sql.TestingUDT.{IntervalData, NullData, NullUDT}
+import org.apache.spark.sql.TestingUDT.IntervalData
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
 import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA, UTC}
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.datasources.{DataSource, FilePartition}
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.v2.avro.AvroScan
@@ -83,6 +85,10 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }, new GenericDatumReader[Any]()).getSchema.toString(false)
   }
 
+  private def getResourceAvroFilePath(name: String): String = {
+    Thread.currentThread().getContextClassLoader.getResource(name).toString
+  }
+
   test("resolve avro data source") {
     val databricksAvro = "com.databricks.spark.avro"
     // By default the backward compatibility for com.databricks.spark.avro is enabled.
@@ -402,18 +408,19 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         StructField("float", FloatType, true),
         StructField("date", DateType, true)
       ))
-      TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
-      val rdd = spark.sparkContext.parallelize(Seq(
-        Row(1f, null),
-        Row(2f, new Date(1451948400000L)),
-        Row(3f, new Date(1460066400500L))
-      ))
-      val df = spark.createDataFrame(rdd, schema)
-      df.write.format("avro").save(dir.toString)
-      assert(spark.read.format("avro").load(dir.toString).count == rdd.count)
-      checkAnswer(
-        spark.read.format("avro").load(dir.toString).select("date"),
-        Seq(Row(null), Row(new Date(1451865600000L)), Row(new Date(1459987200000L))))
+      withDefaultTimeZone(UTC) {
+        val rdd = spark.sparkContext.parallelize(Seq(
+          Row(1f, null),
+          Row(2f, new Date(1451948400000L)),
+          Row(3f, new Date(1460066400500L))
+        ))
+        val df = spark.createDataFrame(rdd, schema)
+        df.write.format("avro").save(dir.toString)
+        assert(spark.read.format("avro").load(dir.toString).count == rdd.count)
+        checkAnswer(
+          spark.read.format("avro").load(dir.toString).select("date"),
+          Seq(Row(null), Row(new Date(1451865600000L)), Row(new Date(1459987200000L))))
+      }
     }
   }
 
@@ -1521,6 +1528,315 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       assert(deprecatedEvents.size === 1)
     }
   }
+
+  // It generates input files for the test below:
+  // "SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps"
+  ignore("SPARK-31855: generate test files for checking compatibility with Spark 2.4") {
+    val resourceDir = "external/avro/src/test/resources"
+    val version = "2_4_6"
+    def save(
+      in: Seq[String],
+      t: String,
+      dstFile: String,
+      options: Map[String, String] = Map.empty): Unit = {
+      withTempDir { dir =>
+        in.toDF("dt")
+          .select($"dt".cast(t))
+          .repartition(1)
+          .write
+          .mode("overwrite")
+          .options(options)
+          .format("avro")
+          .save(dir.getCanonicalPath)
+        Files.copy(
+          dir.listFiles().filter(_.getName.endsWith(".avro")).head.toPath,
+          Paths.get(resourceDir, dstFile),
+          StandardCopyOption.REPLACE_EXISTING)
+      }
+    }
+    withDefaultTimeZone(LA) {
+      withSQLConf(
+        SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId) {
+        save(
+          Seq("1001-01-01"),
+          "date",
+          s"before_1582_date_v$version.avro")
+        save(
+          Seq("1001-01-01 01:02:03.123"),
+          "timestamp",
+          s"before_1582_timestamp_millis_v$version.avro",
+          // scalastyle:off line.size.limit
+          Map("avroSchema" ->
+            s"""
+               |  {
+               |    "namespace": "logical",
+               |    "type": "record",
+               |    "name": "test",
+               |    "fields": [
+               |      {"name": "dt", "type": ["null", {"type": "long","logicalType": "timestamp-millis"}], "default": null}
+               |    ]
+               |  }
+               |""".stripMargin))
+        // scalastyle:on line.size.limit
+        save(
+          Seq("1001-01-01 01:02:03.123456"),
+          "timestamp",
+          s"before_1582_timestamp_micros_v$version.avro")
+      }
+    }
+  }
+
+  test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
+    // test reading the existing 2.4 files and new 3.0 files (with rebase on/off) together.
+    def checkReadMixedFiles(
+        fileName: String,
+        dt: String,
+        dataStr: String,
+        checkDefaultLegacyRead: String => Unit): Unit = {
+      withTempPaths(2) { paths =>
+        paths.foreach(_.delete())
+        val path2_4 = getResourceAvroFilePath(fileName)
+        val path3_0 = paths(0).getCanonicalPath
+        val path3_0_rebase = paths(1).getCanonicalPath
+        if (dt == "date") {
+          val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("dt"))
+
+          // By default we should fail to write ancient datetime values.
+          val e = intercept[SparkException](df.write.format("avro").save(path3_0))
+          assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
+          checkDefaultLegacyRead(path2_4)
+
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+            df.write.format("avro").mode("overwrite").save(path3_0)
+          }
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+            df.write.format("avro").save(path3_0_rebase)
+          }
+
+          // For Avro files written by Spark 3.0, we know the writer info and don't need the config
+          // to guide the rebase behavior.
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
+            checkAnswer(
+              spark.read.format("avro").load(path2_4, path3_0, path3_0_rebase),
+              1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
+          }
+        } else {
+          val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("dt"))
+          val avroSchema =
+            s"""
+              |{
+              |  "type" : "record",
+              |  "name" : "test_schema",
+              |  "fields" : [
+              |    {"name": "dt", "type": {"type": "long", "logicalType": "$dt"}}
+              |  ]
+              |}""".stripMargin
+
+          // By default we should fail to write ancient datetime values.
+          val e = intercept[SparkException] {
+            df.write.format("avro").option("avroSchema", avroSchema).save(path3_0)
+          }
+          assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
+          checkDefaultLegacyRead(path2_4)
+
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+            df.write.format("avro").option("avroSchema", avroSchema).mode("overwrite").save(path3_0)
+          }
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+            df.write.format("avro").option("avroSchema", avroSchema).save(path3_0_rebase)
+          }
+
+          // For Avro files written by Spark 3.0, we know the writer info and don't need the config
+          // to guide the rebase behavior.
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
+            checkAnswer(
+              spark.read.format("avro").load(path2_4, path3_0, path3_0_rebase),
+              1.to(3).map(_ => Row(java.sql.Timestamp.valueOf(dataStr))))
+          }
+        }
+      }
+    }
+
+    def failInRead(path: String): Unit = {
+      val e = intercept[SparkException](spark.read.format("avro").load(path).collect())
+      assert(e.getCause.isInstanceOf[SparkUpgradeException])
+    }
+    def successInRead(path: String): Unit = spark.read.format("avro").load(path).collect()
+    Seq(
+      // By default we should fail to read ancient datetime values when parquet files don't
+      // contain Spark version.
+      "2_4_5" -> failInRead _,
+      "2_4_6" -> successInRead _
+    ).foreach { case (version, checkDefaultRead) =>
+      checkReadMixedFiles(
+        s"before_1582_date_v$version.avro",
+        "date",
+        "1001-01-01",
+        checkDefaultRead)
+      checkReadMixedFiles(
+        s"before_1582_timestamp_micros_v$version.avro",
+        "timestamp-micros",
+        "1001-01-01 01:02:03.123456",
+        checkDefaultRead)
+      checkReadMixedFiles(
+        s"before_1582_timestamp_millis_v$version.avro",
+        "timestamp-millis",
+        "1001-01-01 01:02:03.123",
+        checkDefaultRead)
+    }
+  }
+
+  test("SPARK-31183: rebasing microseconds timestamps in write") {
+    val tsStr = "1001-01-01 01:02:03.123456"
+    val nonRebased = "1001-01-07 01:09:05.123456"
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+        Seq(tsStr).toDF("tsS")
+          .select($"tsS".cast("timestamp").as("ts"))
+          .write.format("avro")
+          .save(path)
+      }
+
+      // The file metadata indicates if it needs rebase or not, so we can always get the correct
+      // result regardless of the "rebase mode" config.
+      Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
+          checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
+        }
+      }
+
+      // Force to not rebase to prove the written datetime values are rebased and we will get
+      // wrong result if we don't rebase while reading.
+      withSQLConf("spark.test.forceNoRebase" -> "true") {
+        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(nonRebased)))
+      }
+    }
+  }
+
+  test("SPARK-31183: rebasing milliseconds timestamps in write") {
+    val tsStr = "1001-01-01 01:02:03.123456"
+    val rebased = "1001-01-01 01:02:03.123"
+    val nonRebased = "1001-01-07 01:09:05.123"
+    Seq(
+      """{"type": "long","logicalType": "timestamp-millis"}""",
+      """"long"""").foreach { tsType =>
+      val timestampSchema = s"""
+        |{
+        |  "namespace": "logical",
+        |  "type": "record",
+        |  "name": "test",
+        |  "fields": [
+        |    {"name": "ts", "type": $tsType}
+        |  ]
+        |}""".stripMargin
+      withTempPath { dir =>
+        val path = dir.getAbsolutePath
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+          Seq(tsStr).toDF("tsS")
+            .select($"tsS".cast("timestamp").as("ts"))
+            .write
+            .option("avroSchema", timestampSchema)
+            .format("avro")
+            .save(path)
+        }
+
+        // The file metadata indicates if it needs rebase or not, so we can always get the correct
+        // result regardless of the "rebase mode" config.
+        Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
+            checkAnswer(
+              spark.read.schema("ts timestamp").format("avro").load(path),
+              Row(Timestamp.valueOf(rebased)))
+          }
+        }
+
+        // Force to not rebase to prove the written datetime values are rebased and we will get
+        // wrong result if we don't rebase while reading.
+        withSQLConf("spark.test.forceNoRebase" -> "true") {
+          checkAnswer(
+            spark.read.schema("ts timestamp").format("avro").load(path),
+            Row(Timestamp.valueOf(nonRebased)))
+        }
+      }
+    }
+  }
+
+  test("SPARK-31183: rebasing dates in write") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+        Seq("1001-01-01").toDF("dateS")
+          .select($"dateS".cast("date").as("date"))
+          .write.format("avro")
+          .save(path)
+      }
+
+      // The file metadata indicates if it needs rebase or not, so we can always get the correct
+      // result regardless of the "rebase mode" config.
+      Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
+          checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
+        }
+      }
+
+      // Force to not rebase to prove the written datetime values are rebased and we will get
+      // wrong result if we don't rebase while reading.
+      withSQLConf("spark.test.forceNoRebase" -> "true") {
+        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-07")))
+      }
+    }
+  }
+
+  private def checkMetaData(path: java.io.File, key: String, expectedValue: String): Unit = {
+    val avroFiles = path.listFiles()
+      .filter(f => f.isFile && !f.getName.startsWith(".") && !f.getName.startsWith("_"))
+    assert(avroFiles.length === 1)
+    val reader = DataFileReader.openReader(avroFiles(0), new GenericDatumReader[GenericRecord]())
+    val value = reader.asInstanceOf[DataFileReader[_]].getMetaString(key)
+    assert(value === expectedValue)
+  }
+
+  test("SPARK-31327: Write Spark version into Avro file metadata") {
+    withTempPath { path =>
+      spark.range(1).repartition(1).write.format("avro").save(path.getCanonicalPath)
+      checkMetaData(path, SPARK_VERSION_METADATA_KEY, SPARK_VERSION_SHORT)
+    }
+  }
+
+  test("SPARK-33089: should propagate Hadoop config from DS options to underlying file system") {
+    withSQLConf(
+      "fs.file.impl" -> classOf[FakeFileSystemRequiringDSOption].getName,
+      "fs.file.impl.disable.cache" -> "true") {
+      val conf = Map("ds_option" -> "value")
+      val path = "file:" + testAvro.stripPrefix("file:")
+      spark.read.format("avro").options(conf).load(path)
+    }
+  }
+
+  test("SPARK-33163: write the metadata key 'org.apache.spark.legacyDateTime'") {
+    def saveTs(dir: java.io.File): Unit = {
+      Seq(Timestamp.valueOf("2020-10-15 01:02:03")).toDF()
+        .repartition(1)
+        .write
+        .format("avro")
+        .save(dir.getAbsolutePath)
+    }
+    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+      withTempPath { dir =>
+        saveTs(dir)
+        checkMetaData(dir, SPARK_LEGACY_DATETIME, "")
+      }
+    }
+    Seq(CORRECTED, EXCEPTION).foreach { mode =>
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> mode.toString) {
+        withTempPath { dir =>
+          saveTs(dir)
+          checkMetaData(dir, SPARK_LEGACY_DATETIME, null)
+        }
+      }
+    }
+  }
 }
 
 class AvroV1Suite extends AvroSuite {
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index aff79b8b8e642..4f16290891c3e 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -50,6 +50,7 @@
       <groupId>com.spotify</groupId>
       <artifactId>docker-client</artifactId>
       <scope>test</scope>
+      <classifier>shaded</classifier>
     </dependency>
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
index 519a5cddb3639..996e697bf62ca 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
@@ -44,7 +44,7 @@ abstract class DatabaseOnDocker {
   val env: Map[String, String]
 
   /**
-   * Wheather or not to use ipc mode for shared memory when starting docker image
+   * Whether or not to use ipc mode for shared memory when starting docker image
    */
   val usesIpc: Boolean
 
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
index 5738307095933..7076e18c404f5 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
@@ -100,6 +100,37 @@ class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite {
         |'the', 'lazy',
         |'dog')
       """.stripMargin).executeUpdate()
+    conn.prepareStatement(
+      """
+        |CREATE TABLE spatials (
+        |point geometry,
+        |line geometry,
+        |circle geometry,
+        |curve geography,
+        |polygon geometry,
+        |curve_polygon geography,
+        |multi_point geometry,
+        |multi_line geometry,
+        |multi_polygon geometry,
+        |geometry_collection geometry)
+      """.stripMargin).executeUpdate()
+    conn.prepareStatement(
+      """
+        |INSERT INTO spatials VALUES (
+        |'POINT(3 4 7 2.5)',
+        |'LINESTRING(1 0, 0 1, -1 0)',
+        |'CIRCULARSTRING(
+        |  -122.358 47.653, -122.348 47.649, -122.348 47.658, -122.358 47.658, -122.358 47.653)',
+        |'COMPOUNDCURVE(
+        |  CIRCULARSTRING(-122.358 47.653, -122.348 47.649,
+        |    -122.348 47.658, -122.358 47.658, -122.358 47.653))',
+        |'POLYGON((-20 -20, -20 20, 20 20, 20 -20, -20 -20), (10 0, 0 10, 0 -10, 10 0))',
+        |'CURVEPOLYGON((-122.3 47, 122.3 47, 125.7 49, 121 38, -122.3 47))',
+        |'MULTIPOINT((2 3), (7 8 9.5))',
+        |'MULTILINESTRING((0 2, 1 1), (1 0, 1 1))',
+        |'MULTIPOLYGON(((2 2, 2 -2, -2 -2, -2 2, 2 2)),((1 1, 3 1, 3 3, 1 3, 1 1)))',
+        |'GEOMETRYCOLLECTION(LINESTRING(1 1, 3 5),POLYGON((-1 -1, -1 -5, -5 -5, -5 -1, -1 -1)))')
+      """.stripMargin).executeUpdate()
   }
 
   test("Basic test") {
@@ -225,4 +256,94 @@ class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite {
     df2.write.jdbc(jdbcUrl, "datescopy", new Properties)
     df3.write.jdbc(jdbcUrl, "stringscopy", new Properties)
   }
+
+  test("SPARK-33813: MsSqlServerDialect should support spatial types") {
+    val df = spark.read.jdbc(jdbcUrl, "spatials", new Properties)
+    val rows = df.collect()
+    assert(rows.length == 1)
+    val row = rows(0)
+    val types = row.toSeq.map(x => x.getClass.toString)
+    assert(types.length == 10)
+    assert(types(0) == "class [B")
+    assert(row.getAs[Array[Byte]](0) ===
+      Array(0, 0, 0, 0, 1, 15, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0,
+        16, 64, 0, 0, 0, 0, 0, 0, 28, 64, 0, 0, 0, 0, 0, 0, 4, 64))
+    assert(types(1) == "class [B")
+    assert(row.getAs[Array[Byte]](1) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        -16, 63, 0, 0, 0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
+        0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 2))
+    assert(types(2) == "class [B")
+    assert(row.getAs[Array[Byte]](2) ===
+      Array[Byte](0, 0, 0, 0, 2, 4, 5, 0, 0, 0, -12, -3, -44, 120, -23, -106,
+        94, -64, -35, 36, 6, -127, -107, -45, 71, 64, -125, -64, -54, -95, 69,
+        -106, 94, -64, 80, -115, -105, 110, 18, -45, 71, 64, -125, -64, -54,
+        -95, 69, -106, 94, -64, 78, 98, 16, 88, 57, -44, 71, 64, -12, -3, -44,
+        120, -23, -106, 94, -64, 78, 98, 16, 88, 57, -44, 71, 64, -12, -3, -44,
+        120, -23, -106, 94, -64, -35, 36, 6, -127, -107, -45, 71, 64, 1, 0, 0,
+        0, 2, 0, 0, 0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 8))
+    assert(types(3) == "class [B")
+    assert(row.getAs[Array[Byte]](3) ===
+      Array[Byte](-26, 16, 0, 0, 2, 4, 5, 0, 0, 0, -35, 36, 6, -127, -107, -45,
+        71, 64, -12, -3, -44, 120, -23, -106, 94, -64, 80, -115, -105, 110, 18,
+        -45, 71, 64, -125, -64, -54, -95, 69, -106, 94, -64, 78, 98, 16, 88, 57,
+        -44, 71, 64, -125, -64, -54, -95, 69, -106, 94, -64, 78, 98, 16, 88, 57,
+        -44, 71, 64, -12, -3, -44, 120, -23, -106, 94, -64, -35, 36, 6, -127, -107,
+        -45, 71, 64, -12, -3, -44, 120, -23, -106, 94, -64, 1, 0, 0, 0, 3, 0, 0,
+        0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 9, 2, 0, 0, 0, 3, 1))
+    assert(types(5) == "class [B")
+    assert(row.getAs[Array[Byte]](4) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0,
+        0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0, 0, 52, 64,
+        0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 52,
+        64, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0,
+        0, 52, -64, 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 36, -64, 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0,
+        0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 3))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](5) ===
+      Array[Byte](-26, 16, 0, 0, 2, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, -128, 71, 64, 51,
+        51, 51, 51, 51, -109, 94, -64, 0, 0, 0, 0, 0, -128, 71, 64, 51, 51, 51, 51,
+        51, -109, 94, 64, 0, 0, 0, 0, 0, -128, 72, 64, -51, -52, -52, -52, -52, 108,
+        95, 64, 0, 0, 0, 0, 0, 0, 67, 64, 0, 0, 0, 0, 0, 64, 94, 64, 0, 0, 0, 0, 0,
+        -128, 71, 64, 51, 51, 51, 51, 51, -109, 94, -64, 1, 0, 0, 0, 1, 0, 0, 0, 0,
+        1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 10))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](6) ===
+      Array[Byte](0, 0, 0, 0, 1, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0,
+        0, 0, 8, 64, 0, 0, 0, 0, 0, 0, 28, 64, 0, 0, 0, 0, 0, 0, 32, 64, 0, 0, 0, 0,
+        0, 0, -8, -1, 0, 0, 0, 0, 0, 0, 35, 64, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0,
+        0, 0, 3, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1,
+        0, 0, 0, 0, 1, 0, 0, 0, 1))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](7) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 64, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0,
+        0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0,
+        0, 0, 0, 0, -16, 63, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 3, 0, 0, 0,
+        -1, -1, -1, -1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 2))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](8) ===
+      Array[Byte](0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0,
+        0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, -64, 0, 0, 0,
+        0, 0, 0, 0, -64, 0, 0, 0, 0, 0, 0, 0, -64, 0, 0, 0, 0, 0, 0, 0, -64, 0, 0,
+        0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0,
+        0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0,
+        0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0, 8, 64, 0,
+        0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0, -16, 63,
+        0, 0, 0, 0, 0, 0, -16, 63, 2, 0, 0, 0, 2, 0, 0, 0, 0, 2, 5, 0, 0, 0, 3, 0,
+        0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 3))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](9) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0,
+        0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0, 20, 64, 0, 0,
+        0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0, -16, -65,
+        0, 0, 0, 0, 0, 0, 20, -64, 0, 0, 0, 0, 0, 0, 20, -64, 0, 0, 0, 0, 0, 0, 20,
+        -64, 0, 0, 0, 0, 0, 0, 20, -64, 0, 0, 0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0,
+        -16, -65, 0, 0, 0, 0, 0, 0, -16, -65, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 0, 0,
+        0, 3, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0,
+        0, 0, 0, 1, 0, 0, 0, 3))
+  }
 }
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
index 599f00def0750..2b676be68dfa4 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
@@ -77,8 +77,20 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
       "'172.16.0.42']::inet[], ARRAY['192.168.0.0/24', '10.1.0.0/16']::cidr[], " +
       """ARRAY['{"a": "foo", "b": "bar"}', '{"a": 1, "b": 2}']::json[], """ +
       """ARRAY['{"a": 1, "b": 2, "c": 3}']::jsonb[])"""
-    )
-      .executeUpdate()
+    ).executeUpdate()
+
+    conn.prepareStatement("CREATE TABLE char_types (" +
+      "c0 char(4), c1 character(4), c2 character varying(4), c3 varchar(4), c4 bpchar)"
+    ).executeUpdate()
+    conn.prepareStatement("INSERT INTO char_types VALUES " +
+      "('abcd', 'efgh', 'ijkl', 'mnop', 'q')").executeUpdate()
+
+    conn.prepareStatement("CREATE TABLE char_array_types (" +
+      "c0 char(4)[], c1 character(4)[], c2 character varying(4)[], c3 varchar(4)[], c4 bpchar[])"
+    ).executeUpdate()
+    conn.prepareStatement("INSERT INTO char_array_types VALUES " +
+      """('{"a", "bcd"}', '{"ef", "gh"}', '{"i", "j", "kl"}', '{"mnop"}', '{"q", "r"}')"""
+    ).executeUpdate()
   }
 
   test("Type mapping for various types") {
@@ -219,4 +231,28 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
     assert(rows(0).getShort(0) === 1)
     assert(rows(0).getShort(1) === 2)
   }
+
+  test("character type tests") {
+    val df = sqlContext.read.jdbc(jdbcUrl, "char_types", new Properties)
+    val row = df.collect()
+    assert(row.length == 1)
+    assert(row(0).length === 5)
+    assert(row(0).getString(0) === "abcd")
+    assert(row(0).getString(1) === "efgh")
+    assert(row(0).getString(2) === "ijkl")
+    assert(row(0).getString(3) === "mnop")
+    assert(row(0).getString(4) === "q")
+  }
+
+  test("SPARK-32576: character array type tests") {
+    val df = sqlContext.read.jdbc(jdbcUrl, "char_array_types", new Properties)
+    val row = df.collect()
+    assert(row.length == 1)
+    assert(row(0).length === 5)
+    assert(row(0).getSeq[String](0) === Seq("a   ", "bcd "))
+    assert(row(0).getSeq[String](1) === Seq("ef  ", "gh  "))
+    assert(row(0).getSeq[String](2) === Seq("i", "j", "kl"))
+    assert(row(0).getSeq[String](3) === Seq("mnop"))
+    assert(row(0).getSeq[String](4) === Seq("q", "r"))
+  }
 }
diff --git a/external/docker/spark-test/base/Dockerfile b/external/docker/spark-test/base/Dockerfile
index 5bec5d3f16548..d4a30c4681cba 100644
--- a/external/docker/spark-test/base/Dockerfile
+++ b/external/docker/spark-test/base/Dockerfile
@@ -15,23 +15,14 @@
 # limitations under the License.
 #
 
-FROM ubuntu:xenial
+FROM ubuntu:20.04
 
 # Upgrade package index
-# install a few other useful packages plus Open Jdk 8
+# install a few other useful packages plus Open Java 11
 # Remove unneeded /var/lib/apt/lists/* after install to reduce the
 # docker image size (by ~30MB)
 RUN apt-get update && \
-    apt-get install -y less openjdk-8-jre-headless iproute2 vim-tiny sudo openssh-server && \
+    apt-get install -y less openjdk-11-jre-headless iproute2 vim-tiny sudo openssh-server && \
     rm -rf /var/lib/apt/lists/*
 
-ENV SCALA_VERSION 2.12.10
-ENV CDH_VERSION cdh4
-ENV SCALA_HOME /opt/scala-$SCALA_VERSION
 ENV SPARK_HOME /opt/spark
-ENV PATH $SPARK_HOME:$SCALA_HOME/bin:$PATH
-
-# Install Scala
-ADD https://www.scala-lang.org/files/archive/scala-$SCALA_VERSION.tgz /
-RUN (cd / && gunzip < scala-$SCALA_VERSION.tgz)|(cd /opt && tar -xvf -)
-RUN rm /scala-$SCALA_VERSION.tgz
diff --git a/external/docker/spark-test/master/default_cmd b/external/docker/spark-test/master/default_cmd
index 5a7da3446f6d2..96a36cd0bb682 100755
--- a/external/docker/spark-test/master/default_cmd
+++ b/external/docker/spark-test/master/default_cmd
@@ -22,7 +22,4 @@ echo "CONTAINER_IP=$IP"
 export SPARK_LOCAL_IP=$IP
 export SPARK_PUBLIC_DNS=$IP
 
-# Avoid the default Docker behavior of mapping our IP address to an unreachable host name
-umount /etc/hosts
-
 /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master -i $IP
diff --git a/external/docker/spark-test/worker/default_cmd b/external/docker/spark-test/worker/default_cmd
index 31b06cb0eb047..2401f5565aa0b 100755
--- a/external/docker/spark-test/worker/default_cmd
+++ b/external/docker/spark-test/worker/default_cmd
@@ -22,7 +22,4 @@ echo "CONTAINER_IP=$IP"
 export SPARK_LOCAL_IP=$IP
 export SPARK_PUBLIC_DNS=$IP
 
-# Avoid the default Docker behavior of mapping our IP address to an unreachable host name
-umount /etc/hosts
-
 /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker $1
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index f2dcf5d217a89..7852552ed5a54 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 693820da6af6b..24f2d9acc1ee6 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
index 6599e7e0fe707..e60054dba9b01 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
@@ -167,7 +167,7 @@ private[kafka010] class KafkaMicroBatchStream(
       limit: Long,
       from: PartitionOffsetMap,
       until: PartitionOffsetMap): PartitionOffsetMap = {
-    val fromNew = kafkaOffsetReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
+    lazy val fromNew = kafkaOffsetReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
     val sizes = until.flatMap {
       case (tp, end) =>
         // If begin isn't defined, something's wrong, but let alert logic in getBatch handle it
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
index 57879c7ca31cf..a1397de9c7a04 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
@@ -159,7 +159,7 @@ private[kafka010] class KafkaSource(
       limit: Long,
       from: Map[TopicPartition, Long],
       until: Map[TopicPartition, Long]): Map[TopicPartition, Long] = {
-    val fromNew = kafkaReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
+    lazy val fromNew = kafkaReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
     val sizes = until.flatMap {
       case (tp, end) =>
         // If begin isn't defined, something's wrong, but let alert logic in getBatch handle it
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
index a5e5d01152db8..ede58bd26ce34 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
@@ -33,10 +33,10 @@ import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, Table, TableCapability, TableProvider}
 import org.apache.spark.sql.connector.read.{Batch, Scan, ScanBuilder}
 import org.apache.spark.sql.connector.read.streaming.{ContinuousStream, MicroBatchStream}
-import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, WriteBuilder}
+import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, SupportsTruncate, WriteBuilder}
 import org.apache.spark.sql.connector.write.streaming.StreamingWrite
 import org.apache.spark.sql.execution.streaming.{Sink, Source}
-import org.apache.spark.sql.internal.connector.SimpleTableProvider
+import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStreamingUpdate}
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.streaming.OutputMode
 import org.apache.spark.sql.types.StructType
@@ -394,7 +394,7 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
       () => new KafkaScan(options)
 
     override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-      new WriteBuilder {
+      new WriteBuilder with SupportsTruncate with SupportsStreamingUpdate {
         private val options = info.options
         private val inputSchema: StructType = info.schema()
         private val topic = Option(options.get(TOPIC_OPTION_KEY)).map(_.trim)
@@ -410,6 +410,9 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
           assert(inputSchema != null)
           new KafkaStreamingWrite(topic, producerParams, inputSchema)
         }
+
+        override def truncate(): WriteBuilder = this
+        override def update(): WriteBuilder = this
       }
     }
   }
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
index 5f23029d9fed3..df51c4f77f079 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
@@ -275,7 +275,7 @@ private[kafka010] class KafkaDataConsumer(
     val fetchedData = getOrRetrieveFetchedData(offset)
 
     logDebug(s"Get $groupId $topicPartition nextOffset ${fetchedData.nextOffsetInFetchedData} " +
-      "requested $offset")
+      s"requested $offset")
 
     // The following loop is basically for `failOnDataLoss = false`. When `failOnDataLoss` is
     // `false`, first, we will try to fetch the record at `offset`. If no such record exists, then
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/package.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/package.scala
index 460bb8bd34ec6..b7e42c00b8cff 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/package.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/package.scala
@@ -29,6 +29,7 @@ package object kafka010 {   // scalastyle:ignore
   private[kafka010] val PRODUCER_CACHE_TIMEOUT =
     ConfigBuilder("spark.kafka.producer.cache.timeout")
       .doc("The expire time to remove the unused producers.")
+      .version("2.2.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10m")
 
@@ -36,6 +37,7 @@ package object kafka010 {   // scalastyle:ignore
     ConfigBuilder("spark.kafka.producer.cache.evictorThreadRunInterval")
       .doc("The interval of time between runs of the idle evictor thread for producer pool. " +
         "When non-positive, no idle evictor thread will be run.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1m")
 
@@ -43,12 +45,14 @@ package object kafka010 {   // scalastyle:ignore
     ConfigBuilder("spark.kafka.consumer.cache.capacity")
       .doc("The maximum number of consumers cached. Please note it's a soft limit" +
         " (check Structured Streaming Kafka integration guide for further details).")
+      .version("3.0.0")
       .intConf
       .createWithDefault(64)
 
   private[kafka010] val CONSUMER_CACHE_JMX_ENABLED =
     ConfigBuilder("spark.kafka.consumer.cache.jmx.enable")
       .doc("Enable or disable JMX for pools created with this configuration instance.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -57,6 +61,7 @@ package object kafka010 {   // scalastyle:ignore
       .doc("The minimum amount of time a consumer may sit idle in the pool before " +
         "it is eligible for eviction by the evictor. " +
         "When non-positive, no consumers will be evicted from the pool due to idle time alone.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("5m")
 
@@ -64,6 +69,7 @@ package object kafka010 {   // scalastyle:ignore
     ConfigBuilder("spark.kafka.consumer.cache.evictorThreadRunInterval")
       .doc("The interval of time between runs of the idle evictor thread for consumer pool. " +
         "When non-positive, no idle evictor thread will be run.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1m")
 
@@ -72,6 +78,7 @@ package object kafka010 {   // scalastyle:ignore
       .doc("The minimum amount of time a fetched data may sit idle in the pool before " +
         "it is eligible for eviction by the evictor. " +
         "When non-positive, no fetched data will be evicted from the pool due to idle time alone.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("5m")
 
@@ -79,6 +86,7 @@ package object kafka010 {   // scalastyle:ignore
     ConfigBuilder("spark.kafka.consumer.fetchedData.cache.evictorThreadRunInterval")
       .doc("The interval of time between runs of the idle evictor thread for fetched data pool. " +
         "When non-positive, no idle evictor thread will be run.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1m")
 }
diff --git a/external/kafka-0-10-sql/src/test/resources/log4j.properties b/external/kafka-0-10-sql/src/test/resources/log4j.properties
index 75e3b53a093f6..daf05723d4e7b 100644
--- a/external/kafka-0-10-sql/src/test/resources/log4j.properties
+++ b/external/kafka-0-10-sql/src/test/resources/log4j.properties
@@ -25,4 +25,6 @@ log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{
 
 # Ignore messages below warning level from Jetty, because it's a bit verbose
 log4j.logger.org.spark-project.jetty=WARN
-
+log4j.logger.org.apache.spark.sql.kafka010.KafkaTestUtils=DEBUG
+log4j.logger.org.apache.directory.server.kerberos.kdc.authentication=DEBUG
+log4j.logger.org.apache.directory.server.core.DefaultDirectoryService=DEBUG
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
index 76c25980fc33f..43cbe28fc1fe1 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
@@ -33,7 +33,7 @@ class KafkaContinuousSourceSuite extends KafkaSourceSuiteBase with KafkaContinuo
     withTable(table) {
       val topic = newTopic()
       testUtils.createTopic(topic)
-      testUtils.withTranscationalProducer { producer =>
+      testUtils.withTransactionalProducer { producer =>
         val df = spark
           .readStream
           .format("kafka")
@@ -99,7 +99,7 @@ class KafkaContinuousSourceSuite extends KafkaSourceSuiteBase with KafkaContinuo
     withTable(table) {
       val topic = newTopic()
       testUtils.createTopic(topic)
-      testUtils.withTranscationalProducer { producer =>
+      testUtils.withTransactionalProducer { producer =>
         val df = spark
           .readStream
           .format("kafka")
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
index 306483825ae3b..702bd4f6ebdad 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
@@ -62,7 +62,7 @@ class KafkaDelegationTokenSuite extends StreamTest with SharedSparkSession with
     }
   }
 
-  test("Roundtrip") {
+  testRetry("Roundtrip", 3) {
     val hadoopConf = new Configuration()
     val manager = new HadoopDelegationTokenManager(spark.sparkContext.conf, hadoopConf, null)
     val credentials = new Credentials()
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
index a4601b91af0d6..2a3e135b0e63e 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
@@ -349,7 +349,8 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
     )
   }
 
-  test("subscribing topic by pattern with topic deletions") {
+  // TODO (SPARK-31731): re-enable it
+  ignore("subscribing topic by pattern with topic deletions") {
     val topicPrefix = newTopic()
     val topic = topicPrefix + "-seems"
     val topic2 = topicPrefix + "-bad"
@@ -566,7 +567,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
     val rows = spark.table("kafkaWatermark").collect()
     assert(rows.length === 1, s"Unexpected results: ${rows.toList}")
     val row = rows(0)
-    // We cannot check the exact window start time as it depands on the time that messages were
+    // We cannot check the exact window start time as it depends on the time that messages were
     // inserted by the producer. So here we just use a low bound to make sure the internal
     // conversion works.
     assert(
@@ -832,7 +833,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
 
     val topicPartition = new TopicPartition(topic, 0)
     // The message values are the same as their offsets to make the test easy to follow
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       testStream(mapped)(
         StartStream(Trigger.ProcessingTime(100), clock),
         waitUntilBatchProcessed,
@@ -955,7 +956,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
 
     val topicPartition = new TopicPartition(topic, 0)
     // The message values are the same as their offsets to make the test easy to follow
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       testStream(mapped)(
         StartStream(Trigger.ProcessingTime(100), clock),
         waitUntilBatchProcessed,
@@ -1046,7 +1047,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
       .load()
       .select($"value".as[String])
 
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       producer.beginTransaction()
       (0 to 3).foreach { i =>
         producer.send(new ProducerRecord[String, String](topic, i.toString)).get()
@@ -1062,7 +1063,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
         // this case, if we forget to reset `FetchedData._nextOffsetInFetchedData` or
         // `FetchedData._offsetAfterPoll` (See SPARK-25495), the next batch will see incorrect
         // values and return wrong results hence fail the test.
-        testUtils.withTranscationalProducer { producer =>
+        testUtils.withTransactionalProducer { producer =>
           producer.beginTransaction()
           (4 to 7).foreach { i =>
             producer.send(new ProducerRecord[String, String](topic, i.toString)).get()
@@ -1775,7 +1776,7 @@ abstract class KafkaSourceSuiteBase extends KafkaSourceTest {
     withTable(table) {
       val topic = newTopic()
       testUtils.createTopic(topic)
-      testUtils.withTranscationalProducer { producer =>
+      testUtils.withTransactionalProducer { producer =>
         val df = spark
           .readStream
           .format("kafka")
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
index 32d056140a0d7..6f5dc0bb081ba 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
@@ -179,7 +179,8 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
       ("3", Seq(("e", "f".getBytes(UTF_8)), ("e", "g".getBytes(UTF_8))))).toDF)
   }
 
-  test("timestamp provided for starting and ending") {
+  // TODO (SPARK-31729): re-enable it
+  ignore("timestamp provided for starting and ending") {
     val (topic, timestamps) = prepareTimestampRelatedUnitTest
 
     // timestamp both presented: starting "first" ending "finalized"
@@ -502,7 +503,7 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
   test("read Kafka transactional messages: read_committed") {
     val topic = newTopic()
     testUtils.createTopic(topic)
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       val df = spark
         .read
         .format("kafka")
@@ -551,7 +552,7 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
   test("read Kafka transactional messages: read_uncommitted") {
     val topic = newTopic()
     testUtils.createTopic(topic)
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       val df = spark
         .read
         .format("kafka")
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
index 5c8c5b1f3b307..4e808a5277a98 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
@@ -314,7 +314,7 @@ class KafkaSinkMicroBatchStreamingSuite extends KafkaSinkStreamingSuiteBase {
     try {
       input.addData("1", "2", "3")
       verifyResult(writer) {
-        assert(writer.lastProgress.sink.numOutputRows == 3L)
+        assert(writer.recentProgress.exists(_.sink.numOutputRows == 3L))
       }
     } finally {
       writer.stop()
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index 7b972fede96e5..942e2ad5e3727 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -27,6 +27,7 @@ import javax.security.auth.login.Configuration
 import scala.collection.JavaConverters._
 import scala.io.Source
 import scala.util.Random
+import scala.util.control.NonFatal
 
 import com.google.common.io.Files
 import kafka.api.Request
@@ -36,7 +37,7 @@ import kafka.zk.KafkaZkClient
 import org.apache.hadoop.minikdc.MiniKdc
 import org.apache.hadoop.security.UserGroupInformation
 import org.apache.kafka.clients.CommonClientConfigs
-import org.apache.kafka.clients.admin.{AdminClient, CreatePartitionsOptions, ListConsumerGroupsResult, NewPartitions, NewTopic}
+import org.apache.kafka.clients.admin._
 import org.apache.kafka.clients.consumer.KafkaConsumer
 import org.apache.kafka.clients.producer._
 import org.apache.kafka.common.TopicPartition
@@ -136,8 +137,30 @@ class KafkaTestUtils(
     val kdcDir = Utils.createTempDir()
     val kdcConf = MiniKdc.createConf()
     kdcConf.setProperty(MiniKdc.DEBUG, "true")
-    kdc = new MiniKdc(kdcConf, kdcDir)
-    kdc.start()
+    // The port for MiniKdc service gets selected in the constructor, but will be bound
+    // to it later in MiniKdc.start() -> MiniKdc.initKDCServer() -> KdcServer.start().
+    // In meantime, when some other service might capture the port during this progress, and
+    // cause BindException.
+    // This makes our tests which have dedicated JVMs and rely on MiniKDC being flaky
+    //
+    // https://issues.apache.org/jira/browse/HADOOP-12656 get fixed in Hadoop 2.8.0.
+    //
+    // The workaround here is to periodically repeat this process with a timeout , since we are
+    // using Hadoop 2.7.4 as default.
+    // https://issues.apache.org/jira/browse/SPARK-31631
+    eventually(timeout(60.seconds), interval(1.second)) {
+      try {
+        kdc = new MiniKdc(kdcConf, kdcDir)
+        kdc.start()
+      } catch {
+        case NonFatal(e) =>
+          if (kdc != null) {
+            kdc.stop()
+            kdc = null
+          }
+          throw e
+      }
+    }
     // TODO https://issues.apache.org/jira/browse/SPARK-30037
     // Need to build spark's own MiniKDC and customize krb5.conf like Kafka
     rewriteKrb5Conf()
@@ -170,6 +193,7 @@ class KafkaTestUtils(
 
     kdc.getKrb5conf.delete()
     Files.write(krb5confStr, kdc.getKrb5conf, StandardCharsets.UTF_8)
+    logDebug(s"krb5.conf file content: $krb5confStr")
   }
 
   private def addedKrb5Config(key: String, value: String): String = {
@@ -309,6 +333,7 @@ class KafkaTestUtils(
     }
     brokerReady = false
     zkReady = false
+    kdcReady = false
 
     if (producer != null) {
       producer.close()
@@ -317,6 +342,7 @@ class KafkaTestUtils(
 
     if (adminClient != null) {
       adminClient.close()
+      adminClient = null
     }
 
     if (server != null) {
@@ -351,6 +377,7 @@ class KafkaTestUtils(
     Configuration.getConfiguration.refresh()
     if (kdc != null) {
       kdc.stop()
+      kdc = null
     }
     UserGroupInformation.reset()
     teardownKrbDebug()
@@ -538,7 +565,7 @@ class KafkaTestUtils(
   }
 
   /** Call `f` with a `KafkaProducer` that has initialized transactions. */
-  def withTranscationalProducer(f: KafkaProducer[String, String] => Unit): Unit = {
+  def withTransactionalProducer(f: KafkaProducer[String, String] => Unit): Unit = {
     val props = producerConfiguration
     props.put("transactional.id", UUID.randomUUID().toString)
     val producer = new KafkaProducer[String, String](props)
@@ -587,7 +614,7 @@ class KafkaTestUtils(
     // ensure that logs from all replicas are deleted if delete topic is marked successful
     assert(servers.forall(server => topicAndPartitions.forall(tp =>
       server.getLogManager().getLog(tp).isEmpty)),
-      s"topic $topic still exists in log mananger")
+      s"topic $topic still exists in log manager")
     // ensure that topic is removed from all cleaner offsets
     assert(servers.forall(server => topicAndPartitions.forall { tp =>
       val checkpoints = server.getLogManager().liveLogDirs.map { logDir =>
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index 01ca96b4f8f40..c5d41b05baf3b 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala b/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
index 49109d363ba8f..370e727bde05e 100644
--- a/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
+++ b/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
@@ -295,8 +295,8 @@ private[spark] object KafkaTokenUtil extends Logging {
       sparkConf: SparkConf,
       params: ju.Map[String, Object],
       clusterConfig: Option[KafkaTokenClusterConf]): Boolean = {
-    if (HadoopDelegationTokenManager.isServiceEnabled(sparkConf, "kafka") &&
-        clusterConfig.isDefined && params.containsKey(SaslConfigs.SASL_JAAS_CONFIG)) {
+    if (clusterConfig.isDefined && params.containsKey(SaslConfigs.SASL_JAAS_CONFIG) &&
+        HadoopDelegationTokenManager.isServiceEnabled(sparkConf, "kafka")) {
       logDebug("Delegation token used by connector, checking if uses the latest token.")
       val connectorJaasParams = params.get(SaslConfigs.SASL_JAAS_CONFIG).asInstanceOf[String]
       getTokenJaasParams(clusterConfig.get) != connectorJaasParams
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index ad7a8b7e23f1d..c53a5392ff917 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/package.scala b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/package.scala
index 3d2921fa900a3..0679a496fc3c7 100644
--- a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/package.scala
+++ b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/package.scala
@@ -26,43 +26,51 @@ package object kafka010 { //scalastyle:ignore
 
   private[spark] val CONSUMER_CACHE_ENABLED =
     ConfigBuilder("spark.streaming.kafka.consumer.cache.enabled")
+      .version("2.2.1")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val CONSUMER_POLL_MS =
     ConfigBuilder("spark.streaming.kafka.consumer.poll.ms")
-    .longConf
-    .createOptional
+      .version("2.0.1")
+      .longConf
+      .createOptional
 
   private[spark] val CONSUMER_CACHE_INITIAL_CAPACITY =
     ConfigBuilder("spark.streaming.kafka.consumer.cache.initialCapacity")
-    .intConf
-    .createWithDefault(16)
+      .version("2.0.1")
+      .intConf
+      .createWithDefault(16)
 
   private[spark] val CONSUMER_CACHE_MAX_CAPACITY =
     ConfigBuilder("spark.streaming.kafka.consumer.cache.maxCapacity")
-    .intConf
-    .createWithDefault(64)
+      .version("2.0.1")
+      .intConf
+      .createWithDefault(64)
 
   private[spark] val CONSUMER_CACHE_LOAD_FACTOR =
     ConfigBuilder("spark.streaming.kafka.consumer.cache.loadFactor")
-    .doubleConf
-    .createWithDefault(0.75)
+      .version("2.0.1")
+      .doubleConf
+      .createWithDefault(0.75)
 
   private[spark] val MAX_RATE_PER_PARTITION =
     ConfigBuilder("spark.streaming.kafka.maxRatePerPartition")
-    .longConf
-    .createWithDefault(0)
+      .version("1.3.0")
+      .longConf
+      .createWithDefault(0)
 
   private[spark] val MIN_RATE_PER_PARTITION =
     ConfigBuilder("spark.streaming.kafka.minRatePerPartition")
-    .longConf
-    .createWithDefault(1)
+      .version("2.4.0")
+      .longConf
+      .createWithDefault(1)
 
   private[spark] val ALLOW_NON_CONSECUTIVE_OFFSETS =
     ConfigBuilder("spark.streaming.kafka.allowNonConsecutiveOffsets")
-    .booleanConf
-    .createWithDefault(false)
+      .version("2.3.1")
+      .booleanConf
+      .createWithDefault(false)
 
 }
 
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
index 925327d9d58e6..72cf3e8118228 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
@@ -332,7 +332,8 @@ class DirectKafkaStreamSuite
   }
 
   // Test to verify the offset ranges can be recovered from the checkpoints
-  test("offset recovery") {
+  // TODO (SPARK-31722): re-enable it
+  ignore("offset recovery") {
     val topic = "recovery"
     kafkaTestUtils.createTopic(topic)
     testDir = Utils.createTempDir()
@@ -418,8 +419,9 @@ class DirectKafkaStreamSuite
     ssc.stop()
   }
 
-    // Test to verify the offsets can be recovered from Kafka
-  test("offset recovery from kafka") {
+  // Test to verify the offsets can be recovered from Kafka
+  // TODO (SPARK-31722): re-enable it
+  ignore("offset recovery from kafka") {
     val topic = "recoveryfromkafka"
     kafkaTestUtils.createTopic(topic)
 
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
index d6123e16dd238..2053d3655d860 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
@@ -42,7 +42,7 @@ class KafkaRDDSuite extends SparkFunSuite with BeforeAndAfterAll {
   private val sparkConf = new SparkConf().setMaster("local[4]")
     .setAppName(this.getClass.getSimpleName)
     // Set a timeout of 10 seconds that's going to be used to fetch topics/partitions from kafka.
-    // Othewise the poll timeout defaults to 2 minutes and causes test cases to run longer.
+    // Otherwise the poll timeout defaults to 2 minutes and causes test cases to run longer.
     .set("spark.streaming.kafka.consumer.poll.ms", "10000")
 
   private var sc: SparkContext = _
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 0ce922349ea66..4f8cc83219dbc 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 7d69764b77de7..2dd1e786d3634 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
index 31ca2fe5c95ff..244873af70de9 100644
--- a/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
+++ b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
@@ -49,7 +49,7 @@
  *
  * Usage: JavaKinesisWordCountASL [app-name] [stream-name] [endpoint-url] [region-name]
  *   [app-name] is the name of the consumer app, used to track the read data in DynamoDB
- *   [stream-name] name of the Kinesis stream (ie. mySparkStream)
+ *   [stream-name] name of the Kinesis stream (i.e. mySparkStream)
  *   [endpoint-url] endpoint of the Kinesis service
  *     (e.g. https://kinesis.us-east-1.amazonaws.com)
  *
@@ -57,7 +57,7 @@
  * Example:
  *      # export AWS keys if necessary
  *      $ export AWS_ACCESS_KEY_ID=[your-access-key]
- *      $ export AWS_SECRET_KEY=<your-secret-key>
+ *      $ export AWS_SECRET_ACCESS_KEY=<your-secret-key>
  *
  *      # run the example
  *      $ SPARK_HOME/bin/run-example   streaming.JavaKinesisWordCountASL myAppName  mySparkStream \
@@ -68,7 +68,7 @@
  *
  * This code uses the DefaultAWSCredentialsProviderChain to find credentials
  * in the following order:
- *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY
+ *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
  *    Java System Properties - aws.accessKeyId and aws.secretKey
  *    Credential profiles file - default location (~/.aws/credentials) shared by all AWS SDKs
  *    Instance profile credentials - delivered through the Amazon EC2 metadata service
diff --git a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
index 49794faab88c4..66c9ed80f3b33 100644
--- a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
+++ b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
@@ -23,28 +23,29 @@
 
   Usage: kinesis_wordcount_asl.py <app-name> <stream-name> <endpoint-url> <region-name>
     <app-name> is the name of the consumer app, used to track the read data in DynamoDB
-    <stream-name> name of the Kinesis stream (ie. mySparkStream)
+    <stream-name> name of the Kinesis stream (i.e. mySparkStream)
     <endpoint-url> endpoint of the Kinesis service
       (e.g. https://kinesis.us-east-1.amazonaws.com)
+    <region-name> region name of the Kinesis endpoint (e.g. us-east-1)
 
 
   Example:
       # export AWS keys if necessary
       $ export AWS_ACCESS_KEY_ID=<your-access-key>
-      $ export AWS_SECRET_KEY=<your-secret-key>
+      $ export AWS_SECRET_ACCESS_KEY=<your-secret-key>
 
       # run the example
-      $ bin/spark-submit -jars external/kinesis-asl/target/scala-*/\
-        spark-streaming-kinesis-asl-assembly_*.jar \
+      $ bin/spark-submit --jars \
+        'external/kinesis-asl-assembly/target/spark-streaming-kinesis-asl-assembly_*.jar' \
         external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \
-        myAppName mySparkStream https://kinesis.us-east-1.amazonaws.com
+        myAppName mySparkStream https://kinesis.us-east-1.amazonaws.com us-east-1
 
   There is a companion helper class called KinesisWordProducerASL which puts dummy data
   onto the Kinesis stream.
 
   This code uses the DefaultAWSCredentialsProviderChain to find credentials
   in the following order:
-      Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY
+      Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
       Java System Properties - aws.accessKeyId and aws.secretKey
       Credential profiles file - default location (~/.aws/credentials) shared by all AWS SDKs
       Instance profile credentials - delivered through the Amazon EC2 metadata service
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
index a5d5ac769b28d..d6a9160eed98e 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
@@ -43,7 +43,7 @@ import org.apache.spark.streaming.kinesis.KinesisInputDStream
  *
  * Usage: KinesisWordCountASL <app-name> <stream-name> <endpoint-url> <region-name>
  *   <app-name> is the name of the consumer app, used to track the read data in DynamoDB
- *   <stream-name> name of the Kinesis stream (ie. mySparkStream)
+ *   <stream-name> name of the Kinesis stream (i.e. mySparkStream)
  *   <endpoint-url> endpoint of the Kinesis service
  *     (e.g. https://kinesis.us-east-1.amazonaws.com)
  *
@@ -51,7 +51,7 @@ import org.apache.spark.streaming.kinesis.KinesisInputDStream
  * Example:
  *      # export AWS keys if necessary
  *      $ export AWS_ACCESS_KEY_ID=<your-access-key>
- *      $ export AWS_SECRET_KEY=<your-secret-key>
+ *      $ export AWS_SECRET_ACCESS_KEY=<your-secret-key>
  *
  *      # run the example
  *      $ SPARK_HOME/bin/run-example  streaming.KinesisWordCountASL myAppName  mySparkStream \
@@ -62,7 +62,7 @@ import org.apache.spark.streaming.kinesis.KinesisInputDStream
  *
  * This code uses the DefaultAWSCredentialsProviderChain to find credentials
  * in the following order:
- *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY
+ *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
  *    Java System Properties - aws.accessKeyId and aws.secretKey
  *    Credential profiles file - default location (~/.aws/credentials) shared by all AWS SDKs
  *    Instance profile credentials - delivered through the Amazon EC2 metadata service
@@ -78,7 +78,7 @@ object KinesisWordCountASL extends Logging {
     if (args.length != 3) {
       System.err.println(
         """
-          |Usage: KinesisWordCountASL <app-name> <stream-name> <endpoint-url> <region-name>
+          |Usage: KinesisWordCountASL <app-name> <stream-name> <endpoint-url>
           |
           |    <app-name> is the name of the consumer app, used to track the read data in DynamoDB
           |    <stream-name> is the name of the Kinesis stream
@@ -167,15 +167,15 @@ object KinesisWordCountASL extends Logging {
  * Usage: KinesisWordProducerASL <stream-name> <endpoint-url> \
  *   <records-per-sec> <words-per-record>
  *
- *   <stream-name> is the name of the Kinesis stream (ie. mySparkStream)
+ *   <stream-name> is the name of the Kinesis stream (i.e. mySparkStream)
  *   <endpoint-url> is the endpoint of the Kinesis service
- *     (ie. https://kinesis.us-east-1.amazonaws.com)
+ *     (i.e. https://kinesis.us-east-1.amazonaws.com)
  *   <records-per-sec> is the rate of records per second to put onto the stream
- *   <words-per-record> is the rate of records per second to put onto the stream
+ *   <words-per-record> is the number of words per record
  *
  * Example:
  *    $ SPARK_HOME/bin/run-example streaming.KinesisWordProducerASL mySparkStream \
- *         https://kinesis.us-east-1.amazonaws.com us-east-1 10 5
+ *         https://kinesis.us-east-1.amazonaws.com 10 5
  */
 object KinesisWordProducerASL {
   def main(args: Array[String]): Unit = {
@@ -183,13 +183,13 @@ object KinesisWordProducerASL {
       System.err.println(
         """
           |Usage: KinesisWordProducerASL <stream-name> <endpoint-url> <records-per-sec>
-                                         <words-per-record>
+          |                              <words-per-record>
           |
           |    <stream-name> is the name of the Kinesis stream
           |    <endpoint-url> is the endpoint of the Kinesis service
           |                   (e.g. https://kinesis.us-east-1.amazonaws.com)
           |    <records-per-sec> is the rate of records per second to put onto the stream
-          |    <words-per-record> is the rate of records per second to put onto the stream
+          |    <words-per-record> is the number of words per record
           |
         """.stripMargin)
 
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala
index c89dedd3366d1..0056438c4eefb 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala
@@ -46,7 +46,7 @@ private class KinesisUtilsPythonHelper {
     // scalastyle:on
     if (!(stsAssumeRoleArn != null && stsSessionName != null && stsExternalId != null)
         && !(stsAssumeRoleArn == null && stsSessionName == null && stsExternalId == null)) {
-      throw new IllegalArgumentException("stsAssumeRoleArn, stsSessionName, and stsExtenalId " +
+      throw new IllegalArgumentException("stsAssumeRoleArn, stsSessionName, and stsExternalId " +
         "must all be defined or all be null")
     }
     if (awsAccessKeyId == null && awsSecretKey != null) {
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index db64b201abc2c..f1dd31eee35a2 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 444568a03d6c7..d760d5c32099c 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
index a5e2fc5c9a74f..8008a89c6cd5f 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
@@ -274,8 +274,8 @@ class PageRankSuite extends SparkFunSuite with LocalSparkContext {
     withSpark { sc =>
       // Check that implementation can handle large vertexIds, SPARK-25149
       val vertexIdOffset = Int.MaxValue.toLong + 1
-      val sourceOffest = 4
-      val source = vertexIdOffset + sourceOffest
+      val sourceOffset = 4
+      val source = vertexIdOffset + sourceOffset
       val numIter = 10
       val vertices = vertexIdOffset until vertexIdOffset + numIter
       val chain1 = vertices.zip(vertices.tail)
@@ -285,7 +285,7 @@ class PageRankSuite extends SparkFunSuite with LocalSparkContext {
       val tol = 0.0001
       val errorTol = 1.0e-1
 
-      val a = resetProb / (1 - Math.pow(1 - resetProb, numIter - sourceOffest))
+      val a = resetProb / (1 - Math.pow(1 - resetProb, numIter - sourceOffset))
       // We expect the rank to decay as (1 - resetProb) ^ distance
       val expectedRanks = sc.parallelize(vertices).map { vid =>
         val rank = if (vid < source) {
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 02734e82ed26e..860f648ff9e57 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index e75e8345cd51d..48ee744806ed2 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java b/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
index 3ff77878f68a8..d5a277ba581a0 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
@@ -364,7 +364,7 @@ public void close() throws IOException {
      *
      * This method allows a short period for the above to happen (same amount of time as the
      * connection timeout, which is configurable). This should be fine for well-behaved
-     * applications, where they close the connection arond the same time the app handle detects the
+     * applications, where they close the connection around the same time the app handle detects the
      * app has finished.
      *
      * In case the connection is not closed within the grace period, this method forcefully closes
diff --git a/licenses-binary/LICENSE-vis.txt b/licenses-binary/LICENSE-vis-timeline.txt
similarity index 64%
rename from licenses-binary/LICENSE-vis.txt
rename to licenses-binary/LICENSE-vis-timeline.txt
index 18b7323059a41..324402de4dc09 100644
--- a/licenses-binary/LICENSE-vis.txt
+++ b/licenses-binary/LICENSE-vis-timeline.txt
@@ -3,11 +3,11 @@ https://github.com/almende/vis
 
 A dynamic, browser-based visualization library.
 
-@version 4.16.1
-@date    2016-04-18
+@version 4.20.1-SNAPSHOT
+@date    2017-10-12
 
 @license
-Copyright (C) 2011-2016 Almende B.V, http://almende.com
+Copyright (C) 2011-2017 Almende B.V, http://almende.com
 
 Vis.js is dual licensed under both
 
@@ -19,4 +19,4 @@ and
 * The MIT License
   http://opensource.org/licenses/MIT
 
-Vis.js may be distributed under either license.
\ No newline at end of file
+Vis.js may be distributed under either license.
diff --git a/licenses/LICENSE-vis.txt b/licenses/LICENSE-vis-timeline.txt
similarity index 64%
rename from licenses/LICENSE-vis.txt
rename to licenses/LICENSE-vis-timeline.txt
index 18b7323059a41..324402de4dc09 100644
--- a/licenses/LICENSE-vis.txt
+++ b/licenses/LICENSE-vis-timeline.txt
@@ -3,11 +3,11 @@ https://github.com/almende/vis
 
 A dynamic, browser-based visualization library.
 
-@version 4.16.1
-@date    2016-04-18
+@version 4.20.1-SNAPSHOT
+@date    2017-10-12
 
 @license
-Copyright (C) 2011-2016 Almende B.V, http://almende.com
+Copyright (C) 2011-2017 Almende B.V, http://almende.com
 
 Vis.js is dual licensed under both
 
@@ -19,4 +19,4 @@ and
 * The MIT License
   http://opensource.org/licenses/MIT
 
-Vis.js may be distributed under either license.
\ No newline at end of file
+Vis.js may be distributed under either license.
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index 2eab868ac0dc8..fd103d80268e3 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
index 00e5b61dbdc18..fcd33a63bd0a2 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
@@ -27,8 +27,9 @@ private[spark] object BLAS extends Serializable {
 
   @transient private var _f2jBLAS: NetlibBLAS = _
   @transient private var _nativeBLAS: NetlibBLAS = _
+  private val nativeL1Threshold: Int = 256
 
-  // For level-1 routines, we use Java implementation.
+  // For level-1 function dspmv, use f2jBLAS for better performance.
   private[ml] def f2jBLAS: NetlibBLAS = {
     if (_f2jBLAS == null) {
       _f2jBLAS = new F2jBLAS
@@ -36,6 +37,14 @@ private[spark] object BLAS extends Serializable {
     _f2jBLAS
   }
 
+  private[ml] def getBLAS(vectorSize: Int): NetlibBLAS = {
+    if (vectorSize < nativeL1Threshold) {
+      f2jBLAS
+    } else {
+      nativeBLAS
+    }
+  }
+
   /**
    * y += a * x
    */
@@ -63,7 +72,7 @@ private[spark] object BLAS extends Serializable {
    */
   private def axpy(a: Double, x: DenseVector, y: DenseVector): Unit = {
     val n = x.size
-    f2jBLAS.daxpy(n, a, x.values, 1, y.values, 1)
+    getBLAS(n).daxpy(n, a, x.values, 1, y.values, 1)
   }
 
   /**
@@ -94,7 +103,7 @@ private[spark] object BLAS extends Serializable {
   private[spark] def axpy(a: Double, X: DenseMatrix, Y: DenseMatrix): Unit = {
     require(X.numRows == Y.numRows && X.numCols == Y.numCols, "Dimension mismatch: " +
       s"size(X) = ${(X.numRows, X.numCols)} but size(Y) = ${(Y.numRows, Y.numCols)}.")
-    f2jBLAS.daxpy(X.numRows * X.numCols, a, X.values, 1, Y.values, 1)
+    getBLAS(X.values.length).daxpy(X.numRows * X.numCols, a, X.values, 1, Y.values, 1)
   }
 
   /**
@@ -123,7 +132,7 @@ private[spark] object BLAS extends Serializable {
    */
   private def dot(x: DenseVector, y: DenseVector): Double = {
     val n = x.size
-    f2jBLAS.ddot(n, x.values, 1, y.values, 1)
+    getBLAS(n).ddot(n, x.values, 1, y.values, 1)
   }
 
   /**
@@ -218,16 +227,16 @@ private[spark] object BLAS extends Serializable {
   def scal(a: Double, x: Vector): Unit = {
     x match {
       case sx: SparseVector =>
-        f2jBLAS.dscal(sx.values.length, a, sx.values, 1)
+        getBLAS(sx.values.length).dscal(sx.values.length, a, sx.values, 1)
       case dx: DenseVector =>
-        f2jBLAS.dscal(dx.values.length, a, dx.values, 1)
+        getBLAS(dx.size).dscal(dx.values.length, a, dx.values, 1)
       case _ =>
         throw new IllegalArgumentException(s"scal doesn't support vector type ${x.getClass}.")
     }
   }
 
   // For level-3 routines, we use the native BLAS.
-  private def nativeBLAS: NetlibBLAS = {
+  private[ml] def nativeBLAS: NetlibBLAS = {
     if (_nativeBLAS == null) {
       _nativeBLAS = NativeBLAS
     }
@@ -374,7 +383,7 @@ private[spark] object BLAS extends Serializable {
       // gemm: alpha is equal to 0 and beta is equal to 1. Returning C.
       return
     } else if (alpha == 0.0) {
-      f2jBLAS.dscal(C.values.length, beta, C.values, 1)
+      getBLAS(C.values.length).dscal(C.values.length, beta, C.values, 1)
     } else {
       A match {
         case sparse: SparseMatrix => gemm(alpha, sparse, B, beta, C)
@@ -480,7 +489,7 @@ private[spark] object BLAS extends Serializable {
     } else {
       // Scale matrix first if `beta` is not equal to 1.0
       if (beta != 1.0) {
-        f2jBLAS.dscal(C.values.length, beta, C.values, 1)
+        getBLAS(C.values.length).dscal(C.values.length, beta, C.values, 1)
       }
       // Perform matrix multiplication and add to C. The rows of A are multiplied by the columns of
       // B, and added to C.
@@ -682,6 +691,7 @@ private[spark] object BLAS extends Serializable {
 
           val xTemp = xValues(k) * alpha
           while (i < indEnd) {
+            val rowIndex = Arows(i)
             yValues(Arows(i)) += Avals(i) * xTemp
             i += 1
           }
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
index 61d35c8f7e303..34e4366701bbe 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
@@ -1103,7 +1103,7 @@ object Matrices {
     DenseMatrix.rand(numRows, numCols, rng)
 
   /**
-   * Generate a `SparseMatrix` consisting of `i.i.d.` gaussian random numbers.
+   * Generate a `SparseMatrix` consisting of `i.i.d.` uniform random numbers.
    * @param numRows number of rows of the matrix
    * @param numCols number of columns of the matrix
    * @param density the desired density for the matrix
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
index 877ac68983348..781f3da313d82 100644
--- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
@@ -23,6 +23,12 @@ import org.apache.spark.ml.util.TestingUtils._
 
 class BLASSuite extends SparkMLFunSuite {
 
+  test("nativeL1Threshold") {
+    assert(getBLAS(128) == BLAS.f2jBLAS)
+    assert(getBLAS(256) == BLAS.nativeBLAS)
+    assert(getBLAS(512) == BLAS.nativeBLAS)
+  }
+
   test("copy") {
     val sx = Vectors.sparse(4, Array(0, 2), Array(1.0, -2.0))
     val dx = Vectors.dense(1.0, 0.0, -2.0, 0.0)
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 11769ef548d7c..135c4ca54a491 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala b/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala
index 1247882d6c1bd..8815eb29bc860 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala
@@ -19,15 +19,13 @@ package org.apache.spark.ml
 
 import scala.annotation.varargs
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.param.{ParamMap, ParamPair}
 import org.apache.spark.sql.Dataset
 
 /**
- * :: DeveloperApi ::
  * Abstract class for estimators that fit models to data.
  */
-@DeveloperApi
 abstract class Estimator[M <: Model[M]] extends PipelineStage {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Model.scala b/mllib/src/main/scala/org/apache/spark/ml/Model.scala
index c581fed177273..709a1124cbb5e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Model.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Model.scala
@@ -17,16 +17,13 @@
 
 package org.apache.spark.ml
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.ml.param.ParamMap
 
 /**
- * :: DeveloperApi ::
  * A fitted model, i.e., a [[Transformer]] produced by an [[Estimator]].
  *
  * @tparam M model type
  */
-@DeveloperApi
 abstract class Model[M <: Model[M]] extends Transformer {
   /**
    * The parent estimator that produced this model.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala b/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
index 69a4dbef138e3..59e95878a5fec 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
@@ -36,14 +36,11 @@ import org.apache.spark.sql.{DataFrame, Dataset}
 import org.apache.spark.sql.types.StructType
 
 /**
- * :: DeveloperApi ::
  * A stage in a pipeline, either an [[Estimator]] or a [[Transformer]].
  */
-@DeveloperApi
 abstract class PipelineStage extends Params with Logging {
 
   /**
-   * :: DeveloperApi ::
    *
    * Check transform validity and derive the output schema from the input schema.
    *
@@ -54,7 +51,6 @@ abstract class PipelineStage extends Params with Logging {
    * Typical implementation should first conduct verification on schema change and parameter
    * validity, including complex parameter interaction checks.
    */
-  @DeveloperApi
   def transformSchema(schema: StructType): StructType
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
index 98dd692cbe55d..febeba7e13fcb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.ml
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.feature.{Instance, LabeledPoint}
 import org.apache.spark.ml.linalg.{Vector, VectorUDT}
 import org.apache.spark.ml.param._
@@ -99,7 +99,6 @@ private[ml] trait PredictorParams extends Params
 }
 
 /**
- * :: DeveloperApi ::
  * Abstraction for prediction problems (regression and classification). It accepts all NumericType
  * labels and will automatically cast it to DoubleType in `fit()`. If this predictor supports
  * weights, it accepts all NumericType weights, which will be automatically casted to DoubleType
@@ -112,7 +111,6 @@ private[ml] trait PredictorParams extends Params
  * @tparam M  Specialization of [[PredictionModel]].  If you subclass this type, use this type
  *            parameter to specify the concrete type for the corresponding model.
  */
-@DeveloperApi
 abstract class Predictor[
     FeaturesType,
     Learner <: Predictor[FeaturesType, Learner, M],
@@ -190,7 +188,6 @@ abstract class Predictor[
 }
 
 /**
- * :: DeveloperApi ::
  * Abstraction for a model for prediction tasks (regression and classification).
  *
  * @tparam FeaturesType  Type of features.
@@ -198,7 +195,6 @@ abstract class Predictor[
  * @tparam M  Specialization of [[PredictionModel]].  If you subclass this type, use this type
  *            parameter to specify the concrete type for the corresponding model.
  */
-@DeveloperApi
 abstract class PredictionModel[FeaturesType, M <: PredictionModel[FeaturesType, M]]
   extends Model[M] with PredictorParams {
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala b/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala
index 7874fc29db6c8..3b60b5ae294ea 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala
@@ -18,8 +18,9 @@
 package org.apache.spark.ml
 
 import scala.annotation.varargs
+import scala.reflect.runtime.universe.TypeTag
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
@@ -28,10 +29,8 @@ import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
 
 /**
- * :: DeveloperApi ::
  * Abstract class for transformers that transform one dataset into another.
  */
-@DeveloperApi
 abstract class Transformer extends PipelineStage {
 
   /**
@@ -74,12 +73,10 @@ abstract class Transformer extends PipelineStage {
 }
 
 /**
- * :: DeveloperApi ::
  * Abstract class for transformers that take one input column, apply transformation, and output the
  * result as a new column.
  */
-@DeveloperApi
-abstract class UnaryTransformer[IN, OUT, T <: UnaryTransformer[IN, OUT, T]]
+abstract class UnaryTransformer[IN: TypeTag, OUT: TypeTag, T <: UnaryTransformer[IN, OUT, T]]
   extends Transformer with HasInputCol with HasOutputCol with Logging {
 
   /** @group setParam */
@@ -118,7 +115,7 @@ abstract class UnaryTransformer[IN, OUT, T <: UnaryTransformer[IN, OUT, T]]
 
   override def transform(dataset: Dataset[_]): DataFrame = {
     val outputSchema = transformSchema(dataset.schema, logging = true)
-    val transformUDF = udf(this.createTransformFunc, outputDataType)
+    val transformUDF = udf(this.createTransformFunc)
     dataset.withColumn($(outputCol), transformUDF(dataset($(inputCol))),
       outputSchema($(outputCol)).metadata)
   }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala
index 21a246e454c83..59d48915c0d58 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala
@@ -19,12 +19,10 @@ package org.apache.spark.ml.attribute
 
 import scala.collection.mutable.ArrayBuffer
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.ml.linalg.VectorUDT
 import org.apache.spark.sql.types.{Metadata, MetadataBuilder, StructField}
 
 /**
- * :: DeveloperApi ::
  * Attributes that describe a vector ML column.
  *
  * @param name name of the attribute group (the ML column name)
@@ -33,7 +31,6 @@ import org.apache.spark.sql.types.{Metadata, MetadataBuilder, StructField}
  * @param attrs optional array of attributes. Attribute will be copied with their corresponding
  *              indices in the array.
  */
-@DeveloperApi
 class AttributeGroup private (
     val name: String,
     val numAttributes: Option[Int],
@@ -188,10 +185,8 @@ class AttributeGroup private (
 }
 
 /**
- * :: DeveloperApi ::
  * Factory methods to create attribute groups.
  */
-@DeveloperApi
 object AttributeGroup {
 
   import AttributeKeys._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala
index 078fecf088282..f1f3a97f8ac1c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala
@@ -17,20 +17,13 @@
 
 package org.apache.spark.ml.attribute
 
-import org.apache.spark.annotation.DeveloperApi
 
 /**
- * :: DeveloperApi ::
  * An enum-like type for attribute types: [[AttributeType$#Numeric]], [[AttributeType$#Nominal]],
  * and [[AttributeType$#Binary]].
  */
-@DeveloperApi
 sealed abstract class AttributeType(val name: String)
 
-/**
- * :: DeveloperApi ::
- */
-@DeveloperApi
 object AttributeType {
 
   /** Numeric type. */
diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
index 756dd677cb579..576c771d83bec 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
@@ -19,14 +19,11 @@ package org.apache.spark.ml.attribute
 
 import scala.annotation.varargs
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.sql.types.{DoubleType, Metadata, MetadataBuilder, NumericType, StructField}
 
 /**
- * :: DeveloperApi ::
  * Abstract class for ML attributes.
  */
-@DeveloperApi
 sealed abstract class Attribute extends Serializable {
 
   name.foreach { n =>
@@ -150,10 +147,6 @@ private[attribute] trait AttributeFactory {
   def fromStructField(field: StructField): Attribute = decodeStructField(field, false)
 }
 
-/**
- * :: DeveloperApi ::
- */
-@DeveloperApi
 object Attribute extends AttributeFactory {
 
   private[attribute] override def fromMetadata(metadata: Metadata): Attribute = {
@@ -182,7 +175,6 @@ object Attribute extends AttributeFactory {
 
 
 /**
- * :: DeveloperApi ::
  * A numeric attribute with optional summary statistics.
  * @param name optional name
  * @param index optional index
@@ -191,7 +183,6 @@ object Attribute extends AttributeFactory {
  * @param std optional standard deviation
  * @param sparsity optional sparsity (ratio of zeros)
  */
-@DeveloperApi
 class NumericAttribute private[ml] (
     override val name: Option[String] = None,
     override val index: Option[Int] = None,
@@ -299,10 +290,8 @@ class NumericAttribute private[ml] (
 }
 
 /**
- * :: DeveloperApi ::
  * Factory methods for numeric attributes.
  */
-@DeveloperApi
 object NumericAttribute extends AttributeFactory {
 
   /** The default numeric attribute. */
@@ -321,7 +310,6 @@ object NumericAttribute extends AttributeFactory {
 }
 
 /**
- * :: DeveloperApi ::
  * A nominal attribute.
  * @param name optional name
  * @param index optional index
@@ -330,7 +318,6 @@ object NumericAttribute extends AttributeFactory {
  *                  defined.
  * @param values optional values. At most one of `numValues` and `values` can be defined.
  */
-@DeveloperApi
 class NominalAttribute private[ml] (
     override val name: Option[String] = None,
     override val index: Option[Int] = None,
@@ -464,10 +451,8 @@ class NominalAttribute private[ml] (
 }
 
 /**
- * :: DeveloperApi ::
  * Factory methods for nominal attributes.
  */
-@DeveloperApi
 object NominalAttribute extends AttributeFactory {
 
   /** The default nominal attribute. */
@@ -487,13 +472,11 @@ object NominalAttribute extends AttributeFactory {
 }
 
 /**
- * :: DeveloperApi ::
  * A binary attribute.
  * @param name optional name
  * @param index optional index
  * @param values optional values. If set, its size must be 2.
  */
-@DeveloperApi
 class BinaryAttribute private[ml] (
     override val name: Option[String] = None,
     override val index: Option[Int] = None,
@@ -566,10 +549,8 @@ class BinaryAttribute private[ml] (
 }
 
 /**
- * :: DeveloperApi ::
  * Factory methods for binary attributes.
  */
-@DeveloperApi
 object BinaryAttribute extends AttributeFactory {
 
   /** The default binary attribute. */
@@ -586,10 +567,8 @@ object BinaryAttribute extends AttributeFactory {
 }
 
 /**
- * :: DeveloperApi ::
  * An unresolved attribute.
  */
-@DeveloperApi
 object UnresolvedAttribute extends Attribute {
 
   override def attrType: AttributeType = AttributeType.Unresolved
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
index ac92299a931bc..1f3f291644f93 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.ml.classification
 
 import org.apache.spark.SparkException
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams}
 import org.apache.spark.ml.feature.{Instance, LabeledPoint}
 import org.apache.spark.ml.linalg.{Vector, VectorUDT}
@@ -62,8 +62,6 @@ private[spark] trait ClassifierParams
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Single-label binary or multiclass classification.
  * Classes are indexed {0, 1, ..., numClasses - 1}.
  *
@@ -71,7 +69,6 @@ private[spark] trait ClassifierParams
  * @tparam E  Concrete Estimator type
  * @tparam M  Concrete Model type
  */
-@DeveloperApi
 abstract class Classifier[
     FeaturesType,
     E <: Classifier[FeaturesType, E, M],
@@ -166,15 +163,12 @@ abstract class Classifier[
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Model produced by a [[Classifier]].
  * Classes are indexed {0, 1, ..., numClasses - 1}.
  *
  * @tparam FeaturesType  Type of input features.  E.g., `Vector`
  * @tparam M  Concrete Model type
  */
-@DeveloperApi
 abstract class ClassificationModel[FeaturesType, M <: ClassificationModel[FeaturesType, M]]
   extends PredictionModel[FeaturesType, M] with ClassifierParams {
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
index d511c1b5dda98..1511cb61b8b30 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
@@ -52,12 +52,16 @@ private[classification] trait FMClassifierParams extends ProbabilisticClassifier
  * FM is able to estimate interactions even in problems with huge sparsity
  * (like advertising and recommendation system).
  * FM formula is:
- * {{{
+ * <blockquote>
+ *   $$
+ *   \begin{align}
  *   y = \sigma\left( w_0 + \sum\limits^n_{i-1} w_i x_i +
  *     \sum\limits^n_{i=1} \sum\limits^n_{j=i+1} \langle v_i, v_j \rangle x_i x_j \right)
- * }}}
+ *   \end{align}
+ *   $$
+ * </blockquote>
  * First two terms denote global bias and linear term (as same as linear regression),
- * and last term denotes pairwise interactions term. {{{v_i}}} describes the i-th variable
+ * and last term denotes pairwise interactions term. v_i describes the i-th variable
  * with k factors.
  *
  * FM classification model uses logistic loss which can be solved by gradient descent method, and
@@ -82,7 +86,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFactorSize(value: Int): this.type = set(factorSize, value)
-  setDefault(factorSize -> 8)
 
   /**
    * Set whether to fit intercept term.
@@ -92,7 +95,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set whether to fit linear term.
@@ -102,7 +104,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitLinear(value: Boolean): this.type = set(fitLinear, value)
-  setDefault(fitLinear -> true)
 
   /**
    * Set the L2 regularization parameter.
@@ -112,7 +113,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the mini-batch fraction parameter.
@@ -122,7 +122,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMiniBatchFraction(value: Double): this.type = set(miniBatchFraction, value)
-  setDefault(miniBatchFraction -> 1.0)
 
   /**
    * Set the standard deviation of initial coefficients.
@@ -132,7 +131,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setInitStd(value: Double): this.type = set(initStd, value)
-  setDefault(initStd -> 0.01)
 
   /**
    * Set the maximum number of iterations.
@@ -142,7 +140,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the initial step size for the first step (like learning rate).
@@ -152,7 +149,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setStepSize(value: Double): this.type = set(stepSize, value)
-  setDefault(stepSize -> 1.0)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -162,7 +158,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Set the solver algorithm used for optimization.
@@ -173,7 +168,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> AdamW)
 
   /**
    * Set the random seed for weight initialization.
@@ -183,7 +177,7 @@ class FMClassifier @Since("3.0.0") (
   @Since("3.0.0")
   def setSeed(value: Long): this.type = set(seed, value)
 
-  override protected[spark] def train(
+  override protected def train(
       dataset: Dataset[_]
     ): FMClassificationModel = instrumented { instr =>
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
index f16648d2abee6..88621b34beb13 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.SparkException
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.optim.aggregator.HingeAggregator
 import org.apache.spark.ml.optim.loss.{L2Regularization, RDDLossFunction}
@@ -41,7 +41,7 @@ import org.apache.spark.storage.StorageLevel
 /** Params for linear SVM Classifier. */
 private[classification] trait LinearSVCParams extends ClassifierParams with HasRegParam
   with HasMaxIter with HasFitIntercept with HasTol with HasStandardization with HasWeightCol
-  with HasAggregationDepth with HasThreshold with HasBlockSize {
+  with HasAggregationDepth with HasThreshold {
 
   /**
    * Param for threshold in binary classification prediction.
@@ -54,6 +54,9 @@ private[classification] trait LinearSVCParams extends ClassifierParams with HasR
    */
   final override val threshold: DoubleParam = new DoubleParam(this, "threshold",
     "threshold in binary classification prediction applied to rawPrediction")
+
+  setDefault(regParam -> 0.0, maxIter -> 100, fitIntercept -> true, tol -> 1E-6,
+    standardization -> true, threshold -> 0.0, aggregationDepth -> 2)
 }
 
 /**
@@ -81,7 +84,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the maximum number of iterations.
@@ -91,7 +93,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Whether to fit an intercept term.
@@ -101,7 +102,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -112,7 +112,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Whether to standardize the training features before fitting the model.
@@ -122,7 +121,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setStandardization(value: Boolean): this.type = set(standardization, value)
-  setDefault(standardization -> true)
 
   /**
    * Set the value of param [[weightCol]].
@@ -141,7 +139,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setThreshold(value: Double): this.type = set(threshold, value)
-  setDefault(threshold -> 0.0)
 
   /**
    * Suggested depth for treeAggregate (greater than or equal to 2).
@@ -153,28 +150,20 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
-
-  /**
-   * Set block size for stacking input data in matrices.
-   * Default is 1024.
-   *
-   * @group expertSetParam
-   */
-  @Since("3.0.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
 
   @Since("2.2.0")
   override def copy(extra: ParamMap): LinearSVC = defaultCopy(extra)
 
   override protected def train(dataset: Dataset[_]): LinearSVCModel = instrumented { instr =>
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+
+    val instances = extractInstances(dataset)
+    if (handlePersistence) instances.persist(StorageLevel.MEMORY_AND_DISK)
+
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol, rawPredictionCol,
-      regParam, maxIter, fitIntercept, tol, standardization, threshold, aggregationDepth, blockSize)
-
-    val sc = dataset.sparkSession.sparkContext
-    val instances = extractInstances(dataset)
+      regParam, maxIter, fitIntercept, tol, standardization, threshold, aggregationDepth)
 
     val (summarizer, labelSummarizer) = instances.treeAggregate(
       (Summarizer.createSummarizerBuffer("mean", "std", "count"), new MultiClassSummarizer))(
@@ -215,33 +204,20 @@ class LinearSVC @Since("2.2.0") (
         throw new SparkException(msg)
       }
 
-      val featuresStd = summarizer.std.compressed
-      val bcFeaturesStd = sc.broadcast(featuresStd)
+      val featuresStd = summarizer.std.toArray
+      val getFeaturesStd = (j: Int) => featuresStd(j)
       val regParamL2 = $(regParam)
+      val bcFeaturesStd = instances.context.broadcast(featuresStd)
       val regularization = if (regParamL2 != 0.0) {
         val shouldApply = (idx: Int) => idx >= 0 && idx < numFeatures
         Some(new L2Regularization(regParamL2, shouldApply,
-          if ($(standardization)) None else Some(featuresStd.apply)))
+          if ($(standardization)) None else Some(getFeaturesStd)))
       } else {
         None
       }
 
-      val standardized = instances.map {
-        case Instance(label, weight, features) =>
-          val featuresStd = bcFeaturesStd.value
-          val array = Array.ofDim[Double](numFeatures)
-          features.foreachNonZero { (i, v) =>
-            val std = featuresStd(i)
-            if (std != 0) array(i) = v / std
-          }
-          Instance(label, weight, Vectors.dense(array))
-      }
-      val blocks = InstanceBlock.blokify(standardized, $(blockSize))
-        .persist(StorageLevel.MEMORY_AND_DISK)
-        .setName(s"training dataset (blockSize=${$(blockSize)})")
-
-      val getAggregatorFunc = new HingeAggregator(numFeatures, $(fitIntercept))(_)
-      val costFun = new RDDLossFunction(blocks, getAggregatorFunc, regularization,
+      val getAggregatorFunc = new HingeAggregator(bcFeaturesStd, $(fitIntercept))(_)
+      val costFun = new RDDLossFunction(instances, getAggregatorFunc, regularization,
         $(aggregationDepth))
 
       def regParamL1Fun = (index: Int) => 0D
@@ -258,7 +234,6 @@ class LinearSVC @Since("2.2.0") (
         scaledObjectiveHistory += state.adjustedValue
       }
 
-      blocks.unpersist()
       bcFeaturesStd.destroy()
       if (state == null) {
         val msg = s"${optimizer.getClass.getName} failed."
@@ -289,6 +264,8 @@ class LinearSVC @Since("2.2.0") (
       (Vectors.dense(coefficientArray), intercept, scaledObjectiveHistory.result())
     }
 
+    if (handlePersistence) instances.unpersist()
+
     copyValues(new LinearSVCModel(uid, coefficientVector, interceptVector))
   }
 }
@@ -319,7 +296,6 @@ class LinearSVCModel private[classification] (
 
   @Since("2.2.0")
   def setThreshold(value: Double): this.type = set(threshold, value)
-  setDefault(threshold, 0.0)
 
   private val margin: Vector => Double = (features) => {
     BLAS.dot(features, coefficients) + intercept
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index 9b5b36257a584..a61eb329977cd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.SparkException
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.optim.aggregator.LogisticAggregator
 import org.apache.spark.ml.optim.loss.{L2Regularization, RDDLossFunction}
@@ -50,8 +50,7 @@ import org.apache.spark.util.VersionUtils
  */
 private[classification] trait LogisticRegressionParams extends ProbabilisticClassifierParams
   with HasRegParam with HasElasticNetParam with HasMaxIter with HasFitIntercept with HasTol
-  with HasStandardization with HasWeightCol with HasThreshold with HasAggregationDepth
-  with HasBlockSize  {
+  with HasStandardization with HasWeightCol with HasThreshold with HasAggregationDepth {
 
   import org.apache.spark.ml.classification.LogisticRegression.supportedFamilyNames
 
@@ -249,6 +248,10 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
       isSet(lowerBoundsOnIntercepts) || isSet(upperBoundsOnIntercepts)
   }
 
+  setDefault(regParam -> 0.0, elasticNetParam -> 0.0, maxIter -> 100, tol -> 1E-6,
+    fitIntercept -> true, family -> "auto", standardization -> true, threshold -> 0.5,
+    aggregationDepth -> 2)
+
   override protected def validateAndTransformSchema(
       schema: StructType,
       fitting: Boolean,
@@ -291,7 +294,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.2.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the ElasticNet mixing parameter.
@@ -307,7 +309,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.4.0")
   def setElasticNetParam(value: Double): this.type = set(elasticNetParam, value)
-  setDefault(elasticNetParam -> 0.0)
 
   /**
    * Set the maximum number of iterations.
@@ -317,7 +318,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.2.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -328,7 +328,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.4.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Whether to fit an intercept term.
@@ -338,7 +337,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.4.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Sets the value of param [[family]].
@@ -348,7 +346,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("2.1.0")
   def setFamily(value: String): this.type = set(family, value)
-  setDefault(family -> "auto")
 
   /**
    * Whether to standardize the training features before fitting the model.
@@ -362,11 +359,9 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.5.0")
   def setStandardization(value: Boolean): this.type = set(standardization, value)
-  setDefault(standardization -> true)
 
   @Since("1.5.0")
   override def setThreshold(value: Double): this.type = super.setThreshold(value)
-  setDefault(threshold -> 0.5)
 
   @Since("1.5.0")
   override def getThreshold: Double = super.getThreshold
@@ -397,7 +392,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("2.1.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   /**
    * Set the lower bounds on coefficients if fitting under bound constrained optimization.
@@ -431,15 +425,6 @@ class LogisticRegression @Since("1.2.0") (
   @Since("2.2.0")
   def setUpperBoundsOnIntercepts(value: Vector): this.type = set(upperBoundsOnIntercepts, value)
 
-  /**
-   * Set block size for stacking input data in matrices.
-   * Default is 1024.
-   *
-   * @group expertSetParam
-   */
-  @Since("3.0.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-
   private def assertBoundConstrainedOptimizationParamsValid(
       numCoefficientSets: Int,
       numFeatures: Int): Unit = {
@@ -492,17 +477,24 @@ class LogisticRegression @Since("1.2.0") (
     this
   }
 
-  override protected[spark] def train(
-      dataset: Dataset[_]): LogisticRegressionModel = instrumented { instr =>
+  override protected[spark] def train(dataset: Dataset[_]): LogisticRegressionModel = {
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    train(dataset, handlePersistence)
+  }
+
+  protected[spark] def train(
+      dataset: Dataset[_],
+      handlePersistence: Boolean): LogisticRegressionModel = instrumented { instr =>
+    val instances = extractInstances(dataset)
+
+    if (handlePersistence) instances.persist(StorageLevel.MEMORY_AND_DISK)
+
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol, rawPredictionCol,
       probabilityCol, regParam, elasticNetParam, standardization, threshold, maxIter, tol,
       fitIntercept)
 
-    val sc = dataset.sparkSession.sparkContext
-    val instances = extractInstances(dataset)
-
     val (summarizer, labelSummarizer) = instances.treeAggregate(
       (Summarizer.createSummarizerBuffer("mean", "std", "count"), new MultiClassSummarizer))(
       seqOp = (c: (SummarizerBuffer, MultiClassSummarizer), instance: Instance) =>
@@ -585,9 +577,8 @@ class LogisticRegression @Since("1.2.0") (
             s"dangerous ground, so the algorithm may not converge.")
         }
 
-        val featuresMean = summarizer.mean.compressed
-        val featuresStd = summarizer.std.compressed
-        val bcFeaturesStd = sc.broadcast(featuresStd)
+        val featuresMean = summarizer.mean.toArray
+        val featuresStd = summarizer.std.toArray
 
         if (!$(fitIntercept) && (0 until numFeatures).exists { i =>
           featuresStd(i) == 0.0 && featuresMean(i) != 0.0 }) {
@@ -599,7 +590,8 @@ class LogisticRegression @Since("1.2.0") (
         val regParamL1 = $(elasticNetParam) * $(regParam)
         val regParamL2 = (1.0 - $(elasticNetParam)) * $(regParam)
 
-        val getAggregatorFunc = new LogisticAggregator(numFeatures, numClasses, $(fitIntercept),
+        val bcFeaturesStd = instances.context.broadcast(featuresStd)
+        val getAggregatorFunc = new LogisticAggregator(bcFeaturesStd, numClasses, $(fitIntercept),
           multinomial = isMultinomial)(_)
         val getFeaturesStd = (j: Int) => if (j >= 0 && j < numCoefficientSets * numFeatures) {
           featuresStd(j / numCoefficientSets)
@@ -615,21 +607,7 @@ class LogisticRegression @Since("1.2.0") (
           None
         }
 
-        val standardized = instances.map {
-          case Instance(label, weight, features) =>
-            val featuresStd = bcFeaturesStd.value
-            val array = Array.ofDim[Double](numFeatures)
-            features.foreachNonZero { (i, v) =>
-              val std = featuresStd(i)
-              if (std != 0) array(i) = v / std
-            }
-            Instance(label, weight, Vectors.dense(array))
-        }
-        val blocks = InstanceBlock.blokify(standardized, $(blockSize))
-          .persist(StorageLevel.MEMORY_AND_DISK)
-          .setName(s"training dataset (blockSize=${$(blockSize)})")
-
-        val costFun = new RDDLossFunction(blocks, getAggregatorFunc, regularization,
+        val costFun = new RDDLossFunction(instances, getAggregatorFunc, regularization,
           $(aggregationDepth))
 
         val numCoeffsPlusIntercepts = numFeaturesPlusIntercept * numCoefficientSets
@@ -823,7 +801,6 @@ class LogisticRegression @Since("1.2.0") (
           state = states.next()
           arrayBuilder += state.adjustedValue
         }
-        blocks.unpersist()
         bcFeaturesStd.destroy()
 
         if (state == null) {
@@ -893,6 +870,8 @@ class LogisticRegression @Since("1.2.0") (
       }
     }
 
+    if (handlePersistence) instances.unpersist()
+
     val model = copyValues(new LogisticRegressionModel(uid, coefficientMatrix, interceptVector,
       numClasses, isMultinomial))
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
index 94681ae9ef796..409a39c5b8ca9 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
@@ -63,6 +63,8 @@ private[classification] trait NaiveBayesParams extends PredictorParams with HasW
 
   /** @group getParam */
   final def getModelType: String = $(modelType)
+
+  setDefault(smoothing -> 1.0, modelType -> NaiveBayes.Multinomial)
 }
 
 // scalastyle:off line.size.limit
@@ -106,7 +108,6 @@ class NaiveBayes @Since("1.5.0") (
    */
   @Since("1.5.0")
   def setSmoothing(value: Double): this.type = set(smoothing, value)
-  setDefault(smoothing -> 1.0)
 
   /**
    * Set the model type using a string (case-sensitive).
@@ -116,7 +117,6 @@ class NaiveBayes @Since("1.5.0") (
    */
   @Since("1.5.0")
   def setModelType(value: String): this.type = set(modelType, value)
-  setDefault(modelType -> Multinomial)
 
   /**
    * Sets the value of param [[weightCol]].
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
index cc5da3446f1f8..9758e3ca72c38 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.ml.classification
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.linalg.{DenseVector, Vector, VectorUDT}
 import org.apache.spark.ml.param.shared._
 import org.apache.spark.ml.util.SchemaUtils
@@ -41,15 +41,12 @@ private[ml] trait ProbabilisticClassifierParams
 
 
 /**
- * :: DeveloperApi ::
- *
  * Single-label binary or multiclass classifier which can output class conditional probabilities.
  *
  * @tparam FeaturesType  Type of input features.  E.g., `Vector`
  * @tparam E  Concrete Estimator type
  * @tparam M  Concrete Model type
  */
-@DeveloperApi
 abstract class ProbabilisticClassifier[
     FeaturesType,
     E <: ProbabilisticClassifier[FeaturesType, E, M],
@@ -65,15 +62,12 @@ abstract class ProbabilisticClassifier[
 
 
 /**
- * :: DeveloperApi ::
- *
  * Model produced by a [[ProbabilisticClassifier]].
  * Classes are indexed {0, 1, ..., numClasses - 1}.
  *
  * @tparam FeaturesType  Type of input features.  E.g., `Vector`
  * @tparam M  Concrete Model type
  */
-@DeveloperApi
 abstract class ProbabilisticClassificationModel[
     FeaturesType,
     M <: ProbabilisticClassificationModel[FeaturesType, M]]
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index 6c7112b80569f..b3f2d223628ea 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -28,9 +28,8 @@ import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.Instrumentation.instrumented
 import org.apache.spark.mllib.clustering.{BisectingKMeans => MLlibBisectingKMeans,
   BisectingKMeansModel => MLlibBisectingKMeansModel}
-import org.apache.spark.mllib.linalg.{Vector => OldVector, Vectors => OldVectors}
+import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
 import org.apache.spark.mllib.linalg.VectorImplicits._
-import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.{DoubleType, IntegerType, StructType}
@@ -71,6 +70,8 @@ private[clustering] trait BisectingKMeansParams extends Params with HasMaxIter
   @Since("2.0.0")
   def getMinDivisibleClusterSize: Double = $(minDivisibleClusterSize)
 
+  setDefault(k -> 4, maxIter -> 20, minDivisibleClusterSize -> 1.0)
+
   /**
    * Validates and transforms the input schema.
    * @param schema input schema
@@ -225,11 +226,6 @@ class BisectingKMeans @Since("2.0.0") (
     @Since("2.0.0") override val uid: String)
   extends Estimator[BisectingKMeansModel] with BisectingKMeansParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 4,
-    maxIter -> 20,
-    minDivisibleClusterSize -> 1.0)
-
   @Since("2.0.0")
   override def copy(extra: ParamMap): BisectingKMeans = defaultCopy(extra)
 
@@ -278,21 +274,6 @@ class BisectingKMeans @Since("2.0.0") (
   override def fit(dataset: Dataset[_]): BisectingKMeansModel = instrumented { instr =>
     transformSchema(dataset.schema, logging = true)
 
-    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
-    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
-      col($(weightCol)).cast(DoubleType)
-    } else {
-      lit(1.0)
-    }
-
-    val instances: RDD[(OldVector, Double)] = dataset
-      .select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w).rdd.map {
-      case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight)
-    }
-    if (handlePersistence) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
-    }
-
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, featuresCol, predictionCol, k, maxIter, seed,
@@ -304,11 +285,18 @@ class BisectingKMeans @Since("2.0.0") (
       .setMinDivisibleClusterSize($(minDivisibleClusterSize))
       .setSeed($(seed))
       .setDistanceMeasure($(distanceMeasure))
-    val parentModel = bkm.runWithWeight(instances, Some(instr))
-    val model = copyValues(new BisectingKMeansModel(uid, parentModel).setParent(this))
-    if (handlePersistence) {
-      instances.unpersist()
+
+    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
+      col($(weightCol)).cast(DoubleType)
+    } else {
+      lit(1.0)
     }
+    val instances = dataset.select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w)
+      .rdd.map { case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight) }
+
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    val parentModel = bkm.runWithWeight(instances, handlePersistence, Some(instr))
+    val model = copyValues(new BisectingKMeansModel(uid, parentModel).setParent(this))
 
     val summary = new BisectingKMeansSummary(
       model.transform(dataset),
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index d779e602545cf..26a866bbecaeb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -59,6 +59,8 @@ private[clustering] trait GaussianMixtureParams extends Params with HasMaxIter w
   @Since("2.0.0")
   def getK: Int = $(k)
 
+  setDefault(k -> 2, maxIter -> 100, tol -> 0.01)
+
   /**
    * Validates and transforms the input schema.
    *
@@ -323,11 +325,6 @@ class GaussianMixture @Since("2.0.0") (
     @Since("2.0.0") override val uid: String)
   extends Estimator[GaussianMixtureModel] with GaussianMixtureParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 2,
-    maxIter -> 100,
-    tol -> 0.01)
-
   @Since("2.0.0")
   override def copy(extra: ParamMap): GaussianMixture = defaultCopy(extra)
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
index a42c920e24987..e182f3d1f5733 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
@@ -31,7 +31,6 @@ import org.apache.spark.ml.util.Instrumentation.instrumented
 import org.apache.spark.mllib.clustering.{DistanceMeasure, KMeans => MLlibKMeans, KMeansModel => MLlibKMeansModel}
 import org.apache.spark.mllib.linalg.{Vector => OldVector, Vectors => OldVectors}
 import org.apache.spark.mllib.linalg.VectorImplicits._
-import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.{DoubleType, IntegerType, StructType}
@@ -86,6 +85,9 @@ private[clustering] trait KMeansParams extends Params with HasMaxIter with HasFe
   @Since("1.5.0")
   def getInitSteps: Int = $(initSteps)
 
+  setDefault(k -> 2, maxIter -> 20, initMode -> MLlibKMeans.K_MEANS_PARALLEL, initSteps -> 2,
+    tol -> 1e-4, distanceMeasure -> DistanceMeasure.EUCLIDEAN)
+
   /**
    * Validates and transforms the input schema.
    * @param schema input schema
@@ -270,14 +272,6 @@ class KMeans @Since("1.5.0") (
     @Since("1.5.0") override val uid: String)
   extends Estimator[KMeansModel] with KMeansParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 2,
-    maxIter -> 20,
-    initMode -> MLlibKMeans.K_MEANS_PARALLEL,
-    initSteps -> 2,
-    tol -> 1e-4,
-    distanceMeasure -> DistanceMeasure.EUCLIDEAN)
-
   @Since("1.5.0")
   override def copy(extra: ParamMap): KMeans = defaultCopy(extra)
 
@@ -334,22 +328,6 @@ class KMeans @Since("1.5.0") (
   override def fit(dataset: Dataset[_]): KMeansModel = instrumented { instr =>
     transformSchema(dataset.schema, logging = true)
 
-    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
-    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
-      col($(weightCol)).cast(DoubleType)
-    } else {
-      lit(1.0)
-    }
-
-    val instances: RDD[(OldVector, Double)] = dataset
-      .select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w).rdd.map {
-      case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight)
-    }
-
-    if (handlePersistence) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
-    }
-
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, featuresCol, predictionCol, k, initMode, initSteps, distanceMeasure,
@@ -362,8 +340,19 @@ class KMeans @Since("1.5.0") (
       .setSeed($(seed))
       .setEpsilon($(tol))
       .setDistanceMeasure($(distanceMeasure))
-    val parentModel = algo.runWithWeight(instances, Option(instr))
+
+    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
+      col($(weightCol)).cast(DoubleType)
+    } else {
+      lit(1.0)
+    }
+    val instances = dataset.select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w)
+      .rdd.map { case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight) }
+
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    val parentModel = algo.runWithWeight(instances, handlePersistence, Some(instr))
     val model = copyValues(new KMeansModel(uid, parentModel).setParent(this))
+
     val summary = new KMeansSummary(
       model.transform(dataset),
       $(predictionCol),
@@ -374,9 +363,6 @@ class KMeans @Since("1.5.0") (
 
     model.setSummary(Some(summary))
     instr.logNamedValue("clusterSizes", summary.clusterSizes)
-    if (handlePersistence) {
-      instances.unpersist()
-    }
     model
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index e30be8c20dcc3..c1b76fb40b2f6 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -26,7 +26,7 @@ import org.json4s.DefaultFormats
 import org.json4s.JsonAST.JObject
 import org.json4s.jackson.JsonMethods._
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.{Estimator, Model}
 import org.apache.spark.ml.linalg._
@@ -199,8 +199,6 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
     " with estimates of the topic mixture distribution for each document (often called \"theta\"" +
     " in the literature).  Returns a vector of zeros for an empty document.")
 
-  setDefault(topicDistributionCol -> "topicDistribution")
-
   /** @group getParam */
   @Since("1.6.0")
   def getTopicDistributionCol: String = $(topicDistributionCol)
@@ -315,6 +313,11 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
   @Since("2.0.0")
   def getKeepLastCheckpoint: Boolean = $(keepLastCheckpoint)
 
+  setDefault(maxIter -> 20, k -> 10, optimizer -> "online", checkpointInterval -> 10,
+    learningOffset -> 1024, learningDecay -> 0.51, subsamplingRate -> 0.05,
+    optimizeDocConcentration -> true, keepLastCheckpoint -> true,
+    topicDistributionCol -> "topicDistribution")
+
   /**
    * Validates and transforms the input schema.
    *
@@ -762,8 +765,6 @@ class DistributedLDAModel private[ml] (
   private var _checkpointFiles: Array[String] = oldDistributedModel.checkpointFiles
 
   /**
-   * :: DeveloperApi ::
-   *
    * If using checkpointing and `LDA.keepLastCheckpoint` is set to true, then there may be
    * saved checkpoint files.  This method is provided so that users can manage those files.
    *
@@ -773,18 +774,14 @@ class DistributedLDAModel private[ml] (
    *
    * @return  Checkpoint files from training
    */
-  @DeveloperApi
   @Since("2.0.0")
   def getCheckpointFiles: Array[String] = _checkpointFiles
 
   /**
-   * :: DeveloperApi ::
-   *
    * Remove any remaining checkpoint files from training.
    *
    * @see [[getCheckpointFiles]]
    */
-  @DeveloperApi
   @Since("2.0.0")
   def deleteCheckpointFiles(): Unit = {
     val hadoopConf = sparkSession.sparkContext.hadoopConfiguration
@@ -869,10 +866,6 @@ class LDA @Since("1.6.0") (
   @Since("1.6.0")
   def this() = this(Identifiable.randomUID("lda"))
 
-  setDefault(maxIter -> 20, k -> 10, optimizer -> "online", checkpointInterval -> 10,
-    learningOffset -> 1024, learningDecay -> 0.51, subsamplingRate -> 0.05,
-    optimizeDocConcentration -> true, keepLastCheckpoint -> true)
-
   /**
    * The features for LDA should be a `Vector` representing the word counts in a document.
    * The vector should be of length vocabSize, with counts for each term (word).
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
index 812a426a062c1..1466b32bef530 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
@@ -91,7 +91,7 @@ private[clustering] trait PowerIterationClusteringParams extends Params with Has
   @Since("2.4.0")
   def getDstCol: String = $(dstCol)
 
-  setDefault(srcCol -> "src", dstCol -> "dst")
+  setDefault(srcCol -> "src", dstCol -> "dst", k -> 2, maxIter -> 20, initMode -> "random")
 }
 
 /**
@@ -111,11 +111,6 @@ class PowerIterationClustering private[clustering] (
     @Since("2.4.0") override val uid: String)
   extends PowerIterationClusteringParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 2,
-    maxIter -> 20,
-    initMode -> "random")
-
   @Since("2.4.0")
   def this() = this(Identifiable.randomUID("PowerIterationClustering"))
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
index 55b910e98d405..cbec87d7dd715 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
@@ -28,7 +28,8 @@ import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.DoubleType
 
 /**
- * Evaluator for binary classification, which expects two input columns: rawPrediction and label.
+ * Evaluator for binary classification, which expects input columns rawPrediction, label and
+ *  an optional weight column.
  * The rawPrediction column can be of type double (binary 0/1 prediction, or probability of label 1)
  * or of type vector (length-2 vector of raw predictions, scores, or label probabilities).
  */
@@ -79,8 +80,6 @@ class BinaryClassificationEvaluator @Since("1.4.0") (@Since("1.4.0") override va
   @Since("3.0.0")
   def setNumBins(value: Int): this.type = set(numBins, value)
 
-  setDefault(numBins -> 1000)
-
   /** @group setParam */
   @Since("1.5.0")
   def setRawPredictionCol(value: String): this.type = set(rawPredictionCol, value)
@@ -93,7 +92,7 @@ class BinaryClassificationEvaluator @Since("1.4.0") (@Since("1.4.0") override va
   @Since("3.0.0")
   def setWeightCol(value: String): this.type = set(weightCol, value)
 
-  setDefault(metricName -> "areaUnderROC")
+  setDefault(metricName -> "areaUnderROC", numBins -> 1000)
 
   @Since("2.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala
index e7b949ddce344..f7a2d0dbe998f 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala
@@ -17,16 +17,14 @@
 
 package org.apache.spark.ml.evaluation
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.param.{ParamMap, Params}
 import org.apache.spark.sql.Dataset
 
 /**
- * :: DeveloperApi ::
  * Abstract class for evaluators that compute metrics from predictions.
  */
 @Since("1.5.0")
-@DeveloperApi
 abstract class Evaluator extends Params {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
index 435708186242f..3bf115e72b431 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
@@ -63,8 +63,6 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("1.5.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
-  setDefault(metricName -> "f1")
-
   /** @group setParam */
   @Since("1.5.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
@@ -81,6 +79,14 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setProbabilityCol(value: String): this.type = set(probabilityCol, value)
 
+  /**
+   * The class whose metric will be computed in `"truePositiveRateByLabel"`,
+   * `"falsePositiveRateByLabel"`, `"precisionByLabel"`, `"recallByLabel"`,
+   * `"fMeasureByLabel"`.
+   * Must be greater than or equal to 0. The default value is 0.
+   *
+   * @group param
+   */
   @Since("3.0.0")
   final val metricLabel: DoubleParam = new DoubleParam(this, "metricLabel",
     "The class whose metric will be computed in " +
@@ -96,8 +102,13 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setMetricLabel(value: Double): this.type = set(metricLabel, value)
 
-  setDefault(metricLabel -> 0.0)
-
+  /**
+   * The beta value, which controls precision vs recall weighting,
+   * used in `"weightedFMeasure"`, `"fMeasureByLabel"`.
+   * Must be greater than 0. The default value is 1.
+   *
+   * @group param
+   */
   @Since("3.0.0")
   final val beta: DoubleParam = new DoubleParam(this, "beta",
     "The beta value, which controls precision vs recall weighting, " +
@@ -112,8 +123,12 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setBeta(value: Double): this.type = set(beta, value)
 
-  setDefault(beta -> 1.0)
-
+  /**
+   * param for eps. log-loss is undefined for p=0 or p=1, so probabilities are clipped to
+   * max(eps, min(1 - eps, p)). Must be in range (0, 0.5). The default value is 1e-15.
+   *
+   * @group param
+   */
   @Since("3.0.0")
   final val eps: DoubleParam = new DoubleParam(this, "eps",
     "log-loss is undefined for p=0 or p=1, so probabilities are clipped to " +
@@ -128,7 +143,7 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setEps(value: Double): this.type = set(eps, value)
 
-  setDefault(eps -> 1e-15)
+  setDefault(metricName -> "f1", eps -> 1e-15, metricLabel -> 0.0, beta -> 1.0)
 
   @Since("2.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
index 5216c40819b06..632e67c59696c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
@@ -34,12 +34,13 @@ import org.apache.spark.sql.types._
  */
 @Since("3.0.0")
 @Experimental
-class MultilabelClassificationEvaluator (override val uid: String)
+class MultilabelClassificationEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String)
   extends Evaluator with HasPredictionCol with HasLabelCol
     with DefaultParamsWritable {
 
   import MultilabelClassificationEvaluator.supportedMetricNames
 
+  @Since("3.0.0")
   def this() = this(Identifiable.randomUID("mlcEval"))
 
   /**
@@ -49,6 +50,7 @@ class MultilabelClassificationEvaluator (override val uid: String)
    * `"microF1Measure"`)
    * @group param
    */
+  @Since("3.0.0")
   final val metricName: Param[String] = {
     val allowedParams = ParamValidators.inArray(supportedMetricNames)
     new Param(this, "metricName", "metric name in evaluation " +
@@ -56,13 +58,19 @@ class MultilabelClassificationEvaluator (override val uid: String)
   }
 
   /** @group getParam */
+  @Since("3.0.0")
   def getMetricName: String = $(metricName)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
-  setDefault(metricName -> "f1Measure")
-
+  /**
+   * param for the class whose metric will be computed in `"precisionByLabel"`, `"recallByLabel"`,
+   * `"f1MeasureByLabel"`.
+   * @group param
+   */
+  @Since("3.0.0")
   final val metricLabel: DoubleParam = new DoubleParam(this, "metricLabel",
     "The class whose metric will be computed in " +
       s"${supportedMetricNames.filter(_.endsWith("ByLabel")).mkString("(", "|", ")")}. " +
@@ -70,20 +78,23 @@ class MultilabelClassificationEvaluator (override val uid: String)
     ParamValidators.gtEq(0.0))
 
   /** @group getParam */
+  @Since("3.0.0")
   def getMetricLabel: Double = $(metricLabel)
 
   /** @group setParam */
   def setMetricLabel(value: Double): this.type = set(metricLabel, value)
 
-  setDefault(metricLabel -> 0.0)
-
   /** @group setParam */
+  @Since("3.0.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setLabelCol(value: String): this.type = set(labelCol, value)
 
+  setDefault(metricLabel -> 0.0, metricName -> "f1Measure")
 
+  @Since("3.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
     val schema = dataset.schema
     SchemaUtils.checkColumnTypes(schema, $(predictionCol),
@@ -113,6 +124,7 @@ class MultilabelClassificationEvaluator (override val uid: String)
     }
   }
 
+  @Since("3.0.0")
   override def isLargerBetter: Boolean = {
     $(metricName) match {
       case "hammingLoss" => false
@@ -120,6 +132,7 @@ class MultilabelClassificationEvaluator (override val uid: String)
     }
   }
 
+  @Since("3.0.0")
   override def copy(extra: ParamMap): MultilabelClassificationEvaluator = defaultCopy(extra)
 
   @Since("3.0.0")
@@ -139,5 +152,6 @@ object MultilabelClassificationEvaluator
     "precisionByLabel", "recallByLabel", "f1MeasureByLabel",
     "microPrecision", "microRecall", "microF1Measure")
 
+  @Since("3.0.0")
   override def load(path: String): MultilabelClassificationEvaluator = super.load(path)
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
index ca3a8ebc1659d..3a2ddf4d97ee3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
@@ -33,11 +33,12 @@ import org.apache.spark.sql.types._
  */
 @Experimental
 @Since("3.0.0")
-class RankingEvaluator (override val uid: String)
+class RankingEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String)
   extends Evaluator with HasPredictionCol with HasLabelCol with DefaultParamsWritable {
 
   import RankingEvaluator.supportedMetricNames
 
+  @Since("3.0.0")
   def this() = this(Identifiable.randomUID("rankEval"))
 
   /**
@@ -45,6 +46,7 @@ class RankingEvaluator (override val uid: String)
    * `"meanAveragePrecisionAtK"`, `"precisionAtK"`, `"ndcgAtK"`, `"recallAtK"`)
    * @group param
    */
+  @Since("3.0.0")
   final val metricName: Param[String] = {
     val allowedParams = ParamValidators.inArray(supportedMetricNames)
     new Param(this, "metricName", "metric name in evaluation " +
@@ -52,13 +54,19 @@ class RankingEvaluator (override val uid: String)
   }
 
   /** @group getParam */
+  @Since("3.0.0")
   def getMetricName: String = $(metricName)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
-  setDefault(metricName -> "meanAveragePrecision")
-
+  /**
+   * param for ranking position value used in `"meanAveragePrecisionAtK"`, `"precisionAtK"`,
+   * `"ndcgAtK"`, `"recallAtK"`. Must be &gt; 0. The default value is 10.
+   * @group param
+   */
+  @Since("3.0.0")
   final val k = new IntParam(this, "k",
     "The ranking position value used in " +
       s"${supportedMetricNames.filter(_.endsWith("AtK")).mkString("(", "|", ")")}  " +
@@ -66,20 +74,24 @@ class RankingEvaluator (override val uid: String)
     ParamValidators.gt(0))
 
   /** @group getParam */
+  @Since("3.0.0")
   def getK: Int = $(k)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setK(value: Int): this.type = set(k, value)
 
-  setDefault(k -> 10)
-
   /** @group setParam */
+  @Since("3.0.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setLabelCol(value: String): this.type = set(labelCol, value)
 
+  setDefault(k -> 10, metricName -> "meanAveragePrecision")
 
+  @Since("3.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
     val schema = dataset.schema
     SchemaUtils.checkColumnTypes(schema, $(predictionCol),
@@ -102,8 +114,10 @@ class RankingEvaluator (override val uid: String)
     }
   }
 
+  @Since("3.0.0")
   override def isLargerBetter: Boolean = true
 
+  @Since("3.0.0")
   override def copy(extra: ParamMap): RankingEvaluator = defaultCopy(extra)
 
   @Since("3.0.0")
@@ -119,5 +133,6 @@ object RankingEvaluator extends DefaultParamsReadable[RankingEvaluator] {
   private val supportedMetricNames = Array("meanAveragePrecision",
     "meanAveragePrecisionAtK", "precisionAtK", "ndcgAtK", "recallAtK")
 
+  @Since("3.0.0")
   override def load(path: String): RankingEvaluator = super.load(path)
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
index 9f32d40d166bd..de256600982e5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
@@ -27,7 +27,8 @@ import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.{DoubleType, FloatType}
 
 /**
- * Evaluator for regression, which expects two input columns: prediction and label.
+ * Evaluator for regression, which expects input columns prediction, label and
+ * an optional weight column.
  */
 @Since("1.4.0")
 final class RegressionEvaluator @Since("1.4.0") (@Since("1.4.0") override val uid: String)
@@ -78,8 +79,6 @@ final class RegressionEvaluator @Since("1.4.0") (@Since("1.4.0") override val ui
   @Since("3.0.0")
   def setThroughOrigin(value: Boolean): this.type = set(throughOrigin, value)
 
-  setDefault(throughOrigin -> false)
-
   /** @group setParam */
   @Since("1.4.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
@@ -92,7 +91,7 @@ final class RegressionEvaluator @Since("1.4.0") (@Since("1.4.0") override val ui
   @Since("3.0.0")
   def setWeightCol(value: String): this.type = set(weightCol, value)
 
-  setDefault(metricName -> "rmse")
+  setDefault(metricName -> "rmse", throughOrigin -> false)
 
   @Since("2.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala
index 76f4f944f11d5..c582d92ec538b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala
@@ -54,7 +54,6 @@ private[feature] trait ChiSqSelectorParams extends Params
     "Number of features that selector will select, ordered by ascending p-value. If the" +
       " number of features is < numTopFeatures, then this will select all features.",
     ParamValidators.gtEq(1))
-  setDefault(numTopFeatures -> 50)
 
   /** @group getParam */
   @Since("1.6.0")
@@ -70,7 +69,6 @@ private[feature] trait ChiSqSelectorParams extends Params
   final val percentile = new DoubleParam(this, "percentile",
     "Percentile of features that selector will select, ordered by ascending p-value.",
     ParamValidators.inRange(0, 1))
-  setDefault(percentile -> 0.1)
 
   /** @group getParam */
   @Since("2.1.0")
@@ -85,7 +83,6 @@ private[feature] trait ChiSqSelectorParams extends Params
   @Since("2.1.0")
   final val fpr = new DoubleParam(this, "fpr", "The highest p-value for features to be kept.",
     ParamValidators.inRange(0, 1))
-  setDefault(fpr -> 0.05)
 
   /** @group getParam */
   @Since("2.1.0")
@@ -100,7 +97,6 @@ private[feature] trait ChiSqSelectorParams extends Params
   @Since("2.2.0")
   final val fdr = new DoubleParam(this, "fdr",
     "The upper bound of the expected false discovery rate.", ParamValidators.inRange(0, 1))
-  setDefault(fdr -> 0.05)
 
   /** @group getParam */
   def getFdr: Double = $(fdr)
@@ -114,7 +110,6 @@ private[feature] trait ChiSqSelectorParams extends Params
   @Since("2.2.0")
   final val fwe = new DoubleParam(this, "fwe",
     "The upper bound of the expected family-wise error rate.", ParamValidators.inRange(0, 1))
-  setDefault(fwe -> 0.05)
 
   /** @group getParam */
   def getFwe: Double = $(fwe)
@@ -129,11 +124,13 @@ private[feature] trait ChiSqSelectorParams extends Params
     "The selector type of the ChisqSelector. " +
       "Supported options: " + OldChiSqSelector.supportedSelectorTypes.mkString(", "),
     ParamValidators.inArray[String](OldChiSqSelector.supportedSelectorTypes))
-  setDefault(selectorType -> OldChiSqSelector.NumTopFeatures)
 
   /** @group getParam */
   @Since("2.1.0")
   def getSelectorType: String = $(selectorType)
+
+  setDefault(numTopFeatures -> 50, percentile -> 0.1, fpr -> 0.05, fdr -> 0.05, fwe -> 0.05,
+    selectorType -> OldChiSqSelector.NumTopFeatures)
 }
 
 /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
index 80bf85936aace..d2bb013448aae 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
@@ -42,14 +42,17 @@ import org.apache.spark.util.VersionUtils.majorMinorVersion
  * otherwise the features will not be mapped evenly to the columns.
  */
 @Since("1.2.0")
-class HashingTF @Since("1.4.0") (@Since("1.4.0") override val uid: String)
+class HashingTF @Since("3.0.0") private[ml] (
+    @Since("1.4.0") override val uid: String,
+    @Since("3.1.0") val hashFuncVersion: Int)
   extends Transformer with HasInputCol with HasOutputCol with HasNumFeatures
     with DefaultParamsWritable {
 
-  private var hashFunc: Any => Int = FeatureHasher.murmur3Hash
-
   @Since("1.2.0")
-  def this() = this(Identifiable.randomUID("hashingTF"))
+  def this() = this(Identifiable.randomUID("hashingTF"), HashingTF.SPARK_3_MURMUR3_HASH)
+
+  @Since("1.4.0")
+  def this(uid: String) = this(uid, hashFuncVersion = HashingTF.SPARK_3_MURMUR3_HASH)
 
   /** @group setParam */
   @Since("1.4.0")
@@ -122,7 +125,12 @@ class HashingTF @Since("1.4.0") (@Since("1.4.0") override val uid: String)
    */
   @Since("3.0.0")
   def indexOf(term: Any): Int = {
-    Utils.nonNegativeMod(hashFunc(term), $(numFeatures))
+    val hashValue = hashFuncVersion match {
+      case HashingTF.SPARK_2_MURMUR3_HASH => OldHashingTF.murmur3Hash(term)
+      case HashingTF.SPARK_3_MURMUR3_HASH => FeatureHasher.murmur3Hash(term)
+      case _ => throw new IllegalArgumentException("Illegal hash function version setting.")
+    }
+    Utils.nonNegativeMod(hashValue, $(numFeatures))
   }
 
   @Since("1.4.1")
@@ -132,27 +140,41 @@ class HashingTF @Since("1.4.0") (@Since("1.4.0") override val uid: String)
   override def toString: String = {
     s"HashingTF: uid=$uid, binary=${$(binary)}, numFeatures=${$(numFeatures)}"
   }
+
+  @Since("3.0.0")
+  override def save(path: String): Unit = {
+    require(hashFuncVersion == HashingTF.SPARK_3_MURMUR3_HASH,
+      "Cannot save model which is loaded from lower version spark saved model. We can address " +
+      "it by (1) use old spark version to save the model, or (2) use new version spark to " +
+      "re-train the pipeline.")
+    super.save(path)
+  }
 }
 
 @Since("1.6.0")
 object HashingTF extends DefaultParamsReadable[HashingTF] {
 
+  private[ml] val SPARK_2_MURMUR3_HASH = 1
+  private[ml] val SPARK_3_MURMUR3_HASH = 2
+
   private class HashingTFReader extends MLReader[HashingTF] {
 
     private val className = classOf[HashingTF].getName
 
     override def load(path: String): HashingTF = {
       val metadata = DefaultParamsReader.loadMetadata(path, sc, className)
-      val hashingTF = new HashingTF(metadata.uid)
-      metadata.getAndSetParams(hashingTF)
 
       // We support loading old `HashingTF` saved by previous Spark versions.
       // Previous `HashingTF` uses `mllib.feature.HashingTF.murmur3Hash`, but new `HashingTF` uses
       // `ml.Feature.FeatureHasher.murmur3Hash`.
       val (majorVersion, _) = majorMinorVersion(metadata.sparkVersion)
-      if (majorVersion < 3) {
-        hashingTF.hashFunc = OldHashingTF.murmur3Hash
+      val hashFuncVersion = if (majorVersion < 3) {
+        SPARK_2_MURMUR3_HASH
+      } else {
+        SPARK_3_MURMUR3_HASH
       }
+      val hashingTF = new HashingTF(metadata.uid, hashFuncVersion = hashFuncVersion)
+      metadata.getAndSetParams(hashingTF)
       hashingTF
     }
   }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala
index 64f1722f5fcb8..ad1010da5c104 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala
@@ -64,6 +64,8 @@ private[feature] trait ImputerParams extends Params with HasInputCol with HasInp
   /** @group getParam */
   def getMissingValue: Double = $(missingValue)
 
+  setDefault(strategy -> Imputer.mean, missingValue -> Double.NaN)
+
   /** Returns the input and output column names corresponding in pair. */
   private[feature] def getInOutCols(): (Array[String], Array[String]) = {
     if (isSet(inputCol)) {
@@ -144,8 +146,6 @@ class Imputer @Since("2.2.0") (@Since("2.2.0") override val uid: String)
   @Since("3.0.0")
   def setRelativeError(value: Double): this.type = set(relativeError, value)
 
-  setDefault(strategy -> Imputer.mean, missingValue -> Double.NaN)
-
   override def fit(dataset: Dataset[_]): ImputerModel = {
     transformSchema(dataset.schema, logging = true)
     val spark = dataset.sparkSession
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
index 5476a86eb9d76..11d0c4689cbba 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
@@ -17,10 +17,7 @@
 
 package org.apache.spark.ml.feature
 
-import scala.collection.mutable
-
-import org.apache.spark.ml.linalg._
-import org.apache.spark.rdd.RDD
+import org.apache.spark.ml.linalg.Vector
 
 /**
  * Class that represents an instance of weighted data point with label and features.
@@ -31,131 +28,6 @@ import org.apache.spark.rdd.RDD
  */
 private[spark] case class Instance(label: Double, weight: Double, features: Vector)
 
-
-/**
- * Class that represents an block of instance.
- * If all weights are 1, then an empty array is stored.
- */
-private[spark] case class InstanceBlock(
-    labels: Array[Double],
-    weights: Array[Double],
-    matrix: Matrix) {
-  require(labels.length == matrix.numRows)
-  require(matrix.isTransposed)
-  if (weights.nonEmpty) {
-    require(labels.length == weights.length)
-  }
-
-  def size: Int = labels.length
-
-  def numFeatures: Int = matrix.numCols
-
-  def instanceIterator: Iterator[Instance] = {
-    if (weights.nonEmpty) {
-      labels.iterator.zip(weights.iterator).zip(matrix.rowIter)
-        .map { case ((label, weight), vec) => Instance(label, weight, vec) }
-    } else {
-      labels.iterator.zip(matrix.rowIter)
-        .map { case (label, vec) => Instance(label, 1.0, vec) }
-    }
-  }
-
-  def getLabel(i: Int): Double = labels(i)
-
-  def labelIter: Iterator[Double] = labels.iterator
-
-  @transient lazy val getWeight: Int => Double = {
-    if (weights.nonEmpty) {
-      (i: Int) => weights(i)
-    } else {
-      (i: Int) => 1.0
-    }
-  }
-
-  def weightIter: Iterator[Double] = {
-    if (weights.nonEmpty) {
-      weights.iterator
-    } else {
-      Iterator.fill(size)(1.0)
-    }
-  }
-
-  // directly get the non-zero iterator of i-th row vector without array copy or slice
-  @transient lazy val getNonZeroIter: Int => Iterator[(Int, Double)] = {
-    matrix match {
-      case dm: DenseMatrix =>
-        (i: Int) =>
-          val start = numFeatures * i
-          Iterator.tabulate(numFeatures)(j =>
-            (j, dm.values(start + j))
-          ).filter(_._2 != 0)
-      case sm: SparseMatrix =>
-        (i: Int) =>
-          val start = sm.colPtrs(i)
-          val end = sm.colPtrs(i + 1)
-          Iterator.tabulate(end - start)(j =>
-            (sm.rowIndices(start + j), sm.values(start + j))
-          ).filter(_._2 != 0)
-    }
-  }
-}
-
-private[spark] object InstanceBlock {
-
-  def fromInstances(instances: Seq[Instance]): InstanceBlock = {
-    val labels = instances.map(_.label).toArray
-    val weights = if (instances.exists(_.weight != 1)) {
-      instances.map(_.weight).toArray
-    } else {
-      Array.emptyDoubleArray
-    }
-    val numRows = instances.length
-    val numCols = instances.head.features.size
-    val denseSize = Matrices.getDenseSize(numCols, numRows)
-    val nnz = instances.iterator.map(_.features.numNonzeros).sum
-    val sparseSize = Matrices.getSparseSize(nnz, numRows + 1)
-    val matrix = if (denseSize < sparseSize) {
-      val values = Array.ofDim[Double](numRows * numCols)
-      var offset = 0
-      var j = 0
-      while (j < numRows) {
-        instances(j).features.foreachNonZero { (i, v) =>
-          values(offset + i) = v
-        }
-        offset += numCols
-        j += 1
-      }
-      new DenseMatrix(numRows, numCols, values, true)
-    } else {
-      val colIndices = mutable.ArrayBuilder.make[Int]
-      val values = mutable.ArrayBuilder.make[Double]
-      val rowPtrs = mutable.ArrayBuilder.make[Int]
-      var rowPtr = 0
-      rowPtrs += 0
-      var j = 0
-      while (j < numRows) {
-        var nnz = 0
-        instances(j).features.foreachNonZero { (i, v) =>
-          colIndices += i
-          values += v
-          nnz += 1
-        }
-        rowPtr += nnz
-        rowPtrs += rowPtr
-        j += 1
-      }
-      new SparseMatrix(numRows, numCols, rowPtrs.result(),
-        colIndices.result(), values.result(), true)
-    }
-    InstanceBlock(labels, weights, matrix)
-  }
-
-  def blokify(instances: RDD[Instance], blockSize: Int): RDD[InstanceBlock] = {
-    instances.mapPartitions(_.grouped(blockSize).map(InstanceBlock.fromInstances))
-  }
-}
-
-
 /**
  * Case class that represents an instance of data point with
  * label, weight, offset and features.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala
index 01741019fb546..6d5c7c50dbacc 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala
@@ -98,7 +98,7 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
 
   override def transform(dataset: Dataset[_]): DataFrame = {
     transformSchema(dataset.schema, logging = true)
-    val transformUDF = udf(hashFunction(_: Vector), DataTypes.createArrayType(new VectorUDT))
+    val transformUDF = udf(hashFunction(_: Vector))
     dataset.withColumn($(outputCol), transformUDF(dataset($(inputCol))))
   }
 
@@ -128,14 +128,13 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
       }
 
       // In the origin dataset, find the hash value that hash the same bucket with the key
-      val sameBucketWithKeyUDF = udf((x: Seq[Vector]) =>
-        sameBucket(x, keyHash), DataTypes.BooleanType)
+      val sameBucketWithKeyUDF = udf((x: Seq[Vector]) => sameBucket(x, keyHash))
 
       modelDataset.filter(sameBucketWithKeyUDF(col($(outputCol))))
     } else {
       // In the origin dataset, find the hash value that is closest to the key
       // Limit the use of hashDist since it's controversial
-      val hashDistUDF = udf((x: Seq[Vector]) => hashDistance(x, keyHash), DataTypes.DoubleType)
+      val hashDistUDF = udf((x: Seq[Vector]) => hashDistance(x, keyHash))
       val hashDistCol = hashDistUDF(col($(outputCol)))
       val modelDatasetWithDist = modelDataset.withColumn(distCol, hashDistCol)
 
@@ -172,7 +171,7 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
     }
 
     // Get the top k nearest neighbor by their distance to the key
-    val keyDistUDF = udf((x: Vector) => keyDistance(x, key), DataTypes.DoubleType)
+    val keyDistUDF = udf((x: Vector) => keyDistance(x, key))
     val modelSubsetWithDistCol = modelSubset.withColumn(distCol, keyDistUDF(col($(inputCol))))
     modelSubsetWithDistCol.sort(distCol).limit(numNearestNeighbors)
   }
@@ -290,7 +289,7 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
       .drop(explodeCols: _*).distinct()
 
     // Add a new column to store the distance of the two rows.
-    val distUDF = udf((x: Vector, y: Vector) => keyDistance(x, y), DataTypes.DoubleType)
+    val distUDF = udf((x: Vector, y: Vector) => keyDistance(x, y))
     val joinedDatasetWithDist = joinedDataset.select(col("*"),
       distUDF(col(s"$leftColName.${$(inputCol)}"), col(s"$rightColName.${$(inputCol)}")).as(distCol)
     )
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
index c84892c974b90..56798a8e61a61 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
@@ -58,6 +58,8 @@ private[feature] trait MinMaxScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getMax: Double = $(max)
 
+  setDefault(min -> 0.0, max -> 1.0)
+
   /** Validates and transforms the input schema. */
   protected def validateAndTransformSchema(schema: StructType): StructType = {
     require($(min) < $(max), s"The specified min(${$(min)}) is larger or equal to max(${$(max)})")
@@ -93,8 +95,6 @@ class MinMaxScaler @Since("1.5.0") (@Since("1.5.0") override val uid: String)
   @Since("1.5.0")
   def this() = this(Identifiable.randomUID("minMaxScal"))
 
-  setDefault(min -> 0.0, max -> 1.0)
-
   /** @group setParam */
   @Since("1.5.0")
   def setInputCol(value: String): this.type = set(inputCol, value)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
index 0ef092f6be463..5a500fefb57ec 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
@@ -53,8 +53,6 @@ private[ml] trait OneHotEncoderBase extends Params with HasHandleInvalid
     "during fitting, invalid data will result in an error.",
     ParamValidators.inArray(OneHotEncoder.supportedHandleInvalids))
 
-  setDefault(handleInvalid, OneHotEncoder.ERROR_INVALID)
-
   /**
    * Whether to drop the last category in the encoded vector (default: true)
    * @group param
@@ -62,12 +60,13 @@ private[ml] trait OneHotEncoderBase extends Params with HasHandleInvalid
   @Since("2.3.0")
   final val dropLast: BooleanParam =
     new BooleanParam(this, "dropLast", "whether to drop the last category")
-  setDefault(dropLast -> true)
 
   /** @group getParam */
   @Since("2.3.0")
   def getDropLast: Boolean = $(dropLast)
 
+  setDefault(handleInvalid -> OneHotEncoder.ERROR_INVALID, dropLast -> true)
+
   /** Returns the input and output column names corresponding in pair. */
   private[feature] def getInOutCols(): (Array[String], Array[String]) = {
     if (isSet(inputCol)) {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
index 216d99d01f2f7..b93c9b1fcd204 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
@@ -46,7 +46,6 @@ private[feature] trait QuantileDiscretizerBase extends Params
   val numBuckets = new IntParam(this, "numBuckets", "Number of buckets (quantiles, or " +
     "categories) into which data points are grouped. Must be >= 2.",
     ParamValidators.gtEq(2))
-  setDefault(numBuckets -> 2)
 
   /** @group getParam */
   def getNumBuckets: Int = getOrDefault(numBuckets)
@@ -82,7 +81,8 @@ private[feature] trait QuantileDiscretizerBase extends Params
     "how to handle invalid entries. Options are skip (filter out rows with invalid values), " +
     "error (throw an error), or keep (keep invalid values in a special additional bucket).",
     ParamValidators.inArray(Bucketizer.supportedHandleInvalids))
-  setDefault(handleInvalid, Bucketizer.ERROR_INVALID)
+
+  setDefault(handleInvalid -> Bucketizer.ERROR_INVALID, numBuckets -> 2)
 }
 
 /**
@@ -236,6 +236,18 @@ final class QuantileDiscretizer @Since("1.6.0") (@Since("1.6.0") override val ui
   private def getDistinctSplits(splits: Array[Double]): Array[Double] = {
     splits(0) = Double.NegativeInfinity
     splits(splits.length - 1) = Double.PositiveInfinity
+
+    // 0.0 and -0.0 are distinct values, array.distinct will preserve both of them.
+    // but 0.0 > -0.0 is False which will break the parameter validation checking.
+    // and in scala <= 2.12, there's bug which will cause array.distinct generate
+    // non-deterministic results when array contains both 0.0 and -0.0
+    // So that here we should first normalize all 0.0 and -0.0 to be 0.0
+    // See https://github.com/scala/bug/issues/11995
+    for (i <- 0 until splits.length) {
+      if (splits(i) == -0.0) {
+        splits(i) = 0.0
+      }
+    }
     val distinctSplits = splits.distinct
     if (splits.length != distinctSplits.length) {
       log.warn(s"Some quantiles were identical. Bucketing to ${distinctSplits.length - 1}" +
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
index 7ccfafa4ac813..b8da020017f12 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
@@ -60,7 +60,6 @@ private[feature] trait RFormulaBase extends HasFeaturesCol with HasLabelCol with
   @Since("2.1.0")
   val forceIndexLabel: BooleanParam = new BooleanParam(this, "forceIndexLabel",
     "Force to index label whether it is numeric or string")
-  setDefault(forceIndexLabel -> false)
 
   /** @group getParam */
   @Since("2.1.0")
@@ -80,7 +79,6 @@ private[feature] trait RFormulaBase extends HasFeaturesCol with HasLabelCol with
     "type. Options are 'skip' (filter out rows with invalid data), error (throw an error), " +
     "or 'keep' (put invalid data in a special additional bucket, at index numLabels).",
     ParamValidators.inArray(StringIndexer.supportedHandleInvalids))
-  setDefault(handleInvalid, StringIndexer.ERROR_INVALID)
 
   /**
    * Param for how to order categories of a string FEATURE column used by `StringIndexer`.
@@ -113,12 +111,14 @@ private[feature] trait RFormulaBase extends HasFeaturesCol with HasLabelCol with
     "The default value is 'frequencyDesc'. When the ordering is set to 'alphabetDesc', " +
     "RFormula drops the same category as R when encoding strings.",
     ParamValidators.inArray(StringIndexer.supportedStringOrderType))
-  setDefault(stringIndexerOrderType, StringIndexer.frequencyDesc)
 
   /** @group getParam */
   @Since("2.3.0")
   def getStringIndexerOrderType: String = $(stringIndexerOrderType)
 
+  setDefault(forceIndexLabel -> false, handleInvalid -> StringIndexer.ERROR_INVALID,
+    stringIndexerOrderType -> StringIndexer.frequencyDesc)
+
   protected def hasLabelCol(schema: StructType): Boolean = {
     schema.map(_.name).contains($(labelCol))
   }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
index 2a1204a40bde0..12aba69528891 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
@@ -50,8 +50,6 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getLower: Double = $(lower)
 
-  setDefault(lower -> 0.25)
-
   /**
    * Upper quantile to calculate quantile range, shared by all features
    * Default: 0.75
@@ -64,8 +62,6 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getUpper: Double = $(upper)
 
-  setDefault(upper -> 0.75)
-
   /**
    * Whether to center the data with median before scaling.
    * It will build a dense output, so take care when applying to sparse input.
@@ -78,8 +74,6 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getWithCentering: Boolean = $(withCentering)
 
-  setDefault(withCentering -> false)
-
   /**
    * Whether to scale the data to quantile range.
    * Default: true
@@ -91,7 +85,7 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getWithScaling: Boolean = $(withScaling)
 
-  setDefault(withScaling -> true)
+  setDefault(withScaling -> true, lower -> 0.25, upper -> 0.75, withCentering -> false)
 
   /** Validates and transforms the input schema. */
   protected def validateAndTransformSchema(schema: StructType): StructType = {
@@ -120,7 +114,7 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
  * Note that NaN values are ignored in the computation of medians and ranges.
  */
 @Since("3.0.0")
-class RobustScaler (override val uid: String)
+class RobustScaler @Since("3.0.0") (@Since("3.0.0") override val uid: String)
   extends Estimator[RobustScalerModel] with RobustScalerParams with DefaultParamsWritable {
 
   import RobustScaler._
@@ -186,7 +180,7 @@ class RobustScaler (override val uid: String)
 object RobustScaler extends DefaultParamsReadable[RobustScaler] {
 
   // compute QuantileSummaries for each feature
-  private[spark] def computeSummaries(
+  private[ml] def computeSummaries(
       vectors: RDD[Vector],
       numFeatures: Int,
       relativeError: Double): RDD[(Int, QuantileSummaries)] = {
@@ -229,9 +223,9 @@ object RobustScaler extends DefaultParamsReadable[RobustScaler] {
  */
 @Since("3.0.0")
 class RobustScalerModel private[ml] (
-    override val uid: String,
-    val range: Vector,
-    val median: Vector)
+    @Since("3.0.0") override val uid: String,
+    @Since("3.0.0") val range: Vector,
+    @Since("3.0.0") val median: Vector)
   extends Model[RobustScalerModel] with RobustScalerParams with MLWritable {
 
   import RobustScalerModel._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
index 9f9f097a26ead..77e3d902e0456 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
@@ -56,8 +56,6 @@ private[feature] trait StringIndexerBase extends Params with HasHandleInvalid wi
     "or 'keep' (put invalid data in a special additional bucket, at index numLabels).",
     ParamValidators.inArray(StringIndexer.supportedHandleInvalids))
 
-  setDefault(handleInvalid, StringIndexer.ERROR_INVALID)
-
   /**
    * Param for how to order labels of string column. The first label after ordering is assigned
    * an index of 0.
@@ -84,6 +82,9 @@ private[feature] trait StringIndexerBase extends Params with HasHandleInvalid wi
   @Since("2.3.0")
   def getStringOrderType: String = $(stringOrderType)
 
+  setDefault(handleInvalid -> StringIndexer.ERROR_INVALID,
+    stringOrderType -> StringIndexer.frequencyDesc)
+
   /** Returns the input and output column names corresponding in pair. */
   private[feature] def getInOutCols(): (Array[String], Array[String]) = {
     ParamValidators.checkSingleVsMultiColumnParams(this, Seq(outputCol), Seq(outputCols))
@@ -107,7 +108,7 @@ private[feature] trait StringIndexerBase extends Params with HasHandleInvalid wi
         s"but got $inputDataType.")
     require(schema.fields.forall(_.name != outputColName),
       s"Output column $outputColName already exists.")
-    NominalAttribute.defaultAttr.withName($(outputCol)).toStructField()
+    NominalAttribute.defaultAttr.withName(outputColName).toStructField()
   }
 
   /** Validates and transforms the input schema. */
@@ -155,7 +156,6 @@ class StringIndexer @Since("1.4.0") (
   /** @group setParam */
   @Since("2.3.0")
   def setStringOrderType(value: String): this.type = set(stringOrderType, value)
-  setDefault(stringOrderType, StringIndexer.frequencyDesc)
 
   /** @group setParam */
   @Since("1.4.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
index 51acbf3806b9d..7bc5e56aaebf2 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
@@ -233,7 +233,7 @@ object VectorAssembler extends DefaultParamsReadable[VectorAssembler] {
         getVectorLengthsFromFirstRow(dataset.na.drop(missingColumns), missingColumns)
       case (true, VectorAssembler.KEEP_INVALID) => throw new RuntimeException(
         s"""Can not infer column lengths with handleInvalid = "keep". Consider using VectorSizeHint
-           |to add metadata for columns: ${columns.mkString("[", ", ", "]")}."""
+           |to add metadata for columns: ${missingColumns.mkString("[", ", ", "]")}."""
           .stripMargin.replaceAll("\n", " "))
       case (_, _) => Map.empty
     }
@@ -288,7 +288,7 @@ object VectorAssembler extends DefaultParamsReadable[VectorAssembler] {
           featureIndex += length
         } else {
           throw new SparkException(
-            s"""Encountered null while assembling a row with handleInvalid = "keep". Consider
+            s"""Encountered null while assembling a row with handleInvalid = "error". Consider
                |removing nulls from dataset or using handleInvalid = "keep" or "skip"."""
               .stripMargin)
         }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
index 866074fb1453e..b7cf4392cd177 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
@@ -60,8 +60,6 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
     "number of categories of the feature).",
     ParamValidators.inArray(VectorIndexer.supportedHandleInvalids))
 
-  setDefault(handleInvalid, VectorIndexer.ERROR_INVALID)
-
   /**
    * Threshold for the number of values a categorical feature can take.
    * If a feature is found to have {@literal >} maxCategories values, then it is declared
@@ -75,10 +73,10 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
       " If a feature is found to have > maxCategories values, then it is declared continuous.",
     ParamValidators.gtEq(2))
 
-  setDefault(maxCategories -> 20)
-
   /** @group getParam */
   def getMaxCategories: Int = $(maxCategories)
+
+  setDefault(maxCategories -> 20, handleInvalid -> VectorIndexer.ERROR_INVALID)
 }
 
 /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
index be91844ba39e6..3590b9118f3b8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
@@ -57,8 +57,6 @@ final class VectorSlicer @Since("1.5.0") (@Since("1.5.0") override val uid: Stri
     "An array of indices to select features from a vector column." +
       " There can be no overlap with names.", VectorSlicer.validIndices)
 
-  setDefault(indices -> Array.emptyIntArray)
-
   /** @group getParam */
   @Since("1.5.0")
   def getIndices: Array[Int] = $(indices)
@@ -79,8 +77,6 @@ final class VectorSlicer @Since("1.5.0") (@Since("1.5.0") override val uid: Stri
     "An array of feature names to select features from a vector column." +
       " There can be no overlap with indices.", VectorSlicer.validNames)
 
-  setDefault(names -> Array.empty[String])
-
   /** @group getParam */
   @Since("1.5.0")
   def getNames: Array[String] = $(names)
@@ -97,6 +93,8 @@ final class VectorSlicer @Since("1.5.0") (@Since("1.5.0") override val uid: Stri
   @Since("1.5.0")
   def setOutputCol(value: String): this.type = set(outputCol, value)
 
+  setDefault(indices -> Array.emptyIntArray, names -> Array.empty[String])
+
   @Since("2.0.0")
   override def transform(dataset: Dataset[_]): DataFrame = {
     // Validity checks
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala
index bbfcbfbe038ef..287fce4243743 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala
@@ -47,7 +47,6 @@ private[feature] trait Word2VecBase extends Params
   final val vectorSize = new IntParam(
     this, "vectorSize", "the dimension of codes after transforming from words (> 0)",
     ParamValidators.gt(0))
-  setDefault(vectorSize -> 100)
 
   /** @group getParam */
   def getVectorSize: Int = $(vectorSize)
@@ -60,7 +59,6 @@ private[feature] trait Word2VecBase extends Params
   final val windowSize = new IntParam(
     this, "windowSize", "the window size (context words from [-window, window]) (> 0)",
     ParamValidators.gt(0))
-  setDefault(windowSize -> 5)
 
   /** @group expertGetParam */
   def getWindowSize: Int = $(windowSize)
@@ -73,7 +71,6 @@ private[feature] trait Word2VecBase extends Params
   final val numPartitions = new IntParam(
     this, "numPartitions", "number of partitions for sentences of words (> 0)",
     ParamValidators.gt(0))
-  setDefault(numPartitions -> 1)
 
   /** @group getParam */
   def getNumPartitions: Int = $(numPartitions)
@@ -86,7 +83,6 @@ private[feature] trait Word2VecBase extends Params
    */
   final val minCount = new IntParam(this, "minCount", "the minimum number of times a token must " +
     "appear to be included in the word2vec model's vocabulary (>= 0)", ParamValidators.gtEq(0))
-  setDefault(minCount -> 5)
 
   /** @group getParam */
   def getMinCount: Int = $(minCount)
@@ -101,13 +97,12 @@ private[feature] trait Word2VecBase extends Params
   final val maxSentenceLength = new IntParam(this, "maxSentenceLength", "Maximum length " +
     "(in words) of each sentence in the input data. Any sentence longer than this threshold will " +
     "be divided into chunks up to the size (> 0)", ParamValidators.gt(0))
-  setDefault(maxSentenceLength -> 1000)
 
   /** @group getParam */
   def getMaxSentenceLength: Int = $(maxSentenceLength)
 
-  setDefault(stepSize -> 0.025)
-  setDefault(maxIter -> 1)
+  setDefault(vectorSize -> 100, windowSize -> 5, numPartitions -> 1, minCount -> 5,
+    maxSentenceLength -> 1000, stepSize -> 0.025, maxIter -> 1)
 
   /**
    * Validate and transform the input schema.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
index 4d001c159eda0..de9d57e400551 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
@@ -29,10 +29,10 @@ import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared.HasPredictionCol
 import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.Instrumentation.instrumented
-import org.apache.spark.mllib.fpm.{AssociationRules => MLlibAssociationRules,
-  FPGrowth => MLlibFPGrowth}
+import org.apache.spark.mllib.fpm.{AssociationRules => MLlibAssociationRules, FPGrowth => MLlibFPGrowth}
 import org.apache.spark.mllib.fpm.FPGrowth.FreqItemset
 import org.apache.spark.sql._
+import org.apache.spark.sql.expressions.SparkUserDefinedFunction
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
 import org.apache.spark.storage.StorageLevel
@@ -50,7 +50,6 @@ private[fpm] trait FPGrowthParams extends Params with HasPredictionCol {
    */
   @Since("2.2.0")
   val itemsCol: Param[String] = new Param[String](this, "itemsCol", "items column name")
-  setDefault(itemsCol -> "items")
 
   /** @group getParam */
   @Since("2.2.0")
@@ -66,7 +65,6 @@ private[fpm] trait FPGrowthParams extends Params with HasPredictionCol {
   val minSupport: DoubleParam = new DoubleParam(this, "minSupport",
     "the minimal support level of a frequent pattern",
     ParamValidators.inRange(0.0, 1.0))
-  setDefault(minSupport -> 0.3)
 
   /** @group getParam */
   @Since("2.2.0")
@@ -95,12 +93,13 @@ private[fpm] trait FPGrowthParams extends Params with HasPredictionCol {
   val minConfidence: DoubleParam = new DoubleParam(this, "minConfidence",
     "minimal confidence for generating Association Rule",
     ParamValidators.inRange(0.0, 1.0))
-  setDefault(minConfidence -> 0.8)
 
   /** @group getParam */
   @Since("2.2.0")
   def getMinConfidence: Double = $(minConfidence)
 
+  setDefault(minSupport -> 0.3, itemsCol -> "items", minConfidence -> 0.8)
+
   /**
    * Validates and transforms the input schema.
    * @param schema input schema
@@ -286,14 +285,17 @@ class FPGrowthModel private[ml] (
 
     val dt = dataset.schema($(itemsCol)).dataType
     // For each rule, examine the input items and summarize the consequents
-    val predictUDF = udf((items: Seq[Any]) => {
+    val predictUDF = SparkUserDefinedFunction((items: Seq[Any]) => {
       if (items != null) {
         val itemset = items.toSet
         brRules.value.filter(_._1.forall(itemset.contains))
           .flatMap(_._2.filter(!itemset.contains(_))).distinct
       } else {
         Seq.empty
-      }}, dt)
+      }},
+      dt,
+      Nil
+    )
     dataset.withColumn($(predictionCol), predictUDF(col($(itemsCol))))
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/functions.scala b/mllib/src/main/scala/org/apache/spark/ml/functions.scala
index 1faf562c4d896..0f03231079866 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/functions.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/functions.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.ml
 
 import org.apache.spark.annotation.Since
-import org.apache.spark.ml.linalg.Vector
+import org.apache.spark.ml.linalg.{SparseVector, Vector}
 import org.apache.spark.mllib.linalg.{Vector => OldVector}
 import org.apache.spark.sql.Column
 import org.apache.spark.sql.functions.udf
@@ -27,7 +27,6 @@ import org.apache.spark.sql.functions.udf
 @Since("3.0.0")
 object functions {
 // scalastyle:on
-
   private val vectorToArrayUdf = udf { vec: Any =>
     vec match {
       case v: Vector => v.toArray
@@ -39,10 +38,37 @@ object functions {
     }
   }.asNonNullable()
 
+  private val vectorToArrayFloatUdf = udf { vec: Any =>
+    vec match {
+      case v: SparseVector =>
+        val data = new Array[Float](v.size)
+        v.foreachActive { (index, value) => data(index) = value.toFloat }
+        data
+      case v: Vector => v.toArray.map(_.toFloat)
+      case v: OldVector => v.toArray.map(_.toFloat)
+      case v => throw new IllegalArgumentException(
+        "function vector_to_array requires a non-null input argument and input type must be " +
+        "`org.apache.spark.ml.linalg.Vector` or `org.apache.spark.mllib.linalg.Vector`, " +
+        s"but got ${ if (v == null) "null" else v.getClass.getName }.")
+    }
+  }.asNonNullable()
+
   /**
    * Converts a column of MLlib sparse/dense vectors into a column of dense arrays.
-   *
+   * @param v: the column of MLlib sparse/dense vectors
+   * @param dtype: the desired underlying data type in the returned array
+   * @return an array&lt;float&gt; if dtype is float32, or array&lt;double&gt; if dtype is float64
    * @since 3.0.0
    */
-  def vector_to_array(v: Column): Column = vectorToArrayUdf(v)
+  def vector_to_array(v: Column, dtype: String = "float64"): Column = {
+    if (dtype == "float64") {
+      vectorToArrayUdf(v)
+    } else if (dtype == "float32") {
+      vectorToArrayFloatUdf(v)
+    } else {
+      throw new IllegalArgumentException(
+        s"Unsupported dtype: $dtype. Valid values: float64, float32."
+      )
+    }
+  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala
index a66ba27a7b9c5..1cab37a25ad57 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala
@@ -17,15 +17,13 @@
 
 package org.apache.spark.ml.linalg
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.sql.types.DataType
 
 /**
- * :: DeveloperApi ::
  * SQL data types for vectors and matrices.
  */
 @Since("2.0.0")
-@DeveloperApi
 object SQLDataTypes {
 
   /** Data type for [[Vector]]. */
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala
index 292187b3e146e..b0906f1b06511 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg._
 
 /**
@@ -32,28 +32,21 @@ import org.apache.spark.ml.linalg._
  *
  * @param bcCoefficients The coefficients corresponding to the features.
  * @param fitIntercept Whether to fit an intercept term.
+ * @param bcFeaturesStd The standard deviation values of the features.
  */
 private[ml] class HingeAggregator(
-    numFeatures: Int,
+    bcFeaturesStd: Broadcast[Array[Double]],
     fitIntercept: Boolean)(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[InstanceBlock, HingeAggregator] {
+  extends DifferentiableLossAggregator[Instance, HingeAggregator] {
 
+  private val numFeatures: Int = bcFeaturesStd.value.length
   private val numFeaturesPlusIntercept: Int = if (fitIntercept) numFeatures + 1 else numFeatures
-  protected override val dim: Int = numFeaturesPlusIntercept
   @transient private lazy val coefficientsArray = bcCoefficients.value match {
     case DenseVector(values) => values
     case _ => throw new IllegalArgumentException(s"coefficients only supports dense vector" +
       s" but got type ${bcCoefficients.value.getClass}.")
   }
-
-  @transient private lazy val linear = {
-    if (fitIntercept) {
-      new DenseVector(coefficientsArray.take(numFeatures))
-    } else {
-      new DenseVector(coefficientsArray)
-    }
-  }
-
+  protected override val dim: Int = numFeaturesPlusIntercept
 
   /**
    * Add a new training instance to this HingeAggregator, and update the loss and gradient
@@ -69,13 +62,16 @@ private[ml] class HingeAggregator(
       require(weight >= 0.0, s"instance weight, $weight has to be >= 0.0")
 
       if (weight == 0.0) return this
+      val localFeaturesStd = bcFeaturesStd.value
       val localCoefficients = coefficientsArray
       val localGradientSumArray = gradientSumArray
 
       val dotProduct = {
         var sum = 0.0
         features.foreachNonZero { (index, value) =>
-          sum += localCoefficients(index) * value
+          if (localFeaturesStd(index) != 0.0) {
+            sum += localCoefficients(index) * value / localFeaturesStd(index)
+          }
         }
         if (fitIntercept) sum += localCoefficients(numFeaturesPlusIntercept - 1)
         sum
@@ -92,7 +88,9 @@ private[ml] class HingeAggregator(
       if (1.0 > labelScaled * dotProduct) {
         val gradientScale = -labelScaled * weight
         features.foreachNonZero { (index, value) =>
-          localGradientSumArray(index) += value * gradientScale
+          if (localFeaturesStd(index) != 0.0) {
+            localGradientSumArray(index) += value * gradientScale / localFeaturesStd(index)
+          }
         }
         if (fitIntercept) {
           localGradientSumArray(localGradientSumArray.length - 1) += gradientScale
@@ -104,78 +102,4 @@ private[ml] class HingeAggregator(
       this
     }
   }
-
-  /**
-   * Add a new training instance block to this HingeAggregator, and update the loss and gradient
-   * of the objective function.
-   *
-   * @param block The InstanceBlock to be added.
-   * @return This HingeAggregator object.
-   */
-  def add(block: InstanceBlock): this.type = {
-    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
-    require(block.weightIter.forall(_ >= 0),
-      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
-
-    if (block.weightIter.forall(_ == 0)) return this
-    val size = block.size
-    val localGradientSumArray = gradientSumArray
-
-    // vec here represents dotProducts
-    val vec = if (fitIntercept && coefficientsArray.last != 0) {
-      val intercept = coefficientsArray.last
-      new DenseVector(Array.fill(size)(intercept))
-    } else {
-      new DenseVector(Array.ofDim[Double](size))
-    }
-
-    if (fitIntercept) {
-      BLAS.gemv(1.0, block.matrix, linear, 1.0, vec)
-    } else {
-      BLAS.gemv(1.0, block.matrix, linear, 0.0, vec)
-    }
-
-    // in-place convert dotProducts to gradient scales
-    // then, vec represents gradient scales
-    var i = 0
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        // Our loss function with {0, 1} labels is max(0, 1 - (2y - 1) (f_w(x)))
-        // Therefore the gradient is -(2y - 1)*x
-        val label = block.getLabel(i)
-        val labelScaled = 2 * label - 1.0
-        val loss = (1.0 - labelScaled * vec(i)) * weight
-        if (loss > 0) {
-          lossSum += loss
-          val gradScale = -labelScaled * weight
-          vec.values(i) = gradScale
-        } else {
-          vec.values(i) = 0.0
-        }
-      } else {
-        vec.values(i) = 0.0
-      }
-      i += 1
-    }
-
-    // predictions are all correct, no gradient signal
-    if (vec.values.forall(_ == 0)) return this
-
-    if (fitIntercept) {
-      // localGradientSumArray is of size numFeatures+1, so can not
-      // be directly used as the output of BLAS.gemv
-      val linearGradSumVec = new DenseVector(Array.ofDim[Double](numFeatures))
-      BLAS.gemv(1.0, block.matrix.transpose, vec, 0.0, linearGradSumVec)
-      linearGradSumVec.foreachNonZero { (i, v) => localGradientSumArray(i) += v }
-      localGradientSumArray(numFeatures) += vec.values.sum
-    } else {
-      val gradSumVec = new DenseVector(localGradientSumArray)
-      BLAS.gemv(1.0, block.matrix.transpose, vec, 1.0, gradSumVec)
-    }
-
-    this
-  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala
index f83621506500e..8a1a41b2950c1 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala
@@ -17,8 +17,8 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
-import org.apache.spark.ml.linalg._
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.linalg.Vector
 
 /**
  * HuberAggregator computes the gradient and loss for a huber loss function,
@@ -62,17 +62,19 @@ import org.apache.spark.ml.linalg._
  *
  * @param fitIntercept Whether to fit an intercept term.
  * @param epsilon The shape parameter to control the amount of robustness.
+ * @param bcFeaturesStd The broadcast standard deviation values of the features.
  * @param bcParameters including three parts: the regression coefficients corresponding
  *                     to the features, the intercept (if fitIntercept is ture)
  *                     and the scale parameter (sigma).
  */
 private[ml] class HuberAggregator(
-    numFeatures: Int,
     fitIntercept: Boolean,
-    epsilon: Double)(bcParameters: Broadcast[Vector])
-  extends DifferentiableLossAggregator[InstanceBlock, HuberAggregator] {
+    epsilon: Double,
+    bcFeaturesStd: Broadcast[Array[Double]])(bcParameters: Broadcast[Vector])
+  extends DifferentiableLossAggregator[Instance, HuberAggregator] {
 
   protected override val dim: Int = bcParameters.value.size
+  private val numFeatures: Int = if (fitIntercept) dim - 2 else dim - 1
   private val sigma: Double = bcParameters.value(dim - 1)
   private val intercept: Double = if (fitIntercept) {
     bcParameters.value(dim - 2)
@@ -80,8 +82,7 @@ private[ml] class HuberAggregator(
     0.0
   }
   // make transient so we do not serialize between aggregation stages
-  @transient private lazy val linear =
-    new DenseVector(bcParameters.value.toArray.take(numFeatures))
+  @transient private lazy val coefficients = bcParameters.value.toArray.slice(0, numFeatures)
 
   /**
    * Add a new training instance to this HuberAggregator, and update the loss and gradient
@@ -97,13 +98,16 @@ private[ml] class HuberAggregator(
       require(weight >= 0.0, s"instance weight, $weight has to be >= 0.0")
 
       if (weight == 0.0) return this
-      val localCoefficients = linear.values
+      val localFeaturesStd = bcFeaturesStd.value
+      val localCoefficients = coefficients
       val localGradientSumArray = gradientSumArray
 
       val margin = {
         var sum = 0.0
         features.foreachNonZero { (index, value) =>
-            sum += localCoefficients(index) * value
+          if (localFeaturesStd(index) != 0.0) {
+            sum += localCoefficients(index) * (value / localFeaturesStd(index))
+          }
         }
         if (fitIntercept) sum += intercept
         sum
@@ -115,7 +119,10 @@ private[ml] class HuberAggregator(
         val linearLossDivSigma = linearLoss / sigma
 
         features.foreachNonZero { (index, value) =>
-          localGradientSumArray(index) -= weight * linearLossDivSigma * value
+          if (localFeaturesStd(index) != 0.0) {
+            localGradientSumArray(index) +=
+              -1.0 * weight * linearLossDivSigma * (value / localFeaturesStd(index))
+          }
         }
         if (fitIntercept) {
           localGradientSumArray(dim - 2) += -1.0 * weight * linearLossDivSigma
@@ -127,7 +134,10 @@ private[ml] class HuberAggregator(
           (sigma + 2.0 * epsilon * math.abs(linearLoss) - sigma * epsilon * epsilon)
 
         features.foreachNonZero { (index, value) =>
-          localGradientSumArray(index) += weight * sign * epsilon * value
+          if (localFeaturesStd(index) != 0.0) {
+            localGradientSumArray(index) +=
+              weight * sign * epsilon * (value / localFeaturesStd(index))
+          }
         }
         if (fitIntercept) {
           localGradientSumArray(dim - 2) += weight * sign * epsilon
@@ -139,75 +149,4 @@ private[ml] class HuberAggregator(
       this
     }
   }
-
-  /**
-   * Add a new training instance block to this HuberAggregator, and update the loss and gradient
-   * of the objective function.
-   *
-   * @param block The instance block of data point to be added.
-   * @return This HuberAggregator object.
-   */
-  def add(block: InstanceBlock): HuberAggregator = {
-    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
-    require(block.weightIter.forall(_ >= 0),
-      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
-
-    if (block.weightIter.forall(_ == 0)) return this
-    val size = block.size
-    val localGradientSumArray = gradientSumArray
-
-    // vec here represents margins or dotProducts
-    val vec = if (fitIntercept && intercept != 0) {
-      new DenseVector(Array.fill(size)(intercept))
-    } else {
-      new DenseVector(Array.ofDim[Double](size))
-    }
-
-    if (fitIntercept) {
-      BLAS.gemv(1.0, block.matrix, linear, 1.0, vec)
-    } else {
-      BLAS.gemv(1.0, block.matrix, linear, 0.0, vec)
-    }
-
-    // in-place convert margins to multipliers
-    // then, vec represents multipliers
-    var i = 0
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        val label = block.getLabel(i)
-        val margin = vec(i)
-        val linearLoss = label - margin
-
-        if (math.abs(linearLoss) <= sigma * epsilon) {
-          lossSum += 0.5 * weight * (sigma + math.pow(linearLoss, 2.0) / sigma)
-          val linearLossDivSigma = linearLoss / sigma
-          val multiplier = -1.0 * weight * linearLossDivSigma
-          vec.values(i) = multiplier
-          localGradientSumArray(dim - 1) += 0.5 * weight * (1.0 - math.pow(linearLossDivSigma, 2.0))
-        } else {
-          lossSum += 0.5 * weight *
-            (sigma + 2.0 * epsilon * math.abs(linearLoss) - sigma * epsilon * epsilon)
-          val sign = if (linearLoss >= 0) -1.0 else 1.0
-          val multiplier = weight * sign * epsilon
-          vec.values(i) = multiplier
-          localGradientSumArray(dim - 1) += 0.5 * weight * (1.0 - epsilon * epsilon)
-        }
-      } else {
-        vec.values(i) = 0.0
-      }
-      i += 1
-    }
-
-    val linearGradSumVec = new DenseVector(Array.ofDim[Double](numFeatures))
-    BLAS.gemv(1.0, block.matrix.transpose, vec, 0.0, linearGradSumVec)
-    linearGradSumVec.foreachNonZero { (i, v) => localGradientSumArray(i) += v }
-    if (fitIntercept) {
-      localGradientSumArray(dim - 2) += vec.values.sum
-    }
-
-    this
-  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala
index a8bda9ca5d245..7a5806dc24aee 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala
@@ -17,8 +17,8 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
-import org.apache.spark.ml.linalg._
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
 
 /**
  * LeastSquaresAggregator computes the gradient and loss for a Least-squared loss function,
@@ -157,25 +157,26 @@ private[ml] class LeastSquaresAggregator(
     labelStd: Double,
     labelMean: Double,
     fitIntercept: Boolean,
-    bcFeaturesStd: Broadcast[Vector],
-    bcFeaturesMean: Broadcast[Vector])(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[InstanceBlock, LeastSquaresAggregator] {
+    bcFeaturesStd: Broadcast[Array[Double]],
+    bcFeaturesMean: Broadcast[Array[Double]])(bcCoefficients: Broadcast[Vector])
+  extends DifferentiableLossAggregator[Instance, LeastSquaresAggregator] {
   require(labelStd > 0.0, s"${this.getClass.getName} requires the label standard " +
     s"deviation to be positive.")
 
-  private val numFeatures = bcFeaturesStd.value.size
+  private val numFeatures = bcFeaturesStd.value.length
   protected override val dim: Int = numFeatures
   // make transient so we do not serialize between aggregation stages
+  @transient private lazy val featuresStd = bcFeaturesStd.value
   @transient private lazy val effectiveCoefAndOffset = {
     val coefficientsArray = bcCoefficients.value.toArray.clone()
     val featuresMean = bcFeaturesMean.value
-    val featuresStd = bcFeaturesStd.value
     var sum = 0.0
     var i = 0
     val len = coefficientsArray.length
     while (i < len) {
       if (featuresStd(i) != 0.0) {
-        sum += coefficientsArray(i) / featuresStd(i) * featuresMean(i)
+        coefficientsArray(i) /=  featuresStd(i)
+        sum += coefficientsArray(i) * featuresMean(i)
       } else {
         coefficientsArray(i) = 0.0
       }
@@ -185,7 +186,7 @@ private[ml] class LeastSquaresAggregator(
     (Vectors.dense(coefficientsArray), offset)
   }
   // do not use tuple assignment above because it will circumvent the @transient tag
-  @transient private lazy val effectiveCoefficientsVec = effectiveCoefAndOffset._1
+  @transient private lazy val effectiveCoefficientsVector = effectiveCoefAndOffset._1
   @transient private lazy val offset = effectiveCoefAndOffset._2
 
   /**
@@ -203,20 +204,16 @@ private[ml] class LeastSquaresAggregator(
 
       if (weight == 0.0) return this
 
-      val localEffectiveCoefficientsVec = effectiveCoefficientsVec
-
-      val diff = {
-        var dot = 0.0
-        features.foreachNonZero { (index, value) =>
-          dot += localEffectiveCoefficientsVec(index) * value
-        }
-        dot - label / labelStd + offset
-      }
+      val diff = BLAS.dot(features, effectiveCoefficientsVector) - label / labelStd + offset
 
       if (diff != 0) {
         val localGradientSumArray = gradientSumArray
+        val localFeaturesStd = featuresStd
         features.foreachNonZero { (index, value) =>
-          localGradientSumArray(index) += weight * diff * value
+          val fStd = localFeaturesStd(index)
+          if (fStd != 0.0) {
+            localGradientSumArray(index) += weight * diff * value / fStd
+          }
         }
         lossSum += weight * diff * diff / 2.0
       }
@@ -224,43 +221,4 @@ private[ml] class LeastSquaresAggregator(
       this
     }
   }
-
-  /**
-   * Add a new training instance block to this LeastSquaresAggregator, and update the loss
-   * and gradient of the objective function.
-   *
-   * @param block The instance block of data point to be added.
-   * @return This LeastSquaresAggregator object.
-   */
-  def add(block: InstanceBlock): LeastSquaresAggregator = {
-    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
-    require(block.weightIter.forall(_ >= 0),
-      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
-
-    if (block.weightIter.forall(_ == 0)) return this
-    val size = block.size
-
-    // vec here represents diffs
-    val vec = new DenseVector(Array.tabulate(size)(i => offset - block.getLabel(i) / labelStd))
-    BLAS.gemv(1.0, block.matrix, effectiveCoefficientsVec, 1.0, vec)
-
-    // in-place convert diffs to multipliers
-    // then, vec represents multipliers
-    var i = 0
-    while (i < size) {
-      val weight = block.getWeight(i)
-      val diff = vec(i)
-      lossSum += weight * diff * diff / 2
-      weightSum += weight
-      val multiplier = weight * diff
-      vec.values(i) = multiplier
-      i += 1
-    }
-
-    val gradSumVec = new DenseVector(gradientSumArray)
-    BLAS.gemv(1.0, block.matrix.transpose, vec, 1.0, gradSumVec)
-
-    this
-  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala
index 76d21995a2c50..f2b3566f8f09e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala
@@ -18,8 +18,8 @@ package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.internal.Logging
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
-import org.apache.spark.ml.linalg._
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.linalg.{DenseVector, Vector}
 import org.apache.spark.mllib.util.MLUtils
 
 /**
@@ -171,6 +171,7 @@ import org.apache.spark.mllib.util.MLUtils
  *
  *
  * @param bcCoefficients The broadcast coefficients corresponding to the features.
+ * @param bcFeaturesStd The broadcast standard deviation values of the features.
  * @param numClasses the number of possible outcomes for k classes classification problem in
  *                   Multinomial Logistic Regression.
  * @param fitIntercept Whether to fit an intercept term.
@@ -182,12 +183,13 @@ import org.apache.spark.mllib.util.MLUtils
  * since this form is optimal for the matrix operations used for prediction.
  */
 private[ml] class LogisticAggregator(
-    numFeatures: Int,
+    bcFeaturesStd: Broadcast[Array[Double]],
     numClasses: Int,
     fitIntercept: Boolean,
     multinomial: Boolean)(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[InstanceBlock, LogisticAggregator] with Logging {
+  extends DifferentiableLossAggregator[Instance, LogisticAggregator] with Logging {
 
+  private val numFeatures = bcFeaturesStd.value.length
   private val numFeaturesPlusIntercept = if (fitIntercept) numFeatures + 1 else numFeatures
   private val coefficientSize = bcCoefficients.value.size
   protected override val dim: Int = coefficientSize
@@ -207,31 +209,6 @@ private[ml] class LogisticAggregator(
       s"got type ${bcCoefficients.value.getClass}.)")
   }
 
-  @transient private lazy val binaryLinear = {
-    if (!multinomial) {
-      if (fitIntercept) {
-        new DenseVector(coefficientsArray.take(numFeatures))
-      } else {
-        new DenseVector(coefficientsArray)
-      }
-    } else {
-      null
-    }
-  }
-
-  @transient private lazy val multinomialLinear = {
-    if (multinomial) {
-      if (fitIntercept) {
-        new DenseMatrix(numClasses, numFeatures, coefficientsArray.take(numClasses * numFeatures))
-      } else {
-        new DenseMatrix(numClasses, numFeatures, coefficientsArray)
-      }
-    } else {
-      null
-    }
-  }
-
-
   if (multinomial && numClasses <= 2) {
     logInfo(s"Multinomial logistic regression for binary classification yields separate " +
       s"coefficients for positive and negative classes. When no regularization is applied, the" +
@@ -242,12 +219,15 @@ private[ml] class LogisticAggregator(
   /** Update gradient and loss using binary loss function. */
   private def binaryUpdateInPlace(features: Vector, weight: Double, label: Double): Unit = {
 
+    val localFeaturesStd = bcFeaturesStd.value
     val localCoefficients = coefficientsArray
     val localGradientArray = gradientSumArray
     val margin = - {
       var sum = 0.0
       features.foreachNonZero { (index, value) =>
-        sum += localCoefficients(index) * value
+        if (localFeaturesStd(index) != 0.0) {
+          sum += localCoefficients(index) * value / localFeaturesStd(index)
+        }
       }
       if (fitIntercept) sum += localCoefficients(numFeaturesPlusIntercept - 1)
       sum
@@ -256,7 +236,9 @@ private[ml] class LogisticAggregator(
     val multiplier = weight * (1.0 / (1.0 + math.exp(margin)) - label)
 
     features.foreachNonZero { (index, value) =>
-      localGradientArray(index) += multiplier * value
+      if (localFeaturesStd(index) != 0.0) {
+        localGradientArray(index) += multiplier * value / localFeaturesStd(index)
+      }
     }
 
     if (fitIntercept) {
@@ -271,61 +253,6 @@ private[ml] class LogisticAggregator(
     }
   }
 
-  /** Update gradient and loss using binary loss function. */
-  private def binaryUpdateInPlace(block: InstanceBlock): Unit = {
-    val size = block.size
-    val localGradientSumArray = gradientSumArray
-
-    // vec here represents margins or negative dotProducts
-    val vec = if (fitIntercept && coefficientsArray.last != 0) {
-      val intercept = coefficientsArray.last
-      new DenseVector(Array.fill(size)(intercept))
-    } else {
-      new DenseVector(Array.ofDim[Double](size))
-    }
-
-    if (fitIntercept) {
-      BLAS.gemv(-1.0, block.matrix, binaryLinear, -1.0, vec)
-    } else {
-      BLAS.gemv(-1.0, block.matrix, binaryLinear, 0.0, vec)
-    }
-
-    // in-place convert margins to multiplier
-    // then, vec represents multiplier
-    var i = 0
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        val label = block.getLabel(i)
-        val margin = vec(i)
-        if (label > 0) {
-          // The following is equivalent to log(1 + exp(margin)) but more numerically stable.
-          lossSum += weight * MLUtils.log1pExp(margin)
-        } else {
-          lossSum += weight * (MLUtils.log1pExp(margin) - margin)
-        }
-        val multiplier = weight * (1.0 / (1.0 + math.exp(margin)) - label)
-        vec.values(i) = multiplier
-      } else {
-        vec.values(i) = 0.0
-      }
-      i += 1
-    }
-
-    if (fitIntercept) {
-      // localGradientSumArray is of size numFeatures+1, so can not
-      // be directly used as the output of BLAS.gemv
-      val linearGradSumVec = new DenseVector(Array.ofDim[Double](numFeatures))
-      BLAS.gemv(1.0, block.matrix.transpose, vec, 0.0, linearGradSumVec)
-      linearGradSumVec.foreachNonZero { (i, v) => localGradientSumArray(i) += v }
-      localGradientSumArray(numFeatures) += vec.values.sum
-    } else {
-      val gradSumVec = new DenseVector(localGradientSumArray)
-      BLAS.gemv(1.0, block.matrix.transpose, vec, 1.0, gradSumVec)
-    }
-  }
-
   /** Update gradient and loss using multinomial (softmax) loss function. */
   private def multinomialUpdateInPlace(features: Vector, weight: Double, label: Double): Unit = {
     // TODO: use level 2 BLAS operations
@@ -333,6 +260,7 @@ private[ml] class LogisticAggregator(
       Note: this can still be used when numClasses = 2 for binary
       logistic regression without pivoting.
      */
+    val localFeaturesStd = bcFeaturesStd.value
     val localCoefficients = coefficientsArray
     val localGradientArray = gradientSumArray
 
@@ -342,10 +270,13 @@ private[ml] class LogisticAggregator(
 
     val margins = new Array[Double](numClasses)
     features.foreachNonZero { (index, value) =>
-      var j = 0
-      while (j < numClasses) {
-        margins(j) += localCoefficients(index * numClasses + j) * value
-        j += 1
+      if (localFeaturesStd(index) != 0.0) {
+        val stdValue = value / localFeaturesStd(index)
+        var j = 0
+        while (j < numClasses) {
+          margins(j) += localCoefficients(index * numClasses + j) * stdValue
+          j += 1
+        }
       }
     }
     var i = 0
@@ -383,10 +314,13 @@ private[ml] class LogisticAggregator(
       multipliers(i) = multipliers(i) / sum - (if (label == i) 1.0 else 0.0)
     }
     features.foreachNonZero { (index, value) =>
-      var j = 0
-      while (j < numClasses) {
-        localGradientArray(index * numClasses + j) += weight * multipliers(j) * value
-        j += 1
+      if (localFeaturesStd(index) != 0.0) {
+        val stdValue = value / localFeaturesStd(index)
+        var j = 0
+        while (j < numClasses) {
+          localGradientArray(index * numClasses + j) += weight * multipliers(j) * stdValue
+          j += 1
+        }
       }
     }
     if (fitIntercept) {
@@ -405,112 +339,6 @@ private[ml] class LogisticAggregator(
     lossSum += weight * loss
   }
 
-  /** Update gradient and loss using multinomial (softmax) loss function. */
-  private def multinomialUpdateInPlace(block: InstanceBlock): Unit = {
-    val size = block.size
-    val localGradientSumArray = gradientSumArray
-
-    // mat here represents margins, shape: S X C
-    val mat = new DenseMatrix(size, numClasses, Array.ofDim[Double](size * numClasses))
-
-    if (fitIntercept) {
-      val intercept = coefficientsArray.takeRight(numClasses)
-      var i = 0
-      while (i < size) {
-        var j = 0
-        while (j < numClasses) {
-          mat.update(i, j, intercept(j))
-          j += 1
-        }
-        i += 1
-      }
-      BLAS.gemm(1.0, block.matrix, multinomialLinear.transpose, 1.0, mat)
-    } else {
-      BLAS.gemm(1.0, block.matrix, multinomialLinear.transpose, 0.0, mat)
-    }
-
-    // in-place convert margins to multipliers
-    // then, mat represents multipliers
-    var i = 0
-    val tmp = Array.ofDim[Double](numClasses)
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        val label = block.getLabel(i)
-
-        var maxMargin = Double.NegativeInfinity
-        var j = 0
-        while (j < numClasses) {
-          tmp(j) = mat(i, j)
-          maxMargin = math.max(maxMargin, tmp(j))
-          j += 1
-        }
-
-        // marginOfLabel is margins(label) in the formula
-        val marginOfLabel = tmp(label.toInt)
-
-        var sum = 0.0
-        j = 0
-        while (j < numClasses) {
-          if (maxMargin > 0) tmp(j) -= maxMargin
-          val exp = math.exp(tmp(j))
-          sum += exp
-          tmp(j) = exp
-          j += 1
-        }
-
-        j = 0
-        while (j < numClasses) {
-          val multiplier = weight * (tmp(j) / sum - (if (label == j) 1.0 else 0.0))
-          mat.update(i, j, multiplier)
-          j += 1
-        }
-
-        if (maxMargin > 0) {
-          lossSum += weight * (math.log(sum) - marginOfLabel + maxMargin)
-        } else {
-          lossSum += weight * (math.log(sum) - marginOfLabel)
-        }
-      } else {
-        var j = 0
-        while (j < numClasses) {
-          mat.update(i, j, 0.0)
-          j += 1
-        }
-      }
-      i += 1
-    }
-
-    // block.matrix:                  S X F, unknown type
-    // mat (multipliers):             S X C, dense
-    // gradSumMat(gradientSumArray):  C X FPI (numFeaturesPlusIntercept), dense
-    block.matrix match {
-      case dm: DenseMatrix if !fitIntercept =>
-        // If fitIntercept==false, gradientSumArray += mat.T X matrix
-        // GEMM requires block.matrix is dense
-        val gradSumMat = new DenseMatrix(numClasses, numFeatures, localGradientSumArray)
-        BLAS.gemm(1.0, mat.transpose, dm, 1.0, gradSumMat)
-
-      case _ =>
-        // Otherwise, use linearGradSumMat (F X C) as a temp matrix:
-        // linearGradSumMat = matrix.T X mat
-        val linearGradSumMat = new DenseMatrix(numFeatures, numClasses,
-          Array.ofDim[Double](numFeatures * numClasses))
-        BLAS.gemm(1.0, block.matrix.transpose, mat, 0.0, linearGradSumMat)
-        linearGradSumMat.foreachActive { (i, j, v) =>
-          if (v != 0) localGradientSumArray(i * numClasses + j) += v
-        }
-
-        if (fitIntercept) {
-          val start = numClasses * numFeatures
-          mat.foreachActive { (i, j, v) =>
-            if (v != 0) localGradientSumArray(start + j) += v
-          }
-        }
-    }
-  }
-
   /**
    * Add a new training instance to this LogisticAggregator, and update the loss and gradient
    * of the objective function.
@@ -535,28 +363,4 @@ private[ml] class LogisticAggregator(
       this
     }
   }
-
-  /**
-   * Add a new training instance block to this LogisticAggregator, and update the loss and gradient
-   * of the objective function.
-   *
-   * @param block The instance block of data point to be added.
-   * @return This LogisticAggregator object.
-   */
-  def add(block: InstanceBlock): this.type = {
-    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
-    require(block.weightIter.forall(_ >= 0),
-      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
-
-    if (block.weightIter.forall(_ == 0)) return this
-
-    if (multinomial) {
-      multinomialUpdateInPlace(block)
-    } else {
-      binaryUpdateInPlace(block)
-    }
-
-    this
-  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
index 4c50f1e3292bc..53ca35ccd0073 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
@@ -29,12 +29,11 @@ import org.json4s._
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.SparkException
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.linalg.{JsonMatrixConverter, JsonVectorConverter, Matrix, Vector}
 import org.apache.spark.ml.util.Identifiable
 
 /**
- * :: DeveloperApi ::
  * A param with self-contained documentation and optionally default value. Primitive-typed param
  * should use the specialized versions, which are more friendly to Java users.
  *
@@ -45,7 +44,6 @@ import org.apache.spark.ml.util.Identifiable
  *                See [[ParamValidators]] for factory methods for common validation functions.
  * @tparam T param value type
  */
-@DeveloperApi
 class Param[T](val parent: String, val name: String, val doc: String, val isValid: T => Boolean)
   extends Serializable {
 
@@ -159,11 +157,9 @@ private[ml] object Param {
 }
 
 /**
- * :: DeveloperApi ::
  * Factory methods for common validation functions for `Param.isValid`.
  * The numerical methods only support Int, Long, Float, and Double.
  */
-@DeveloperApi
 object ParamValidators {
 
   /** (private[param]) Default validation always return true */
@@ -323,10 +319,8 @@ object ParamValidators {
 // specialize primitive-typed params because Java doesn't recognize scala.Double, scala.Int, ...
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Double]` for Java.
  */
-@DeveloperApi
 class DoubleParam(parent: String, name: String, doc: String, isValid: Double => Boolean)
   extends Param[Double](parent, name, doc, isValid) {
 
@@ -383,10 +377,8 @@ private[param] object DoubleParam {
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Int]` for Java.
  */
-@DeveloperApi
 class IntParam(parent: String, name: String, doc: String, isValid: Int => Boolean)
   extends Param[Int](parent, name, doc, isValid) {
 
@@ -412,10 +404,8 @@ class IntParam(parent: String, name: String, doc: String, isValid: Int => Boolea
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Float]` for Java.
  */
-@DeveloperApi
 class FloatParam(parent: String, name: String, doc: String, isValid: Float => Boolean)
   extends Param[Float](parent, name, doc, isValid) {
 
@@ -473,10 +463,8 @@ private object FloatParam {
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Long]` for Java.
  */
-@DeveloperApi
 class LongParam(parent: String, name: String, doc: String, isValid: Long => Boolean)
   extends Param[Long](parent, name, doc, isValid) {
 
@@ -502,10 +490,8 @@ class LongParam(parent: String, name: String, doc: String, isValid: Long => Bool
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Boolean]` for Java.
  */
-@DeveloperApi
 class BooleanParam(parent: String, name: String, doc: String) // No need for isValid
   extends Param[Boolean](parent, name, doc) {
 
@@ -525,10 +511,8 @@ class BooleanParam(parent: String, name: String, doc: String) // No need for isV
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Array[String]]` for Java.
  */
-@DeveloperApi
 class StringArrayParam(parent: Params, name: String, doc: String, isValid: Array[String] => Boolean)
   extends Param[Array[String]](parent, name, doc, isValid) {
 
@@ -550,10 +534,8 @@ class StringArrayParam(parent: Params, name: String, doc: String, isValid: Array
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Array[Double]]` for Java.
  */
-@DeveloperApi
 class DoubleArrayParam(parent: Params, name: String, doc: String, isValid: Array[Double] => Boolean)
   extends Param[Array[Double]](parent, name, doc, isValid) {
 
@@ -580,10 +562,8 @@ class DoubleArrayParam(parent: Params, name: String, doc: String, isValid: Array
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Array[Array[Double]]]` for Java.
  */
-@DeveloperApi
 class DoubleArrayArrayParam(
     parent: Params,
     name: String,
@@ -619,10 +599,8 @@ class DoubleArrayArrayParam(
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Array[Int]]` for Java.
  */
-@DeveloperApi
 class IntArrayParam(parent: Params, name: String, doc: String, isValid: Array[Int] => Boolean)
   extends Param[Array[Int]](parent, name, doc, isValid) {
 
@@ -657,11 +635,9 @@ case class ParamPair[T] @Since("1.2.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * Trait for components that take parameters. This also provides an internal param map to store
  * parameter values attached to the instance.
  */
-@DeveloperApi
 trait Params extends Identifiable with Serializable {
 
   /**
@@ -915,13 +891,11 @@ private[ml] object Params {
 }
 
 /**
- * :: DeveloperApi ::
  * Java-friendly wrapper for [[Params]].
  * Java developers who need to extend [[Params]] should use this class instead.
  * If you need to extend an abstract class which already extends [[Params]], then that abstract
  * class should be Java-friendly as well.
  */
-@DeveloperApi
 abstract class JavaParams extends Params
 
 /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
index 3d1fab8692af7..7fd5f5938b565 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
@@ -104,10 +104,10 @@ private[shared] object SharedParamsCodeGen {
         isValid = "ParamValidators.inArray(Array(\"euclidean\", \"cosine\"))"),
       ParamDesc[String]("validationIndicatorCol", "name of the column that indicates whether " +
         "each row is for training or for validation. False indicates training; true indicates " +
-        "validation"),
+        "validation."),
       ParamDesc[Int]("blockSize", "block size for stacking input data in matrices. Data is " +
         "stacked within partitions. If block size is more than remaining data in a partition " +
-        "then it is adjusted to the size of this data", Some("1024"),
+        "then it is adjusted to the size of this data.",
         isValid = "ParamValidators.gt(0)", isExpertParam = true)
     )
 
@@ -207,7 +207,6 @@ private[shared] object SharedParamsCodeGen {
       | * Trait for shared param $name$defaultValueDoc. This trait may be changed or
       | * removed between minor versions.
       | */
-      |@DeveloperApi
       |trait Has$Name extends Params {
       |
       |  /**
@@ -244,7 +243,6 @@ private[shared] object SharedParamsCodeGen {
         |
         |package org.apache.spark.ml.param.shared
         |
-        |import org.apache.spark.annotation.DeveloperApi
         |import org.apache.spark.ml.param._
         |
         |// DO NOT MODIFY THIS FILE! It was generated by SharedParamsCodeGen.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
index 7fe8ccd973a72..60203eba61ea5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.ml.param.shared
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.ml.param._
 
 // DO NOT MODIFY THIS FILE! It was generated by SharedParamsCodeGen.
@@ -28,7 +27,6 @@ import org.apache.spark.ml.param._
  * Trait for shared param regParam. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasRegParam extends Params {
 
   /**
@@ -45,7 +43,6 @@ trait HasRegParam extends Params {
  * Trait for shared param maxIter. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasMaxIter extends Params {
 
   /**
@@ -62,7 +59,6 @@ trait HasMaxIter extends Params {
  * Trait for shared param featuresCol (default: "features"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasFeaturesCol extends Params {
 
   /**
@@ -81,7 +77,6 @@ trait HasFeaturesCol extends Params {
  * Trait for shared param labelCol (default: "label"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasLabelCol extends Params {
 
   /**
@@ -100,7 +95,6 @@ trait HasLabelCol extends Params {
  * Trait for shared param predictionCol (default: "prediction"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasPredictionCol extends Params {
 
   /**
@@ -119,7 +113,6 @@ trait HasPredictionCol extends Params {
  * Trait for shared param rawPredictionCol (default: "rawPrediction"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasRawPredictionCol extends Params {
 
   /**
@@ -138,7 +131,6 @@ trait HasRawPredictionCol extends Params {
  * Trait for shared param probabilityCol (default: "probability"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasProbabilityCol extends Params {
 
   /**
@@ -157,7 +149,6 @@ trait HasProbabilityCol extends Params {
  * Trait for shared param varianceCol. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasVarianceCol extends Params {
 
   /**
@@ -174,7 +165,6 @@ trait HasVarianceCol extends Params {
  * Trait for shared param threshold. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasThreshold extends Params {
 
   /**
@@ -191,7 +181,6 @@ trait HasThreshold extends Params {
  * Trait for shared param thresholds. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasThresholds extends Params {
 
   /**
@@ -208,7 +197,6 @@ trait HasThresholds extends Params {
  * Trait for shared param inputCol. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasInputCol extends Params {
 
   /**
@@ -225,7 +213,6 @@ trait HasInputCol extends Params {
  * Trait for shared param inputCols. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasInputCols extends Params {
 
   /**
@@ -242,7 +229,6 @@ trait HasInputCols extends Params {
  * Trait for shared param outputCol (default: uid + "__output"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasOutputCol extends Params {
 
   /**
@@ -261,7 +247,6 @@ trait HasOutputCol extends Params {
  * Trait for shared param outputCols. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasOutputCols extends Params {
 
   /**
@@ -278,7 +263,6 @@ trait HasOutputCols extends Params {
  * Trait for shared param numFeatures (default: 262144). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasNumFeatures extends Params {
 
   /**
@@ -297,7 +281,6 @@ trait HasNumFeatures extends Params {
  * Trait for shared param checkpointInterval. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasCheckpointInterval extends Params {
 
   /**
@@ -314,7 +297,6 @@ trait HasCheckpointInterval extends Params {
  * Trait for shared param fitIntercept (default: true). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasFitIntercept extends Params {
 
   /**
@@ -333,7 +315,6 @@ trait HasFitIntercept extends Params {
  * Trait for shared param handleInvalid. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasHandleInvalid extends Params {
 
   /**
@@ -350,7 +331,6 @@ trait HasHandleInvalid extends Params {
  * Trait for shared param standardization (default: true). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasStandardization extends Params {
 
   /**
@@ -369,7 +349,6 @@ trait HasStandardization extends Params {
  * Trait for shared param seed (default: this.getClass.getName.hashCode.toLong). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasSeed extends Params {
 
   /**
@@ -388,7 +367,6 @@ trait HasSeed extends Params {
  * Trait for shared param elasticNetParam. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasElasticNetParam extends Params {
 
   /**
@@ -405,7 +383,6 @@ trait HasElasticNetParam extends Params {
  * Trait for shared param tol. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasTol extends Params {
 
   /**
@@ -422,7 +399,6 @@ trait HasTol extends Params {
  * Trait for shared param relativeError (default: 0.001). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasRelativeError extends Params {
 
   /**
@@ -441,7 +417,6 @@ trait HasRelativeError extends Params {
  * Trait for shared param stepSize. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasStepSize extends Params {
 
   /**
@@ -458,7 +433,6 @@ trait HasStepSize extends Params {
  * Trait for shared param weightCol. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasWeightCol extends Params {
 
   /**
@@ -475,7 +449,6 @@ trait HasWeightCol extends Params {
  * Trait for shared param solver. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasSolver extends Params {
 
   /**
@@ -492,7 +465,6 @@ trait HasSolver extends Params {
  * Trait for shared param aggregationDepth (default: 2). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasAggregationDepth extends Params {
 
   /**
@@ -511,7 +483,6 @@ trait HasAggregationDepth extends Params {
  * Trait for shared param collectSubModels (default: false). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasCollectSubModels extends Params {
 
   /**
@@ -530,7 +501,6 @@ trait HasCollectSubModels extends Params {
  * Trait for shared param loss. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasLoss extends Params {
 
   /**
@@ -547,7 +517,6 @@ trait HasLoss extends Params {
  * Trait for shared param distanceMeasure (default: "euclidean"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasDistanceMeasure extends Params {
 
   /**
@@ -566,33 +535,29 @@ trait HasDistanceMeasure extends Params {
  * Trait for shared param validationIndicatorCol. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasValidationIndicatorCol extends Params {
 
   /**
-   * Param for name of the column that indicates whether each row is for training or for validation. False indicates training; true indicates validation.
+   * Param for name of the column that indicates whether each row is for training or for validation. False indicates training; true indicates validation..
    * @group param
    */
-  final val validationIndicatorCol: Param[String] = new Param[String](this, "validationIndicatorCol", "name of the column that indicates whether each row is for training or for validation. False indicates training; true indicates validation")
+  final val validationIndicatorCol: Param[String] = new Param[String](this, "validationIndicatorCol", "name of the column that indicates whether each row is for training or for validation. False indicates training; true indicates validation.")
 
   /** @group getParam */
   final def getValidationIndicatorCol: String = $(validationIndicatorCol)
 }
 
 /**
- * Trait for shared param blockSize (default: 1024). This trait may be changed or
+ * Trait for shared param blockSize. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasBlockSize extends Params {
 
   /**
-   * Param for block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data.
+   * Param for block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data..
    * @group expertParam
    */
-  final val blockSize: IntParam = new IntParam(this, "blockSize", "block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data", ParamValidators.gt(0))
-
-  setDefault(blockSize, 1024)
+  final val blockSize: IntParam = new IntParam(this, "blockSize", "block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data.", ParamValidators.gt(0))
 
   /** @group expertGetParam */
   final def getBlockSize: Int = $(blockSize)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeClassificationWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeClassifierWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeClassificationWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeClassifierWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeRegressorWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeRegressionWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeRegressorWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/GBTClassificationWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/GBTClassifierWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/GBTClassificationWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/GBTClassifierWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressorWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressionWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressorWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassificationWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassifierWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassificationWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassifierWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressorWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressionWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressorWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
index 002146f89e79a..0fa17c188209a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
@@ -32,7 +32,7 @@ import org.json4s.DefaultFormats
 import org.json4s.JsonDSL._
 
 import org.apache.spark.{Dependency, Partitioner, ShuffleDependency, SparkContext, SparkException}
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.{Estimator, Model}
 import org.apache.spark.ml.linalg.BLAS
@@ -731,21 +731,17 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel]
 
 
 /**
- * :: DeveloperApi ::
  * An implementation of ALS that supports generic ID types, specialized for Int and Long. This is
  * exposed as a developer API for users who do need other ID types. But it is not recommended
  * because it increases the shuffle size and memory requirement during training. For simplicity,
  * users and items must have the same type. The number of distinct users/items should be smaller
  * than 2 billion.
  */
-@DeveloperApi
 object ALS extends DefaultParamsReadable[ALS] with Logging {
 
   /**
-   * :: DeveloperApi ::
    * Rating class for better code readability.
    */
-  @DeveloperApi
   case class Rating[@specialized(Int, Long) ID](user: ID, item: ID, rating: Float)
 
   @Since("1.6.0")
@@ -908,7 +904,6 @@ object ALS extends DefaultParamsReadable[ALS] with Logging {
   }
 
   /**
-   * :: DeveloperApi ::
    * Implementation of the ALS algorithm.
    *
    * This implementation of the ALS factorization algorithm partitions the two sets of factors among
@@ -933,7 +928,6 @@ object ALS extends DefaultParamsReadable[ALS] with Logging {
    * "block" as referring to a subset of an RDD containing the ratings rather than a contiguous
    * submatrix of the ratings matrix.
    */
-  @DeveloperApi
   def train[ID: ClassTag]( // scalastyle:ignore
       ratings: RDD[Rating[ID]],
       rank: Int = 10,
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
index 8c95d25f3b5b8..86989c54bde3c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
@@ -59,7 +59,6 @@ private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
   /** @group getParam */
   @Since("1.6.0")
   def getCensorCol: String = $(censorCol)
-  setDefault(censorCol -> "censor")
 
   /**
    * Param for quantile probabilities array.
@@ -75,7 +74,6 @@ private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
   /** @group getParam */
   @Since("1.6.0")
   def getQuantileProbabilities: Array[Double] = $(quantileProbabilities)
-  setDefault(quantileProbabilities -> Array(0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99))
 
   /**
    * Param for quantiles column name.
@@ -94,6 +92,10 @@ private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
     isDefined(quantilesCol) && $(quantilesCol).nonEmpty
   }
 
+  setDefault(censorCol -> "censor",
+    quantileProbabilities -> Array(0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99),
+    fitIntercept -> true, maxIter -> 100, tol -> 1E-6, aggregationDepth -> 2)
+
   /**
    * Validates and transforms the input schema with the provided param map.
    * @param schema input schema
@@ -150,7 +152,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("1.6.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set the maximum number of iterations.
@@ -159,7 +160,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("1.6.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -169,7 +169,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("1.6.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Suggested depth for treeAggregate (greater than or equal to 2).
@@ -180,7 +179,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("2.1.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   /**
    * Extract [[featuresCol]], [[labelCol]] and [[censorCol]] from input dataset,
@@ -194,8 +192,8 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
       }
   }
 
-  @Since("3.0.0")
-  override def train(dataset: Dataset[_]): AFTSurvivalRegressionModel = instrumented { instr =>
+  override protected def train(
+      dataset: Dataset[_]): AFTSurvivalRegressionModel = instrumented { instr =>
     val instances = extractAFTPoints(dataset)
     val handlePersistence = dataset.storageLevel == StorageLevel.NONE
     if (handlePersistence) instances.persist(StorageLevel.MEMORY_AND_DISK)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
index f7810eb17cf59..ce9881750c558 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
@@ -113,6 +113,10 @@ private[ml] trait FactorizationMachinesParams extends PredictorParams
     "The solver algorithm for optimization. Supported options: " +
       s"${supportedSolvers.mkString(", ")}. (Default adamW)",
     ParamValidators.inArray[String](supportedSolvers))
+
+  setDefault(factorSize -> 8, fitIntercept -> true, fitLinear -> true, regParam -> 0.0,
+    miniBatchFraction -> 1.0, initStd -> 0.01, maxIter -> 100, stepSize -> 1.0, tol -> 1E-6,
+    solver -> AdamW)
 }
 
 private[ml] trait FactorizationMachines extends FactorizationMachinesParams {
@@ -277,12 +281,16 @@ private[regression] trait FMRegressorParams extends FactorizationMachinesParams
  * FM is able to estimate interactions even in problems with huge sparsity
  * (like advertising and recommendation system).
  * FM formula is:
- * {{{
+ * <blockquote>
+ *   $$
+ *   \begin{align}
  *   y = w_0 + \sum\limits^n_{i-1} w_i x_i +
  *     \sum\limits^n_{i=1} \sum\limits^n_{j=i+1} \langle v_i, v_j \rangle x_i x_j
- * }}}
+ *   \end{align}
+ *   $$
+ * </blockquote>
  * First two terms denote global bias and linear term (as same as linear regression),
- * and last term denotes pairwise interactions term. {{{v_i}}} describes the i-th variable
+ * and last term denotes pairwise interactions term. v_i describes the i-th variable
  * with k factors.
  *
  * FM regression model uses MSE loss which can be solved by gradient descent method, and
@@ -305,7 +313,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFactorSize(value: Int): this.type = set(factorSize, value)
-  setDefault(factorSize -> 8)
 
   /**
    * Set whether to fit intercept term.
@@ -315,7 +322,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set whether to fit linear term.
@@ -325,7 +331,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitLinear(value: Boolean): this.type = set(fitLinear, value)
-  setDefault(fitLinear -> true)
 
   /**
    * Set the L2 regularization parameter.
@@ -335,7 +340,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the mini-batch fraction parameter.
@@ -345,7 +349,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMiniBatchFraction(value: Double): this.type = set(miniBatchFraction, value)
-  setDefault(miniBatchFraction -> 1.0)
 
   /**
    * Set the standard deviation of initial coefficients.
@@ -355,7 +358,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setInitStd(value: Double): this.type = set(initStd, value)
-  setDefault(initStd -> 0.01)
 
   /**
    * Set the maximum number of iterations.
@@ -365,7 +367,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the initial step size for the first step (like learning rate).
@@ -375,7 +376,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setStepSize(value: Double): this.type = set(stepSize, value)
-  setDefault(stepSize -> 1.0)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -385,7 +385,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Set the solver algorithm used for optimization.
@@ -396,7 +395,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> AdamW)
 
   /**
    * Set the random seed for weight initialization.
@@ -406,7 +404,7 @@ class FMRegressor @Since("3.0.0") (
   @Since("3.0.0")
   def setSeed(value: Long): this.type = set(seed, value)
 
-  override protected[spark] def train(
+  override protected def train(
       dataset: Dataset[_]
     ): FMRegressionModel = instrumented { instr =>
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
index fa41a98749f32..8fda44c756f58 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
@@ -180,6 +180,9 @@ private[regression] trait GeneralizedLinearRegressionBase extends PredictorParam
       s"${supportedSolvers.mkString(", ")}. (Default irls)",
     ParamValidators.inArray[String](supportedSolvers))
 
+  setDefault(family -> Gaussian.name, variancePower -> 0.0, maxIter -> 25, tol -> 1E-6,
+    regParam -> 0.0, solver -> IRLS)
+
   @Since("2.0.0")
   override def validateAndTransformSchema(
       schema: StructType,
@@ -256,7 +259,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setFamily(value: String): this.type = set(family, value)
-  setDefault(family -> Gaussian.name)
 
   /**
    * Sets the value of param [[variancePower]].
@@ -267,7 +269,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.2.0")
   def setVariancePower(value: Double): this.type = set(variancePower, value)
-  setDefault(variancePower -> 0.0)
 
   /**
    * Sets the value of param [[linkPower]].
@@ -304,7 +305,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 25)
 
   /**
    * Sets the convergence tolerance of iterations.
@@ -315,7 +315,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Sets the regularization parameter for L2 regularization.
@@ -331,7 +330,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Sets the value of param [[weightCol]].
@@ -363,7 +361,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> IRLS)
 
   /**
    * Sets the link prediction (linear predictor) column name.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
index fc59da8a9c165..354c583ecc9ae 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
@@ -28,7 +28,7 @@ import org.apache.spark.SparkException
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.{PipelineStage, PredictorParams}
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg.{Vector, Vectors}
 import org.apache.spark.ml.linalg.BLAS._
 import org.apache.spark.ml.optim.WeightedLeastSquares
@@ -55,7 +55,7 @@ import org.apache.spark.util.VersionUtils.majorMinorVersion
 private[regression] trait LinearRegressionParams extends PredictorParams
     with HasRegParam with HasElasticNetParam with HasMaxIter with HasTol
     with HasFitIntercept with HasStandardization with HasWeightCol with HasSolver
-    with HasAggregationDepth with HasLoss with HasBlockSize {
+    with HasAggregationDepth with HasLoss {
 
   import LinearRegression._
 
@@ -104,6 +104,10 @@ private[regression] trait LinearRegressionParams extends PredictorParams
   @Since("2.3.0")
   def getEpsilon: Double = $(epsilon)
 
+  setDefault(regParam -> 0.0, fitIntercept -> true, standardization -> true,
+    elasticNetParam -> 0.0, maxIter -> 100, tol -> 1E-6, solver -> Auto,
+    aggregationDepth -> 2, loss -> SquaredError, epsilon -> 1.35)
+
   override protected def validateAndTransformSchema(
       schema: StructType,
       fitting: Boolean,
@@ -190,7 +194,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.3.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set if we should fit the intercept.
@@ -200,7 +203,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.5.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Whether to standardize the training features before fitting the model.
@@ -216,7 +218,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.5.0")
   def setStandardization(value: Boolean): this.type = set(standardization, value)
-  setDefault(standardization -> true)
 
   /**
    * Set the ElasticNet mixing parameter.
@@ -232,7 +233,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.4.0")
   def setElasticNetParam(value: Double): this.type = set(elasticNetParam, value)
-  setDefault(elasticNetParam -> 0.0)
 
   /**
    * Set the maximum number of iterations.
@@ -242,7 +242,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.3.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -253,7 +252,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.4.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Whether to over-/under-sample training instances according to the given weights in weightCol.
@@ -282,7 +280,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.6.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> Auto)
 
   /**
    * Suggested depth for treeAggregate (greater than or equal to 2).
@@ -294,7 +291,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("2.1.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   /**
    * Sets the value of param [[loss]].
@@ -304,7 +300,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("2.3.0")
   def setLoss(value: String): this.type = set(loss, value)
-  setDefault(loss -> SquaredError)
 
   /**
    * Sets the value of param [[epsilon]].
@@ -314,16 +309,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("2.3.0")
   def setEpsilon(value: Double): this.type = set(epsilon, value)
-  setDefault(epsilon -> 1.35)
-
-  /**
-   * Set block size for stacking input data in matrices.
-   * Default is 1024.
-   *
-   * @group expertSetParam
-   */
-  @Since("3.0.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
 
   override protected def train(dataset: Dataset[_]): LinearRegressionModel = instrumented { instr =>
     // Extract the number of features before deciding optimization solver.
@@ -363,6 +348,9 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
       return lrModel.setSummary(Some(trainingSummary))
     }
 
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    if (handlePersistence) instances.persist(StorageLevel.MEMORY_AND_DISK)
+
     val (featuresSummarizer, ySummarizer) = instances.treeAggregate(
       (Summarizer.createSummarizerBuffer("mean", "std"),
         Summarizer.createSummarizerBuffer("mean", "std", "count")))(
@@ -398,6 +386,7 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
             s"will be zeros and the intercept will be the mean of the label; as a result, " +
             s"training is not needed.")
         }
+        if (handlePersistence) instances.unpersist()
         val coefficients = Vectors.sparse(numFeatures, Seq.empty)
         val intercept = yMean
 
@@ -426,8 +415,8 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
     // if y is constant (rawYStd is zero), then y cannot be scaled. In this case
     // setting yStd=abs(yMean) ensures that y is not scaled anymore in l-bfgs algorithm.
     val yStd = if (rawYStd > 0) rawYStd else math.abs(yMean)
-    val featuresMean = featuresSummarizer.mean.compressed
-    val featuresStd = featuresSummarizer.std.compressed
+    val featuresMean = featuresSummarizer.mean.toArray
+    val featuresStd = featuresSummarizer.std.toArray
     val bcFeaturesMean = instances.context.broadcast(featuresMean)
     val bcFeaturesStd = instances.context.broadcast(featuresStd)
 
@@ -447,36 +436,23 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
     val effectiveL1RegParam = $(elasticNetParam) * effectiveRegParam
     val effectiveL2RegParam = (1.0 - $(elasticNetParam)) * effectiveRegParam
 
+    val getFeaturesStd = (j: Int) => if (j >= 0 && j < numFeatures) featuresStd(j) else 0.0
     val regularization = if (effectiveL2RegParam != 0.0) {
       val shouldApply = (idx: Int) => idx >= 0 && idx < numFeatures
       Some(new L2Regularization(effectiveL2RegParam, shouldApply,
-        if ($(standardization)) None else Some(featuresStd.apply)))
+        if ($(standardization)) None else Some(getFeaturesStd)))
     } else {
       None
     }
 
-    val standardized = instances.map {
-      case Instance(label, weight, features) =>
-        val featuresStd = bcFeaturesStd.value
-        val array = Array.ofDim[Double](numFeatures)
-        features.foreachNonZero { (i, v) =>
-          val std = featuresStd(i)
-          if (std != 0) array(i) = v / std
-        }
-        Instance(label, weight, Vectors.dense(array))
-    }
-    val blocks = InstanceBlock.blokify(standardized, $(blockSize))
-      .persist(StorageLevel.MEMORY_AND_DISK)
-      .setName(s"training dataset (blockSize=${$(blockSize)})")
-
     val costFun = $(loss) match {
       case SquaredError =>
         val getAggregatorFunc = new LeastSquaresAggregator(yStd, yMean, $(fitIntercept),
           bcFeaturesStd, bcFeaturesMean)(_)
-        new RDDLossFunction(blocks, getAggregatorFunc, regularization, $(aggregationDepth))
+        new RDDLossFunction(instances, getAggregatorFunc, regularization, $(aggregationDepth))
       case Huber =>
-        val getAggregatorFunc = new HuberAggregator(numFeatures, $(fitIntercept), $(epsilon))(_)
-        new RDDLossFunction(blocks, getAggregatorFunc, regularization, $(aggregationDepth))
+        val getAggregatorFunc = new HuberAggregator($(fitIntercept), $(epsilon), bcFeaturesStd)(_)
+        new RDDLossFunction(instances, getAggregatorFunc, regularization, $(aggregationDepth))
     }
 
     val optimizer = $(loss) match {
@@ -542,7 +518,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
         throw new SparkException(msg)
       }
 
-      blocks.unpersist()
       bcFeaturesMean.destroy()
       bcFeaturesStd.destroy()
 
@@ -576,7 +551,7 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
             after the coefficients are converged. See the following discussion for detail.
             http://stats.stackexchange.com/questions/13617/how-is-the-intercept-computed-in-glmnet
             */
-            yMean - dot(Vectors.dense(rawCoefficients), featuresMean)
+            yMean - dot(Vectors.dense(rawCoefficients), Vectors.dense(featuresMean))
           case Huber => parameters(numFeatures)
         }
       } else {
@@ -591,6 +566,8 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
       (Vectors.dense(rawCoefficients).compressed, interceptValue, scaleValue, arrayBuilder.result())
     }
 
+    if (handlePersistence) instances.unpersist()
+
     val model = copyValues(new LinearRegressionModel(uid, coefficients, intercept, scale))
     // Handle possible missing or invalid prediction columns
     val (summaryModel, predictionColName) = model.findSummaryModelAndPredictionCol()
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala
index 314cf422be87e..c28dac6850e85 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.ml.regression
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams}
 
 
@@ -28,7 +27,6 @@ import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams}
  * @tparam Learner  Concrete Estimator type
  * @tparam M  Concrete Model type
  */
-@DeveloperApi
 abstract class Regressor[
     FeaturesType,
     Learner <: Regressor[FeaturesType, Learner, M],
@@ -39,14 +37,11 @@ abstract class Regressor[
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Model produced by a `Regressor`.
  *
  * @tparam FeaturesType  Type of input features.  E.g., [[org.apache.spark.mllib.linalg.Vector]]
  * @tparam M  Concrete Model type.
  */
-@DeveloperApi
 abstract class RegressionModel[FeaturesType, M <: RegressionModel[FeaturesType, M]]
   extends PredictionModel[FeaturesType, M] with PredictorParams {
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala b/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala
index c3321447e3c96..4944e0c061091 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala
@@ -91,8 +91,8 @@ private[image] class ImageFileFormat extends FileFormat with DataSourceRegister
         if (requiredSchema.isEmpty) {
           filteredResult.map(_ => emptyUnsafeRow)
         } else {
-          val converter = RowEncoder(requiredSchema)
-          filteredResult.map(row => converter.toRow(row))
+          val toRow = RowEncoder(requiredSchema).createSerializer()
+          filteredResult.map(row => toRow(row))
         }
       }
     }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
index 6ead4df87fb54..df64de4b10075 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
@@ -100,8 +100,8 @@ private[libsvm] class LibSVMFileFormat
         "though the input. If you know the number in advance, please specify it via " +
         "'numFeatures' option to avoid the extra scan.")
 
-      val paths = files.map(_.getPath.toUri.toString)
-      val parsed = MLUtils.parseLibSVMFile(sparkSession, paths)
+      val paths = files.map(_.getPath.toString)
+      val parsed = MLUtils.parseLibSVMFile(sparkSession, paths, options)
       MLUtils.computeNumFeatures(parsed)
     }
 
@@ -166,7 +166,7 @@ private[libsvm] class LibSVMFileFormat
             LabeledPoint(label, Vectors.sparse(numFeatures, indices, values))
           }
 
-      val converter = RowEncoder(dataSchema)
+      val toRow = RowEncoder(dataSchema).createSerializer()
       val fullOutput = dataSchema.map { f =>
         AttributeReference(f.name, f.dataType, f.nullable, f.metadata)()
       }
@@ -178,7 +178,7 @@ private[libsvm] class LibSVMFileFormat
 
       points.map { pt =>
         val features = if (isSparse) pt.features.toSparse else pt.features.toDense
-        requiredColumns(converter.toRow(Row(pt.label, features)))
+        requiredColumns(toRow(Row(pt.label, features)))
       }
     }
   }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index 162641f605264..67b9166a0f44d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -31,8 +31,10 @@ import org.apache.spark.ml.util.{DefaultParamsReader, DefaultParamsWriter}
 import org.apache.spark.ml.util.DefaultParamsReader.Metadata
 import org.apache.spark.mllib.tree.impurity.ImpurityCalculator
 import org.apache.spark.mllib.tree.model.{DecisionTreeModel => OldDecisionTreeModel}
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Dataset, SparkSession}
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.functions.{col, lit, struct}
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.util.VersionUtils
 import org.apache.spark.util.collection.OpenHashMap
 
 /**
@@ -401,8 +403,13 @@ private[ml] object DecisionTreeModelReadWrite {
     }
 
     val dataPath = new Path(path, "data").toString
-    val data = sparkSession.read.parquet(dataPath).as[NodeData]
-    buildTreeFromNodes(data.collect(), impurityType)
+    var df = sparkSession.read.parquet(dataPath)
+    val (major, minor) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
+    if (major.toInt < 3) {
+      df = df.withColumn("rawCount", lit(-1L))
+    }
+
+    buildTreeFromNodes(df.as[NodeData].collect(), impurityType)
   }
 
   /**
@@ -497,25 +504,36 @@ private[ml] object EnsembleModelReadWrite {
     }
 
     val treesMetadataPath = new Path(path, "treesMetadata").toString
-    val treesMetadataRDD: RDD[(Int, (Metadata, Double))] = sql.read.parquet(treesMetadataPath)
-      .select("treeID", "metadata", "weights").as[(Int, String, Double)].rdd.map {
-      case (treeID: Int, json: String, weights: Double) =>
+    val treesMetadataRDD = sql.read.parquet(treesMetadataPath)
+      .select("treeID", "metadata", "weights")
+      .as[(Int, String, Double)].rdd
+      .map { case (treeID: Int, json: String, weights: Double) =>
         treeID -> ((DefaultParamsReader.parseMetadata(json, treeClassName), weights))
-    }
+      }
 
     val treesMetadataWeights = treesMetadataRDD.sortByKey().values.collect()
     val treesMetadata = treesMetadataWeights.map(_._1)
     val treesWeights = treesMetadataWeights.map(_._2)
 
     val dataPath = new Path(path, "data").toString
-    val nodeData: Dataset[EnsembleNodeData] =
-      sql.read.parquet(dataPath).as[EnsembleNodeData]
-    val rootNodesRDD: RDD[(Int, Node)] =
-      nodeData.rdd.map(d => (d.treeID, d.nodeData)).groupByKey().map {
-        case (treeID: Int, nodeData: Iterable[NodeData]) =>
-          treeID -> DecisionTreeModelReadWrite.buildTreeFromNodes(nodeData.toArray, impurityType)
+    var df = sql.read.parquet(dataPath)
+    val (major, minor) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
+    if (major.toInt < 3) {
+      val newNodeDataCol = df.schema("nodeData").dataType match {
+        case StructType(fields) =>
+          val cols = fields.map(f => col(s"nodeData.${f.name}")) :+ lit(-1L).as("rawCount")
+          struct(cols: _*)
+      }
+      df = df.withColumn("nodeData", newNodeDataCol)
+    }
+
+    val rootNodesRDD = df.as[EnsembleNodeData].rdd
+      .map(d => (d.treeID, d.nodeData))
+      .groupByKey()
+      .map { case (treeID: Int, nodeData: Iterable[NodeData]) =>
+        treeID -> DecisionTreeModelReadWrite.buildTreeFromNodes(nodeData.toArray, impurityType)
       }
-    val rootNodes: Array[Node] = rootNodesRDD.sortByKey().values.collect()
+    val rootNodes = rootNodesRDD.sortByKey().values.collect()
     (metadata, treesMetadata.zip(rootNodes), treesWeights)
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
index 26a639eea2992..19ea8ae4775d8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
@@ -36,7 +36,7 @@ import org.apache.spark.sql.types.{DataType, DoubleType, StructType}
 /**
  * Parameters for Decision Tree-based algorithms.
  *
- * Note: Marked as private and DeveloperApi since this may be made public in the future.
+ * Note: Marked as private since this may be made public in the future.
  */
 private[ml] trait DecisionTreeParams extends PredictorParams
   with HasCheckpointInterval with HasSeed with HasWeightCol {
@@ -47,6 +47,7 @@ private[ml] trait DecisionTreeParams extends PredictorParams
    * (default = "")
    * @group param
    */
+  @Since("3.0.0")
   final val leafCol: Param[String] =
     new Param[String](this, "leafCol", "Leaf indices column name. " +
       "Predicted leaf index of each instance in each tree by preorder")
@@ -139,9 +140,11 @@ private[ml] trait DecisionTreeParams extends PredictorParams
     cacheNodeIds -> false, checkpointInterval -> 10)
 
   /** @group setParam */
+  @Since("3.0.0")
   final def setLeafCol(value: String): this.type = set(leafCol, value)
 
   /** @group getParam */
+  @Since("3.0.0")
   final def getLeafCol: String = $(leafCol)
 
   /** @group getParam */
@@ -317,7 +320,7 @@ private[spark] object TreeEnsembleParams {
 /**
  * Parameters for Decision Tree-based ensemble algorithms.
  *
- * Note: Marked as private and DeveloperApi since this may be made public in the future.
+ * Note: Marked as private since this may be made public in the future.
  */
 private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
 
@@ -330,8 +333,6 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
     "Fraction of the training data used for learning each decision tree, in range (0, 1].",
     ParamValidators.inRange(0, 1, lowerInclusive = false, upperInclusive = true))
 
-  setDefault(subsamplingRate -> 1.0)
-
   /** @group getParam */
   final def getSubsamplingRate: Double = $(subsamplingRate)
 
@@ -383,10 +384,10 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
       || Try(value.toInt).filter(_ > 0).isSuccess
       || Try(value.toDouble).filter(_ > 0).filter(_ <= 1.0).isSuccess)
 
-  setDefault(featureSubsetStrategy -> "auto")
-
   /** @group getParam */
   final def getFeatureSubsetStrategy: String = $(featureSubsetStrategy).toLowerCase(Locale.ROOT)
+
+  setDefault(subsamplingRate -> 1.0, featureSubsetStrategy -> "auto")
 }
 
 /**
@@ -445,8 +446,6 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams {
     new IntParam(this, "numTrees", "Number of trees to train (at least 1)",
     ParamValidators.gtEq(1))
 
-  setDefault(numTrees -> 20)
-
   /** @group getParam */
   final def getNumTrees: Int = $(numTrees)
 
@@ -458,11 +457,11 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams {
   final val bootstrap: BooleanParam = new BooleanParam(this, "bootstrap",
     "Whether bootstrap samples are used when building trees.")
 
-  setDefault(bootstrap -> true)
-
   /** @group getParam */
   @Since("3.0.0")
   final def getBootstrap: Boolean = $(bootstrap)
+
+  setDefault(numTrees -> 20, bootstrap -> true)
 }
 
 private[ml] trait RandomForestClassifierParams
@@ -474,7 +473,7 @@ private[ml] trait RandomForestRegressorParams
 /**
  * Parameters for Gradient-Boosted Tree algorithms.
  *
- * Note: Marked as private and DeveloperApi since this may be made public in the future.
+ * Note: Marked as private since this may be made public in the future.
  */
 private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter with HasStepSize
   with HasValidationIndicatorCol {
@@ -515,9 +514,7 @@ private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter with HasS
     "(a.k.a. learning rate) in interval (0, 1] for shrinking the contribution of each estimator.",
     ParamValidators.inRange(0, 1, lowerInclusive = false, upperInclusive = true))
 
-  setDefault(maxIter -> 20, stepSize -> 0.1, validationTol -> 0.01)
-
-  setDefault(featureSubsetStrategy -> "all")
+  setDefault(maxIter -> 20, stepSize -> 0.1, validationTol -> 0.01, featureSubsetStrategy -> "all")
 
   /** (private[ml]) Create a BoostingStrategy instance to use with the old API. */
   private[ml] def getOldBoostingStrategy(
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala b/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala
index bd213e7362e94..653ffb765f610 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala
@@ -19,18 +19,13 @@ package org.apache.spark.ml.util
 
 import java.util.UUID
 
-import org.apache.spark.annotation.DeveloperApi
-
 
 /**
- * :: DeveloperApi ::
- *
  * Trait for an object with an immutable unique ID that identifies itself and its derivatives.
  *
  * WARNING: There have not yet been final discussions on this API, so it may be broken in future
  *          releases.
  */
-@DeveloperApi
 trait Identifiable {
 
   /**
@@ -41,10 +36,7 @@ trait Identifiable {
   override def toString: String = uid
 }
 
-/**
- * :: DeveloperApi ::
- */
-@DeveloperApi
+
 object Identifiable {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
index dd0c40f33ff2a..fec05ccf15c7c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
@@ -31,7 +31,7 @@ import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.{SparkContext, SparkException}
-import org.apache.spark.annotation.{DeveloperApi, Since, Unstable}
+import org.apache.spark.annotation.{Since, Unstable}
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml._
 import org.apache.spark.ml.classification.{OneVsRest, OneVsRestModel}
@@ -301,8 +301,6 @@ trait GeneralMLWritable extends MLWritable {
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Helper trait for making simple `Params` types writable.  If a `Params` class stores
  * all data as [[org.apache.spark.ml.param.Param]] values, then extending this trait will provide
  * a default implementation of writing saved instances of the class.
@@ -311,7 +309,6 @@ trait GeneralMLWritable extends MLWritable {
  *
  * @see `DefaultParamsReadable`, the counterpart to this trait
  */
-@DeveloperApi
 trait DefaultParamsWritable extends MLWritable { self: Params =>
 
   override def write: MLWriter = new DefaultParamsWriter(this)
@@ -360,8 +357,6 @@ trait MLReadable[T] {
 
 
 /**
- * :: DeveloperApi ::
- *
  * Helper trait for making simple `Params` types readable.  If a `Params` class stores
  * all data as [[org.apache.spark.ml.param.Param]] values, then extending this trait will provide
  * a default implementation of reading saved instances of the class.
@@ -371,7 +366,6 @@ trait MLReadable[T] {
  * @tparam T ML instance type
  * @see `DefaultParamsWritable`, the counterpart to this trait
  */
-@DeveloperApi
 trait DefaultParamsReadable[T] extends MLReadable[T] {
 
   override def read: MLReader[T] = new DefaultParamsReader[T]
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala
index f88f3fce61b33..21eb17dfaacb3 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala
@@ -339,8 +339,10 @@ class LogisticRegressionWithLBFGS
         // Convert our input into a DataFrame
         val spark = SparkSession.builder().sparkContext(input.context).getOrCreate()
         val df = spark.createDataFrame(input.map(_.asML))
+        // Determine if we should cache the DF
+        val handlePersistence = input.getStorageLevel == StorageLevel.NONE
         // Train our model
-        val mlLogisticRegressionModel = lr.train(df)
+        val mlLogisticRegressionModel = lr.train(df, handlePersistence)
         // convert the model
         val weights = Vectors.dense(mlLogisticRegressionModel.coefficients.toArray)
         createModel(weights, mlLogisticRegressionModel.intercept)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
index 7c12697be95c8..7255f12846525 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
@@ -153,30 +153,25 @@ class BisectingKMeans private (
     this
   }
 
-  private[spark] def run(
-      input: RDD[Vector],
-      instr: Option[Instrumentation]): BisectingKMeansModel = {
-    val instances: RDD[(Vector, Double)] = input.map {
-      case (point) => (point, 1.0)
-    }
-    runWithWeight(instances, None)
-  }
-
   private[spark] def runWithWeight(
-      input: RDD[(Vector, Double)],
+      instances: RDD[(Vector, Double)],
+      handlePersistence: Boolean,
       instr: Option[Instrumentation]): BisectingKMeansModel = {
-    val d = input.map(_._1.size).first
+    val d = instances.map(_._1.size).first
     logInfo(s"Feature dimension: $d.")
 
-    val dMeasure: DistanceMeasure = DistanceMeasure.decodeFromString(this.distanceMeasure)
-    // Compute and cache vector norms for fast distance computation.
-    val norms = input.map(d => Vectors.norm(d._1, 2.0))
-    val vectors = input.zip(norms).map {
-      case ((x, weight), norm) => new VectorWithNorm(x, norm, weight)
-    }
-    if (input.getStorageLevel == StorageLevel.NONE) {
+    val dMeasure = DistanceMeasure.decodeFromString(this.distanceMeasure)
+    val norms = instances.map(d => Vectors.norm(d._1, 2.0))
+    val vectors = instances.zip(norms)
+      .map { case ((x, weight), norm) => new VectorWithNorm(x, norm, weight) }
+
+    if (handlePersistence) {
       vectors.persist(StorageLevel.MEMORY_AND_DISK)
+    } else {
+      // Compute and cache vector norms for fast distance computation.
+      norms.persist(StorageLevel.MEMORY_AND_DISK)
     }
+
     var assignments = vectors.map(v => (ROOT_INDEX, v))
     var activeClusters = summarize(d, assignments, dMeasure)
     instr.foreach(_.logNumExamples(activeClusters.values.map(_.size).sum))
@@ -244,13 +239,11 @@ class BisectingKMeans private (
       }
       level += 1
     }
-    if (preIndices != null) {
-      preIndices.unpersist()
-    }
-    if (indices != null) {
-      indices.unpersist()
-    }
-    vectors.unpersist()
+
+    if (preIndices != null) { preIndices.unpersist() }
+    if (indices != null) { indices.unpersist() }
+    if (handlePersistence) { vectors.unpersist() } else { norms.unpersist() }
+
     val clusters = activeClusters ++ inactiveClusters
     val root = buildTree(clusters, dMeasure)
     val totalCost = root.leafNodes.map(_.cost).sum
@@ -264,7 +257,9 @@ class BisectingKMeans private (
    */
   @Since("1.6.0")
   def run(input: RDD[Vector]): BisectingKMeansModel = {
-    run(input, None)
+    val instances = input.map(point => (point, 1.0))
+    val handlePersistence = input.getStorageLevel == StorageLevel.NONE
+    runWithWeight(instances, handlePersistence, None)
   }
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
index a3cf7f96478af..b03151247d8e7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
@@ -209,30 +209,27 @@ class KMeans private (
    */
   @Since("0.8.0")
   def run(data: RDD[Vector]): KMeansModel = {
-    val instances: RDD[(Vector, Double)] = data.map {
-      case (point) => (point, 1.0)
-    }
-    runWithWeight(instances, None)
+    val instances = data.map(point => (point, 1.0))
+    val handlePersistence = data.getStorageLevel == StorageLevel.NONE
+    runWithWeight(instances, handlePersistence, None)
   }
 
   private[spark] def runWithWeight(
-      data: RDD[(Vector, Double)],
+      instances: RDD[(Vector, Double)],
+      handlePersistence: Boolean,
       instr: Option[Instrumentation]): KMeansModel = {
+    val norms = instances.map { case (v, _) => Vectors.norm(v, 2.0) }
+    val vectors = instances.zip(norms)
+      .map { case ((v, w), norm) => new VectorWithNorm(v, norm, w) }
 
-    // Compute squared norms and cache them.
-    val norms = data.map { case (v, _) =>
-      Vectors.norm(v, 2.0)
-    }
-
-    val zippedData = data.zip(norms).map { case ((v, w), norm) =>
-      new VectorWithNorm(v, norm, w)
-    }
-
-    if (data.getStorageLevel == StorageLevel.NONE) {
-      zippedData.persist(StorageLevel.MEMORY_AND_DISK)
+    if (handlePersistence) {
+      vectors.persist(StorageLevel.MEMORY_AND_DISK)
+    } else {
+      // Compute squared norms and cache them.
+      norms.persist(StorageLevel.MEMORY_AND_DISK)
     }
-    val model = runAlgorithmWithWeight(zippedData, instr)
-    zippedData.unpersist()
+    val model = runAlgorithmWithWeight(vectors, instr)
+    if (handlePersistence) { vectors.unpersist() } else { norms.unpersist() }
 
     model
   }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala
index e858ac9844a24..8c43c0b468526 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala
@@ -21,7 +21,7 @@ import java.util.Locale
 
 import breeze.linalg.{DenseVector => BDV}
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaPairRDD
 import org.apache.spark.graphx._
 import org.apache.spark.internal.Logging
@@ -281,21 +281,15 @@ class LDA private (
 
 
   /**
-   * :: DeveloperApi ::
-   *
    * LDAOptimizer used to perform the actual calculation
    */
   @Since("1.4.0")
-  @DeveloperApi
   def getOptimizer: LDAOptimizer = ldaOptimizer
 
   /**
-   * :: DeveloperApi ::
-   *
    * LDAOptimizer used to perform the actual calculation (default = EMLDAOptimizer)
    */
   @Since("1.4.0")
-  @DeveloperApi
   def setOptimizer(optimizer: LDAOptimizer): this.type = {
     this.ldaOptimizer = optimizer
     this
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
index dc90f6c3e3885..b2742ee6ecb5b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
@@ -23,7 +23,7 @@ import breeze.linalg.{all, normalize, sum, DenseMatrix => BDM, DenseVector => BD
 import breeze.numerics.{abs, exp, trigamma}
 import breeze.stats.distributions.{Gamma, RandBasis}
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.graphx._
 import org.apache.spark.graphx.util.PeriodicGraphCheckpointer
 import org.apache.spark.internal.Logging
@@ -33,13 +33,10 @@ import org.apache.spark.storage.StorageLevel
 
 
 /**
- * :: DeveloperApi ::
- *
  * An LDAOptimizer specifies which optimization/learning/inference algorithm to use, and it can
  * hold optimizer-specific parameters for users to set.
  */
 @Since("1.4.0")
-@DeveloperApi
 trait LDAOptimizer {
 
   /*
@@ -62,8 +59,6 @@ trait LDAOptimizer {
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Optimizer for EM algorithm which stores data + parameter graph, plus algorithm parameters.
  *
  * Currently, the underlying implementation uses Expectation-Maximization (EM), implemented
@@ -78,7 +73,6 @@ trait LDAOptimizer {
  *    "On Smoothing and Inference for Topic Models."  UAI, 2009.
  */
 @Since("1.4.0")
-@DeveloperApi
 final class EMLDAOptimizer extends LDAOptimizer {
 
   import LDA._
@@ -253,8 +247,6 @@ final class EMLDAOptimizer extends LDAOptimizer {
 
 
 /**
- * :: DeveloperApi ::
- *
  * An online optimizer for LDA. The Optimizer implements the Online variational Bayes LDA
  * algorithm, which processes a subset of the corpus on each iteration, and updates the term-topic
  * distribution adaptively.
@@ -263,7 +255,6 @@ final class EMLDAOptimizer extends LDAOptimizer {
  *   Hoffman, Blei and Bach, "Online Learning for Latent Dirichlet Allocation." NIPS, 2010.
  */
 @Since("1.4.0")
-@DeveloperApi
 final class OnlineLDAOptimizer extends LDAOptimizer with Logging {
 
   // LDA common parameters
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala
index 21e01ef36814f..78c974e22f2cf 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.feature
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.feature.{StandardScalerModel => NewStandardScalerModel}
 import org.apache.spark.mllib.linalg.{DenseVector, SparseVector, Vector, Vectors}
@@ -96,22 +96,14 @@ class StandardScalerModel @Since("1.3.0") (
   @Since("1.3.0")
   def this(std: Vector) = this(std, null)
 
-  /**
-   * :: DeveloperApi ::
-   */
   @Since("1.3.0")
-  @DeveloperApi
   def setWithMean(withMean: Boolean): this.type = {
     require(!(withMean && this.mean == null), "cannot set withMean to true while mean is null")
     this.withMean = withMean
     this
   }
 
-  /**
-   * :: DeveloperApi ::
-   */
   @Since("1.3.0")
-  @DeveloperApi
   def setWithStd(withStd: Boolean): this.type = {
     require(!(withStd && this.std == null),
       "cannot set withStd to true while std is null")
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala
index 9db725097ae90..ac58ffe3e5509 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala
@@ -17,17 +17,15 @@
 
 package org.apache.spark.mllib.feature
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.mllib.linalg.Vector
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Trait for transformation of a vector
  */
 @Since("1.1.0")
-@DeveloperApi
 trait VectorTransformer extends Serializable {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
index 1f5558dc2a50e..3371b0565acee 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
@@ -29,8 +29,9 @@ private[spark] object BLAS extends Serializable with Logging {
 
   @transient private var _f2jBLAS: NetlibBLAS = _
   @transient private var _nativeBLAS: NetlibBLAS = _
+  private val nativeL1Threshold: Int = 256
 
-  // For level-1 routines, we use Java implementation.
+  // For level-1 function dspmv, use f2jBLAS for better performance.
   private[mllib] def f2jBLAS: NetlibBLAS = {
     if (_f2jBLAS == null) {
       _f2jBLAS = new F2jBLAS
@@ -38,6 +39,14 @@ private[spark] object BLAS extends Serializable with Logging {
     _f2jBLAS
   }
 
+  private[mllib] def getBLAS(vectorSize: Int): NetlibBLAS = {
+    if (vectorSize < nativeL1Threshold) {
+      f2jBLAS
+    } else {
+      nativeBLAS
+    }
+  }
+
   /**
    * y += a * x
    */
@@ -65,7 +74,7 @@ private[spark] object BLAS extends Serializable with Logging {
    */
   private def axpy(a: Double, x: DenseVector, y: DenseVector): Unit = {
     val n = x.size
-    f2jBLAS.daxpy(n, a, x.values, 1, y.values, 1)
+    getBLAS(n).daxpy(n, a, x.values, 1, y.values, 1)
   }
 
   /**
@@ -96,7 +105,7 @@ private[spark] object BLAS extends Serializable with Logging {
   private[spark] def axpy(a: Double, X: DenseMatrix, Y: DenseMatrix): Unit = {
     require(X.numRows == Y.numRows && X.numCols == Y.numCols, "Dimension mismatch: " +
       s"size(X) = ${(X.numRows, X.numCols)} but size(Y) = ${(Y.numRows, Y.numCols)}.")
-    f2jBLAS.daxpy(X.numRows * X.numCols, a, X.values, 1, Y.values, 1)
+    getBLAS(X.values.length).daxpy(X.numRows * X.numCols, a, X.values, 1, Y.values, 1)
   }
 
   /**
@@ -125,7 +134,7 @@ private[spark] object BLAS extends Serializable with Logging {
    */
   private def dot(x: DenseVector, y: DenseVector): Double = {
     val n = x.size
-    f2jBLAS.ddot(n, x.values, 1, y.values, 1)
+    getBLAS(n).ddot(n, x.values, 1, y.values, 1)
   }
 
   /**
@@ -220,16 +229,16 @@ private[spark] object BLAS extends Serializable with Logging {
   def scal(a: Double, x: Vector): Unit = {
     x match {
       case sx: SparseVector =>
-        f2jBLAS.dscal(sx.values.length, a, sx.values, 1)
+        getBLAS(sx.values.length).dscal(sx.values.length, a, sx.values, 1)
       case dx: DenseVector =>
-        f2jBLAS.dscal(dx.values.length, a, dx.values, 1)
+        getBLAS(dx.size).dscal(dx.values.length, a, dx.values, 1)
       case _ =>
         throw new IllegalArgumentException(s"scal doesn't support vector type ${x.getClass}.")
     }
   }
 
   // For level-3 routines, we use the native BLAS.
-  private def nativeBLAS: NetlibBLAS = {
+  private[mllib] def nativeBLAS: NetlibBLAS = {
     if (_nativeBLAS == null) {
       _nativeBLAS = NativeBLAS
     }
@@ -356,7 +365,7 @@ private[spark] object BLAS extends Serializable with Logging {
     if (alpha == 0.0 && beta == 1.0) {
       logDebug("gemm: alpha is equal to 0 and beta is equal to 1. Returning C.")
     } else if (alpha == 0.0) {
-      f2jBLAS.dscal(C.values.length, beta, C.values, 1)
+      getBLAS(C.values.length).dscal(C.values.length, beta, C.values, 1)
     } else {
       A match {
         case sparse: SparseMatrix => gemm(alpha, sparse, B, beta, C)
@@ -462,7 +471,7 @@ private[spark] object BLAS extends Serializable with Logging {
     } else {
       // Scale matrix first if `beta` is not equal to 1.0
       if (beta != 1.0) {
-        f2jBLAS.dscal(C.values.length, beta, C.values, 1)
+        getBLAS(C.values.length).dscal(C.values.length, beta, C.values, 1)
       }
       // Perform matrix multiplication and add to C. The rows of A are multiplied by the columns of
       // B, and added to C.
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
index 83187d65530ad..57edc965112ef 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
@@ -1055,7 +1055,7 @@ object Matrices {
     DenseMatrix.rand(numRows, numCols, rng)
 
   /**
-   * Generate a `SparseMatrix` consisting of `i.i.d.` gaussian random numbers.
+   * Generate a `SparseMatrix` consisting of `i.i.d.` uniform random numbers.
    * @param numRows number of rows of the matrix
    * @param numCols number of columns of the matrix
    * @param density the desired density for the matrix
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
index 20e26cee9e0d6..07b9d91c1f59b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
@@ -786,11 +786,15 @@ class RowMatrix @Since("1.0.0") (
    * Based on the formulae: (numPartitions)^(1/depth) * objectSize <= DriverMaxResultSize
    * @param aggregatedObjectSizeInBytes the size, in megabytes, of the object being tree aggregated
    */
-  private[spark] def getTreeAggregateIdealDepth(aggregatedObjectSizeInBytes: Long) = {
+  private[spark] def getTreeAggregateIdealDepth(aggregatedObjectSizeInBytes: Long): Int = {
     require(aggregatedObjectSizeInBytes > 0,
       "Cannot compute aggregate depth heuristic based on a zero-size object to aggregate")
 
     val maxDriverResultSizeInBytes = rows.conf.get[Long](MAX_RESULT_SIZE)
+    if (maxDriverResultSizeInBytes <= 0) {
+      // Unlimited result size, so 1 is OK
+      return 1
+    }
 
     require(maxDriverResultSizeInBytes > aggregatedObjectSizeInBytes,
       s"Cannot aggregate object of size $aggregatedObjectSizeInBytes Bytes, "
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala
index 46a18295d839b..cdeed33c92381 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala
@@ -17,16 +17,13 @@
 
 package org.apache.spark.mllib.optimization
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.mllib.linalg.{DenseVector, Vector, Vectors}
 import org.apache.spark.mllib.linalg.BLAS.{axpy, dot, scal}
 import org.apache.spark.mllib.util.MLUtils
 
 /**
- * :: DeveloperApi ::
  * Class used to compute the gradient for a loss function, given a single data point.
  */
-@DeveloperApi
 abstract class Gradient extends Serializable {
   /**
    * Compute the gradient and loss given the features of a single data point.
@@ -58,7 +55,6 @@ abstract class Gradient extends Serializable {
 }
 
 /**
- * :: DeveloperApi ::
  * Compute gradient and loss for a multinomial logistic loss function, as used
  * in multi-class classification (it is also used in binary logistic regression).
  *
@@ -162,7 +158,6 @@ abstract class Gradient extends Serializable {
  *                   Multinomial Logistic Regression. By default, it is binary logistic regression
  *                   so numClasses will be set to 2.
  */
-@DeveloperApi
 class LogisticGradient(numClasses: Int) extends Gradient {
 
   def this() = this(2)
@@ -275,13 +270,11 @@ class LogisticGradient(numClasses: Int) extends Gradient {
 }
 
 /**
- * :: DeveloperApi ::
  * Compute gradient and loss for a Least-squared loss function, as used in linear regression.
  * This is correct for the averaged least squares loss function (mean squared error)
  *              L = 1/2n ||A weights-y||^2
  * See also the documentation for the precise formulation.
  */
-@DeveloperApi
 class LeastSquaresGradient extends Gradient {
   override def compute(data: Vector, label: Double, weights: Vector): (Vector, Double) = {
     val diff = dot(data, weights) - label
@@ -303,13 +296,11 @@ class LeastSquaresGradient extends Gradient {
 }
 
 /**
- * :: DeveloperApi ::
  * Compute gradient and loss for a Hinge loss function, as used in SVM binary classification.
  * See also the documentation for the precise formulation.
  *
  * @note This assumes that the labels are {0,1}
  */
-@DeveloperApi
 class HingeGradient extends Gradient {
   override def compute(data: Vector, label: Double, weights: Vector): (Vector, Double) = {
     val dotProduct = dot(data, weights)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
index ffe3964981a18..d9031661fbc85 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
@@ -21,7 +21,6 @@ import scala.collection.mutable.ArrayBuffer
 
 import breeze.linalg.{norm, DenseVector => BDV}
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
 import org.apache.spark.rdd.RDD
@@ -124,13 +123,11 @@ class GradientDescent private[spark] (private var gradient: Gradient, private va
   }
 
   /**
-   * :: DeveloperApi ::
    * Runs gradient descent on the given training data.
    * @param data training data
    * @param initialWeights initial weights
    * @return solution vector
    */
-  @DeveloperApi
   def optimize(data: RDD[(Double, Vector)], initialWeights: Vector): Vector = {
     val (weights, _) = GradientDescent.runMiniBatchSGD(
       data,
@@ -148,10 +145,8 @@ class GradientDescent private[spark] (private var gradient: Gradient, private va
 }
 
 /**
- * :: DeveloperApi ::
  * Top-level method to run gradient descent.
  */
-@DeveloperApi
 object GradientDescent extends Logging {
   /**
    * Run stochastic gradient descent (SGD) in parallel using mini batches.
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
index 65c1159723c80..1ee9241104f87 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
@@ -22,21 +22,18 @@ import scala.collection.mutable
 import breeze.linalg.{DenseVector => BDV}
 import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS}
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
 import org.apache.spark.mllib.linalg.BLAS.axpy
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Class used to solve an optimization problem using Limited-memory BFGS.
  * Reference: <a href="http://en.wikipedia.org/wiki/Limited-memory_BFGS">
  * Wikipedia on Limited-memory BFGS</a>
  * @param gradient Gradient function to be used.
  * @param updater Updater to be used to update weights after every iteration.
  */
-@DeveloperApi
 class LBFGS(private var gradient: Gradient, private var updater: Updater)
   extends Optimizer with Logging {
 
@@ -154,10 +151,8 @@ class LBFGS(private var gradient: Gradient, private var updater: Updater)
 }
 
 /**
- * :: DeveloperApi ::
  * Top-level method to run L-BFGS.
  */
-@DeveloperApi
 object LBFGS extends Logging {
   /**
    * Run Limited-memory BFGS (L-BFGS) in parallel.
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala
index d8e56720967d8..582b520e08eee 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala
@@ -17,15 +17,12 @@
 
 package org.apache.spark.mllib.optimization
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.mllib.linalg.Vector
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Trait for optimization problem solvers.
  */
-@DeveloperApi
 trait Optimizer extends Serializable {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala
index 142f0ec6b9021..1865dd96ce72d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala
@@ -21,11 +21,9 @@ import scala.math._
 
 import breeze.linalg.{axpy => brzAxpy, norm => brzNorm, Vector => BV}
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
 
 /**
- * :: DeveloperApi ::
  * Class used to perform steps (weight update) using Gradient Descent methods.
  *
  * For general minimization problems, or for regularized problems of the form
@@ -37,7 +35,6 @@ import org.apache.spark.mllib.linalg.{Vector, Vectors}
  * The updater is responsible to also perform the update coming from the
  * regularization term R(w) (if any regularization is used).
  */
-@DeveloperApi
 abstract class Updater extends Serializable {
   /**
    * Compute an updated value for weights given the gradient, stepSize, iteration number and
@@ -62,11 +59,9 @@ abstract class Updater extends Serializable {
 }
 
 /**
- * :: DeveloperApi ::
  * A simple updater for gradient descent *without* any regularization.
  * Uses a step-size decreasing with the square root of the number of iterations.
  */
-@DeveloperApi
 class SimpleUpdater extends Updater {
   override def compute(
       weightsOld: Vector,
@@ -83,7 +78,6 @@ class SimpleUpdater extends Updater {
 }
 
 /**
- * :: DeveloperApi ::
  * Updater for L1 regularized problems.
  *          R(w) = ||w||_1
  * Uses a step-size decreasing with the square root of the number of iterations.
@@ -101,7 +95,6 @@ class SimpleUpdater extends Updater {
  *
  * Equivalently, set weight component to signum(w) * max(0.0, abs(w) - shrinkageVal)
  */
-@DeveloperApi
 class L1Updater extends Updater {
   override def compute(
       weightsOld: Vector,
@@ -128,12 +121,10 @@ class L1Updater extends Updater {
 }
 
 /**
- * :: DeveloperApi ::
  * Updater for L2 regularized problems.
  *          R(w) = 1/2 ||w||^2
  * Uses a step-size decreasing with the square root of the number of iterations.
  */
-@DeveloperApi
 class SquaredL2Updater extends Updater {
   override def compute(
       weightsOld: Vector,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala b/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala
index 340386c7e7bec..32471b0a64d9e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala
@@ -23,16 +23,14 @@ import javax.xml.transform.stream.StreamResult
 import org.jpmml.model.JAXBUtil
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.pmml.export.PMMLModelExportFactory
 
 /**
- * :: DeveloperApi ::
  * Export model to the PMML format
  * Predictive Model Markup Language (PMML) is an XML-based file format
  * developed by the Data Mining Group (www.dmg.org).
  */
-@DeveloperApi
 @Since("1.4.0")
 trait PMMLExportable {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala
index d3b548832bb21..3a2ecf17d88d7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala
@@ -19,14 +19,12 @@ package org.apache.spark.mllib.random
 
 import org.apache.commons.math3.distribution._
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.util.random.{Pseudorandom, XORShiftRandom}
 
 /**
- * :: DeveloperApi ::
  * Trait for random data generators that generate i.i.d. data.
  */
-@DeveloperApi
 @Since("1.1.0")
 trait RandomDataGenerator[T] extends Pseudorandom with Serializable {
 
@@ -45,10 +43,8 @@ trait RandomDataGenerator[T] extends Pseudorandom with Serializable {
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from U[0.0, 1.0]
  */
-@DeveloperApi
 @Since("1.1.0")
 class UniformGenerator extends RandomDataGenerator[Double] {
 
@@ -68,10 +64,8 @@ class UniformGenerator extends RandomDataGenerator[Double] {
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the standard normal distribution.
  */
-@DeveloperApi
 @Since("1.1.0")
 class StandardNormalGenerator extends RandomDataGenerator[Double] {
 
@@ -91,12 +85,10 @@ class StandardNormalGenerator extends RandomDataGenerator[Double] {
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the Poisson distribution with the given mean.
  *
  * @param mean mean for the Poisson distribution.
  */
-@DeveloperApi
 @Since("1.1.0")
 class PoissonGenerator @Since("1.1.0") (
     @Since("1.1.0") val mean: Double) extends RandomDataGenerator[Double] {
@@ -116,12 +108,10 @@ class PoissonGenerator @Since("1.1.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the exponential distribution with the given mean.
  *
  * @param mean mean for the exponential distribution.
  */
-@DeveloperApi
 @Since("1.3.0")
 class ExponentialGenerator @Since("1.3.0") (
     @Since("1.3.0") val mean: Double) extends RandomDataGenerator[Double] {
@@ -141,13 +131,11 @@ class ExponentialGenerator @Since("1.3.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the gamma distribution with the given shape and scale.
  *
  * @param shape shape for the gamma distribution.
  * @param scale scale for the gamma distribution
  */
-@DeveloperApi
 @Since("1.3.0")
 class GammaGenerator @Since("1.3.0") (
     @Since("1.3.0") val shape: Double,
@@ -168,14 +156,12 @@ class GammaGenerator @Since("1.3.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the log normal distribution with the
  * given mean and standard deviation.
  *
  * @param mean mean for the log normal distribution.
  * @param std standard deviation for the log normal distribution
  */
-@DeveloperApi
 @Since("1.3.0")
 class LogNormalGenerator @Since("1.3.0") (
     @Since("1.3.0") val mean: Double,
@@ -196,14 +182,12 @@ class LogNormalGenerator @Since("1.3.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the Weibull distribution with the
  * given shape and scale parameter.
  *
  * @param alpha shape parameter for the Weibull distribution.
  * @param beta scale parameter for the Weibull distribution.
  */
-@DeveloperApi
 class WeibullGenerator(
     val alpha: Double,
     val beta: Double) extends RandomDataGenerator[Double] {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala
index 258b1763bba86..eacccfcc4d59f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala
@@ -20,7 +20,7 @@ package org.apache.spark.mllib.random
 import scala.reflect.ClassTag
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.{JavaDoubleRDD, JavaRDD, JavaSparkContext}
 import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
 import org.apache.spark.mllib.linalg.Vector
@@ -372,7 +372,6 @@ object RandomRDDs {
 
 
   /**
-   * :: DeveloperApi ::
    * Generates an RDD comprised of `i.i.d.` samples produced by the input RandomDataGenerator.
    *
    * @param sc SparkContext used to create the RDD.
@@ -382,7 +381,6 @@ object RandomRDDs {
    * @param seed Random seed (default: a random long integer).
    * @return RDD[T] comprised of `i.i.d.` samples produced by generator.
    */
-  @DeveloperApi
   @Since("1.1.0")
   def randomRDD[T: ClassTag](
       sc: SparkContext,
@@ -394,7 +392,6 @@ object RandomRDDs {
   }
 
   /**
-   * :: DeveloperApi ::
    * Generates an RDD comprised of `i.i.d.` samples produced by the input RandomDataGenerator.
    *
    * @param jsc JavaSparkContext used to create the RDD.
@@ -404,7 +401,6 @@ object RandomRDDs {
    * @param seed Random seed (default: a random long integer).
    * @return RDD[T] comprised of `i.i.d.` samples produced by generator.
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaRDD[T](
       jsc: JavaSparkContext,
@@ -418,10 +414,8 @@ object RandomRDDs {
   }
 
   /**
-   * :: DeveloperApi ::
    * `RandomRDDs.randomJavaRDD` with the default seed.
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaRDD[T](
     jsc: JavaSparkContext,
@@ -432,10 +426,8 @@ object RandomRDDs {
   }
 
   /**
-   * :: DeveloperApi ::
    * `RandomRDDs.randomJavaRDD` with the default seed & numPartitions
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaRDD[T](
       jsc: JavaSparkContext,
@@ -831,7 +823,6 @@ object RandomRDDs {
 
 
   /**
-   * :: DeveloperApi ::
    * Generates an RDD[Vector] with vectors containing `i.i.d.` samples produced by the
    * input RandomDataGenerator.
    *
@@ -843,7 +834,6 @@ object RandomRDDs {
    * @param seed Random seed (default: a random long integer).
    * @return RDD[Vector] with vectors containing `i.i.d.` samples produced by generator.
    */
-  @DeveloperApi
   @Since("1.1.0")
   def randomVectorRDD(sc: SparkContext,
       generator: RandomDataGenerator[Double],
@@ -856,10 +846,8 @@ object RandomRDDs {
   }
 
   /**
-   * :: DeveloperApi ::
    * Java-friendly version of `RandomRDDs.randomVectorRDD`.
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaVectorRDD(
       jsc: JavaSparkContext,
@@ -871,11 +859,9 @@ object RandomRDDs {
     randomVectorRDD(jsc.sc, generator, numRows, numCols, numPartitions, seed).toJavaRDD()
   }
 
-  /**
-   * :: DeveloperApi ::
+  /** ::
    * `RandomRDDs.randomJavaVectorRDD` with the default seed.
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaVectorRDD(
       jsc: JavaSparkContext,
@@ -887,10 +873,8 @@ object RandomRDDs {
   }
 
   /**
-   * :: DeveloperApi ::
    * `RandomRDDs.randomJavaVectorRDD` with the default number of partitions and the default seed.
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaVectorRDD(
       jsc: JavaSparkContext,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala b/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala
index e28e1af5b0a26..8f78bcc15347f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala
@@ -20,15 +20,12 @@ package org.apache.spark.mllib.rdd
 import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.rdd.RDD
 import org.apache.spark.util.BoundedPriorityQueue
 
 /**
- * :: DeveloperApi ::
  * Machine learning specific Pair RDD functions.
  */
-@DeveloperApi
 class MLPairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) extends Serializable {
   /**
    * Returns the top k (largest) elements for each key from this RDD as defined by the specified
@@ -51,10 +48,6 @@ class MLPairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) extends Se
   }
 }
 
-/**
- * :: DeveloperApi ::
- */
-@DeveloperApi
 object MLPairRDDFunctions {
   /** Implicit conversion from a pair RDD to MLPairRDDFunctions. */
   implicit def fromPairRDD[K: ClassTag, V: ClassTag](rdd: RDD[(K, V)]): MLPairRDDFunctions[K, V] =
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala b/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala
index 32e6ecf6308e0..1a13cbb5191c6 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala
@@ -20,14 +20,11 @@ package org.apache.spark.mllib.rdd
 import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Machine learning specific RDD functions.
  */
-@DeveloperApi
 class RDDFunctions[T: ClassTag](self: RDD[T]) extends Serializable {
 
   /**
@@ -54,10 +51,7 @@ class RDDFunctions[T: ClassTag](self: RDD[T]) extends Serializable {
 
 }
 
-/**
- * :: DeveloperApi ::
- */
-@DeveloperApi
+
 object RDDFunctions {
 
   /** Implicit conversion from an RDD to RDDFunctions. */
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala
index f3f15ba0d0f2c..1ae0b580a3716 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.recommendation
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.recommendation.{ALS => NewALS}
@@ -195,12 +195,10 @@ class ALS private (
   }
 
   /**
-   * :: DeveloperApi ::
    * Sets storage level for intermediate RDDs (user/product in/out links). The default value is
    * `MEMORY_AND_DISK`. Users can change it to a serialized storage, e.g., `MEMORY_AND_DISK_SER` and
    * set `spark.rdd.compress` to `true` to reduce the space requirement, at the cost of speed.
    */
-  @DeveloperApi
   @Since("1.1.0")
   def setIntermediateRDDStorageLevel(storageLevel: StorageLevel): this.type = {
     require(storageLevel != StorageLevel.NONE,
@@ -210,13 +208,11 @@ class ALS private (
   }
 
   /**
-   * :: DeveloperApi ::
    * Sets storage level for final RDDs (user/product used in MatrixFactorizationModel). The default
    * value is `MEMORY_AND_DISK`. Users can change it to a serialized storage, e.g.
    * `MEMORY_AND_DISK_SER` and set `spark.rdd.compress` to `true` to reduce the space requirement,
    * at the cost of speed.
    */
-  @DeveloperApi
   @Since("1.3.0")
   def setFinalRDDStorageLevel(storageLevel: StorageLevel): this.type = {
     this.finalRDDStorageLevel = storageLevel
@@ -224,14 +220,12 @@ class ALS private (
   }
 
   /**
-   * :: DeveloperApi ::
    * Set period (in iterations) between checkpoints (default = 10). Checkpointing helps with
    * recovery (when nodes fail) and StackOverflow exceptions caused by long lineage. It also helps
    * with eliminating temporary shuffle files on disk, which can be important when there are many
    * ALS iterations. If the checkpoint directory is not set in [[org.apache.spark.SparkContext]],
    * this setting is ignored.
    */
-  @DeveloperApi
   @Since("1.4.0")
   def setCheckpointInterval(checkpointInterval: Int): this.type = {
     this.checkpointInterval = checkpointInterval
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala
index ef3ff2b98a5e6..6d0b3fa791d6e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.mllib.regression
 
 import org.apache.spark.SparkException
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.feature.StandardScaler
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
@@ -28,7 +28,6 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.storage.StorageLevel
 
 /**
- * :: DeveloperApi ::
  * GeneralizedLinearModel (GLM) represents a model trained using
  * GeneralizedLinearAlgorithm. GLMs consist of a weight vector and
  * an intercept.
@@ -38,7 +37,6 @@ import org.apache.spark.storage.StorageLevel
  *
  */
 @Since("0.8.0")
-@DeveloperApi
 abstract class GeneralizedLinearModel @Since("1.0.0") (
     @Since("1.0.0") val weights: Vector,
     @Since("0.8.0") val intercept: Double)
@@ -94,13 +92,11 @@ abstract class GeneralizedLinearModel @Since("1.0.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * GeneralizedLinearAlgorithm implements methods to train a Generalized Linear Model (GLM).
  * This class should be extended with an Optimizer to create a new GLM.
  *
  */
 @Since("0.8.0")
-@DeveloperApi
 abstract class GeneralizedLinearAlgorithm[M <: GeneralizedLinearModel]
   extends Logging with Serializable {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala
index f44c8fe351459..eb83f9a6b75af 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala
@@ -19,7 +19,7 @@ package org.apache.spark.mllib.regression
 
 import scala.reflect.ClassTag
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.linalg.Vector
@@ -27,7 +27,6 @@ import org.apache.spark.streaming.api.java.{JavaDStream, JavaPairDStream}
 import org.apache.spark.streaming.dstream.DStream
 
 /**
- * :: DeveloperApi ::
  * StreamingLinearAlgorithm implements methods for continuously
  * training a generalized linear model on streaming data,
  * and using it for prediction on (possibly different) streaming data.
@@ -56,7 +55,6 @@ import org.apache.spark.streaming.dstream.DStream
  *
  */
 @Since("1.1.0")
-@DeveloperApi
 abstract class StreamingLinearAlgorithm[
     M <: GeneralizedLinearModel,
     A <: GeneralizedLinearAlgorithm[M]] extends Logging {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala
index 00c1da66d7df6..829cce6eac73e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.stat
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
 
 /**
- * :: DeveloperApi ::
  * MultivariateOnlineSummarizer implements [[MultivariateStatisticalSummary]] to compute the mean,
  * variance, minimum, maximum, counts, and nonzero counts for instances in sparse or dense vector
  * format in an online fashion.
@@ -41,7 +40,6 @@ import org.apache.spark.mllib.linalg.{Vector, Vectors}
  * Reliability weights (Wikipedia)</a>.
  */
 @Since("1.1.0")
-@DeveloperApi
 class MultivariateOnlineSummarizer extends MultivariateStatisticalSummary with Serializable {
 
   private var n = 0
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala
index 9a746dcf35556..0b6341b227012 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala
@@ -19,12 +19,11 @@ package org.apache.spark.mllib.stat.distribution
 
 import breeze.linalg.{diag, eigSym, max, DenseMatrix => DBM, DenseVector => DBV, Vector => BV}
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.{Matrices, Matrix, Vector, Vectors}
 import org.apache.spark.mllib.util.MLUtils
 
 /**
- * :: DeveloperApi ::
  * This class provides basic functionality for a Multivariate Gaussian (Normal) Distribution. In
  * the event that the covariance matrix is singular, the density will be computed in a
  * reduced dimensional subspace under which the distribution is supported.
@@ -35,7 +34,6 @@ import org.apache.spark.mllib.util.MLUtils
  * @param sigma The covariance matrix of the distribution
  */
 @Since("1.3.0")
-@DeveloperApi
 class MultivariateGaussian @Since("1.3.0") (
     @Since("1.3.0") val mu: Vector,
     @Since("1.3.0") val sigma: Matrix) extends Serializable {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala
index f01a98e74886b..f62d3810ed3ed 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.tree.impurity
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 /**
  * Class for calculating entropy during multiclass classification.
@@ -28,14 +28,12 @@ object Entropy extends Impurity {
   private[tree] def log2(x: Double) = scala.math.log(x) / scala.math.log(2)
 
   /**
-   * :: DeveloperApi ::
    * information calculation for multiclass classification
    * @param counts Array[Double] with counts for each label
    * @param totalCount sum of counts for all labels
    * @return information value, or 0 if totalCount = 0
    */
   @Since("1.1.0")
-  @DeveloperApi
   override def calculate(counts: Array[Double], totalCount: Double): Double = {
     if (totalCount == 0) {
       return 0
@@ -55,7 +53,6 @@ object Entropy extends Impurity {
   }
 
   /**
-   * :: DeveloperApi ::
    * variance calculation
    * @param count number of instances
    * @param sum sum of labels
@@ -63,7 +60,6 @@ object Entropy extends Impurity {
    * @return information value, or 0 if count = 0
    */
   @Since("1.0.0")
-  @DeveloperApi
   override def calculate(count: Double, sum: Double, sumSquares: Double): Double =
     throw new UnsupportedOperationException("Entropy.calculate")
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala
index 913ffbbb2457a..5983118c05754 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.tree.impurity
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 /**
  * Class for calculating the Gini impurity
@@ -28,14 +28,12 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
 object Gini extends Impurity {
 
   /**
-   * :: DeveloperApi ::
    * information calculation for multiclass classification
    * @param counts Array[Double] with counts for each label
    * @param totalCount sum of counts for all labels
    * @return information value, or 0 if totalCount = 0
    */
   @Since("1.1.0")
-  @DeveloperApi
   override def calculate(counts: Array[Double], totalCount: Double): Double = {
     if (totalCount == 0) {
       return 0
@@ -52,7 +50,6 @@ object Gini extends Impurity {
   }
 
   /**
-   * :: DeveloperApi ::
    * variance calculation
    * @param count number of instances
    * @param sum sum of labels
@@ -60,7 +57,6 @@ object Gini extends Impurity {
    * @return information value, or 0 if count = 0
    */
   @Since("1.0.0")
-  @DeveloperApi
   override def calculate(count: Double, sum: Double, sumSquares: Double): Double =
     throw new UnsupportedOperationException("Gini.calculate")
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala
index 491473490eba7..9b1d8d717d40f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala
@@ -19,7 +19,7 @@ package org.apache.spark.mllib.tree.impurity
 
 import java.util.Locale
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 /**
  * Trait for calculating information gain.
@@ -31,18 +31,15 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
 trait Impurity extends Serializable {
 
   /**
-   * :: DeveloperApi ::
    * information calculation for multiclass classification
    * @param counts Array[Double] with counts for each label
    * @param totalCount sum of counts for all labels
    * @return information value, or 0 if totalCount = 0
    */
   @Since("1.1.0")
-  @DeveloperApi
   def calculate(counts: Array[Double], totalCount: Double): Double
 
   /**
-   * :: DeveloperApi ::
    * information calculation for regression
    * @param count number of instances
    * @param sum sum of labels
@@ -50,7 +47,6 @@ trait Impurity extends Serializable {
    * @return information value, or 0 if count = 0
    */
   @Since("1.0.0")
-  @DeveloperApi
   def calculate(count: Double, sum: Double, sumSquares: Double): Double
 }
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala
index a07b919271f71..f5b2f8d514c7e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.tree.impurity
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 /**
  * Class for calculating variance during regression
@@ -26,27 +26,22 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
 object Variance extends Impurity {
 
   /**
-   * :: DeveloperApi ::
    * information calculation for multiclass classification
    * @param counts Array[Double] with counts for each label
    * @param totalCount sum of counts for all labels
    * @return information value, or 0 if totalCount = 0
    */
   @Since("1.1.0")
-  @DeveloperApi
   override def calculate(counts: Array[Double], totalCount: Double): Double =
      throw new UnsupportedOperationException("Variance.calculate")
 
   /**
-   * :: DeveloperApi ::
    * variance calculation
    * @param count number of instances
    * @param sum sum of labels
    * @param sumSquares summation of squares of the labels
    * @return information value, or 0 if count = 0
    */
-  @Since("1.0.0")
-  @DeveloperApi
   override def calculate(count: Double, sum: Double, sumSquares: Double): Double = {
     if (count == 0) {
       return 0
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala
index 9b60d018d0eda..48e576afdf675 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.tree.loss
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 
 /**
- * :: DeveloperApi ::
  * Class for absolute error loss calculation (for regression).
  *
  * The absolute (L1) error is defined as:
@@ -29,7 +28,6 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
  * where y is the label and F(x) is the model prediction for features x.
  */
 @Since("1.2.0")
-@DeveloperApi
 object AbsoluteError extends Loss {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala
index 9339f0a23c1bd..49f1215c77ad2 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.tree.loss
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.util.MLUtils
 
 /**
- * :: DeveloperApi ::
  * Class for log loss calculation (for classification).
  * This uses twice the binomial negative log likelihood, called "deviance" in Friedman (1999).
  *
@@ -30,7 +29,6 @@ import org.apache.spark.mllib.util.MLUtils
  * where y is a label in {-1, 1} and F(x) is the model prediction for features x.
  */
 @Since("1.2.0")
-@DeveloperApi
 object LogLoss extends ClassificationLoss {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala
index e7ffb3f8f53c1..f589e0ce5d4a1 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala
@@ -17,17 +17,15 @@
 
 package org.apache.spark.mllib.tree.loss
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.mllib.tree.model.TreeEnsembleModel
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Trait for adding "pluggable" loss functions for the gradient boosting algorithm.
  */
 @Since("1.2.0")
-@DeveloperApi
 trait Loss extends Serializable {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala
index 4eb6810c46b20..3ff59fc64ce7f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.tree.loss
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 
 /**
- * :: DeveloperApi ::
  * Class for squared error loss calculation.
  *
  * The squared (L2) error is defined as:
@@ -29,7 +28,6 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
  * where y is the label and F(x) is the model prediction for features x.
  */
 @Since("1.2.0")
-@DeveloperApi
 object SquaredError extends Loss {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala
index f3dbfd96e1815..54d95a3ffad1b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.tree.model
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.tree.impurity.ImpurityCalculator
 
 /**
- * :: DeveloperApi ::
  * Information gain statistics for each split
  * @param gain information gain value
  * @param impurity current node impurity
@@ -31,7 +30,6 @@ import org.apache.spark.mllib.tree.impurity.ImpurityCalculator
  * @param rightPredict right node predict
  */
 @Since("1.0.0")
-@DeveloperApi
 class InformationGainStats(
     val gain: Double,
     val impurity: Double,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala
index 5fd053647aa46..bac36ce6937b0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala
@@ -17,13 +17,12 @@
 
 package org.apache.spark.mllib.tree.model
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.linalg.Vector
 import org.apache.spark.mllib.tree.configuration.FeatureType._
 
 /**
- * :: DeveloperApi ::
  * Node in a decision tree.
  *
  * About node indexing:
@@ -40,7 +39,6 @@ import org.apache.spark.mllib.tree.configuration.FeatureType._
  * @param stats information gain stats
  */
 @Since("1.0.0")
-@DeveloperApi
 class Node @Since("1.2.0") (
     @Since("1.0.0") val id: Int,
     @Since("1.0.0") var predict: Predict,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala
index 1dbdd2d860efd..44bc625d217f1 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala
@@ -17,16 +17,14 @@
 
 package org.apache.spark.mllib.tree.model
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 /**
- * :: DeveloperApi ::
  * Predicted value for a node
  * @param predict predicted value
  * @param prob probability of the label (classification only)
  */
 @Since("1.2.0")
-@DeveloperApi
 class Predict @Since("1.2.0") (
     @Since("1.2.0") val predict: Double,
     @Since("1.2.0") val prob: Double = 0.0) extends Serializable {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala
index bda5e662779c7..dbada8f983b99 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.tree.model
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.tree.configuration.FeatureType.FeatureType
 
 /**
- * :: DeveloperApi ::
  * Split applied to a feature
  * @param feature feature index
  * @param threshold Threshold for continuous feature.
@@ -30,7 +29,6 @@ import org.apache.spark.mllib.tree.configuration.FeatureType.FeatureType
  * @param categories Split left if categorical feature value is in this set, else right.
  */
 @Since("1.0.0")
-@DeveloperApi
 case class Split(
     @Since("1.0.0") feature: Int,
     @Since("1.0.0") threshold: Double,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
index 810f528c71906..6b3afdd262e99 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
@@ -25,7 +25,7 @@ import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.linalg.Vector
@@ -174,7 +174,6 @@ class GradientBoostedTreesModel @Since("1.2.0") (
 object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] {
 
   /**
-   * :: DeveloperApi ::
    * Compute the initial predictions and errors for a dataset for the first
    * iteration of gradient boosting.
    * @param data: training data.
@@ -185,7 +184,6 @@ object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] {
    *         corresponding to every sample.
    */
   @Since("1.4.0")
-  @DeveloperApi
   def computeInitialPredictionAndError(
       data: RDD[LabeledPoint],
       initTreeWeight: Double,
@@ -199,7 +197,6 @@ object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] {
   }
 
   /**
-   * :: DeveloperApi ::
    * Update a zipped predictionError RDD
    * (as obtained with computeInitialPredictionAndError)
    * @param data: training data.
@@ -211,7 +208,6 @@ object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] {
    *         corresponding to each sample.
    */
   @Since("1.4.0")
-  @DeveloperApi
   def updatePredictionError(
     data: RDD[LabeledPoint],
     predictionAndError: RDD[(Double, Double)],
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala
index 2c712d8f821a0..1a8c8807f91d4 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala
@@ -17,16 +17,14 @@
 
 package org.apache.spark.mllib.util
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * A collection of methods used to validate data before applying ML algorithms.
  */
-@DeveloperApi
 @Since("0.8.0")
 object DataValidators extends Logging {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
index b6eb10e9de00a..a409abc125c0c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
@@ -20,16 +20,14 @@ package org.apache.spark.mllib.util
 import scala.util.Random
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Generate test data for KMeans. This class first chooses k cluster centers
  * from a d-dimensional Gaussian distribution scaled by factor r and then creates a Gaussian
  * cluster with scale 1 around each center.
  */
-@DeveloperApi
 @Since("0.8.0")
 object KMeansDataGenerator {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
index 30c77d2af8fbc..b044c86f1069a 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
@@ -23,18 +23,16 @@ import scala.util.Random
 import com.github.fommil.netlib.BLAS.{getInstance => blas}
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.{BLAS, Vectors}
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Generate sample data used for Linear Data. This class generates
  * uniformly random values for every feature and adds Gaussian noise with mean `eps` to the
  * response variable `Y`.
  */
-@DeveloperApi
 @Since("0.8.0")
 object LinearDataGenerator {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala
index 7e9d9465441c9..bca4ad4a3a60e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala
@@ -20,17 +20,15 @@ package org.apache.spark.mllib.util
 import scala.util.Random
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.Vectors
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Generate test data for LogisticRegression. This class chooses positive labels
  * with probability `probOne` and scales features for positive examples by `eps`.
  */
-@DeveloperApi
 @Since("0.8.0")
 object LogisticRegressionDataGenerator {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
index 7a308a5ec25c0..de21db8d44d7c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
@@ -22,12 +22,11 @@ import java.{util => ju}
 import scala.util.Random
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.{BLAS, DenseMatrix}
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Generate RDD(s) containing data for Matrix Factorization.
  *
  * This method samples training entries according to the oversampling factor
@@ -50,7 +49,6 @@ import org.apache.spark.rdd.RDD
  *   test           (Boolean) Whether to create testing RDD.
  *   testSampFact   (Double) Percentage of training data to use as test data.
  */
-@DeveloperApi
 @Since("0.8.0")
 object MFDataGenerator {
   @Since("0.8.0")
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
index 9198334ba02a1..2065bd4dc9dec 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
@@ -105,12 +105,15 @@ object MLUtils extends Logging {
   }
 
   private[spark] def parseLibSVMFile(
-      sparkSession: SparkSession, paths: Seq[String]): RDD[(Double, Array[Int], Array[Double])] = {
+      sparkSession: SparkSession,
+      paths: Seq[String],
+      options: Map[String, String]): RDD[(Double, Array[Int], Array[Double])] = {
     val lines = sparkSession.baseRelationToDataFrame(
       DataSource.apply(
         sparkSession,
         paths = paths,
-        className = classOf[TextFileFormat].getName
+        className = classOf[TextFileFormat].getName,
+        options = options ++ Map(DataSource.GLOB_PATHS_KEY -> "false")
       ).resolveRelation(checkFilesExist = false))
       .select("value")
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
index 9f6ba025aedde..9fffa508afbfb 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
@@ -22,17 +22,15 @@ import scala.util.Random
 import com.github.fommil.netlib.BLAS.{getInstance => blas}
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.Vectors
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Generate sample data used for SVM. This class generates uniform random values
  * for the features and adds Gaussian noise with weight 0.1 to generate labels.
  */
-@DeveloperApi
 @Since("0.8.0")
 object SVMDataGenerator {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala
index e8889bfd7c381..8f2d8b9014c68 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala
@@ -24,17 +24,14 @@ import org.json4s._
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.sql.catalyst.ScalaReflection
 import org.apache.spark.sql.types.{DataType, StructField, StructType}
 
 /**
- * :: DeveloperApi ::
- *
  * Trait for models and transformers which may be saved as files.
  * This should be inherited by the class which implements model instances.
  */
-@DeveloperApi
 @Since("1.3.0")
 trait Saveable {
 
@@ -57,12 +54,9 @@ trait Saveable {
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Trait for classes which can load models and transformers from files.
  * This should be inherited by an object paired with the model class.
  */
-@DeveloperApi
 @Since("1.3.0")
 trait Loader[M <: Saveable] {
 
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/.part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/.part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..3ac562a8b2b84
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/.part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet
new file mode 100644
index 0000000000000..09c38d602c105
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..22b1eb8089ce3
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..ef92265f89fff
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.DecisionTreeClassificationModel","timestamp":1608687929358,"sparkVersion":"2.4.7","uid":"dtc_bc7ad285bb73","paramMap":{},"defaultParamMap":{"impurity":"gini","maxDepth":5,"labelCol":"label","maxMemoryInMB":256,"featuresCol":"features","predictionCol":"prediction","minInfoGain":0.0,"seed":159147643,"rawPredictionCol":"rawPrediction","minInstancesPerNode":1,"cacheNodeIds":false,"probabilityCol":"probability","maxBins":32,"checkpointInterval":10},"numFeatures":692,"numClasses":2}
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/.part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/.part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..f6465e27d902b
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/.part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet
new file mode 100644
index 0000000000000..2904f8433b78f
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..bbad108766e14
Binary files /dev/null and b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..2895223cffde6
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.regression.DecisionTreeRegressionModel","timestamp":1608687932847,"sparkVersion":"2.4.7","uid":"dtr_c16a90fcdaf8","paramMap":{},"defaultParamMap":{"labelCol":"label","checkpointInterval":10,"minInfoGain":0.0,"maxMemoryInMB":256,"minInstancesPerNode":1,"maxBins":32,"seed":926680331,"cacheNodeIds":false,"maxDepth":5,"predictionCol":"prediction","featuresCol":"features","impurity":"variance"},"numFeatures":692}
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/.part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/.part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..13fc4ed696eff
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/.part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet
new file mode 100644
index 0000000000000..5682038fcc764
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..a810dd9107fe7
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..675fea29ba9e2
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.GBTClassificationModel","timestamp":1608687932103,"sparkVersion":"2.4.7","uid":"gbtc_81db008b4f25","paramMap":{"maxIter":2},"defaultParamMap":{"seed":-1287390502,"maxMemoryInMB":256,"stepSize":0.1,"validationTol":0.01,"maxBins":32,"checkpointInterval":10,"predictionCol":"prediction","lossType":"logistic","rawPredictionCol":"rawPrediction","featuresCol":"features","cacheNodeIds":false,"maxIter":20,"featureSubsetStrategy":"all","impurity":"gini","minInstancesPerNode":1,"minInfoGain":0.0,"maxDepth":5,"subsamplingRate":1.0,"labelCol":"label","probabilityCol":"probability"},"numFeatures":692,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/.part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/.part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..101c207119388
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/.part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet
new file mode 100644
index 0000000000000..e232de3400423
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/.part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/.part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..c35b81f9cb25f
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/.part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet
new file mode 100644
index 0000000000000..ba26a44f46f11
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..7dc6e149db71e
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..a9a712e626510
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.regression.GBTRegressionModel","timestamp":1608687942434,"sparkVersion":"2.4.7","uid":"gbtr_0a74cb2536ff","paramMap":{"maxIter":2},"defaultParamMap":{"impurity":"variance","maxMemoryInMB":256,"maxDepth":5,"subsamplingRate":1.0,"validationTol":0.01,"labelCol":"label","maxIter":20,"checkpointInterval":10,"minInfoGain":0.0,"predictionCol":"prediction","stepSize":0.1,"cacheNodeIds":false,"lossType":"squared","seed":-131597770,"featureSubsetStrategy":"all","featuresCol":"features","minInstancesPerNode":1,"maxBins":32},"numFeatures":692,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/.part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/.part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..b681b9f615bf5
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/.part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet
new file mode 100644
index 0000000000000..9a7e77acf5464
Binary files /dev/null and b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/.part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/.part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..5bb3a22617543
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/.part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet
new file mode 100644
index 0000000000000..d9ec35a4de580
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..58bda6df9cca7
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..07748b070ee84
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.RandomForestClassificationModel","timestamp":1608687930713,"sparkVersion":"2.4.7","uid":"rfc_db1adb353f1e","paramMap":{"numTrees":2},"defaultParamMap":{"impurity":"gini","predictionCol":"prediction","numTrees":20,"maxDepth":5,"featureSubsetStrategy":"auto","subsamplingRate":1.0,"featuresCol":"features","checkpointInterval":10,"rawPredictionCol":"rawPrediction","cacheNodeIds":false,"labelCol":"label","seed":207336481,"probabilityCol":"probability","maxBins":32,"minInstancesPerNode":1,"minInfoGain":0.0,"maxMemoryInMB":256},"numFeatures":692,"numClasses":2,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/.part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/.part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..729c5bb30d7fe
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/.part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet
new file mode 100644
index 0000000000000..610882198c790
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/.part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/.part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..52cf21f6701ca
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/.part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet
new file mode 100644
index 0000000000000..75a3f0459864c
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000..1a72b8e029844
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/.part-00000.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..cccbb8f9f3b05
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.regression.RandomForestRegressionModel","timestamp":1608687933536,"sparkVersion":"2.4.7","uid":"rfr_d946d96b7ff0","paramMap":{"numTrees":2},"defaultParamMap":{"numTrees":20,"featureSubsetStrategy":"auto","maxDepth":5,"minInstancesPerNode":1,"labelCol":"label","cacheNodeIds":false,"checkpointInterval":10,"featuresCol":"features","maxMemoryInMB":256,"predictionCol":"prediction","minInfoGain":0.0,"subsamplingRate":1.0,"impurity":"variance","seed":235498149,"maxBins":32},"numFeatures":692,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000..3b7b044936a89
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/._SUCCESS.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/.part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/.part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000..8081f8842670e
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/.part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet.crc differ
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet
new file mode 100644
index 0000000000000..093c3468db301
Binary files /dev/null and b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet differ
diff --git a/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
index 2f5062c689fc7..3dd9a7d8ec85d 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
@@ -34,9 +34,8 @@ class FunctionsSuite extends MLTest {
       (Vectors.sparse(3, Seq((0, 2.0), (2, 3.0))), OldVectors.sparse(3, Seq((0, 20.0), (2, 30.0))))
     ).toDF("vec", "oldVec")
 
-      val result = df.select(vector_to_array('vec), vector_to_array('oldVec))
-      .as[(Seq[Double], Seq[Double])]
-      .collect().toSeq
+    val result = df.select(vector_to_array('vec), vector_to_array('oldVec))
+                   .as[(Seq[Double], Seq[Double])].collect().toSeq
 
     val expected = Seq(
       (Seq(1.0, 2.0, 3.0), Seq(10.0, 20.0, 30.0)),
@@ -50,7 +49,6 @@ class FunctionsSuite extends MLTest {
       (null, null, 0)
     ).toDF("vec", "oldVec", "label")
 
-
     for ((colName, valType) <- Seq(
         ("vec", "null"), ("oldVec", "null"), ("label", "java.lang.Integer"))) {
       val thrown1 = intercept[SparkException] {
@@ -61,5 +59,32 @@ class FunctionsSuite extends MLTest {
         "`org.apache.spark.ml.linalg.Vector` or `org.apache.spark.mllib.linalg.Vector`, " +
         s"but got ${valType}"))
     }
+
+    val df3 = Seq(
+      (Vectors.dense(1.0, 2.0, 3.0), OldVectors.dense(10.0, 20.0, 30.0)),
+      (Vectors.sparse(3, Seq((0, 2.0), (2, 3.0))), OldVectors.sparse(3, Seq((0, 20.0), (2, 30.0))))
+    ).toDF("vec", "oldVec")
+    val dfArrayFloat = df3.select(
+      vector_to_array('vec, dtype = "float32"), vector_to_array('oldVec, dtype = "float32"))
+
+    // Check values are correct
+    val result3 = dfArrayFloat.as[(Seq[Float], Seq[Float])].collect().toSeq
+
+    val expected3 = Seq(
+      (Seq(1.0, 2.0, 3.0), Seq(10.0, 20.0, 30.0)),
+      (Seq(2.0, 0.0, 3.0), Seq(20.0, 0.0, 30.0))
+    )
+    assert(result3 === expected3)
+
+    // Check data types are correct
+    assert(dfArrayFloat.schema.simpleString ===
+      "struct<UDF(vec):array<float>,UDF(oldVec):array<float>>")
+
+    val thrown2 = intercept[IllegalArgumentException] {
+      df3.select(
+        vector_to_array('vec, dtype = "float16"), vector_to_array('oldVec, dtype = "float16"))
+    }
+    assert(thrown2.getMessage.contains(
+      s"Unsupported dtype: float16. Valid values: float64, float32."))
   }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
index d1ade85cea049..13efdf13d9e54 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
@@ -446,6 +446,18 @@ class DecisionTreeClassifierSuite extends MLTest with DefaultReadWriteTest {
 
     testDefaultReadWrite(model)
   }
+
+  test("SPARK-33398: Load DecisionTreeClassificationModel prior to Spark 3.0") {
+    val path = testFile("ml-models/dtc-2.4.7")
+    val model = DecisionTreeClassificationModel.load(path)
+    assert(model.numClasses === 2)
+    assert(model.numFeatures === 692)
+    assert(model.numNodes === 5)
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private[ml] object DecisionTreeClassifierSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
index a2208edcb8394..37e695ef88b14 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
@@ -545,6 +545,20 @@ class GBTClassifierSuite extends MLTest with DefaultReadWriteTest {
     testEstimatorAndModelReadWrite(gbt, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load GBTClassificationModel prior to Spark 3.0") {
+    val path = testFile("ml-models/gbtc-2.4.7")
+    val model = GBTClassificationModel.load(path)
+    assert(model.numClasses === 2)
+    assert(model.numFeatures === 692)
+    assert(model.getNumTrees === 2)
+    assert(model.totalNumNodes === 22)
+    assert(model.trees.map(_.numNodes) === Array(5, 17))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object GBTClassifierSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
index 2b63dc259a14f..c2072cea11859 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
@@ -179,7 +179,7 @@ class LinearSVCSuite extends MLTest with DefaultReadWriteTest {
   test("sparse coefficients in HingeAggregator") {
     val bcCoefficients = spark.sparkContext.broadcast(Vectors.sparse(2, Array(0), Array(1.0)))
     val bcFeaturesStd = spark.sparkContext.broadcast(Array(1.0))
-    val agg = new HingeAggregator(1, true)(bcCoefficients)
+    val agg = new HingeAggregator(bcFeaturesStd, true)(bcCoefficients)
     val thrown = withClue("LinearSVCAggregator cannot handle sparse coefficients") {
       intercept[IllegalArgumentException] {
         agg.add(Instance(1.0, 1.0, Vectors.dense(1.0)))
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
index 9e359ba098bfb..6d31e6efc7e1c 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
@@ -542,7 +542,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
   test("sparse coefficients in LogisticAggregator") {
     val bcCoefficientsBinary = spark.sparkContext.broadcast(Vectors.sparse(2, Array(0), Array(1.0)))
     val bcFeaturesStd = spark.sparkContext.broadcast(Array(1.0))
-    val binaryAgg = new LogisticAggregator(1, 2,
+    val binaryAgg = new LogisticAggregator(bcFeaturesStd, 2,
       fitIntercept = true, multinomial = false)(bcCoefficientsBinary)
     val thrownBinary = withClue("binary logistic aggregator cannot handle sparse coefficients") {
       intercept[IllegalArgumentException] {
@@ -552,7 +552,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
     assert(thrownBinary.getMessage.contains("coefficients only supports dense"))
 
     val bcCoefficientsMulti = spark.sparkContext.broadcast(Vectors.sparse(6, Array(0), Array(1.0)))
-    val multinomialAgg = new LogisticAggregator(1, 3,
+    val multinomialAgg = new LogisticAggregator(bcFeaturesStd, 3,
       fitIntercept = true, multinomial = true)(bcCoefficientsMulti)
     val thrown = withClue("multinomial logistic aggregator cannot handle sparse coefficients") {
       intercept[IllegalArgumentException] {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala
index 902af71e42f86..7cf4e65baf557 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala
@@ -240,6 +240,6 @@ class MultilayerPerceptronClassifierSuite extends MLTest with DefaultReadWriteTe
 
     val metadata = spark.read.json(s"$mlpPath/metadata")
     val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
-    assert(sparkVersionStr == "2.4.4")
+    assert(sparkVersionStr === "2.4.4")
   }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
index e30e93ad4628c..28b245f7f05de 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
@@ -25,10 +25,10 @@ import org.apache.spark.ml.param.ParamsSuite
 import org.apache.spark.ml.tree._
 import org.apache.spark.ml.tree.impl.TreeTests
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
+import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.regression.{LabeledPoint => OldLabeledPoint}
 import org.apache.spark.mllib.tree.{EnsembleTestHelper, RandomForest => OldRandomForest}
 import org.apache.spark.mllib.tree.configuration.{Algo => OldAlgo}
-import org.apache.spark.mllib.util.TestingUtils._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Row}
 
@@ -319,6 +319,20 @@ class RandomForestClassifierSuite extends MLTest with DefaultReadWriteTest {
     testEstimatorAndModelReadWrite(rf, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load RandomForestClassificationModel prior to Spark 3.0") {
+    val path = testFile("ml-models/rfc-2.4.7")
+    val model = RandomForestClassificationModel.load(path)
+    assert(model.numClasses === 2)
+    assert(model.numFeatures === 692)
+    assert(model.getNumTrees === 2)
+    assert(model.totalNumNodes === 10)
+    assert(model.trees.map(_.numNodes) === Array(3, 7))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object RandomForestClassifierSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
index 722302e5a165f..861bf1e0b1292 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
@@ -99,7 +99,11 @@ class HashingTFSuite extends MLTest with DefaultReadWriteTest {
 
     val metadata = spark.read.json(s"$hashingTFPath/metadata")
     val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
-    assert(sparkVersionStr == "2.4.4")
+    assert(sparkVersionStr === "2.4.4")
+
+    intercept[IllegalArgumentException] {
+      loadedHashingTF.save(hashingTFPath)
+    }
   }
 
   test("read/write") {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
index d780bdf5f5dc8..5a74490058398 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
@@ -42,36 +42,5 @@ class InstanceSuite extends SparkFunSuite{
       val o2 = ser.deserialize[OffsetInstance](ser.serialize(o))
       assert(o === o2)
     }
-
-    val block1 = InstanceBlock.fromInstances(Seq(instance1))
-    val block2 = InstanceBlock.fromInstances(Seq(instance1, instance2))
-    Seq(block1, block2).foreach { o =>
-      val o2 = ser.deserialize[InstanceBlock](ser.serialize(o))
-      assert(o.labels === o2.labels)
-      assert(o.weights === o2.weights)
-      assert(o.matrix === o2.matrix)
-    }
-  }
-
-  test("InstanceBlock: check correctness") {
-    val instance1 = Instance(19.0, 2.0, Vectors.dense(1.0, 7.0))
-    val instance2 = Instance(17.0, 1.0, Vectors.dense(0.0, 5.0).toSparse)
-    val instances = Seq(instance1, instance2)
-
-    val block = InstanceBlock.fromInstances(instances)
-    assert(block.size === 2)
-    assert(block.numFeatures === 2)
-    block.instanceIterator.zipWithIndex.foreach {
-      case (instance, i) =>
-        assert(instance.label === instances(i).label)
-        assert(instance.weight === instances(i).weight)
-        assert(instance.features.toArray === instances(i).features.toArray)
-    }
-    Seq(0, 1).foreach { i =>
-      val nzIter = block.getNonZeroIter(i)
-      val vec = Vectors.sparse(2, nzIter.toSeq)
-      assert(vec.toArray === instances(i).features.toArray)
-    }
   }
-
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala
index 76a4acd798e34..1d052fbebd92d 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala
@@ -76,9 +76,8 @@ private[ml] object LSHTest {
 
     // Perform a cross join and label each pair of same_bucket and distance
     val pairs = transformedData.as("a").crossJoin(transformedData.as("b"))
-    val distUDF = udf((x: Vector, y: Vector) => model.keyDistance(x, y), DataTypes.DoubleType)
-    val sameBucket = udf((x: Seq[Vector], y: Seq[Vector]) => model.hashDistance(x, y) == 0.0,
-      DataTypes.BooleanType)
+    val distUDF = udf((x: Vector, y: Vector) => model.keyDistance(x, y))
+    val sameBucket = udf((x: Seq[Vector], y: Seq[Vector]) => model.hashDistance(x, y) == 0.0)
     val result = pairs
       .withColumn("same_bucket", sameBucket(col(s"a.$outputCol"), col(s"b.$outputCol")))
       .withColumn("distance", distUDF(col(s"a.$inputCol"), col(s"b.$inputCol")))
@@ -110,7 +109,7 @@ private[ml] object LSHTest {
     val model = lsh.fit(dataset)
 
     // Compute expected
-    val distUDF = udf((x: Vector) => model.keyDistance(x, key), DataTypes.DoubleType)
+    val distUDF = udf((x: Vector) => model.keyDistance(x, key))
     val expected = dataset.sort(distUDF(col(model.getInputCol))).limit(k)
 
     // Compute actual
@@ -148,7 +147,7 @@ private[ml] object LSHTest {
     val inputCol = model.getInputCol
 
     // Compute expected
-    val distUDF = udf((x: Vector, y: Vector) => model.keyDistance(x, y), DataTypes.DoubleType)
+    val distUDF = udf((x: Vector, y: Vector) => model.keyDistance(x, y))
     val expected = datasetA.as("a").crossJoin(datasetB.as("b"))
       .filter(distUDF(col(s"a.$inputCol"), col(s"b.$inputCol")) < threshold)
 
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala
index 6f6ab26cbac43..682b87a0f68d3 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala
@@ -512,4 +512,22 @@ class QuantileDiscretizerSuite extends MLTest with DefaultReadWriteTest {
     assert(observedNumBuckets === numBuckets,
       "Observed number of buckets does not equal expected number of buckets.")
   }
+
+  test("[SPARK-31676] QuantileDiscretizer raise error parameter splits given invalid value") {
+    import scala.util.Random
+    val rng = new Random(3)
+
+    val a1 = Array.tabulate(200)(_ => rng.nextDouble * 2.0 - 1.0) ++
+      Array.fill(20)(0.0) ++ Array.fill(20)(-0.0)
+
+    val df1 = sc.parallelize(a1, 2).toDF("id")
+
+    val qd = new QuantileDiscretizer()
+      .setInputCol("id")
+      .setOutputCol("out")
+      .setNumBuckets(200)
+      .setRelativeError(0.0)
+
+    qd.fit(df1) // assert no exception raised here.
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
index b5ce2bace98f6..c8247b9c8f3bf 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
@@ -96,6 +96,23 @@ class StringIndexerSuite extends MLTest with DefaultReadWriteTest {
     }
   }
 
+  test("StringIndexer.transformSchema)") {
+    val idxToStr = new StringIndexer().setInputCol("input").setOutputCol("output")
+    val inSchema = StructType(Seq(StructField("input", StringType)))
+    val outSchema = idxToStr.transformSchema(inSchema)
+    assert(outSchema("output").dataType === DoubleType)
+  }
+
+  test("StringIndexer.transformSchema multi col") {
+    val idxToStr = new StringIndexer().setInputCols(Array("input", "input2")).
+      setOutputCols(Array("output", "output2"))
+    val inSchema = StructType(Seq(StructField("input", StringType),
+      StructField("input2", StringType)))
+    val outSchema = idxToStr.transformSchema(inSchema)
+    assert(outSchema("output").dataType === DoubleType)
+    assert(outSchema("output2").dataType === DoubleType)
+  }
+
   test("StringIndexerUnseen") {
     val data = Seq((0, "a"), (1, "b"), (4, "b"))
     val data2 = Seq((0, "a"), (1, "b"), (2, "c"), (3, "d"))
@@ -466,6 +483,6 @@ class StringIndexerSuite extends MLTest with DefaultReadWriteTest {
 
     val metadata = spark.read.json(s"$modelPath/metadata")
     val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
-    assert(sparkVersionStr == "2.4.4")
+    assert(sparkVersionStr === "2.4.4")
   }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
index a4d388fd321db..4957f6f1f46a7 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
@@ -261,4 +261,15 @@ class VectorAssemblerSuite
     val output = vectorAssembler.transform(dfWithNullsAndNaNs)
     assert(output.select("a").limit(1).collect().head == Row(Vectors.sparse(0, Seq.empty)))
   }
+
+  test("SPARK-31671: should give explicit error message when can not infer column lengths") {
+    val df = Seq(
+      (Vectors.dense(1.0), Vectors.dense(2.0))
+    ).toDF("n1", "n2")
+    val hintedDf = new VectorSizeHint().setInputCol("n1").setSize(1).transform(df)
+    val assembler = new VectorAssembler()
+      .setInputCols(Array("n1", "n2")).setOutputCol("features")
+    assert(!intercept[RuntimeException](assembler.setHandleInvalid("keep").transform(hintedDf))
+      .getMessage.contains("n1"), "should only show no vector size columns' name")
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala
index c02a0a5e5e7d0..61b48ffa10944 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.util.MLlibTestSparkContext
@@ -32,21 +32,21 @@ class HingeAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    instances = standardize(Array(
+    instances = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
       Instance(0.0, 0.3, Vectors.dense(4.0, 0.5))
-    ))
-    instancesConstantFeature = standardize(Array(
+    )
+    instancesConstantFeature = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
       Instance(1.0, 0.3, Vectors.dense(1.0, 0.5))
-    ))
-    instancesConstantFeatureFiltered = standardize(Array(
+    )
+    instancesConstantFeatureFiltered = Array(
       Instance(0.0, 0.1, Vectors.dense(2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0)),
       Instance(2.0, 0.3, Vectors.dense(0.5))
-    ))
+    )
   }
 
    /** Get summary statistics for some data and create a new HingeAggregator. */
@@ -54,23 +54,12 @@ class HingeAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
       instances: Array[Instance],
       coefficients: Vector,
       fitIntercept: Boolean): HingeAggregator = {
-    val bcCoefficients = spark.sparkContext.broadcast(coefficients)
-    new HingeAggregator(instances.head.features.size, fitIntercept)(bcCoefficients)
-  }
-
-  private def standardize(instances: Array[Instance]): Array[Instance] = {
-    val (featuresSummarizer, _) =
+    val (featuresSummarizer, ySummarizer) =
       DifferentiableLossAggregatorSuite.getClassificationSummarizers(instances)
-    val stdArray = featuresSummarizer.variance.toArray.map(math.sqrt)
-    val numFeatures = stdArray.length
-    instances.map { case Instance(label, weight, features) =>
-      val standardized = Array.ofDim[Double](numFeatures)
-      features.foreachNonZero { (i, v) =>
-        val std = stdArray(i)
-        if (std != 0) standardized(i) = v / std
-      }
-      Instance(label, weight, Vectors.dense(standardized).compressed)
-    }
+    val featuresStd = featuresSummarizer.variance.toArray.map(math.sqrt)
+    val bcFeaturesStd = spark.sparkContext.broadcast(featuresStd)
+    val bcCoefficients = spark.sparkContext.broadcast(coefficients)
+    new HingeAggregator(bcFeaturesStd, fitIntercept)(bcCoefficients)
   }
 
   test("aggregator add method input size") {
@@ -171,21 +160,4 @@ class HingeAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
     assert(aggConstantFeatureBinary.gradient(1) == aggConstantFeatureBinaryFiltered.gradient(0))
   }
 
-  test("add instance block") {
-    val coefArray = Array(1.0, 2.0)
-    val intercept = 1.0
-
-    val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ Array(intercept)),
-      fitIntercept = true)
-    instances.foreach(agg.add)
-
-    val agg2 = getNewAggregator(instances, Vectors.dense(coefArray ++ Array(intercept)),
-      fitIntercept = true)
-    val block = InstanceBlock.fromInstances(instances)
-    agg2.add(block)
-
-    assert(agg.loss ~== agg2.loss relTol 1e-8)
-    assert(agg.gradient ~== agg2.gradient relTol 1e-8)
-  }
-
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HuberAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HuberAggregatorSuite.scala
index 7c544e99f88b5..718ffa230a749 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HuberAggregatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HuberAggregatorSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.util.MLlibTestSparkContext
@@ -32,21 +32,21 @@ class HuberAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    instances = standardize(Array(
+    instances = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(4.0, 0.5))
-    ))
-    instancesConstantFeature = standardize(Array(
+    )
+    instancesConstantFeature = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(1.0, 0.5))
-    ))
-    instancesConstantFeatureFiltered = standardize(Array(
+    )
+    instancesConstantFeatureFiltered = Array(
       Instance(0.0, 0.1, Vectors.dense(2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0)),
       Instance(2.0, 0.3, Vectors.dense(0.5))
-    ))
+    )
   }
 
   /** Get summary statistics for some data and create a new HuberAggregator. */
@@ -56,28 +56,10 @@ class HuberAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
       fitIntercept: Boolean,
       epsilon: Double): HuberAggregator = {
     val (featuresSummarizer, _) = getRegressionSummarizers(instances)
-    val numFeatures = featuresSummarizer.variance.size
+    val featuresStd = featuresSummarizer.variance.toArray.map(math.sqrt)
+    val bcFeaturesStd = spark.sparkContext.broadcast(featuresStd)
     val bcParameters = spark.sparkContext.broadcast(parameters)
-    new HuberAggregator(numFeatures, fitIntercept, epsilon)(bcParameters)
-  }
-
-  private def standardize(
-      instances: Array[Instance],
-      std: Array[Double] = null): Array[Instance] = {
-    val stdArray = if (std == null) {
-      getRegressionSummarizers(instances)._1.variance.toArray.map(math.sqrt)
-    } else {
-      std
-    }
-    val numFeatures = stdArray.length
-    instances.map { case Instance(label, weight, features) =>
-      val standardized = Array.ofDim[Double](numFeatures)
-      features.foreachNonZero { (i, v) =>
-        val std = stdArray(i)
-        if (std != 0) standardized(i) = v / std
-      }
-      Instance(label, weight, Vectors.dense(standardized).compressed)
-    }
+    new HuberAggregator(fitIntercept, epsilon, bcFeaturesStd)(bcParameters)
   }
 
   test("aggregator add method should check input size") {
@@ -173,15 +155,9 @@ class HuberAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
     val parametersFiltered = Vectors.dense(2.0, 3.0, 4.0)
     val aggConstantFeature = getNewAggregator(instancesConstantFeature, parameters,
       fitIntercept = true, epsilon = 1.35)
-    // std of instancesConstantFeature
-    val stdConstantFeature = getRegressionSummarizers(instancesConstantFeature)
-      ._1.variance.toArray.map(math.sqrt)
-    // Since 3.0.0, we start to standardize input outside of gradient computation,
-    // so here we use std of instancesConstantFeature to standardize instances
-    standardize(instances, stdConstantFeature).foreach(aggConstantFeature.add)
-
     val aggConstantFeatureFiltered = getNewAggregator(instancesConstantFeatureFiltered,
       parametersFiltered, fitIntercept = true, epsilon = 1.35)
+    instances.foreach(aggConstantFeature.add)
     instancesConstantFeatureFiltered.foreach(aggConstantFeatureFiltered.add)
     // constant features should not affect gradient
     def validateGradient(grad: Vector, gradFiltered: Vector): Unit = {
@@ -191,19 +167,4 @@ class HuberAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
 
     validateGradient(aggConstantFeature.gradient, aggConstantFeatureFiltered.gradient)
   }
-
-  test("add instance block") {
-    val paramWithIntercept = Vectors.dense(1.0, 2.0, 3.0, 4.0)
-    val agg1 = getNewAggregator(instances, paramWithIntercept,
-      fitIntercept = true, epsilon = 1.35)
-    instances.foreach(agg1.add)
-
-    val agg2 = getNewAggregator(instances, paramWithIntercept,
-      fitIntercept = true, epsilon = 1.35)
-    val block = InstanceBlock.fromInstances(instances)
-    agg2.add(block)
-
-    assert(agg1.loss ~== agg2.loss relTol 1e-8)
-    assert(agg1.gradient ~== agg2.gradient relTol 1e-8)
-  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregatorSuite.scala
index 5eb4e41c58262..35b6944624707 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregatorSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.util.MLlibTestSparkContext
@@ -32,21 +32,21 @@ class LeastSquaresAggregatorSuite extends SparkFunSuite with MLlibTestSparkConte
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    instances = standardize(Array(
+    instances = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(4.0, 0.5))
-    ))
-    instancesConstantFeature = standardize(Array(
+    )
+    instancesConstantFeature = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(1.0, 0.5))
-    ))
-    instancesConstantLabel = standardize(Array(
+    )
+    instancesConstantLabel = Array(
       Instance(1.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
       Instance(1.0, 0.3, Vectors.dense(4.0, 0.5))
-    ))
+    )
   }
 
   /** Get summary statistics for some data and create a new LeastSquaresAggregator. */
@@ -57,34 +57,15 @@ class LeastSquaresAggregatorSuite extends SparkFunSuite with MLlibTestSparkConte
     val (featuresSummarizer, ySummarizer) = getRegressionSummarizers(instances)
     val yStd = math.sqrt(ySummarizer.variance(0))
     val yMean = ySummarizer.mean(0)
-    val featuresStd = Vectors.dense(featuresSummarizer.variance.toArray.map(math.sqrt))
+    val featuresStd = featuresSummarizer.variance.toArray.map(math.sqrt)
     val bcFeaturesStd = spark.sparkContext.broadcast(featuresStd)
-    val featuresMean = featuresSummarizer.mean.asML
-    val bcFeaturesMean = spark.sparkContext.broadcast(featuresMean.compressed)
-    val bcCoefficients = spark.sparkContext.broadcast(coefficients.compressed)
+    val featuresMean = featuresSummarizer.mean
+    val bcFeaturesMean = spark.sparkContext.broadcast(featuresMean.toArray)
+    val bcCoefficients = spark.sparkContext.broadcast(coefficients)
     new LeastSquaresAggregator(yStd, yMean, fitIntercept, bcFeaturesStd,
       bcFeaturesMean)(bcCoefficients)
   }
 
-  private def standardize(
-      instances: Array[Instance],
-      std: Array[Double] = null): Array[Instance] = {
-    val stdArray = if (std == null) {
-      getRegressionSummarizers(instances)._1.variance.toArray.map(math.sqrt)
-    } else {
-      std
-    }
-    val numFeatures = stdArray.length
-    instances.map { case Instance(label, weight, features) =>
-      val standardized = Array.ofDim[Double](numFeatures)
-      features.foreachNonZero { (i, v) =>
-        val std = stdArray(i)
-        if (std != 0) standardized(i) = v / std
-      }
-      Instance(label, weight, Vectors.dense(standardized).compressed)
-    }
-  }
-
   test("aggregator add method input size") {
     val coefficients = Vectors.dense(1.0, 2.0)
     val agg = getNewAggregator(instances, coefficients, fitIntercept = true)
@@ -164,15 +145,9 @@ class LeastSquaresAggregatorSuite extends SparkFunSuite with MLlibTestSparkConte
 
   test("check with zero standard deviation") {
     val coefficients = Vectors.dense(1.0, 2.0)
-    // aggConstantFeature contains std of instancesConstantFeature, and the std of dim=0 is 0
     val aggConstantFeature = getNewAggregator(instancesConstantFeature, coefficients,
       fitIntercept = true)
-    // std of instancesConstantFeature
-    val stdConstantFeature = getRegressionSummarizers(instancesConstantFeature)
-      ._1.variance.toArray.map(math.sqrt)
-    // Since 3.0.0, we start to standardize input outside of gradient computation,
-    // so here we use std of instancesConstantFeature to standardize instances
-    standardize(instances, stdConstantFeature).foreach(aggConstantFeature.add)
+    instances.foreach(aggConstantFeature.add)
     // constant features should not affect gradient
     assert(aggConstantFeature.gradient(0) === 0.0)
 
@@ -182,17 +157,4 @@ class LeastSquaresAggregatorSuite extends SparkFunSuite with MLlibTestSparkConte
       }
     }
   }
-
-  test("add instance block") {
-    val coefficients = Vectors.dense(1.0, 2.0)
-    val agg1 = getNewAggregator(instances, coefficients, fitIntercept = true)
-    instances.foreach(agg1.add)
-
-    val agg2 = getNewAggregator(instances, coefficients, fitIntercept = true)
-    val block = InstanceBlock.fromInstances(instances)
-    agg2.add(block)
-
-    assert(agg1.loss ~== agg2.loss relTol 1e-8)
-    assert(agg1.gradient ~== agg2.gradient relTol 1e-8)
-  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala
index 83718076dde7b..e699adcc14c03 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg.{BLAS, Matrices, Vector, Vectors}
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.util.MLlibTestSparkContext
@@ -32,21 +32,21 @@ class LogisticAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    instances = standardize(Array(
+    instances = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(4.0, 0.5))
-    ))
-    instancesConstantFeature = standardize(Array(
+    )
+    instancesConstantFeature = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(1.0, 0.5))
-    ))
-    instancesConstantFeatureFiltered = standardize(Array(
+    )
+    instancesConstantFeatureFiltered = Array(
       Instance(0.0, 0.1, Vectors.dense(2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0)),
       Instance(2.0, 0.3, Vectors.dense(0.5))
-    ))
+    )
   }
 
   /** Get summary statistics for some data and create a new LogisticAggregator. */
@@ -55,27 +55,13 @@ class LogisticAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
       coefficients: Vector,
       fitIntercept: Boolean,
       isMultinomial: Boolean): LogisticAggregator = {
-    val (_, ySummarizer) =
+    val (featuresSummarizer, ySummarizer) =
       DifferentiableLossAggregatorSuite.getClassificationSummarizers(instances)
     val numClasses = ySummarizer.histogram.length
-    val numFeatures = instances.head.features.size
+    val featuresStd = featuresSummarizer.variance.toArray.map(math.sqrt)
+    val bcFeaturesStd = spark.sparkContext.broadcast(featuresStd)
     val bcCoefficients = spark.sparkContext.broadcast(coefficients)
-    new LogisticAggregator(numFeatures, numClasses, fitIntercept, isMultinomial)(bcCoefficients)
-  }
-
-  private def standardize(instances: Array[Instance]): Array[Instance] = {
-    val (featuresSummarizer, _) =
-      DifferentiableLossAggregatorSuite.getClassificationSummarizers(instances)
-    val stdArray = featuresSummarizer.variance.toArray.map(math.sqrt)
-    val numFeatures = stdArray.length
-    instances.map { case Instance(label, weight, features) =>
-      val standardized = Array.ofDim[Double](numFeatures)
-      features.foreachNonZero { (i, v) =>
-        val std = stdArray(i)
-        if (std != 0) standardized(i) = v / std
-      }
-      Instance(label, weight, Vectors.dense(standardized).compressed)
-    }
+    new LogisticAggregator(bcFeaturesStd, numClasses, fitIntercept, isMultinomial)(bcCoefficients)
   }
 
   test("aggregator add method input size") {
@@ -291,24 +277,4 @@ class LogisticAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
     validateGradient(aggConstantFeatureBinary.gradient,
       aggConstantFeatureBinaryFiltered.gradient, 1)
   }
-
-  test("add instance block") {
-    val binaryInstances = instances.map { instance =>
-      if (instance.label <= 1.0) instance else Instance(0.0, instance.weight, instance.features)
-    }
-    val coefArray = Array(1.0, 2.0)
-    val intercept = 1.0
-
-    val agg = getNewAggregator(binaryInstances, Vectors.dense(coefArray ++ Array(intercept)),
-      fitIntercept = true, isMultinomial = false)
-    binaryInstances.foreach(agg.add)
-
-    val agg2 = getNewAggregator(binaryInstances, Vectors.dense(coefArray ++ Array(intercept)),
-      fitIntercept = true, isMultinomial = false)
-    val block = InstanceBlock.fromInstances(binaryInstances)
-    agg2.add(block)
-
-    assert(agg.loss ~== agg2.loss relTol 1e-8)
-    assert(agg.gradient ~== agg2.gradient relTol 1e-8)
-  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
index 49ebcb385640e..9cb0345400bc4 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.ml.regression
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.ml.feature.LabeledPoint
-import org.apache.spark.ml.linalg.Vector
+import org.apache.spark.ml.linalg.{Vector, Vectors}
 import org.apache.spark.ml.tree.impl.TreeTests
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
 import org.apache.spark.ml.util.TestingUtils._
@@ -236,6 +236,20 @@ class DecisionTreeRegressorSuite extends MLTest with DefaultReadWriteTest {
       TreeTests.allParamSettings ++ Map("maxDepth" -> 0),
       TreeTests.allParamSettings ++ Map("maxDepth" -> 0), checkModelData)
   }
+
+  test("SPARK-33398: Load DecisionTreeRegressionModel prior to Spark 3.0") {
+    val path = testFile("ml-models/dtr-2.4.7")
+    val model = DecisionTreeRegressionModel.load(path)
+    assert(model.numFeatures === 692)
+    assert(model.numNodes === 5)
+    assert(model.featureImportances ~==
+      Vectors.sparse(692, Array(100, 434),
+        Array(0.03987240829346093, 0.960127591706539)) absTol 1e-4)
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private[ml] object DecisionTreeRegressorSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala
index 04b0d4b8470f3..7d84df6326397 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala
@@ -370,6 +370,18 @@ class GBTRegressorSuite extends MLTest with DefaultReadWriteTest {
     testEstimatorAndModelReadWrite(gbt, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load GBTRegressionModel prior to Spark 3.0") {
+    val path = testFile("ml-models/gbtr-2.4.7")
+    val model = GBTRegressionModel.load(path)
+    assert(model.numFeatures === 692)
+    assert(model.totalNumNodes === 6)
+    assert(model.trees.map(_.numNodes) === Array(5, 1))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object GBTRegressorSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala
index 31dc6d379e76c..4df8366ec2443 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala
@@ -221,6 +221,18 @@ class RandomForestRegressorSuite extends MLTest with DefaultReadWriteTest{
     testEstimatorAndModelReadWrite(rf, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load RandomForestRegressionModel prior to Spark 3.0") {
+    val path = testFile("ml-models/rfr-2.4.7")
+    val model = RandomForestRegressionModel.load(path)
+    assert(model.numFeatures === 692)
+    assert(model.totalNumNodes === 8)
+    assert(model.trees.map(_.numNodes) === Array(5, 3))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object RandomForestRegressorSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
index 263ad26657545..cc0ca308cb668 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
@@ -27,12 +27,13 @@ import org.apache.spark.ml.attribute.AttributeGroup
 import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector, Vectors}
 import org.apache.spark.ml.linalg.SQLDataTypes.VectorType
 import org.apache.spark.mllib.util.MLlibTestSparkContext
-import org.apache.spark.sql.{Row, SaveMode}
+import org.apache.spark.sql.{FakeFileSystemRequiringDSOption, Row, SaveMode}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.types.{DoubleType, StructField, StructType}
 import org.apache.spark.util.Utils
 
 
-class LibSVMRelationSuite extends SparkFunSuite with MLlibTestSparkContext {
+class LibSVMRelationSuite extends SparkFunSuite with MLlibTestSparkContext with SQLHelper {
   // Path for dataset
   var path: String = _
 
@@ -191,4 +192,33 @@ class LibSVMRelationSuite extends SparkFunSuite with MLlibTestSparkContext {
       spark.sql("DROP TABLE IF EXISTS libsvmTable")
     }
   }
+
+  test("SPARK-32815: Test LibSVM data source on file paths with glob metacharacters") {
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath
+      // test libsvm writer / reader without specifying schema
+      val svmFileName = "[abc]"
+      val escapedSvmFileName = "\\[abc\\]"
+      val rawData = new java.util.ArrayList[Row]()
+      rawData.add(Row(1.0, Vectors.sparse(2, Seq((0, 2.0), (1, 3.0)))))
+      val struct = new StructType()
+        .add("labelFoo", DoubleType, false)
+        .add("featuresBar", VectorType, false)
+      val df = spark.createDataFrame(rawData, struct)
+      df.write.format("libsvm").save(s"$basePath/$svmFileName")
+      val df2 = spark.read.format("libsvm").load(s"$basePath/$escapedSvmFileName")
+      val row1 = df2.first()
+      val v = row1.getAs[SparseVector](1)
+      assert(v == Vectors.sparse(2, Seq((0, 2.0), (1, 3.0))))
+    }
+  }
+
+  test("SPARK-33101: should propagate Hadoop config from DS options to underlying file system") {
+    withSQLConf(
+      "fs.file.impl" -> classOf[FakeFileSystemRequiringDSOption].getName,
+      "fs.file.impl.disable.cache" -> "true") {
+      val df = spark.read.option("ds_option", "value").format("libsvm").load(path)
+      assert(df.columns(0) == "label")
+    }
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala b/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala
index 4d9e664850c12..dd0139b94f098 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala
@@ -63,6 +63,9 @@ trait DefaultReadWriteTest extends TempDirectory { self: Suite =>
           (instance.getOrDefault(p), newInstance.getOrDefault(p)) match {
             case (Array(values), Array(newValues)) =>
               assert(values === newValues, s"Values do not match on param ${p.name}.")
+            case (value: Double, newValue: Double) =>
+              assert(value.isNaN && newValue.isNaN || value == newValue,
+                s"Values do not match on param ${p.name}.")
             case (value, newValue) =>
               assert(value === newValue, s"Values do not match on param ${p.name}.")
           }
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
index 47dac3ec29a5c..4de10842c52d1 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
@@ -107,9 +107,9 @@ class NaiveBayesSuite extends SparkFunSuite with MLlibTestSparkContext {
     val modelIndex = piData.indices.zip(model.labels.map(_.toInt))
     try {
       for (i <- modelIndex) {
-        assert(math.exp(piData(i._2)) ~== math.exp(model.pi(i._1)) absTol 0.05)
+        assert(piData(i._2) ~== model.pi(i._1) relTol 0.2)
         for (j <- thetaData(i._2).indices) {
-          assert(math.exp(thetaData(i._2)(j)) ~== math.exp(model.theta(i._1)(j)) absTol 0.05)
+          assert(thetaData(i._2)(j) ~== model.theta(i._1)(j) relTol 0.2)
         }
       }
     } catch {
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
index 6e68c1c9d36c8..12ab2ac3cc698 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
@@ -23,6 +23,12 @@ import org.apache.spark.mllib.util.TestingUtils._
 
 class BLASSuite extends SparkFunSuite {
 
+  test("nativeL1Threshold") {
+    assert(getBLAS(128) == BLAS.f2jBLAS)
+    assert(getBLAS(256) == BLAS.nativeBLAS)
+    assert(getBLAS(512) == BLAS.nativeBLAS)
+  }
+
   test("copy") {
     val sx = Vectors.sparse(4, Array(0, 2), Array(1.0, -2.0))
     val dx = Vectors.dense(1.0, 0.0, -2.0, 0.0)
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
index 5f19e466ecad0..3caa8f6d5b1e5 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
@@ -38,12 +38,14 @@ object UDTSerializationBenchmark extends BenchmarkBase {
       val iters = 1e2.toInt
       val numRows = 1e3.toInt
 
-      val encoder = ExpressionEncoder[Vector].resolveAndBind()
+      val encoder = ExpressionEncoder[Vector]().resolveAndBind()
+      val toRow = encoder.createSerializer()
+      val fromRow = encoder.createDeserializer()
 
       val vectors = (1 to numRows).map { i =>
         Vectors.dense(Array.fill(1e5.toInt)(1.0 * i))
       }.toArray
-      val rows = vectors.map(encoder.toRow)
+      val rows = vectors.map(toRow)
 
       val benchmark = new Benchmark("VectorUDT de/serialization", numRows, iters, output = output)
 
@@ -51,7 +53,7 @@ object UDTSerializationBenchmark extends BenchmarkBase {
         var sum = 0
         var i = 0
         while (i < numRows) {
-          sum += encoder.toRow(vectors(i)).numFields
+          sum += toRow(vectors(i)).numFields
           i += 1
         }
       }
@@ -60,7 +62,7 @@ object UDTSerializationBenchmark extends BenchmarkBase {
         var sum = 0
         var i = 0
         while (i < numRows) {
-          sum += encoder.fromRow(rows(i)).numActives
+          sum += fromRow(rows(i)).numActives
           i += 1
         }
       }
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala
index 0a4b11935580a..adc4eeef91bb1 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala
@@ -25,6 +25,7 @@ import breeze.linalg.{norm => brzNorm, svd => brzSvd, DenseMatrix => BDM, DenseV
 import breeze.numerics.abs
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.internal.config.MAX_RESULT_SIZE
 import org.apache.spark.mllib.linalg.{Matrices, Vector, Vectors}
 import org.apache.spark.mllib.random.RandomRDDs
 import org.apache.spark.mllib.util.{LocalClusterSparkContext, MLlibTestSparkContext}
@@ -121,6 +122,20 @@ class RowMatrixSuite extends SparkFunSuite with MLlibTestSparkContext {
     assert(objectBiggerThanResultSize.getMessage.contains("it's bigger than maxResultSize"))
   }
 
+  test("SPARK-33043: getTreeAggregateIdealDepth with unlimited driver size") {
+    val originalMaxResultSize = sc.conf.get[Long](MAX_RESULT_SIZE)
+    sc.conf.set(MAX_RESULT_SIZE, 0L)
+    try {
+      val nbPartitions = 100
+      val vectors = sc.emptyRDD[Vector]
+        .repartition(nbPartitions)
+      val rowMat = new RowMatrix(vectors)
+      assert(rowMat.getTreeAggregateIdealDepth(700 * 1024 * 1024) === 1)
+    } finally {
+      sc.conf.set(MAX_RESULT_SIZE, originalMaxResultSize)
+    }
+  }
+
   test("similar columns") {
     val colMags = Vectors.dense(math.sqrt(126), math.sqrt(66), math.sqrt(94))
     val expected = BDM(
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala
index 8e2d7d10f2ce2..b8342f84be44b 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala
@@ -31,7 +31,7 @@ class StreamingLinearRegressionSuite
   with TestSuiteBase {
 
   // use longer wait time to ensure job completion
-  override def maxWaitTimeMillis: Int = 20000
+  override def maxWaitTimeMillis: Int = 60000
 
   // Assert that two values are equal within tolerance epsilon
   def assertEqual(v1: Double, v2: Double, epsilon: Double): Unit = {
diff --git a/pom.xml b/pom.xml
index 7c23444054efc..2ec79c63e1021 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.0-SNAPSHOT</version>
+  <version>3.0.2-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
@@ -116,8 +116,9 @@
     <maven.compiler.source>${java.version}</maven.compiler.source>
     <maven.compiler.target>${java.version}</maven.compiler.target>
     <maven.version>3.6.3</maven.version>
+    <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
     <sbt.project.name>spark</sbt.project.name>
-    <slf4j.version>1.7.16</slf4j.version>
+    <slf4j.version>1.7.30</slf4j.version>
     <log4j.version>1.2.17</log4j.version>
     <hadoop.version>2.7.4</hadoop.version>
     <protobuf.version>2.5.0</protobuf.version>
@@ -127,29 +128,33 @@
     <hive.group>org.apache.hive</hive.group>
     <hive.classifier>core</hive.classifier>
     <!-- Version used in Maven Hive dependency -->
-    <hive.version>2.3.6</hive.version>
-    <hive23.version>2.3.6</hive23.version>
+    <hive.version>2.3.7</hive.version>
+    <hive23.version>2.3.7</hive23.version>
     <!-- Version used for internal directory structure -->
     <hive.version.short>2.3</hive.version.short>
     <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
-    <kafka.version>2.4.0</kafka.version>
+    <kafka.version>2.4.1</kafka.version>
     <derby.version>10.12.1.1</derby.version>
     <parquet.version>1.10.1</parquet.version>
-    <orc.version>1.5.9</orc.version>
+    <orc.version>1.5.10</orc.version>
     <orc.classifier></orc.classifier>
     <hive.parquet.group>com.twitter</hive.parquet.group>
     <hive.parquet.version>1.6.0</hive.parquet.version>
-    <jetty.version>9.4.18.v20190429</jetty.version>
+    <jetty.version>9.4.34.v20201102</jetty.version>
     <javaxservlet.version>3.1.0</javaxservlet.version>
     <chill.version>0.9.5</chill.version>
     <ivy.version>2.4.0</ivy.version>
     <oro.version>2.0.8</oro.version>
+    <!--
+    If you changes codahale.metrics.version, you also need to change
+    the link to metrics.dropwizard.io in docs/monitoring.md.
+    -->
     <codahale.metrics.version>4.1.1</codahale.metrics.version>
     <avro.version>1.8.2</avro.version>
     <avro.mapred.classifier>hadoop2</avro.mapred.classifier>
     <aws.kinesis.client.version>1.12.0</aws.kinesis.client.version>
     <!-- Should be consistent with Kinesis client dependency -->
-    <aws.java.sdk.version>1.11.271</aws.java.sdk.version>
+    <aws.java.sdk.version>1.11.655</aws.java.sdk.version>
     <!-- the producer is used in tests -->
     <aws.kinesis.producer.version>0.12.8</aws.kinesis.producer.version>
     <!--  org.apache.httpcomponents/httpclient-->
@@ -162,14 +167,16 @@
     <commons.collections.version>3.2.2</commons.collections.version>
     <scala.version>2.12.10</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
+    <scalatest-maven-plugin.version>2.0.0</scalatest-maven-plugin.version>
     <scalafmt.parameters>--test</scalafmt.parameters>
     <!-- for now, not running scalafmt as part of default verify pipeline -->
     <scalafmt.skip>true</scalafmt.skip>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
     <fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
-    <snappy.version>1.1.7.3</snappy.version>
+    <snappy.version>1.1.8.2</snappy.version>
     <netlib.java.version>1.1.2</netlib.java.version>
     <commons-codec.version>1.10</commons-codec.version>
+    <commons-compress.version>1.20</commons-compress.version>
     <commons-io.version>2.4</commons-io.version>
     <!-- org.apache.commons/commons-lang/-->
     <commons-lang2.version>2.6</commons-lang2.version>
@@ -178,7 +185,8 @@
     <!-- org.apache.commons/commons-pool2/-->
     <commons-pool2.version>2.6.2</commons-pool2.version>
     <datanucleus-core.version>4.1.17</datanucleus-core.version>
-    <janino.version>3.0.15</janino.version>
+    <guava.version>14.0.1</guava.version>
+    <janino.version>3.0.16</janino.version>
     <jersey.version>2.30</jersey.version>
     <joda.version>2.10.5</joda.version>
     <jodd.version>3.5.2</jodd.version>
@@ -193,7 +201,7 @@
     -->
     <paranamer.version>2.8</paranamer.version>
     <maven-antrun.version>1.8</maven-antrun.version>
-    <commons-crypto.version>1.0.0</commons-crypto.version>
+    <commons-crypto.version>1.1.0</commons-crypto.version>
     <!--
     If you are changing Arrow version specification, please check ./python/pyspark/sql/utils.py,
     and ./python/setup.py too.
@@ -203,6 +211,9 @@
     <leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
 
     <test.java.home>${java.home}</test.java.home>
+
+    <!-- Some UI tests require Chrome and Chrome driver installed so those tests are disabled by default. -->
+    <test.default.exclude.tags>org.apache.spark.tags.ChromeUITest</test.default.exclude.tags>
     <test.exclude.tags></test.exclude.tags>
     <test.include.tags></test.include.tags>
 
@@ -242,8 +253,11 @@
       things breaking.
     -->
     <spark.test.home>${session.executionRootDirectory}</spark.test.home>
+    <spark.test.webdriver.chrome.driver></spark.test.webdriver.chrome.driver>
 
     <CodeCacheSize>1g</CodeCacheSize>
+    <!-- Needed for consistent times -->
+    <maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
   </properties>
   <repositories>
     <repository>
@@ -253,7 +267,7 @@
         See https://storage-download.googleapis.com/maven-central/index.html
       -->
       <name>GCS Maven Central mirror</name>
-      <url>https://maven-central.storage-download.googleapis.com/repos/central/data/</url>
+      <url>https://maven-central.storage-download.googleapis.com/maven2/</url>
       <releases>
         <enabled>true</enabled>
       </releases>
@@ -284,7 +298,7 @@
         See https://storage-download.googleapis.com/maven-central/index.html
       -->
       <name>GCS Maven Central mirror</name>
-      <url>https://maven-central.storage-download.googleapis.com/repos/central/data/</url>
+      <url>https://maven-central.storage-download.googleapis.com/maven2/</url>
       <releases>
         <enabled>true</enabled>
       </releases>
@@ -444,7 +458,7 @@
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
-        <version>14.0.1</version>
+        <version>${guava.version}</version>
         <scope>provided</scope>
       </dependency>
       <dependency>
@@ -516,6 +530,11 @@
         <artifactId>commons-codec</artifactId>
         <version>${commons-codec.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-compress</artifactId>
+        <version>${commons-compress.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-math3</artifactId>
@@ -612,7 +631,6 @@
         <groupId>xml-apis</groupId>
         <artifactId>xml-apis</artifactId>
         <version>1.4.01</version>
-        <scope>test</scope>
       </dependency>
       <dependency>
         <groupId>org.slf4j</groupId>
@@ -699,7 +717,7 @@
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-all</artifactId>
-        <version>4.1.42.Final</version>
+        <version>4.1.47.Final</version>
       </dependency>
       <dependency>
         <groupId>org.apache.derby</groupId>
@@ -932,8 +950,9 @@
       <dependency>
         <groupId>com.spotify</groupId>
         <artifactId>docker-client</artifactId>
-        <version>5.0.2</version>
+        <version>8.14.1</version>
         <scope>test</scope>
+        <classifier>shaded</classifier>
         <exclusions>
           <exclusion>
             <artifactId>guava</artifactId>
@@ -1083,6 +1102,12 @@
           </exclusion>
         </exclusions>
       </dependency>
+      <!-- Managed up to match Hadoop in HADOOP-16530 -->
+      <dependency>
+        <groupId>xerces</groupId>
+        <artifactId>xercesImpl</artifactId>
+        <version>2.12.0</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.avro</groupId>
         <artifactId>avro</artifactId>
@@ -1200,6 +1225,10 @@
             <groupId>com.sun.jersey.contribs</groupId>
             <artifactId>*</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>jdk.tools</groupId>
+            <artifactId>jdk.tools</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -1334,6 +1363,10 @@
             <groupId>com.zaxxer</groupId>
             <artifactId>HikariCP-java7</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>com.microsoft.sqlserver</groupId>
+            <artifactId>mssql-jdbc</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -1448,6 +1481,11 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service</artifactId>
           </exclusion>
+          <exclusion>
+            <!-- All classes are covered by spark's hive-thriftserver module -->
+            <groupId>${hive.group}</groupId>
+            <artifactId>hive-service-rpc</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>${hive.group}</groupId>
             <artifactId>hive-shims</artifactId>
@@ -1504,6 +1542,11 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service</artifactId>
           </exclusion>
+          <exclusion>
+            <!-- All classes are covered by spark's hive-thriftserver module -->
+            <groupId>${hive.group}</groupId>
+            <artifactId>hive-service-rpc</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>${hive.group}</groupId>
             <artifactId>hive-shims</artifactId>
@@ -1757,6 +1800,11 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service</artifactId>
           </exclusion>
+          <exclusion>
+            <!-- All classes are covered by spark's hive-thriftserver module -->
+            <groupId>${hive.group}</groupId>
+            <artifactId>hive-service-rpc</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>${hive.group}</groupId>
             <artifactId>hive-shims</artifactId>
@@ -1907,6 +1955,11 @@
             <artifactId>groovy-all</artifactId>
           </exclusion>
           <!-- Begin of Hive 2.3 exclusion -->
+          <exclusion>
+            <!-- All classes are covered by spark's hive-thriftserver module -->
+            <groupId>${hive.group}</groupId>
+            <artifactId>hive-service-rpc</artifactId>
+          </exclusion>
           <!-- parquet-hadoop-bundle:1.8.1 conflict with 1.10.1 -->
           <exclusion>
             <groupId>org.apache.parquet</groupId>
@@ -2240,7 +2293,7 @@
       <dependency>
         <groupId>com.univocity</groupId>
         <artifactId>univocity-parsers</artifactId>
-        <version>2.8.3</version>
+        <version>2.9.0</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hive</groupId>
@@ -2303,11 +2356,39 @@
             </execution>
           </executions>
         </plugin>
-        <plugin>
-          <groupId>org.codehaus.mojo</groupId>
-          <artifactId>build-helper-maven-plugin</artifactId>
-          <version>3.0.0</version>
-        </plugin>
+	<plugin>
+	  <groupId>org.codehaus.mojo</groupId>
+	  <artifactId>build-helper-maven-plugin</artifactId>
+	  <version>3.2.0</version>
+	  <executions>
+	    <execution>
+	      <id>module-timestamp-property</id>
+	      <phase>validate</phase>
+	      <goals>
+		<goal>timestamp-property</goal>
+	      </goals>
+	      <configuration>
+		<name>module.build.timestamp</name>
+		<pattern>${maven.build.timestamp.format}</pattern>
+		<timeSource>current</timeSource>
+		<timeZone>America/Los_Angeles</timeZone>
+	      </configuration>
+	    </execution>
+	    <execution>
+	      <id>local-timestamp-property</id>
+	      <phase>validate</phase>
+	      <goals>
+		<goal>timestamp-property</goal>
+	      </goals>
+	      <configuration>
+		<name>local.build.timestamp</name>
+		<pattern>${maven.build.timestamp.format}</pattern>
+		<timeSource>build</timeSource>
+		<timeZone>America/Los_Angeles</timeZone>
+	      </configuration>
+	    </execution>
+	  </executions>
+	</plugin>
         <plugin>
           <groupId>net.alchim31.maven</groupId>
           <artifactId>scala-maven-plugin</artifactId>
@@ -2440,7 +2521,7 @@
         <plugin>
           <groupId>org.scalatest</groupId>
           <artifactId>scalatest-maven-plugin</artifactId>
-          <version>2.0.0</version>
+          <version>${scalatest-maven-plugin.version}</version>
           <!-- Note config is repeated in surefire config -->
           <configuration>
             <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
@@ -2469,10 +2550,11 @@
               <spark.ui.enabled>false</spark.ui.enabled>
               <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
               <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
+              <spark.test.webdriver.chrome.driver>${spark.test.webdriver.chrome.driver}</spark.test.webdriver.chrome.driver>
               <!-- Needed by sql/hive tests. -->
               <test.src.tables>__not_used__</test.src.tables>
             </systemProperties>
-            <tagsToExclude>${test.exclude.tags}</tagsToExclude>
+            <tagsToExclude>${test.exclude.tags},${test.default.exclude.tags}</tagsToExclude>
             <tagsToInclude>${test.include.tags}</tagsToInclude>
           </configuration>
           <executions>
@@ -2575,7 +2657,7 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>1.6.0</version>
+          <version>${exec-maven-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 65ffa228eddec..b79962446d84b 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -36,6 +36,12 @@ object MimaExcludes {
 
   // Exclude rules for 3.0.x
   lazy val v30excludes = v24excludes ++ Seq(
+    // [SPARK-23429][CORE] Add executor memory metrics to heartbeat and expose in executors REST API
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.apply"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.copy"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.this"),
+    ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate$"),
+
     // [SPARK-29306] Add support for Stage level scheduling for executors
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.productElement"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.productArity"),
@@ -47,9 +53,7 @@ object MimaExcludes {
     // [SPARK-29399][core] Remove old ExecutorPlugin interface.
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ExecutorPlugin"),
 
-    // [SPARK-][SQL][CORE][MLLIB] Remove more old deprecated items in Spark 3
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLContext.createExternalTable"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.createExternalTable"),
+    // [SPARK-28980][SQL][CORE][MLLIB] Remove more old deprecated items in Spark 3
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.KMeans.train"),
     ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.mllib.clustering.KMeans.train"),
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.classification.LogisticRegressionWithSGD$"),
@@ -74,8 +78,12 @@ object MimaExcludes {
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.KMeans.getRuns"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.KMeans.setRuns"),
 
+    // [SPARK-26580][SQL][ML][FOLLOW-UP] Throw exception when use untyped UDF by default
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.UnaryTransformer.this"),
+
     // [SPARK-27090][CORE] Removing old LEGACY_DRIVER_IDENTIFIER ("<driver>")
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.SparkContext.LEGACY_DRIVER_IDENTIFIER"),
+
     // [SPARK-25838] Remove formatVersion from Saveable
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.DistributedLDAModel.formatVersion"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.LocalLDAModel.formatVersion"),
@@ -208,14 +216,6 @@ object MimaExcludes {
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.setMinInstancesPerNode"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.setNumTrees"),
 
-    // [SPARK-26124] Update plugins, including MiMa
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsPushDownRequiredColumns.build"),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsReportStatistics.fullSchema"),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsReportStatistics.planInputPartitions"),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsReportPartitioning.fullSchema"),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsReportPartitioning.planInputPartitions"),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsPushDownFilters.build"),
-
     // [SPARK-26090] Resolve most miscellaneous deprecation and build warnings for Spark 3
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.stat.test.BinarySampleBeanInfo"),
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.regression.LabeledPointBeanInfo"),
@@ -228,13 +228,6 @@ object MimaExcludes {
     ProblemFilters.exclude[FinalMethodProblem]("org.apache.spark.ml.feature.QuantileDiscretizer.relativeError"),
     ProblemFilters.exclude[FinalMethodProblem]("org.apache.spark.ml.feature.QuantileDiscretizer.getRelativeError"),
 
-    // [SPARK-25959] GBTClassifier picks wrong impurity stats on loading
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.tree.HasVarianceImpurity.org$apache$spark$ml$tree$HasVarianceImpurity$_setter_$impurity_="),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.tree.HasVarianceImpurity.org$apache$spark$ml$tree$HasVarianceImpurity$_setter_$impurity_="),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.tree.HasVarianceImpurity.org$apache$spark$ml$tree$HasVarianceImpurity$_setter_$impurity_="),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.tree.HasVarianceImpurity.org$apache$spark$ml$tree$HasVarianceImpurity$_setter_$impurity_="),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.tree.HasVarianceImpurity.org$apache$spark$ml$tree$HasVarianceImpurity$_setter_$impurity_="),
-
     // [SPARK-28968][ML] Add HasNumFeatures in the scala side
     ProblemFilters.exclude[FinalMethodProblem]("org.apache.spark.ml.feature.FeatureHasher.getNumFeatures"),
     ProblemFilters.exclude[FinalMethodProblem]("org.apache.spark.ml.feature.FeatureHasher.numFeatures"),
@@ -248,14 +241,6 @@ object MimaExcludes {
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.ShuffleWriteMetrics.shuffleWriteTime"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.ShuffleWriteMetrics.shuffleRecordsWritten"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.AccumulableInfo.apply"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.functions.approxCountDistinct"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.functions.toRadians"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.functions.toDegrees"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.functions.monotonicallyIncreasingId"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLContext.clearActive"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLContext.getOrCreate"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLContext.setActive"),
-    ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.SQLContext.this"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.evaluation.MulticlassMetrics.fMeasure"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.evaluation.MulticlassMetrics.recall"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.evaluation.MulticlassMetrics.precision"),
@@ -313,12 +298,6 @@ object MimaExcludes {
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ml.feature.OneHotEncoderEstimator"),
     ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.ml.feature.OneHotEncoder"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.transform"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.getInputCol"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.getOutputCol"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.inputCol"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.setInputCol"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.setOutputCol"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.outputCol"),
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ml.feature.OneHotEncoderEstimator$"),
 
     // [SPARK-30329][ML] add iterator/foreach methods for Vectors
@@ -352,14 +331,6 @@ object MimaExcludes {
     (problem: Problem) => problem match {
       case MissingClassProblem(cls) =>
         !cls.fullName.startsWith("org.apache.spark.sql.sources.v2")
-      case MissingTypesProblem(newCls, _) =>
-        !newCls.fullName.startsWith("org.apache.spark.sql.sources.v2")
-      case InheritedNewAbstractMethodProblem(cls, _) =>
-        !cls.fullName.startsWith("org.apache.spark.sql.sources.v2")
-      case DirectMissingMethodProblem(meth) =>
-        !meth.owner.fullName.startsWith("org.apache.spark.sql.sources.v2")
-      case ReversedMissingMethodProblem(meth) =>
-        !meth.owner.fullName.startsWith("org.apache.spark.sql.sources.v2")
       case _ => true
     },
 
@@ -444,13 +415,8 @@ object MimaExcludes {
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.ProcessingTime"),
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.ProcessingTime$"),
 
-    // [SPARK-28556][SQL] QueryExecutionListener should also notify Error
-    ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.util.QueryExecutionListener.onFailure"),
-    ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.util.QueryExecutionListener.onFailure"),
-
     // [SPARK-25382][SQL][PYSPARK] Remove ImageSchema.readImages in 3.0
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.image.ImageSchema.readImages"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.image.ImageSchema.readImages"),
 
     // [SPARK-25341][CORE] Support rolling back a shuffle map stage and re-generate the shuffle files
     ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.shuffle.sort.UnsafeShuffleWriter.this"),
@@ -492,17 +458,14 @@ object MimaExcludes {
     ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.AFTSurvivalRegression.setPredictionCol"),
 
     // [SPARK-29543][SS][UI] Init structured streaming ui
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryListener#QueryStartedEvent.this")
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryListener#QueryStartedEvent.this"),
+
+    // [SPARK-30667][CORE] Add allGather method to BarrierTaskContext
+    ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.RequestToSync")
   )
 
   // Exclude rules for 2.4.x
   lazy val v24excludes = v23excludes ++ Seq(
-    // [SPARK-23429][CORE] Add executor memory metrics to heartbeat and expose in executors REST API
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.apply"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.copy"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.this"),
-    ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate$"),
-
     // [SPARK-25248] add package private methods to TaskContext
     ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.markTaskFailed"),
     ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.markInterrupted"),
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 707c31d2248eb..a774a1100fd58 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -226,7 +226,7 @@ object SparkBuild extends PomBuild {
     resolvers := Seq(
       // Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central.
       // See https://storage-download.googleapis.com/maven-central/index.html for more info.
-      "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/repos/central/data/",
+      "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/",
       DefaultMavenRepository,
       Resolver.mavenLocal,
       Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
@@ -475,10 +475,14 @@ object SparkParallelTestGrouping {
     "org.apache.spark.ml.classification.LogisticRegressionSuite",
     "org.apache.spark.ml.classification.LinearSVCSuite",
     "org.apache.spark.sql.SQLQueryTestSuite",
+    "org.apache.spark.sql.hive.client.HadoopVersionInfoSuite",
+    "org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperationSuite",
     "org.apache.spark.sql.hive.thriftserver.ThriftServerQueryTestSuite",
     "org.apache.spark.sql.hive.thriftserver.SparkSQLEnvSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.ThriftServerPageSuite",
-    "org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2ListenerSuite"
+    "org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2ListenerSuite",
+    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextSuite",
+    "org.apache.spark.sql.kafka010.KafkaDelegationTokenSuite"
   )
 
   private val DEFAULT_TEST_GROUP = "default_test_group"
@@ -618,9 +622,13 @@ object KubernetesIntegrationTests {
  * Overrides to work around sbt's dependency resolution being different from Maven's.
  */
 object DependencyOverrides {
+  lazy val guavaVersion = sys.props.get("guava.version").getOrElse("14.0.1")
   lazy val settings = Seq(
-    dependencyOverrides += "com.google.guava" % "guava" % "14.0.1",
-    dependencyOverrides += "jline" % "jline" % "2.14.6")
+    dependencyOverrides += "com.google.guava" % "guava" % guavaVersion,
+    dependencyOverrides += "commons-io" % "commons-io" % "2.4",
+    dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0",
+    dependencyOverrides += "jline" % "jline" % "2.14.6",
+    dependencyOverrides += "org.apache.avro" % "avro" % "1.8.2")
 }
 
 /**
@@ -819,8 +827,10 @@ object Unidoc {
       .map(_.filterNot(_.getName.contains("$")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/deploy")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/examples")))
+      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/internal")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/memory")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/network")))
+      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/rpc")))
       .map(_.filterNot(f =>
         f.getCanonicalPath.contains("org/apache/spark/shuffle") &&
         !f.getCanonicalPath.contains("org/apache/spark/shuffle/api")))
@@ -837,6 +847,7 @@ object Unidoc {
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/hive/test")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalog/v2/utils")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/hive")))
+      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/v2/avro")))
   }
 
   private def ignoreClasspaths(classpaths: Seq[Classpath]): Seq[Classpath] = {
@@ -852,10 +863,10 @@ object Unidoc {
 
     unidocProjectFilter in(ScalaUnidoc, unidoc) :=
       inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
-        yarn, tags, streamingKafka010, sqlKafka010, avro),
+        yarn, tags, streamingKafka010, sqlKafka010),
     unidocProjectFilter in(JavaUnidoc, unidoc) :=
       inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
-        yarn, tags, streamingKafka010, sqlKafka010, avro),
+        yarn, tags, streamingKafka010, sqlKafka010),
 
     unidocAllClasspaths in (ScalaUnidoc, unidoc) := {
       ignoreClasspaths((unidocAllClasspaths in (ScalaUnidoc, unidoc)).value)
@@ -959,6 +970,9 @@ object TestSettings {
       "2.12"
     }
      */
+
+  private val defaultExcludedTags = Seq("org.apache.spark.tags.ChromeUITest")
+
   lazy val settings = Seq (
     // Fork new JVMs for tests and set Java options for those
     fork := true,
@@ -996,10 +1010,23 @@ object TestSettings {
       sys.props.get("test.exclude.tags").map { tags =>
         tags.split(",").flatMap { tag => Seq("-l", tag) }.toSeq
       }.getOrElse(Nil): _*),
+    testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest,
+      sys.props.get("test.default.exclude.tags").map(tags => tags.split(",").toSeq)
+        .map(tags => tags.filter(!_.trim.isEmpty)).getOrElse(defaultExcludedTags)
+        .flatMap(tag => Seq("-l", tag)): _*),
     testOptions in Test += Tests.Argument(TestFrameworks.JUnit,
       sys.props.get("test.exclude.tags").map { tags =>
         Seq("--exclude-categories=" + tags)
       }.getOrElse(Nil): _*),
+    // Include tags defined in a system property
+    testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest,
+      sys.props.get("test.include.tags").map { tags =>
+        tags.split(",").flatMap { tag => Seq("-n", tag) }.toSeq
+      }.getOrElse(Nil): _*),
+    testOptions in Test += Tests.Argument(TestFrameworks.JUnit,
+      sys.props.get("test.include.tags").map { tags =>
+        Seq("--include-categories=" + tags)
+      }.getOrElse(Nil): _*),
     // Show full stack trace and duration in test cases.
     testOptions in Test += Tests.Argument("-oDF"),
     testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
diff --git a/python/README.md b/python/README.md
index 430efe5d724db..7e2cd1396aaf1 100644
--- a/python/README.md
+++ b/python/README.md
@@ -29,4 +29,4 @@ The Python packaging for Spark is not intended to replace all of the other use c
 
 ## Python Requirements
 
-At its core PySpark depends on Py4J (currently version 0.10.8.1), but some additional sub-packages have their own extra requirements for some features (including numpy, pandas, and pyarrow).
+At its core PySpark depends on Py4J, but some additional sub-packages have their own extra requirements for some features (including numpy, pandas, and pyarrow).
diff --git a/python/docs/Makefile b/python/docs/Makefile
index 66d3fc425daa3..b11e3589065c3 100644
--- a/python/docs/Makefile
+++ b/python/docs/Makefile
@@ -7,7 +7,7 @@ SPHINXBUILD   ?= sphinx-build
 PAPER         ?=
 BUILDDIR      ?= _build
 
-export PYTHONPATH=$(realpath ..):$(realpath ../lib/py4j-0.10.8.1-src.zip)
+export PYTHONPATH=$(realpath ..):$(realpath ../lib/py4j-0.10.9-src.zip)
 
 # User-friendly check for sphinx-build
 ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
diff --git a/python/docs/index.rst b/python/docs/index.rst
index 0e7b62361802a..6e059264e6bbb 100644
--- a/python/docs/index.rst
+++ b/python/docs/index.rst
@@ -16,6 +16,7 @@ Contents:
    pyspark.streaming
    pyspark.ml
    pyspark.mllib
+   pyspark.resource
 
 
 Core classes:
diff --git a/python/docs/make2.bat b/python/docs/make2.bat
index 05d22eb5cdd23..2d7d929755a04 100644
--- a/python/docs/make2.bat
+++ b/python/docs/make2.bat
@@ -2,6 +2,8 @@
 
 REM Command file for Sphinx documentation
 
+set PYTHONPATH=..;..\lib\py4j-0.10.9-src.zip
+
 
 if "%SPHINXBUILD%" == "" (
 	set SPHINXBUILD=sphinx-build
diff --git a/python/docs/pyspark.resource.rst b/python/docs/pyspark.resource.rst
new file mode 100644
index 0000000000000..7f3a79b9e5b52
--- /dev/null
+++ b/python/docs/pyspark.resource.rst
@@ -0,0 +1,11 @@
+pyspark.resource module
+=======================
+
+Module Contents
+---------------
+
+.. automodule:: pyspark.resource
+    :members:
+    :undoc-members:
+    :inherited-members:
+
diff --git a/python/docs/pyspark.rst b/python/docs/pyspark.rst
index 0df12c49ad033..402d6ce9eb016 100644
--- a/python/docs/pyspark.rst
+++ b/python/docs/pyspark.rst
@@ -11,6 +11,7 @@ Subpackages
     pyspark.streaming
     pyspark.ml
     pyspark.mllib
+    pyspark.resource
 
 Contents
 --------
diff --git a/python/docs/pyspark.sql.rst b/python/docs/pyspark.sql.rst
index b69562e845920..406ada701941a 100644
--- a/python/docs/pyspark.sql.rst
+++ b/python/docs/pyspark.sql.rst
@@ -1,8 +1,8 @@
 pyspark.sql module
 ==================
 
-Module Context
---------------
+Module Contents
+---------------
 
 .. automodule:: pyspark.sql
     :members:
diff --git a/python/lib/py4j-0.10.8.1-src.zip b/python/lib/py4j-0.10.8.1-src.zip
deleted file mode 100644
index 1b5dede8f2d62..0000000000000
Binary files a/python/lib/py4j-0.10.8.1-src.zip and /dev/null differ
diff --git a/python/lib/py4j-0.10.9-src.zip b/python/lib/py4j-0.10.9-src.zip
new file mode 100644
index 0000000000000..2c49836147030
Binary files /dev/null and b/python/lib/py4j-0.10.9-src.zip differ
diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index 76a5bd0d645ba..ee153af18c88c 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -54,7 +54,6 @@
 from pyspark.storagelevel import StorageLevel
 from pyspark.accumulators import Accumulator, AccumulatorParam
 from pyspark.broadcast import Broadcast
-from pyspark.resourceinformation import ResourceInformation
 from pyspark.serializers import MarshalSerializer, PickleSerializer
 from pyspark.status import *
 from pyspark.taskcontext import TaskContext, BarrierTaskContext, BarrierTaskInfo
@@ -113,11 +112,11 @@ def wrapper(self, *args, **kwargs):
 
 
 # for back compatibility
-from pyspark.sql import SQLContext, Row
+from pyspark.sql import SQLContext, HiveContext, Row
 
 __all__ = [
     "SparkConf", "SparkContext", "SparkFiles", "RDD", "StorageLevel", "Broadcast",
     "Accumulator", "AccumulatorParam", "MarshalSerializer", "PickleSerializer",
     "StatusTracker", "SparkJobInfo", "SparkStageInfo", "Profiler", "BasicProfiler", "TaskContext",
-    "RDDBarrier", "BarrierTaskContext", "BarrierTaskInfo", "ResourceInformation",
+    "RDDBarrier", "BarrierTaskContext", "BarrierTaskInfo",
 ]
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 6cc343e3e495c..25c2a5bfc0725 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -25,6 +25,7 @@
 from tempfile import NamedTemporaryFile
 
 from py4j.protocol import Py4JError
+from py4j.java_gateway import is_instance_of
 
 from pyspark import accumulators
 from pyspark.accumulators import Accumulator
@@ -35,12 +36,12 @@
 from pyspark.serializers import PickleSerializer, BatchedSerializer, UTF8Deserializer, \
     PairDeserializer, AutoBatchedSerializer, NoOpSerializer, ChunkedStream
 from pyspark.storagelevel import StorageLevel
-from pyspark.resourceinformation import ResourceInformation
+from pyspark.resource import ResourceInformation
 from pyspark.rdd import RDD, _load_from_socket, ignore_unicode_prefix
+from pyspark.taskcontext import TaskContext
 from pyspark.traceback_utils import CallSite, first_spark_call
 from pyspark.status import StatusTracker
 from pyspark.profiler import ProfilerCollector, BasicProfiler
-from pyspark.util import _warn_pin_thread
 
 if sys.version > '3':
     xrange = range
@@ -118,6 +119,11 @@ def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None,
             ...
         ValueError:...
         """
+        if (conf is not None and
+                conf.get("spark.executor.allowSparkContext", "true").lower() != "true"):
+            # In order to prevent SparkContext from being created in executors.
+            SparkContext._assert_on_driver()
+
         self._callsite = first_spark_call() or CallSite(None, None, None)
         if gateway is not None and gateway.gateway_parameters.auth_token is None:
             raise ValueError(
@@ -205,6 +211,8 @@ def _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize,
         # data via a socket.
         # scala's mangled names w/ $ in them require special treatment.
         self._encryption_enabled = self._jvm.PythonUtils.isEncryptionEnabled(self._jsc)
+        os.environ["SPARK_BUFFER_SIZE"] = \
+            str(self._jvm.PythonUtils.getSparkBufferSize(self._jsc))
 
         self.pythonExec = os.environ.get("PYSPARK_PYTHON", 'python')
         self.pythonVer = "%d.%d" % sys.version_info[:2]
@@ -864,8 +872,21 @@ def union(self, rdds):
         first_jrdd_deserializer = rdds[0]._jrdd_deserializer
         if any(x._jrdd_deserializer != first_jrdd_deserializer for x in rdds):
             rdds = [x._reserialize() for x in rdds]
-        cls = SparkContext._jvm.org.apache.spark.api.java.JavaRDD
-        jrdds = SparkContext._gateway.new_array(cls, len(rdds))
+        gw = SparkContext._gateway
+        jvm = SparkContext._jvm
+        jrdd_cls = jvm.org.apache.spark.api.java.JavaRDD
+        jpair_rdd_cls = jvm.org.apache.spark.api.java.JavaPairRDD
+        jdouble_rdd_cls = jvm.org.apache.spark.api.java.JavaDoubleRDD
+        if is_instance_of(gw, rdds[0]._jrdd, jrdd_cls):
+            cls = jrdd_cls
+        elif is_instance_of(gw, rdds[0]._jrdd, jpair_rdd_cls):
+            cls = jpair_rdd_cls
+        elif is_instance_of(gw, rdds[0]._jrdd, jdouble_rdd_cls):
+            cls = jdouble_rdd_cls
+        else:
+            cls_name = rdds[0]._jrdd.getClass().getCanonicalName()
+            raise TypeError("Unsupported Java RDD class %s" % cls_name)
+        jrdds = gw.new_array(cls, len(rdds))
         for i in range(0, len(rdds)):
             jrdds[i] = rdds[i]._jrdd
         return RDD(self._jsc.union(jrdds), self, rdds[0]._jrdd_deserializer)
@@ -1012,17 +1033,9 @@ def setJobGroup(self, groupId, description, interruptOnCancel=False):
         .. note:: Currently, setting a group ID (set to local properties) with multiple threads
             does not properly work. Internally threads on PVM and JVM are not synced, and JVM
             thread can be reused for multiple threads on PVM, which fails to isolate local
-            properties for each thread on PVM.
-
-            To work around this, you can set `PYSPARK_PIN_THREAD` to
-            `'true'` (see SPARK-22340). However, note that it cannot inherit the local properties
-            from the parent thread although it isolates each thread on PVM and JVM with its own
-            local properties.
-
-            To work around this, you should manually copy and set the local
-            properties from the parent thread to the child thread when you create another thread.
+            properties for each thread on PVM. To work around this, You can use
+            :meth:`RDD.collectWithJobGroup` for now.
         """
-        _warn_pin_thread("setJobGroup")
         self._jsc.setJobGroup(groupId, description, interruptOnCancel)
 
     def setLocalProperty(self, key, value):
@@ -1033,17 +1046,9 @@ def setLocalProperty(self, key, value):
         .. note:: Currently, setting a local property with multiple threads does not properly work.
             Internally threads on PVM and JVM are not synced, and JVM thread
             can be reused for multiple threads on PVM, which fails to isolate local properties
-            for each thread on PVM.
-
-            To work around this, you can set `PYSPARK_PIN_THREAD` to
-            `'true'` (see SPARK-22340). However, note that it cannot inherit the local properties
-            from the parent thread although it isolates each thread on PVM and JVM with its own
-            local properties.
-
-            To work around this, you should manually copy and set the local
-            properties from the parent thread to the child thread when you create another thread.
+            for each thread on PVM. To work around this, You can use
+            :meth:`RDD.collectWithJobGroup`.
         """
-        _warn_pin_thread("setLocalProperty")
         self._jsc.setLocalProperty(key, value)
 
     def getLocalProperty(self, key):
@@ -1060,17 +1065,9 @@ def setJobDescription(self, value):
         .. note:: Currently, setting a job description (set to local properties) with multiple
             threads does not properly work. Internally threads on PVM and JVM are not synced,
             and JVM thread can be reused for multiple threads on PVM, which fails to isolate
-            local properties for each thread on PVM.
-
-            To work around this, you can set `PYSPARK_PIN_THREAD` to
-            `'true'` (see SPARK-22340). However, note that it cannot inherit the local properties
-            from the parent thread although it isolates each thread on PVM and JVM with its own
-            local properties.
-
-            To work around this, you should manually copy and set the local
-            properties from the parent thread to the child thread when you create another thread.
+            local properties for each thread on PVM. To work around this, You can use
+            :meth:`RDD.collectWithJobGroup` for now.
         """
-        _warn_pin_thread("setJobDescription")
         self._jsc.setJobDescription(value)
 
     def sparkUser(self):
@@ -1156,6 +1153,16 @@ def resources(self):
             resources[name] = ResourceInformation(name, addrs)
         return resources
 
+    @staticmethod
+    def _assert_on_driver():
+        """
+        Called to ensure that SparkContext is created only on the Driver.
+
+        Throws an exception if a SparkContext is about to be created in executors.
+        """
+        if TaskContext.get() is not None:
+            raise Exception("SparkContext should only be created and accessed on the driver.")
+
 
 def _test():
     import atexit
diff --git a/python/pyspark/find_spark_home.py b/python/pyspark/find_spark_home.py
index 9c4ed46598632..52f6ea9a37100 100755
--- a/python/pyspark/find_spark_home.py
+++ b/python/pyspark/find_spark_home.py
@@ -40,6 +40,7 @@ def is_spark_home(path):
     paths = ["../", os.path.dirname(os.path.realpath(__file__))]
 
     # Add the path of the PySpark module if it exists
+    import_error_raised = False
     if sys.version < "3":
         import imp
         try:
@@ -49,7 +50,7 @@ def is_spark_home(path):
             paths.append(os.path.join(module_home, "../../"))
         except ImportError:
             # Not pip installed no worries
-            pass
+            import_error_raised = True
     else:
         from importlib.util import find_spec
         try:
@@ -59,7 +60,7 @@ def is_spark_home(path):
             paths.append(os.path.join(module_home, "../../"))
         except ImportError:
             # Not pip installed no worries
-            pass
+            import_error_raised = True
 
     # Normalize the paths
     paths = [os.path.abspath(p) for p in paths]
@@ -68,6 +69,19 @@ def is_spark_home(path):
         return next(path for path in paths if is_spark_home(path))
     except StopIteration:
         print("Could not find valid SPARK_HOME while searching {0}".format(paths), file=sys.stderr)
+        if import_error_raised:
+            print(
+                "\nDid you install PySpark via a package manager such as pip or Conda? If so,\n"
+                "PySpark was not found in your Python environment. It is possible your\n"
+                "Python environment does not properly bind with your package manager.\n"
+                "\nPlease check your default 'python' and if you set PYSPARK_PYTHON and/or\n"
+                "PYSPARK_DRIVER_PYTHON environment variables, and see if you can import\n"
+                "PySpark, for example, 'python -c 'import pyspark'.\n"
+                "\nIf you cannot import, you can install by using the Python executable directly,\n"
+                "for example, 'python -m pip install pyspark [--user]'. Otherwise, you can also\n"
+                "explicitly set the Python executable, that has PySpark installed, to\n"
+                "PYSPARK_PYTHON or PYSPARK_DRIVER_PYTHON environment variables, for example,\n"
+                "'PYSPARK_PYTHON=python3 pyspark'.\n", file=sys.stderr)
         sys.exit(-1)
 
 if __name__ == "__main__":
diff --git a/python/pyspark/ml/base.py b/python/pyspark/ml/base.py
index 542cb25172ead..3368a9d36c783 100644
--- a/python/pyspark/ml/base.py
+++ b/python/pyspark/ml/base.py
@@ -96,8 +96,6 @@ def fitMultiple(self, dataset, paramMaps):
         :return: A thread safe iterable which contains one model for each param map. Each
                  call to `next(modelIterator)` will return `(index, model)` where model was fit
                  using `paramMaps[index]`. `index` values may not be sequential.
-
-        .. note:: DeveloperApi
         """
         estimator = self.copy()
 
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index 5ee42318afd45..1392bc763dd94 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -27,9 +27,9 @@
     _HasVarianceImpurity, _TreeClassifierParams, _TreeEnsembleParams
 from pyspark.ml.regression import _FactorizationMachinesParams, DecisionTreeRegressionModel
 from pyspark.ml.util import *
-from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, \
+from pyspark.ml.wrapper import JavaParams, \
     JavaPredictor, _JavaPredictorParams, JavaPredictionModel, JavaWrapper
-from pyspark.ml.common import inherit_doc, _java2py, _py2java
+from pyspark.ml.common import inherit_doc
 from pyspark.ml.linalg import Vectors
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import udf, when
@@ -165,8 +165,7 @@ def predictProbability(self, value):
 
 
 class _LinearSVCParams(_JavaClassifierParams, HasRegParam, HasMaxIter, HasFitIntercept, HasTol,
-                       HasStandardization, HasWeightCol, HasAggregationDepth, HasThreshold,
-                       HasBlockSize):
+                       HasStandardization, HasWeightCol, HasAggregationDepth, HasThreshold):
     """
     Params for :py:class:`LinearSVC` and :py:class:`LinearSVCModel`.
 
@@ -179,6 +178,11 @@ class _LinearSVCParams(_JavaClassifierParams, HasRegParam, HasMaxIter, HasFitInt
                       " all predictions 0.0 and -Inf will make all predictions 1.0.",
                       typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_LinearSVCParams, self).__init__(*args)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, fitIntercept=True,
+                         standardization=True, threshold=0.0, aggregationDepth=2)
+
 
 @inherit_doc
 class LinearSVC(JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadable):
@@ -215,8 +219,6 @@ class LinearSVC(JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadable
     LinearSVCModel...
     >>> model.getThreshold()
     0.5
-    >>> model.getBlockSize()
-    1024
     >>> model.coefficients
     DenseVector([0.0, -0.2792, -0.1833])
     >>> model.intercept
@@ -247,6 +249,8 @@ class LinearSVC(JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadable
     True
     >>> model.intercept == model2.intercept
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 2.2.0
     """
@@ -255,19 +259,16 @@ class LinearSVC(JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadable
     def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction",
                  fitIntercept=True, standardization=True, threshold=0.0, weightCol=None,
-                 aggregationDepth=2, blockSize=1024):
+                 aggregationDepth=2):
         """
         __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction", \
                  fitIntercept=True, standardization=True, threshold=0.0, weightCol=None, \
-                 aggregationDepth=2, blockSize=1024):
+                 aggregationDepth=2):
         """
         super(LinearSVC, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.LinearSVC", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, fitIntercept=True,
-                         standardization=True, threshold=0.0, aggregationDepth=2,
-                         blockSize=1024)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -276,12 +277,12 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
     def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction",
                   fitIntercept=True, standardization=True, threshold=0.0, weightCol=None,
-                  aggregationDepth=2, blockSize=1024):
+                  aggregationDepth=2):
         """
         setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction", \
                   fitIntercept=True, standardization=True, threshold=0.0, weightCol=None, \
-                  aggregationDepth=2, blockSize=1024):
+                  aggregationDepth=2):
         Sets params for Linear SVM Classifier.
         """
         kwargs = self._input_kwargs
@@ -346,13 +347,6 @@ def setAggregationDepth(self, value):
         """
         return self._set(aggregationDepth=value)
 
-    @since("3.0.0")
-    def setBlockSize(self, value):
-        """
-        Sets the value of :py:attr:`blockSize`.
-        """
-        return self._set(blockSize=value)
-
 
 class LinearSVCModel(JavaClassificationModel, _LinearSVCParams, JavaMLWritable, JavaMLReadable):
     """
@@ -388,7 +382,7 @@ def intercept(self):
 class _LogisticRegressionParams(_JavaProbabilisticClassifierParams, HasRegParam,
                                 HasElasticNetParam, HasMaxIter, HasFitIntercept, HasTol,
                                 HasStandardization, HasWeightCol, HasAggregationDepth,
-                                HasThreshold, HasBlockSize):
+                                HasThreshold):
     """
     Params for :py:class:`LogisticRegression` and :py:class:`LogisticRegressionModel`.
 
@@ -438,6 +432,10 @@ class _LogisticRegressionParams(_JavaProbabilisticClassifierParams, HasRegParam,
                                     "classes for multinomial regression.",
                                     typeConverter=TypeConverters.toVector)
 
+    def __init__(self, *args):
+        super(_LogisticRegressionParams, self).__init__(*args)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1E-6, threshold=0.5, family="auto")
+
     @since("1.4.0")
     def setThreshold(self, value):
         """
@@ -570,8 +568,6 @@ class LogisticRegression(JavaProbabilisticClassifier, _LogisticRegressionParams,
     10
     >>> blor.clear(blor.maxIter)
     >>> blorModel = blor.fit(bdf)
-    >>> blorModel.getBlockSize()
-    1024
     >>> blorModel.setFeaturesCol("features")
     LogisticRegressionModel...
     >>> blorModel.setProbabilityCol("newProbability")
@@ -629,6 +625,8 @@ class LogisticRegression(JavaProbabilisticClassifier, _LogisticRegressionParams,
     True
     >>> model2
     LogisticRegressionModel: uid=..., numClasses=2, numFeatures=2
+    >>> blorModel.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 1.3.0
     """
@@ -640,7 +638,7 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
                  rawPredictionCol="rawPrediction", standardization=True, weightCol=None,
                  aggregationDepth=2, family="auto",
                  lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None,
-                 lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, blockSize=1024):
+                 lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None):
 
         """
         __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
@@ -649,14 +647,12 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
                  rawPredictionCol="rawPrediction", standardization=True, weightCol=None, \
                  aggregationDepth=2, family="auto", \
                  lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None, \
-                 lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, blockSize=1024):
+                 lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None):
         If the threshold and thresholds Params are both set, they must be equivalent.
         """
         super(LogisticRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.LogisticRegression", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1E-6, threshold=0.5, family="auto",
-                         blockSize=1024)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
         self._checkThresholdConsistency()
@@ -669,7 +665,7 @@ def setParams(self, featuresCol="features", labelCol="label", predictionCol="pre
                   rawPredictionCol="rawPrediction", standardization=True, weightCol=None,
                   aggregationDepth=2, family="auto",
                   lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None,
-                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, blockSize=1024):
+                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None):
         """
         setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \
@@ -677,7 +673,7 @@ def setParams(self, featuresCol="features", labelCol="label", predictionCol="pre
                   rawPredictionCol="rawPrediction", standardization=True, weightCol=None, \
                   aggregationDepth=2, family="auto", \
                   lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None, \
-                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, blockSize=1024):
+                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None):
         Sets params for logistic regression.
         If the threshold and thresholds Params are both set, they must be equivalent.
         """
@@ -772,13 +768,6 @@ def setAggregationDepth(self, value):
         """
         return self._set(aggregationDepth=value)
 
-    @since("3.0.0")
-    def setBlockSize(self, value):
-        """
-        Sets the value of :py:attr:`blockSize`.
-        """
-        return self._set(blockSize=value)
-
 
 class LogisticRegressionModel(JavaProbabilisticClassificationModel, _LogisticRegressionParams,
                               JavaMLWritable, JavaMLReadable, HasTrainingSummary):
@@ -852,7 +841,10 @@ def evaluate(self, dataset):
         if not isinstance(dataset, DataFrame):
             raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
         java_blr_summary = self._call_java("evaluate", dataset)
-        return BinaryLogisticRegressionSummary(java_blr_summary)
+        if self.numClasses <= 2:
+            return BinaryLogisticRegressionSummary(java_blr_summary)
+        else:
+            return LogisticRegressionSummary(java_blr_summary)
 
 
 class LogisticRegressionSummary(JavaWrapper):
@@ -1148,7 +1140,12 @@ class _DecisionTreeClassifierParams(_DecisionTreeParams, _TreeClassifierParams):
     """
     Params for :py:class:`DecisionTreeClassifier` and :py:class:`DecisionTreeClassificationModel`.
     """
-    pass
+
+    def __init__(self, *args):
+        super(_DecisionTreeClassifierParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="gini", leafCol="", minWeightFractionPerNode=0.0)
 
 
 @inherit_doc
@@ -1215,7 +1212,8 @@ class DecisionTreeClassifier(JavaProbabilisticClassifier, _DecisionTreeClassifie
     >>> model2 = DecisionTreeClassificationModel.load(model_path)
     >>> model.featureImportances == model2.featureImportances
     True
-
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> df3 = spark.createDataFrame([
     ...     (1.0, 0.2, Vectors.dense(1.0)),
     ...     (1.0, 0.8, Vectors.dense(1.0)),
@@ -1247,9 +1245,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(DecisionTreeClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.DecisionTreeClassifier", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="gini", leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1383,7 +1378,14 @@ class _RandomForestClassifierParams(_RandomForestParams, _TreeClassifierParams):
     """
     Params for :py:class:`RandomForestClassifier` and :py:class:`RandomForestClassificationModel`.
     """
-    pass
+
+    def __init__(self, *args):
+        super(_RandomForestClassifierParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="gini", numTrees=20, featureSubsetStrategy="auto",
+                         subsamplingRate=1.0, leafCol="", minWeightFractionPerNode=0.0,
+                         bootstrap=True)
 
 
 @inherit_doc
@@ -1455,6 +1457,8 @@ class RandomForestClassifier(JavaProbabilisticClassifier, _RandomForestClassifie
     >>> model2 = RandomForestClassificationModel.load(model_path)
     >>> model.featureImportances == model2.featureImportances
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -1477,11 +1481,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(RandomForestClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.RandomForestClassifier", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="gini", numTrees=20, featureSubsetStrategy="auto",
-                         subsamplingRate=1.0, leafCol="", minWeightFractionPerNode=0.0,
-                         bootstrap=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1651,6 +1650,14 @@ class _GBTClassifierParams(_GBTParams, _HasVarianceImpurity):
                      "Supported options: " + ", ".join(supportedLossTypes),
                      typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_GBTClassifierParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         lossType="logistic", maxIter=20, stepSize=0.1, subsamplingRate=1.0,
+                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
+                         leafCol="", minWeightFractionPerNode=0.0)
+
     @since("1.4.0")
     def getLossType(self):
         """
@@ -1742,6 +1749,8 @@ class GBTClassifier(JavaProbabilisticClassifier, _GBTClassifierParams,
     True
     >>> model.treeWeights == model2.treeWeights
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.trees
     [DecisionTreeRegressionModel...depth=..., DecisionTreeRegressionModel...]
     >>> validation = spark.createDataFrame([(0.0, Vectors.dense(-1.0),)],
@@ -1778,11 +1787,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(GBTClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.GBTClassifier", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         lossType="logistic", maxIter=20, stepSize=0.1, subsamplingRate=1.0,
-                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
-                         leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1980,6 +1984,10 @@ class _NaiveBayesParams(_JavaPredictorParams, HasWeightCol):
                       "and gaussian.",
                       typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_NaiveBayesParams, self).__init__(*args)
+        self._setDefault(smoothing=1.0, modelType="multinomial")
+
     @since("1.5.0")
     def getSmoothing(self):
         """
@@ -2063,6 +2071,8 @@ class NaiveBayes(JavaProbabilisticClassifier, _NaiveBayesParams, HasThresholds,
     True
     >>> model.theta == model2.theta
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> nb = nb.setThresholds([0.01, 10.00])
     >>> model3 = nb.fit(df)
     >>> result = model3.transform(test0).head()
@@ -2098,7 +2108,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(NaiveBayes, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.NaiveBayes", self.uid)
-        self._setDefault(smoothing=1.0, modelType="multinomial")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -2190,6 +2199,10 @@ class _MultilayerPerceptronParams(_JavaProbabilisticClassifierParams, HasSeed, H
     initialWeights = Param(Params._dummy(), "initialWeights", "The initial weights of the model.",
                            typeConverter=TypeConverters.toVector)
 
+    def __init__(self, *args):
+        super(_MultilayerPerceptronParams, self).__init__(*args)
+        self._setDefault(maxIter=100, tol=1E-6, blockSize=128, stepSize=0.03, solver="l-bfgs")
+
     @since("1.6.0")
     def getLayers(self):
         """
@@ -2269,6 +2282,8 @@ class MultilayerPerceptronClassifier(JavaProbabilisticClassifier, _MultilayerPer
     True
     >>> model.weights == model2.weights
     True
+    >>> model.transform(testDF).take(1) == model2.transform(testDF).take(1)
+    True
     >>> mlp2 = mlp2.setInitialWeights(list(range(0, 12)))
     >>> model3 = mlp2.fit(df)
     >>> model3.weights != model2.weights
@@ -2293,7 +2308,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(MultilayerPerceptronClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.MultilayerPerceptronClassifier", self.uid)
-        self._setDefault(maxIter=100, tol=1E-6, blockSize=128, stepSize=0.03, solver="l-bfgs")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -2444,6 +2458,8 @@ class OneVsRest(Estimator, _OneVsRestParams, HasParallelism, JavaMLReadable, Jav
     >>> model2 = OneVsRestModel.load(model_path)
     >>> model2.transform(test0).head().newPrediction
     0.0
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.transform(test2).columns
     ['features', 'rawPrediction', 'newPrediction']
 
@@ -2619,50 +2635,6 @@ def _to_java(self):
         _java_obj.setRawPredictionCol(self.getRawPredictionCol())
         return _java_obj
 
-    def _make_java_param_pair(self, param, value):
-        """
-        Makes a Java param pair.
-        """
-        sc = SparkContext._active_spark_context
-        param = self._resolveParam(param)
-        _java_obj = JavaParams._new_java_obj("org.apache.spark.ml.classification.OneVsRest",
-                                             self.uid)
-        java_param = _java_obj.getParam(param.name)
-        if isinstance(value, JavaParams):
-            # used in the case of an estimator having another estimator as a parameter
-            # the reason why this is not in _py2java in common.py is that importing
-            # Estimator and Model in common.py results in a circular import with inherit_doc
-            java_value = value._to_java()
-        else:
-            java_value = _py2java(sc, value)
-        return java_param.w(java_value)
-
-    def _transfer_param_map_to_java(self, pyParamMap):
-        """
-        Transforms a Python ParamMap into a Java ParamMap.
-        """
-        paramMap = JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
-        for param in self.params:
-            if param in pyParamMap:
-                pair = self._make_java_param_pair(param, pyParamMap[param])
-                paramMap.put([pair])
-        return paramMap
-
-    def _transfer_param_map_from_java(self, javaParamMap):
-        """
-        Transforms a Java ParamMap into a Python ParamMap.
-        """
-        sc = SparkContext._active_spark_context
-        paramMap = dict()
-        for pair in javaParamMap.toList():
-            param = pair.param()
-            if self.hasParam(str(param.name())):
-                if param.name() == "classifier":
-                    paramMap[self.getParam(param.name())] = JavaParams._from_java(pair.value())
-                else:
-                    paramMap[self.getParam(param.name())] = _java2py(sc, pair.value())
-        return paramMap
-
 
 class OneVsRestModel(Model, _OneVsRestParams, JavaMLReadable, JavaMLWritable):
     """
@@ -2869,6 +2841,17 @@ class FMClassifier(JavaProbabilisticClassifier, _FactorizationMachinesParams, Ja
     DenseVector([14.8232])
     >>> model.factors
     DenseMatrix(1, 2, [0.0163, -0.0051], 1)
+    >>> model_path = temp_path + "/fm_model"
+    >>> model.save(model_path)
+    >>> model2 = FMClassificationModel.load(model_path)
+    >>> model2.intercept
+    -7.316665276826291
+    >>> model2.linear
+    DenseVector([14.8232])
+    >>> model2.factors
+    DenseMatrix(1, 2, [0.0163, -0.0051], 1)
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 3.0.0
     """
@@ -2889,9 +2872,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(FMClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.FMClassifier", self.uid)
-        self._setDefault(factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
-                         miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
-                         tol=1e-6, solver="adamW")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index 7465cef7d64d4..2060bbc625a02 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -108,6 +108,10 @@ class _GaussianMixtureParams(HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionC
     k = Param(Params._dummy(), "k", "Number of independent Gaussians in the mixture model. " +
               "Must be > 1.", typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_GaussianMixtureParams, self).__init__(*args)
+        self._setDefault(k=2, tol=0.01, maxIter=100, aggregationDepth=2)
+
     @since("2.0.0")
     def getK(self):
         """
@@ -321,6 +325,8 @@ class GaussianMixture(JavaEstimator, _GaussianMixtureParams, JavaMLWritable, Jav
     Row(mean=DenseVector([0.825, 0.8675]))
     >>> model2.gaussiansDF.select("cov").head()
     Row(cov=DenseMatrix(2, 2, [0.0056, -0.0051, -0.0051, 0.0046], False))
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
     >>> gm2.setWeightCol("weight")
     GaussianMixture...
 
@@ -339,7 +345,6 @@ def __init__(self, featuresCol="features", predictionCol="prediction", k=2,
         super(GaussianMixture, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.GaussianMixture",
                                             self.uid)
-        self._setDefault(k=2, tol=0.01, maxIter=100, aggregationDepth=2)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -493,6 +498,11 @@ class _KMeansParams(HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionCol, HasTo
     initSteps = Param(Params._dummy(), "initSteps", "The number of steps for k-means|| " +
                       "initialization mode. Must be > 0.", typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_KMeansParams, self).__init__(*args)
+        self._setDefault(k=2, initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20,
+                         distanceMeasure="euclidean")
+
     @since("1.5.0")
     def getK(self):
         """
@@ -622,6 +632,8 @@ class KMeans(JavaEstimator, _KMeansParams, JavaMLWritable, JavaMLReadable):
     array([ True,  True], dtype=bool)
     >>> model.clusterCenters()[1] == model2.clusterCenters()[1]
     array([ True,  True], dtype=bool)
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
 
     .. versionadded:: 1.5.0
     """
@@ -637,8 +649,6 @@ def __init__(self, featuresCol="features", predictionCol="prediction", k=2,
         """
         super(KMeans, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.KMeans", self.uid)
-        self._setDefault(k=2, initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20,
-                         distanceMeasure="euclidean")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -747,6 +757,10 @@ class _BisectingKMeansParams(HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionC
                                     "proportion of points (if < 1.0) of a divisible cluster.",
                                     typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_BisectingKMeansParams, self).__init__(*args)
+        self._setDefault(maxIter=20, k=4, minDivisibleClusterSize=1.0)
+
     @since("2.0.0")
     def getK(self):
         """
@@ -795,7 +809,7 @@ def computeCost(self, dataset):
         Computes the sum of squared distances between the input points
         and their corresponding cluster centers.
 
-        ..note:: Deprecated in 3.0.0. It will be removed in future versions. Use
+        .. note:: Deprecated in 3.0.0. It will be removed in future versions. Use
            ClusteringEvaluator instead. You can also get the cost on the training dataset in the
            summary.
         """
@@ -897,6 +911,8 @@ class BisectingKMeans(JavaEstimator, _BisectingKMeansParams, JavaMLWritable, Jav
     array([ True,  True], dtype=bool)
     >>> model.clusterCenters()[1] == model2.clusterCenters()[1]
     array([ True,  True], dtype=bool)
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
 
     .. versionadded:: 2.0.0
     """
@@ -913,7 +929,6 @@ def __init__(self, featuresCol="features", predictionCol="prediction", maxIter=2
         super(BisectingKMeans, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.BisectingKMeans",
                                             self.uid)
-        self._setDefault(maxIter=20, k=4, minDivisibleClusterSize=1.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1056,6 +1071,13 @@ class _LDAParams(HasMaxIter, HasFeaturesCol, HasSeed, HasCheckpointInterval):
                                " partition is lost, so set this bit with care.",
                                TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_LDAParams, self).__init__(*args)
+        self._setDefault(maxIter=20, checkpointInterval=10,
+                         k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,
+                         subsamplingRate=0.05, optimizeDocConcentration=True,
+                         topicDistributionCol="topicDistribution", keepLastCheckpoint=True)
+
     @since("2.0.0")
     def getK(self):
         """
@@ -1366,6 +1388,8 @@ class LDA(JavaEstimator, _LDAParams, JavaMLReadable, JavaMLWritable):
     >>> local_model_path = temp_path + "/lda_local_model"
     >>> localModel.save(local_model_path)
     >>> sameLocalModel = LocalLDAModel.load(local_model_path)
+    >>> model.transform(df).take(1) == sameLocalModel.transform(df).take(1)
+    True
 
     .. versionadded:: 2.0.0
     """
@@ -1385,10 +1409,6 @@ def __init__(self, featuresCol="features", maxIter=20, seed=None, checkpointInte
         """
         super(LDA, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.LDA", self.uid)
-        self._setDefault(maxIter=20, checkpointInterval=10,
-                         k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,
-                         subsamplingRate=0.05, optimizeDocConcentration=True,
-                         topicDistributionCol="topicDistribution", keepLastCheckpoint=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1581,6 +1601,10 @@ class _PowerIterationClusteringParams(HasMaxIter, HasWeightCol):
                    "Name of the input column for destination vertex IDs.",
                    typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_PowerIterationClusteringParams, self).__init__(*args)
+        self._setDefault(k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst")
+
     @since("2.4.0")
     def getK(self):
         """
@@ -1654,6 +1678,8 @@ class PowerIterationClustering(_PowerIterationClusteringParams, JavaParams, Java
     2
     >>> pic2.getMaxIter()
     40
+    >>> pic2.assignClusters(df).take(6) == assignments.take(6)
+    True
 
     .. versionadded:: 2.4.0
     """
@@ -1668,7 +1694,6 @@ def __init__(self, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst
         super(PowerIterationClustering, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.clustering.PowerIterationClustering", self.uid)
-        self._setDefault(k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py
index 556a2f85c708d..265f02c1a03ac 100644
--- a/python/pyspark/ml/evaluation.py
+++ b/python/pyspark/ml/evaluation.py
@@ -110,7 +110,8 @@ def isLargerBetter(self):
 class BinaryClassificationEvaluator(JavaEvaluator, HasLabelCol, HasRawPredictionCol, HasWeightCol,
                                     JavaMLReadable, JavaMLWritable):
     """
-    Evaluator for binary classification, which expects two input columns: rawPrediction and label.
+    Evaluator for binary classification, which expects input columns rawPrediction, label
+    and an optional weight column.
     The rawPrediction column can be of type double (binary 0/1 prediction, or probability of label
     1) or of type vector (length-2 vector of raw predictions, scores, or label probabilities).
 
@@ -409,9 +410,9 @@ class MulticlassClassificationEvaluator(JavaEvaluator, HasLabelCol, HasPredictio
     """
     metricName = Param(Params._dummy(), "metricName",
                        "metric name in evaluation "
-                       "(f1|accuracy|weightedPrecision|weightedRecall|weightedTruePositiveRate|"
-                       "weightedFalsePositiveRate|weightedFMeasure|truePositiveRateByLabel|"
-                       "falsePositiveRateByLabel|precisionByLabel|recallByLabel|fMeasureByLabel|"
+                       "(f1|accuracy|weightedPrecision|weightedRecall|weightedTruePositiveRate| "
+                       "weightedFalsePositiveRate|weightedFMeasure|truePositiveRateByLabel| "
+                       "falsePositiveRateByLabel|precisionByLabel|recallByLabel|fMeasureByLabel| "
                        "logLoss|hammingLoss)",
                        typeConverter=TypeConverters.toString)
     metricLabel = Param(Params._dummy(), "metricLabel",
diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py
index 4c25bb495fdfb..ded263e07a26f 100755
--- a/python/pyspark/ml/feature.py
+++ b/python/pyspark/ml/feature.py
@@ -94,6 +94,8 @@ class Binarizer(JavaTransformer, HasThreshold, HasThresholds, HasInputCol, HasOu
     >>> loadedBinarizer = Binarizer.load(binarizerPath)
     >>> loadedBinarizer.getThreshold() == binarizer.getThreshold()
     True
+    >>> loadedBinarizer.transform(df).take(1) == binarizer.transform(df).take(1)
+    True
     >>> df2 = spark.createDataFrame([(0.5, 0.3)], ["values1", "values2"])
     >>> binarizer2 = Binarizer(thresholds=[0.0, 1.0])
     >>> binarizer2.setInputCols(["values1", "values2"]).setOutputCols(["output1", "output2"])
@@ -197,6 +199,10 @@ class _LSHParams(HasInputCol, HasOutputCol):
                           "and decreasing it improves the running performance.",
                           typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_LSHParams, self).__init__(*args)
+        self._setDefault(numHashTables=1)
+
     def getNumHashTables(self):
         """
         Gets the value of numHashTables or its default value.
@@ -392,7 +398,6 @@ def __init__(self, inputCol=None, outputCol=None, seed=None, numHashTables=1,
         super(BucketedRandomProjectionLSH, self).__init__()
         self._java_obj = \
             self._new_java_obj("org.apache.spark.ml.feature.BucketedRandomProjectionLSH", self.uid)
-        self._setDefault(numHashTables=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -479,6 +484,8 @@ class Bucketizer(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols, HasOu
     >>> loadedBucketizer = Bucketizer.load(bucketizerPath)
     >>> loadedBucketizer.getSplits() == bucketizer.getSplits()
     True
+    >>> loadedBucketizer.transform(df).take(1) == bucketizer.transform(df).take(1)
+    True
     >>> bucketed = bucketizer.setHandleInvalid("skip").transform(df).collect()
     >>> len(bucketed)
     4
@@ -735,6 +742,8 @@ class CountVectorizer(JavaEstimator, _CountVectorizerParams, JavaMLReadable, Jav
     >>> loadedModel = CountVectorizerModel.load(modelPath)
     >>> loadedModel.vocabulary == model.vocabulary
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     >>> fromVocabModel = CountVectorizerModel.from_vocabulary(["a", "b", "c"],
     ...     inputCol="raw", outputCol="vectors")
     >>> fromVocabModel.transform(df).show(truncate=False)
@@ -922,6 +931,8 @@ class DCT(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWrit
     >>> dctPath = temp_path + "/dct"
     >>> dct.save(dctPath)
     >>> loadedDtc = DCT.load(dctPath)
+    >>> loadedDtc.transform(df1).take(1) == dct.transform(df1).take(1)
+    True
     >>> loadedDtc.getInverse()
     False
 
@@ -1005,6 +1016,8 @@ class ElementwiseProduct(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReada
     >>> loadedEp = ElementwiseProduct.load(elementwiseProductPath)
     >>> loadedEp.getScalingVec() == ep.getScalingVec()
     True
+    >>> loadedEp.transform(df).take(1) == ep.transform(df).take(1)
+    True
 
     .. versionadded:: 1.5.0
     """
@@ -1203,6 +1216,8 @@ class HashingTF(JavaTransformer, HasInputCol, HasOutputCol, HasNumFeatures, Java
     >>> loadedHashingTF = HashingTF.load(hashingTFPath)
     >>> loadedHashingTF.getNumFeatures() == hashingTF.getNumFeatures()
     True
+    >>> loadedHashingTF.transform(df).take(1) == hashingTF.transform(df).take(1)
+    True
     >>> hashingTF.indexOf("b")
     5
 
@@ -1287,6 +1302,10 @@ class _IDFParams(HasInputCol, HasOutputCol):
                        "minimum number of documents in which a term should appear for filtering",
                        typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_IDFParams, self).__init__(*args)
+        self._setDefault(minDocFreq=0)
+
     @since("1.4.0")
     def getMinDocFreq(self):
         """
@@ -1347,7 +1366,6 @@ def __init__(self, minDocFreq=0, inputCol=None, outputCol=None):
         """
         super(IDF, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IDF", self.uid)
-        self._setDefault(minDocFreq=0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1447,6 +1465,10 @@ class _ImputerParams(HasInputCol, HasInputCols, HasOutputCol, HasOutputCols, Has
                          "The placeholder for the missing values. All occurrences of missingValue "
                          "will be imputed.", typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_ImputerParams, self).__init__(*args)
+        self._setDefault(strategy="mean", missingValue=float("nan"), relativeError=0.001)
+
     @since("2.2.0")
     def getStrategy(self):
         """
@@ -1467,7 +1489,7 @@ class Imputer(JavaEstimator, _ImputerParams, JavaMLReadable, JavaMLWritable):
     """
     Imputation estimator for completing missing values, either using the mean or the median
     of the columns in which the missing values are located. The input columns should be of
-    DoubleType or FloatType. Currently Imputer does not support categorical features and
+    numeric type. Currently Imputer does not support categorical features and
     possibly creates incorrect values for a categorical feature.
 
     Note that the mean/median value is computed after filtering out missing values.
@@ -1584,7 +1606,6 @@ def __init__(self, strategy="mean", missingValue=float("nan"), inputCols=None,
         """
         super(Imputer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Imputer", self.uid)
-        self._setDefault(strategy="mean", missingValue=float("nan"), relativeError=0.001)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1813,6 +1834,8 @@ class MaxAbsScaler(JavaEstimator, _MaxAbsScalerParams, JavaMLReadable, JavaMLWri
     >>> loadedModel = MaxAbsScalerModel.load(modelPath)
     >>> loadedModel.maxAbs == model.maxAbs
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 2.0.0
     """
@@ -1953,7 +1976,6 @@ def __init__(self, inputCol=None, outputCol=None, seed=None, numHashTables=1):
         """
         super(MinHashLSH, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinHashLSH", self.uid)
-        self._setDefault(numHashTables=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -2004,6 +2026,10 @@ class _MinMaxScalerParams(HasInputCol, HasOutputCol):
     max = Param(Params._dummy(), "max", "Upper bound of the output feature range",
                 typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_MinMaxScalerParams, self).__init__(*args)
+        self._setDefault(min=0.0, max=1.0)
+
     @since("1.6.0")
     def getMin(self):
         """
@@ -2067,6 +2093,8 @@ class MinMaxScaler(JavaEstimator, _MinMaxScalerParams, JavaMLReadable, JavaMLWri
     True
     >>> loadedModel.originalMax == model.originalMax
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
@@ -2078,7 +2106,6 @@ def __init__(self, min=0.0, max=1.0, inputCol=None, outputCol=None):
         """
         super(MinMaxScaler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinMaxScaler", self.uid)
-        self._setDefault(min=0.0, max=1.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -2212,6 +2239,8 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWr
     >>> loadedNGram = NGram.load(ngramPath)
     >>> loadedNGram.getN() == ngram.getN()
     True
+    >>> loadedNGram.transform(df).take(1) == ngram.transform(df).take(1)
+    True
 
     .. versionadded:: 1.5.0
     """
@@ -2292,6 +2321,8 @@ class Normalizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, Jav
     >>> loadedNormalizer = Normalizer.load(normalizerPath)
     >>> loadedNormalizer.getP() == normalizer.getP()
     True
+    >>> loadedNormalizer.transform(df).take(1) == normalizer.transform(df).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -2365,6 +2396,10 @@ class _OneHotEncoderParams(HasInputCol, HasInputCols, HasOutputCol, HasOutputCol
     dropLast = Param(Params._dummy(), "dropLast", "whether to drop the last category",
                      typeConverter=TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_OneHotEncoderParams, self).__init__(*args)
+        self._setDefault(handleInvalid="error", dropLast=True)
+
     @since("2.3.0")
     def getDropLast(self):
         """
@@ -2425,6 +2460,8 @@ class OneHotEncoder(JavaEstimator, _OneHotEncoderParams, JavaMLReadable, JavaMLW
     >>> loadedModel = OneHotEncoderModel.load(modelPath)
     >>> loadedModel.categorySizes == model.categorySizes
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 2.3.0
     """
@@ -2439,7 +2476,6 @@ def __init__(self, inputCols=None, outputCols=None, handleInvalid="error", dropL
         super(OneHotEncoder, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.feature.OneHotEncoder", self.uid)
-        self._setDefault(handleInvalid="error", dropLast=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -2586,6 +2622,8 @@ class PolynomialExpansion(JavaTransformer, HasInputCol, HasOutputCol, JavaMLRead
     >>> loadedPx = PolynomialExpansion.load(polyExpansionPath)
     >>> loadedPx.getDegree() == px.getDegree()
     True
+    >>> loadedPx.transform(df).take(1) == px.transform(df).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -2882,6 +2920,11 @@ class _RobustScalerParams(HasInputCol, HasOutputCol, HasRelativeError):
     withScaling = Param(Params._dummy(), "withScaling", "Whether to scale the data to "
                         "quantile range", typeConverter=TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_RobustScalerParams, self).__init__(*args)
+        self._setDefault(lower=0.25, upper=0.75, withCentering=False, withScaling=True,
+                         relativeError=0.001)
+
     @since("3.0.0")
     def getLower(self):
         """
@@ -2957,6 +3000,8 @@ class RobustScaler(JavaEstimator, _RobustScalerParams, JavaMLReadable, JavaMLWri
     True
     >>> loadedModel.range == model.range
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 3.0.0
     """
@@ -2970,8 +3015,6 @@ def __init__(self, lower=0.25, upper=0.75, withCentering=False, withScaling=True
         """
         super(RobustScaler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RobustScaler", self.uid)
-        self._setDefault(lower=0.25, upper=0.75, withCentering=False, withScaling=True,
-                         relativeError=0.001)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -3117,6 +3160,8 @@ class RegexTokenizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable,
     True
     >>> loadedReTokenizer.getGaps() == reTokenizer.getGaps()
     True
+    >>> loadedReTokenizer.transform(df).take(1) == reTokenizer.transform(df).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -3241,6 +3286,8 @@ class SQLTransformer(JavaTransformer, JavaMLReadable, JavaMLWritable):
     >>> loadedSqlTrans = SQLTransformer.load(sqlTransformerPath)
     >>> loadedSqlTrans.getStatement() == sqlTrans.getStatement()
     True
+    >>> loadedSqlTrans.transform(df).take(1) == sqlTrans.transform(df).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
@@ -3295,6 +3342,10 @@ class _StandardScalerParams(HasInputCol, HasOutputCol):
     withStd = Param(Params._dummy(), "withStd", "Scale to unit standard deviation",
                     typeConverter=TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_StandardScalerParams, self).__init__(*args)
+        self._setDefault(withMean=False, withStd=True)
+
     @since("1.4.0")
     def getWithMean(self):
         """
@@ -3352,6 +3403,8 @@ class StandardScaler(JavaEstimator, _StandardScalerParams, JavaMLReadable, JavaM
     True
     >>> loadedModel.mean == model.mean
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -3363,7 +3416,6 @@ def __init__(self, withMean=False, withStd=True, inputCol=None, outputCol=None):
         """
         super(StandardScaler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StandardScaler", self.uid)
-        self._setDefault(withMean=False, withStd=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -3512,6 +3564,8 @@ class StringIndexer(JavaEstimator, _StringIndexerParams, JavaMLReadable, JavaMLW
     >>> loadedModel = StringIndexerModel.load(modelPath)
     >>> loadedModel.labels == model.labels
     True
+    >>> loadedModel.transform(stringIndDf).take(1) == model.transform(stringIndDf).take(1)
+    True
     >>> indexToStringPath = temp_path + "/index-to-string"
     >>> inverter.save(indexToStringPath)
     >>> loadedInverter = IndexToString.load(indexToStringPath)
@@ -3711,6 +3765,15 @@ def labels(self):
         """
         return self._call_java("labels")
 
+    @property
+    @since("3.0.2")
+    def labelsArray(self):
+        """
+        Array of ordered list of labels, corresponding to indices to be assigned
+        for each input column.
+        """
+        return self._call_java("labelsArray")
+
 
 @inherit_doc
 class IndexToString(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
@@ -3802,6 +3865,8 @@ class StopWordsRemover(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols,
     True
     >>> loadedRemover.getCaseSensitive() == remover.getCaseSensitive()
     True
+    >>> loadedRemover.transform(df).take(1) == remover.transform(df).take(1)
+    True
     >>> df2 = spark.createDataFrame([(["a", "b", "c"], ["a", "b"])], ["text1", "text2"])
     >>> remover2 = StopWordsRemover(stopWords=["b"])
     >>> remover2.setInputCols(["text1", "text2"]).setOutputCols(["words1", "words2"])
@@ -4114,6 +4179,10 @@ class _VectorIndexerParams(HasInputCol, HasOutputCol, HasHandleInvalid):
                           "of categories of the feature).",
                           typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_VectorIndexerParams, self).__init__(*args)
+        self._setDefault(maxCategories=20, handleInvalid="error")
+
     @since("1.4.0")
     def getMaxCategories(self):
         """
@@ -4194,6 +4263,8 @@ class VectorIndexer(JavaEstimator, _VectorIndexerParams, JavaMLReadable, JavaMLW
     True
     >>> loadedModel.categoryMaps == model.categoryMaps
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     >>> dfWithInvalid = spark.createDataFrame([(Vectors.dense([3.0, 1.0]),)], ["a"])
     >>> indexer.getHandleInvalid()
     'error'
@@ -4214,7 +4285,6 @@ def __init__(self, maxCategories=20, inputCol=None, outputCol=None, handleInvali
         """
         super(VectorIndexer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorIndexer", self.uid)
-        self._setDefault(maxCategories=20, handleInvalid="error")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -4337,6 +4407,8 @@ class VectorSlicer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, J
     True
     >>> loadedVs.getNames() == vs.getNames()
     True
+    >>> loadedVs.transform(df).take(1) == vs.transform(df).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
@@ -4436,6 +4508,11 @@ class _Word2VecParams(HasStepSize, HasMaxIter, HasSeed, HasInputCol, HasOutputCo
                               "be divided into chunks up to the size.",
                               typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_Word2VecParams, self).__init__(*args)
+        self._setDefault(vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1,
+                         windowSize=5, maxSentenceLength=1000)
+
     @since("1.4.0")
     def getVectorSize(self):
         """
@@ -4530,6 +4607,8 @@ class Word2Vec(JavaEstimator, _Word2VecParams, JavaMLReadable, JavaMLWritable):
     True
     >>> loadedModel.getVectors().first().vector == model.getVectors().first().vector
     True
+    >>> loadedModel.transform(doc).take(1) == model.transform(doc).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -4543,8 +4622,6 @@ def __init__(self, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025,
         """
         super(Word2Vec, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Word2Vec", self.uid)
-        self._setDefault(vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1,
-                         windowSize=5, maxSentenceLength=1000)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -4736,6 +4813,8 @@ class PCA(JavaEstimator, _PCAParams, JavaMLReadable, JavaMLWritable):
     True
     >>> loadedModel.explainedVariance == model.explainedVariance
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 1.5.0
     """
@@ -4852,6 +4931,11 @@ class _RFormulaParams(HasFeaturesCol, HasLabelCol, HasHandleInvalid):
                           "additional bucket, at index numLabels).",
                           typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_RFormulaParams, self).__init__(*args)
+        self._setDefault(forceIndexLabel=False, stringIndexerOrderType="frequencyDesc",
+                         handleInvalid="error")
+
     @since("1.5.0")
     def getFormula(self):
         """
@@ -4954,8 +5038,6 @@ def __init__(self, formula=None, featuresCol="features", labelCol="label",
         """
         super(RFormula, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RFormula", self.uid)
-        self._setDefault(forceIndexLabel=False, stringIndexerOrderType="frequencyDesc",
-                         handleInvalid="error")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -5064,6 +5146,11 @@ class _ChiSqSelectorParams(HasFeaturesCol, HasOutputCol, HasLabelCol):
     fwe = Param(Params._dummy(), "fwe", "The upper bound of the expected family-wise error rate.",
                 typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_ChiSqSelectorParams, self).__init__(*args)
+        self._setDefault(numTopFeatures=50, selectorType="numTopFeatures", percentile=0.1,
+                         fpr=0.05, fdr=0.05, fwe=0.05)
+
     @since("2.1.0")
     def getSelectorType(self):
         """
@@ -5160,6 +5247,8 @@ class ChiSqSelector(JavaEstimator, _ChiSqSelectorParams, JavaMLReadable, JavaMLW
     >>> loadedModel = ChiSqSelectorModel.load(modelPath)
     >>> loadedModel.selectedFeatures == model.selectedFeatures
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 2.0.0
     """
@@ -5175,8 +5264,6 @@ def __init__(self, numTopFeatures=50, featuresCol="features", outputCol=None,
         """
         super(ChiSqSelector, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ChiSqSelector", self.uid)
-        self._setDefault(numTopFeatures=50, selectorType="numTopFeatures", percentile=0.1,
-                         fpr=0.05, fdr=0.05, fwe=0.05)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py
index 7d933daf9e032..0e302d53d97e7 100644
--- a/python/pyspark/ml/fpm.py
+++ b/python/pyspark/ml/fpm.py
@@ -56,6 +56,11 @@ class _FPGrowthParams(HasPredictionCol):
         "but will affect the association rules generation.",
         typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_FPGrowthParams, self).__init__(*args)
+        self._setDefault(minSupport=0.3, minConfidence=0.8,
+                         itemsCol="items", predictionCol="prediction")
+
     def getItemsCol(self):
         """
         Gets the value of itemsCol or its default value.
@@ -194,6 +199,11 @@ class FPGrowth(JavaEstimator, _FPGrowthParams, JavaMLWritable, JavaMLReadable):
     >>> new_data = spark.createDataFrame([(["t", "s"], )], ["items"])
     >>> sorted(fpm.transform(new_data).first().newPrediction)
     [u'x', u'y', u'z']
+    >>> model_path = temp_path + "/fpm_model"
+    >>> fpm.save(model_path)
+    >>> model2 = FPGrowthModel.load(model_path)
+    >>> fpm.transform(data).take(1) == model2.transform(data).take(1)
+    True
 
     .. versionadded:: 2.2.0
     """
@@ -206,8 +216,6 @@ def __init__(self, minSupport=0.3, minConfidence=0.8, itemsCol="items",
         """
         super(FPGrowth, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.FPGrowth", self.uid)
-        self._setDefault(minSupport=0.3, minConfidence=0.8,
-                         itemsCol="items", predictionCol="prediction")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/functions.py b/python/pyspark/ml/functions.py
index 2b4d8ddcd00a8..65b0558b282a4 100644
--- a/python/pyspark/ml/functions.py
+++ b/python/pyspark/ml/functions.py
@@ -19,11 +19,17 @@
 from pyspark.sql.column import Column, _to_java_column
 
 
-@since(3.0)
-def vector_to_array(col):
+@since("3.0.0")
+def vector_to_array(col, dtype="float64"):
     """
     Converts a column of MLlib sparse/dense vectors into a column of dense arrays.
 
+    :param col: A string of the column name or a Column
+    :param dtype: The data type of the output array. Valid values: "float64" or "float32".
+    :return: The converted column of dense arrays.
+
+    .. versionadded:: 3.0.0
+
     >>> from pyspark.ml.linalg import Vectors
     >>> from pyspark.ml.functions import vector_to_array
     >>> from pyspark.mllib.linalg import Vectors as OldVectors
@@ -32,14 +38,26 @@ def vector_to_array(col):
     ...     (Vectors.sparse(3, [(0, 2.0), (2, 3.0)]),
     ...      OldVectors.sparse(3, [(0, 20.0), (2, 30.0)]))],
     ...     ["vec", "oldVec"])
-    >>> df.select(vector_to_array("vec").alias("vec"),
-    ...           vector_to_array("oldVec").alias("oldVec")).collect()
+    >>> df1 = df.select(vector_to_array("vec").alias("vec"),
+    ...                 vector_to_array("oldVec").alias("oldVec"))
+    >>> df1.collect()
+    [Row(vec=[1.0, 2.0, 3.0], oldVec=[10.0, 20.0, 30.0]),
+     Row(vec=[2.0, 0.0, 3.0], oldVec=[20.0, 0.0, 30.0])]
+    >>> df2 = df.select(vector_to_array("vec", "float32").alias("vec"),
+    ...                 vector_to_array("oldVec", "float32").alias("oldVec"))
+    >>> df2.collect()
     [Row(vec=[1.0, 2.0, 3.0], oldVec=[10.0, 20.0, 30.0]),
      Row(vec=[2.0, 0.0, 3.0], oldVec=[20.0, 0.0, 30.0])]
+    >>> df1.schema.fields
+    [StructField(vec,ArrayType(DoubleType,false),false),
+    StructField(oldVec,ArrayType(DoubleType,false),false)]
+    >>> df2.schema.fields
+    [StructField(vec,ArrayType(FloatType,false),false),
+    StructField(oldVec,ArrayType(FloatType,false),false)]
     """
     sc = SparkContext._active_spark_context
     return Column(
-        sc._jvm.org.apache.spark.ml.functions.vector_to_array(_to_java_column(col)))
+        sc._jvm.org.apache.spark.ml.functions.vector_to_array(_to_java_column(col), dtype))
 
 
 def _test():
diff --git a/python/pyspark/ml/param/__init__.py b/python/pyspark/ml/param/__init__.py
index fe61f9f0fffd6..f83875736375b 100644
--- a/python/pyspark/ml/param/__init__.py
+++ b/python/pyspark/ml/param/__init__.py
@@ -76,8 +76,6 @@ def __eq__(self, other):
 
 class TypeConverters(object):
     """
-    .. note:: DeveloperApi
-
     Factory methods for common type conversion functions for `Param.typeConverter`.
 
     .. versionadded:: 2.0.0
@@ -428,6 +426,12 @@ def _resolveParam(self, param):
         else:
             raise ValueError("Cannot resolve %r as a param." % param)
 
+    def _testOwnParam(self, param_parent, param_name):
+        """
+        Test the ownership. Return True or False
+        """
+        return self.uid == param_parent and self.hasParam(param_name)
+
     @staticmethod
     def _dummy():
         """
diff --git a/python/pyspark/ml/param/_shared_params_code_gen.py b/python/pyspark/ml/param/_shared_params_code_gen.py
index fb4d55d57a2df..2086e831f4282 100644
--- a/python/pyspark/ml/param/_shared_params_code_gen.py
+++ b/python/pyspark/ml/param/_shared_params_code_gen.py
@@ -167,7 +167,7 @@ def get$Name(self):
          None, "TypeConverters.toString"),
         ("blockSize", "block size for stacking input data in matrices. Data is stacked within "
          "partitions. If block size is more than remaining data in a partition then it is "
-         "adjusted to the size of this data.", "1024", "TypeConverters.toInt")]
+         "adjusted to the size of this data.", None, "TypeConverters.toInt")]
 
     code = []
     for name, doc, defaultValueStr, typeConverter in shared:
diff --git a/python/pyspark/ml/param/shared.py b/python/pyspark/ml/param/shared.py
index 456463580878c..24fb0d3e2554d 100644
--- a/python/pyspark/ml/param/shared.py
+++ b/python/pyspark/ml/param/shared.py
@@ -591,7 +591,6 @@ class HasBlockSize(Params):
 
     def __init__(self):
         super(HasBlockSize, self).__init__()
-        self._setDefault(blockSize=1024)
 
     def getBlockSize(self):
         """
diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py
index 097530230cbca..09e0748ffbb3b 100644
--- a/python/pyspark/ml/pipeline.py
+++ b/python/pyspark/ml/pipeline.py
@@ -324,8 +324,6 @@ def _to_java(self):
 @inherit_doc
 class PipelineSharedReadWrite():
     """
-    .. note:: DeveloperApi
-
     Functions for :py:class:`MLReader` and :py:class:`MLWriter` shared between
     :py:class:`Pipeline` and :py:class:`PipelineModel`
 
diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py
index fe571e25c05f5..62b856046234a 100644
--- a/python/pyspark/ml/recommendation.py
+++ b/python/pyspark/ml/recommendation.py
@@ -46,6 +46,10 @@ class _ALSModelParams(HasPredictionCol, HasBlockSize):
                               "Supported values: 'nan', 'drop'.",
                               typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_ALSModelParams, self).__init__(*args)
+        self._setDefault(blockSize=4096)
+
     @since("1.4.0")
     def getUserCol(self):
         """
@@ -99,6 +103,14 @@ class _ALSParams(_ALSModelParams, HasMaxIter, HasRegParam, HasCheckpointInterval
                               "StorageLevel for ALS model factors.",
                               typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_ALSParams, self).__init__(*args)
+        self._setDefault(rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10,
+                         implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item",
+                         ratingCol="rating", nonnegative=False, checkpointInterval=10,
+                         intermediateStorageLevel="MEMORY_AND_DISK",
+                         finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan")
+
     @since("1.4.0")
     def getRank(self):
         """
@@ -275,6 +287,8 @@ class ALS(JavaEstimator, _ALSParams, JavaMLWritable, JavaMLReadable):
     True
     >>> sorted(model.itemFactors.collect()) == sorted(model2.itemFactors.collect())
     True
+    >>> model.transform(test).take(1) == model2.transform(test).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -290,16 +304,10 @@ def __init__(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemB
                  implicitPrefs=false, alpha=1.0, userCol="user", itemCol="item", seed=None, \
                  ratingCol="rating", nonnegative=false, checkpointInterval=10, \
                  intermediateStorageLevel="MEMORY_AND_DISK", \
-                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", lockSize=4096)
+                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096)
         """
         super(ALS, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.recommendation.ALS", self.uid)
-        self._setDefault(rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10,
-                         implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item",
-                         ratingCol="rating", nonnegative=False, checkpointInterval=10,
-                         intermediateStorageLevel="MEMORY_AND_DISK",
-                         finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan",
-                         blockSize=4096)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index a74ba3485bcd9..550efb27c1e97 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -53,7 +53,7 @@ class JavaRegressor(JavaPredictor, _JavaPredictorParams):
 class JavaRegressionModel(JavaPredictionModel, _JavaPredictorParams):
     """
     Java Model produced by a ``_JavaRegressor``.
-    To be mixed in with class:`pyspark.ml.JavaModel`
+    To be mixed in with :class:`pyspark.ml.JavaModel`
 
     .. versionadded:: 3.0.0
     """
@@ -62,7 +62,7 @@ class JavaRegressionModel(JavaPredictionModel, _JavaPredictorParams):
 
 class _LinearRegressionParams(_JavaPredictorParams, HasRegParam, HasElasticNetParam, HasMaxIter,
                               HasTol, HasFitIntercept, HasStandardization, HasWeightCol, HasSolver,
-                              HasAggregationDepth, HasLoss, HasBlockSize):
+                              HasAggregationDepth, HasLoss):
     """
     Params for :py:class:`LinearRegression` and :py:class:`LinearRegressionModel`.
 
@@ -79,6 +79,10 @@ class _LinearRegressionParams(_JavaPredictorParams, HasRegParam, HasElasticNetPa
                     "robustness. Must be > 1.0. Only valid when loss is huber",
                     typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_LinearRegressionParams, self).__init__(*args)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, loss="squaredError", epsilon=1.35)
+
     @since("2.3.0")
     def getEpsilon(self):
         """
@@ -124,8 +128,6 @@ class LinearRegression(JavaRegressor, _LinearRegressionParams, JavaMLWritable, J
     >>> lr.setRegParam(0.0)
     LinearRegression...
     >>> model = lr.fit(df)
-    >>> model.getBlockSize()
-    1024
     >>> model.setFeaturesCol("features")
     LinearRegressionModel...
     >>> model.setPredictionCol("newPrediction")
@@ -160,6 +162,8 @@ class LinearRegression(JavaRegressor, _LinearRegressionParams, JavaMLWritable, J
     True
     >>> model.intercept == model2.intercept
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.numFeatures
     1
     >>> model.write().format("pmml").save(model_path + "_2")
@@ -171,18 +175,16 @@ class LinearRegression(JavaRegressor, _LinearRegressionParams, JavaMLWritable, J
     def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True,
                  standardization=True, solver="auto", weightCol=None, aggregationDepth=2,
-                 loss="squaredError", epsilon=1.35, blockSize=1024):
+                 loss="squaredError", epsilon=1.35):
         """
         __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \
                  standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \
-                 loss="squaredError", epsilon=1.35, blockSize=1024)
+                 loss="squaredError", epsilon=1.35)
         """
         super(LinearRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.LinearRegression", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, loss="squaredError", epsilon=1.35,
-                         blockSize=1024)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -191,12 +193,12 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
     def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True,
                   standardization=True, solver="auto", weightCol=None, aggregationDepth=2,
-                  loss="squaredError", epsilon=1.35, blockSize=1024):
+                  loss="squaredError", epsilon=1.35):
         """
         setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \
                   standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \
-                  loss="squaredError", epsilon=1.35, blockSize=1024)
+                  loss="squaredError", epsilon=1.35)
         Sets params for linear regression.
         """
         kwargs = self._input_kwargs
@@ -272,13 +274,6 @@ def setLoss(self, value):
         """
         return self._set(lossType=value)
 
-    @since("3.0.0")
-    def setBlockSize(self, value):
-        """
-        Sets the value of :py:attr:`blockSize`.
-        """
-        return self._set(blockSize=value)
-
 
 class LinearRegressionModel(JavaRegressionModel, _LinearRegressionParams, GeneralJavaMLWritable,
                             JavaMLReadable, HasTrainingSummary):
@@ -594,6 +589,10 @@ class _IsotonicRegressionParams(HasFeaturesCol, HasLabelCol, HasPredictionCol, H
         "The index of the feature if featuresCol is a vector column, no effect otherwise.",
         typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_IsotonicRegressionParams, self).__init__(*args)
+        self._setDefault(isotonic=True, featureIndex=0)
+
     def getIsotonic(self):
         """
         Gets the value of isotonic or its default value.
@@ -643,6 +642,8 @@ class IsotonicRegression(JavaEstimator, _IsotonicRegressionParams, HasWeightCol,
     True
     >>> model.predictions == model2.predictions
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
@@ -656,7 +657,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(IsotonicRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.IsotonicRegression", self.uid)
-        self._setDefault(isotonic=True, featureIndex=0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -783,7 +783,11 @@ class _DecisionTreeRegressorParams(_DecisionTreeParams, _TreeRegressorParams, Ha
     .. versionadded:: 3.0.0
     """
 
-    pass
+    def __init__(self, *args):
+        super(_DecisionTreeRegressorParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="variance", leafCol="", minWeightFractionPerNode=0.0)
 
 
 @inherit_doc
@@ -839,6 +843,8 @@ class DecisionTreeRegressor(JavaRegressor, _DecisionTreeRegressorParams, JavaMLW
     True
     >>> model.depth == model2.depth
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.transform(test1).head().variance
     0.0
 
@@ -870,9 +876,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(DecisionTreeRegressor, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.DecisionTreeRegressor", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="variance", leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1027,7 +1030,14 @@ class _RandomForestRegressorParams(_RandomForestParams, _TreeRegressorParams):
 
     .. versionadded:: 3.0.0
     """
-    pass
+
+    def __init__(self, *args):
+        super(_RandomForestRegressorParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="variance", subsamplingRate=1.0, numTrees=20,
+                         featureSubsetStrategy="auto", leafCol="", minWeightFractionPerNode=0.0,
+                         bootstrap=True)
 
 
 @inherit_doc
@@ -1088,6 +1098,8 @@ class RandomForestRegressor(JavaRegressor, _RandomForestRegressorParams, JavaMLW
     >>> model2 = RandomForestRegressionModel.load(model_path)
     >>> model.featureImportances == model2.featureImportances
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -1110,11 +1122,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(RandomForestRegressor, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.RandomForestRegressor", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="variance", subsamplingRate=1.0, numTrees=20,
-                         featureSubsetStrategy="auto", leafCol="", minWeightFractionPerNode=0.0,
-                         bootstrap=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1285,6 +1292,14 @@ class _GBTRegressorParams(_GBTParams, _TreeRegressorParams):
                      "Supported options: " + ", ".join(supportedLossTypes),
                      typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_GBTRegressorParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0,
+                         checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1,
+                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
+                         leafCol="", minWeightFractionPerNode=0.0)
+
     @since("1.4.0")
     def getLossType(self):
         """
@@ -1348,6 +1363,8 @@ class GBTRegressor(JavaRegressor, _GBTRegressorParams, JavaMLWritable, JavaMLRea
     True
     >>> model.treeWeights == model2.treeWeights
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.trees
     [DecisionTreeRegressionModel...depth=..., DecisionTreeRegressionModel...]
     >>> validation = spark.createDataFrame([(0.0, Vectors.dense(-1.0))],
@@ -1382,11 +1399,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         """
         super(GBTRegressor, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.GBTRegressor", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0,
-                         checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1,
-                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
-                         leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1605,6 +1617,12 @@ class _AFTSurvivalRegressionParams(_JavaPredictorParams, HasMaxIter, HasTol, Has
         "corresponding quantileProbabilities if it is set.",
         typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_AFTSurvivalRegressionParams, self).__init__(*args)
+        self._setDefault(censorCol="censor",
+                         quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99],
+                         maxIter=100, tol=1E-6)
+
     @since("1.6.0")
     def getCensorCol(self):
         """
@@ -1677,6 +1695,8 @@ class AFTSurvivalRegression(JavaRegressor, _AFTSurvivalRegressionParams,
     True
     >>> model.scale == model2.scale
     True
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
@@ -1695,9 +1715,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(AFTSurvivalRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.AFTSurvivalRegression", self.uid)
-        self._setDefault(censorCol="censor",
-                         quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99],
-                         maxIter=100, tol=1E-6)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1857,6 +1874,11 @@ class _GeneralizedLinearRegressionParams(_JavaPredictorParams, HasFitIntercept,
                       "or empty, we treat all instance offsets as 0.0",
                       typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_GeneralizedLinearRegressionParams, self).__init__(*args)
+        self._setDefault(family="gaussian", maxIter=25, tol=1e-6, regParam=0.0, solver="irls",
+                         variancePower=0.0, aggregationDepth=2)
+
     @since("2.0.0")
     def getFamily(self):
         """
@@ -1971,6 +1993,8 @@ class GeneralizedLinearRegression(JavaRegressor, _GeneralizedLinearRegressionPar
     True
     >>> model.coefficients[0] == model2.coefficients[0]
     True
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
 
     .. versionadded:: 2.0.0
     """
@@ -1989,8 +2013,6 @@ def __init__(self, labelCol="label", featuresCol="features", predictionCol="pred
         super(GeneralizedLinearRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.GeneralizedLinearRegression", self.uid)
-        self._setDefault(family="gaussian", maxIter=25, tol=1e-6, regParam=0.0, solver="irls",
-                         variancePower=0.0, aggregationDepth=2)
         kwargs = self._input_kwargs
 
         self.setParams(**kwargs)
@@ -2364,6 +2386,12 @@ class _FactorizationMachinesParams(_JavaPredictorParams, HasMaxIter, HasStepSize
     solver = Param(Params._dummy(), "solver", "The solver algorithm for optimization. Supported " +
                    "options: gd, adamW. (Default adamW)", typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_FactorizationMachinesParams, self).__init__(*args)
+        self._setDefault(factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
+                         miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
+                         tol=1e-6, solver="adamW")
+
     @since("3.0.0")
     def getFactorSize(self):
         """
@@ -2437,6 +2465,17 @@ class FMRegressor(JavaRegressor, _FactorizationMachinesParams, JavaMLWritable, J
     DenseVector([0.9978])
     >>> model.factors
     DenseMatrix(1, 2, [0.0173, 0.0021], 1)
+    >>> model_path = temp_path + "/fm_model"
+    >>> model.save(model_path)
+    >>> model2 = FMRegressionModel.load(model_path)
+    >>> model2.intercept
+    -0.0032501766849261557
+    >>> model2.linear
+    DenseVector([0.9978])
+    >>> model2.factors
+    DenseMatrix(1, 2, [0.0173, 0.0021], 1)
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 3.0.0
     """
@@ -2455,9 +2494,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(FMRegressor, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.FMRegressor", self.uid)
-        self._setDefault(factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
-                         miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
-                         tol=1e-6, solver="adamW")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/tests/test_feature.py b/python/pyspark/ml/tests/test_feature.py
index 4c6bfa696b110..cdd5ad50f89b1 100644
--- a/python/pyspark/ml/tests/test_feature.py
+++ b/python/pyspark/ml/tests/test_feature.py
@@ -236,6 +236,7 @@ def test_string_indexer_from_labels(self):
         model = StringIndexerModel.from_labels(["a", "b", "c"], inputCol="label",
                                                outputCol="indexed", handleInvalid="keep")
         self.assertEqual(model.labels, ["a", "b", "c"])
+        self.assertEqual(model.labelsArray, [("a", "b", "c")])
 
         df1 = self.spark.createDataFrame([
             (0, "a"),
diff --git a/python/pyspark/ml/tests/test_param.py b/python/pyspark/ml/tests/test_param.py
index 777b4930ce8c9..2b119ea866236 100644
--- a/python/pyspark/ml/tests/test_param.py
+++ b/python/pyspark/ml/tests/test_param.py
@@ -366,15 +366,14 @@ def test_java_params(self):
                 if not name.endswith('Model') and not name.endswith('Params') \
                         and issubclass(cls, JavaParams) and not inspect.isabstract(cls) \
                         and not name.startswith('Java') and name != '_LSH':
-                    # NOTE: disable check_params_exist until there is parity with Scala API
-                    check_params(self, cls(), check_params_exist=False)
+                    check_params(self, cls(), check_params_exist=True)
 
         # Additional classes that need explicit construction
         from pyspark.ml.feature import CountVectorizerModel, StringIndexerModel
         check_params(self, CountVectorizerModel.from_vocabulary(['a'], 'input'),
-                     check_params_exist=False)
+                     check_params_exist=True)
         check_params(self, StringIndexerModel.from_labels(['a', 'b'], 'input'),
-                     check_params_exist=False)
+                     check_params_exist=True)
 
 
 if __name__ == "__main__":
diff --git a/python/pyspark/ml/tests/test_persistence.py b/python/pyspark/ml/tests/test_persistence.py
index d4edcc26e17ac..2f6d451851b4b 100644
--- a/python/pyspark/ml/tests/test_persistence.py
+++ b/python/pyspark/ml/tests/test_persistence.py
@@ -21,19 +21,78 @@
 import unittest
 
 from pyspark.ml import Transformer
-from pyspark.ml.classification import DecisionTreeClassifier, LogisticRegression, OneVsRest, \
-    OneVsRestModel
+from pyspark.ml.classification import DecisionTreeClassifier, FMClassifier, \
+    FMClassificationModel, LogisticRegression, MultilayerPerceptronClassifier, \
+    MultilayerPerceptronClassificationModel, OneVsRest, OneVsRestModel
 from pyspark.ml.clustering import KMeans
 from pyspark.ml.feature import Binarizer, HashingTF, PCA
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.param import Params
 from pyspark.ml.pipeline import Pipeline, PipelineModel
-from pyspark.ml.regression import DecisionTreeRegressor, LinearRegression
+from pyspark.ml.regression import DecisionTreeRegressor, GeneralizedLinearRegression, \
+    GeneralizedLinearRegressionModel, \
+    LinearRegression
 from pyspark.ml.util import DefaultParamsReadable, DefaultParamsWriter
 from pyspark.ml.wrapper import JavaParams
 from pyspark.testing.mlutils import MockUnaryTransformer, SparkSessionTestCase
 
 
+class TestDefaultSolver(SparkSessionTestCase):
+
+    def test_multilayer_load(self):
+        df = self.spark.createDataFrame([(0.0, Vectors.dense([0.0, 0.0])),
+                                         (1.0, Vectors.dense([0.0, 1.0])),
+                                         (1.0, Vectors.dense([1.0, 0.0])),
+                                         (0.0, Vectors.dense([1.0, 1.0]))],
+                                        ["label",  "features"])
+
+        mlp = MultilayerPerceptronClassifier(layers=[2, 2, 2], seed=123)
+        model = mlp.fit(df)
+        self.assertEqual(model.getSolver(), "l-bfgs")
+        transformed1 = model.transform(df)
+        path = tempfile.mkdtemp()
+        model_path = path + "/mlp"
+        model.save(model_path)
+        model2 = MultilayerPerceptronClassificationModel.load(model_path)
+        self.assertEqual(model2.getSolver(), "l-bfgs")
+        transformed2 = model2.transform(df)
+        self.assertEqual(transformed1.take(4), transformed2.take(4))
+
+    def test_fm_load(self):
+        df = self.spark.createDataFrame([(1.0, Vectors.dense(1.0)),
+                                         (0.0, Vectors.sparse(1, [], []))],
+                                        ["label",  "features"])
+        fm = FMClassifier(factorSize=2, maxIter=50, stepSize=2.0)
+        model = fm.fit(df)
+        self.assertEqual(model.getSolver(), "adamW")
+        transformed1 = model.transform(df)
+        path = tempfile.mkdtemp()
+        model_path = path + "/fm"
+        model.save(model_path)
+        model2 = FMClassificationModel.load(model_path)
+        self.assertEqual(model2.getSolver(), "adamW")
+        transformed2 = model2.transform(df)
+        self.assertEqual(transformed1.take(2), transformed2.take(2))
+
+    def test_glr_load(self):
+        df = self.spark.createDataFrame([(1.0, Vectors.dense(0.0, 0.0)),
+                                         (1.0, Vectors.dense(1.0, 2.0)),
+                                         (2.0, Vectors.dense(0.0, 0.0)),
+                                         (2.0, Vectors.dense(1.0, 1.0))],
+                                        ["label",  "features"])
+        glr = GeneralizedLinearRegression(family="gaussian", link="identity", linkPredictionCol="p")
+        model = glr.fit(df)
+        self.assertEqual(model.getSolver(), "irls")
+        transformed1 = model.transform(df)
+        path = tempfile.mkdtemp()
+        model_path = path + "/glr"
+        model.save(model_path)
+        model2 = GeneralizedLinearRegressionModel.load(model_path)
+        self.assertEqual(model2.getSolver(), "irls")
+        transformed2 = model2.transform(df)
+        self.assertEqual(transformed1.take(4), transformed2.take(4))
+
+
 class PersistenceTest(SparkSessionTestCase):
 
     def test_linear_regression(self):
diff --git a/python/pyspark/ml/tests/test_training_summary.py b/python/pyspark/ml/tests/test_training_summary.py
index 1d19ebf9a34a0..b5054095d190b 100644
--- a/python/pyspark/ml/tests/test_training_summary.py
+++ b/python/pyspark/ml/tests/test_training_summary.py
@@ -21,7 +21,8 @@
 if sys.version > '3':
     basestring = str
 
-from pyspark.ml.classification import LogisticRegression
+from pyspark.ml.classification import BinaryLogisticRegressionSummary, LogisticRegression, \
+    LogisticRegressionSummary
 from pyspark.ml.clustering import BisectingKMeans, GaussianMixture, KMeans
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.regression import GeneralizedLinearRegression, LinearRegression
@@ -149,6 +150,7 @@ def test_binary_logistic_regression_summary(self):
         # test evaluation (with training dataset) produces a summary with same values
         # one check is enough to verify a summary is returned, Scala version runs full test
         sameSummary = model.evaluate(df)
+        self.assertTrue(isinstance(sameSummary, BinaryLogisticRegressionSummary))
         self.assertAlmostEqual(sameSummary.areaUnderROC, s.areaUnderROC)
 
     def test_multiclass_logistic_regression_summary(self):
@@ -187,6 +189,8 @@ def test_multiclass_logistic_regression_summary(self):
         # test evaluation (with training dataset) produces a summary with same values
         # one check is enough to verify a summary is returned, Scala version runs full test
         sameSummary = model.evaluate(df)
+        self.assertTrue(isinstance(sameSummary, LogisticRegressionSummary))
+        self.assertFalse(isinstance(sameSummary, BinaryLogisticRegressionSummary))
         self.assertAlmostEqual(sameSummary.accuracy, s.accuracy)
 
     def test_gaussian_mixture_summary(self):
diff --git a/python/pyspark/ml/tests/test_tuning.py b/python/pyspark/ml/tests/test_tuning.py
index 9d8ba37c60da4..b266ced7deb07 100644
--- a/python/pyspark/ml/tests/test_tuning.py
+++ b/python/pyspark/ml/tests/test_tuning.py
@@ -18,7 +18,8 @@
 import tempfile
 import unittest
 
-from pyspark.ml import Estimator, Model
+from pyspark.ml.feature import HashingTF, Tokenizer
+from pyspark.ml import Estimator, Pipeline, Model
 from pyspark.ml.classification import LogisticRegression, LogisticRegressionModel, OneVsRest
 from pyspark.ml.evaluation import BinaryClassificationEvaluator, \
     MulticlassClassificationEvaluator, RegressionEvaluator
@@ -72,7 +73,21 @@ def test_addGrid(self):
                     .build())
 
 
-class CrossValidatorTests(SparkSessionTestCase):
+class ValidatorTestUtilsMixin:
+    def assert_param_maps_equal(self, paramMaps1, paramMaps2):
+        self.assertEqual(len(paramMaps1), len(paramMaps2))
+        for paramMap1, paramMap2 in zip(paramMaps1, paramMaps2):
+            self.assertEqual(set(paramMap1.keys()), set(paramMap2.keys()))
+            for param in paramMap1.keys():
+                v1 = paramMap1[param]
+                v2 = paramMap2[param]
+                if isinstance(v1, Params):
+                    self.assertEqual(v1.uid, v2.uid)
+                else:
+                    self.assertEqual(v1, v2)
+
+
+class CrossValidatorTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
 
     def test_copy(self):
         dataset = self.spark.createDataFrame([
@@ -88,15 +103,48 @@ def test_copy(self):
         grid = (ParamGridBuilder()
                 .addGrid(iee.inducedError, [100.0, 0.0, 10000.0])
                 .build())
-        cv = CrossValidator(estimator=iee, estimatorParamMaps=grid, evaluator=evaluator)
+        cv = CrossValidator(
+            estimator=iee,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True,
+            numFolds=2
+        )
         cvCopied = cv.copy()
-        self.assertEqual(cv.getEstimator().uid, cvCopied.getEstimator().uid)
+        for param in [
+            lambda x: x.getEstimator().uid,
+            # SPARK-32092: CrossValidator.copy() needs to copy all existing params
+            lambda x: x.getNumFolds(),
+            lambda x: x.getCollectSubModels(),
+            lambda x: x.getParallelism(),
+            lambda x: x.getSeed()
+        ]:
+            self.assertEqual(param(cv), param(cvCopied))
 
         cvModel = cv.fit(dataset)
         cvModelCopied = cvModel.copy()
         for index in range(len(cvModel.avgMetrics)):
             self.assertTrue(abs(cvModel.avgMetrics[index] - cvModelCopied.avgMetrics[index])
                             < 0.0001)
+        # SPARK-32092: CrossValidatorModel.copy() needs to copy all existing params
+        for param in [
+            lambda x: x.getNumFolds(),
+            lambda x: x.getSeed()
+        ]:
+            self.assertEqual(param(cvModel), param(cvModelCopied))
+
+        cvModel.avgMetrics[0] = 'foo'
+        self.assertNotEqual(
+            cvModelCopied.avgMetrics[0],
+            'foo',
+            "Changing the original avgMetrics should not affect the copied model"
+        )
+        cvModel.subModels[0][0].getInducedError = lambda: 'foo'
+        self.assertNotEqual(
+            cvModelCopied.subModels[0][0].getInducedError(),
+            'foo',
+            "Changing the original subModels should not affect the copied model"
+        )
 
     def test_fit_minimize_metric(self):
         dataset = self.spark.createDataFrame([
@@ -165,16 +213,38 @@ def test_save_load_trained_model(self):
         lr = LogisticRegression()
         grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
         evaluator = BinaryClassificationEvaluator()
-        cv = CrossValidator(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
+        cv = CrossValidator(
+            estimator=lr,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True,
+            numFolds=4,
+            seed=42
+        )
         cvModel = cv.fit(dataset)
         lrModel = cvModel.bestModel
 
-        cvModelPath = temp_path + "/cvModel"
-        lrModel.save(cvModelPath)
-        loadedLrModel = LogisticRegressionModel.load(cvModelPath)
+        lrModelPath = temp_path + "/lrModel"
+        lrModel.save(lrModelPath)
+        loadedLrModel = LogisticRegressionModel.load(lrModelPath)
         self.assertEqual(loadedLrModel.uid, lrModel.uid)
         self.assertEqual(loadedLrModel.intercept, lrModel.intercept)
 
+        # SPARK-32092: Saving and then loading CrossValidatorModel should not change the params
+        cvModelPath = temp_path + "/cvModel"
+        cvModel.save(cvModelPath)
+        loadedCvModel = CrossValidatorModel.load(cvModelPath)
+        for param in [
+            lambda x: x.getNumFolds(),
+            lambda x: x.getSeed(),
+            lambda x: len(x.subModels)
+        ]:
+            self.assertEqual(param(cvModel), param(loadedCvModel))
+
+        self.assertTrue(all(
+            loadedCvModel.isSet(param) for param in loadedCvModel.params
+        ))
+
     def test_save_load_simple_estimator(self):
         temp_path = tempfile.mkdtemp()
         dataset = self.spark.createDataFrame(
@@ -197,7 +267,7 @@ def test_save_load_simple_estimator(self):
         loadedCV = CrossValidator.load(cvPath)
         self.assertEqual(loadedCV.getEstimator().uid, cv.getEstimator().uid)
         self.assertEqual(loadedCV.getEvaluator().uid, cv.getEvaluator().uid)
-        self.assertEqual(loadedCV.getEstimatorParamMaps(), cv.getEstimatorParamMaps())
+        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), cv.getEstimatorParamMaps())
 
         # test save/load of CrossValidatorModel
         cvModelPath = temp_path + "/cvModel"
@@ -292,6 +362,7 @@ def test_save_load_nested_estimator(self):
         cvPath = temp_path + "/cv"
         cv.save(cvPath)
         loadedCV = CrossValidator.load(cvPath)
+        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedCV.getEstimator().uid, cv.getEstimator().uid)
         self.assertEqual(loadedCV.getEvaluator().uid, cv.getEvaluator().uid)
 
@@ -308,10 +379,90 @@ def test_save_load_nested_estimator(self):
         cvModelPath = temp_path + "/cvModel"
         cvModel.save(cvModelPath)
         loadedModel = CrossValidatorModel.load(cvModelPath)
+        self.assert_param_maps_equal(loadedModel.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedModel.bestModel.uid, cvModel.bestModel.uid)
 
+    def test_save_load_pipeline_estimator(self):
+        temp_path = tempfile.mkdtemp()
+        training = self.spark.createDataFrame([
+            (0, "a b c d e spark", 1.0),
+            (1, "b d", 0.0),
+            (2, "spark f g h", 1.0),
+            (3, "hadoop mapreduce", 0.0),
+            (4, "b spark who", 1.0),
+            (5, "g d a y", 0.0),
+            (6, "spark fly", 1.0),
+            (7, "was mapreduce", 0.0),
+        ], ["id", "text", "label"])
+
+        # Configure an ML pipeline, which consists of tree stages: tokenizer, hashingTF, and lr.
+        tokenizer = Tokenizer(inputCol="text", outputCol="words")
+        hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
+
+        ova = OneVsRest(classifier=LogisticRegression())
+        lr1 = LogisticRegression().setMaxIter(5)
+        lr2 = LogisticRegression().setMaxIter(10)
+
+        pipeline = Pipeline(stages=[tokenizer, hashingTF, ova])
+
+        paramGrid = ParamGridBuilder() \
+            .addGrid(hashingTF.numFeatures, [10, 100]) \
+            .addGrid(ova.classifier, [lr1, lr2]) \
+            .build()
+
+        crossval = CrossValidator(estimator=pipeline,
+                                  estimatorParamMaps=paramGrid,
+                                  evaluator=MulticlassClassificationEvaluator(),
+                                  numFolds=2)  # use 3+ folds in practice
+        cvPath = temp_path + "/cv"
+        crossval.save(cvPath)
+        loadedCV = CrossValidator.load(cvPath)
+        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedCV.getEstimator().uid, crossval.getEstimator().uid)
 
-class TrainValidationSplitTests(SparkSessionTestCase):
+        # Run cross-validation, and choose the best set of parameters.
+        cvModel = crossval.fit(training)
+
+        # test save/load of CrossValidatorModel
+        cvModelPath = temp_path + "/cvModel"
+        cvModel.save(cvModelPath)
+        loadedModel = CrossValidatorModel.load(cvModelPath)
+        self.assertEqual(loadedModel.bestModel.uid, cvModel.bestModel.uid)
+        self.assertEqual(len(loadedModel.bestModel.stages), len(cvModel.bestModel.stages))
+        for loadedStage, originalStage in zip(loadedModel.bestModel.stages,
+                                              cvModel.bestModel.stages):
+            self.assertEqual(loadedStage.uid, originalStage.uid)
+
+        # Test nested pipeline
+        nested_pipeline = Pipeline(stages=[tokenizer, Pipeline(stages=[hashingTF, ova])])
+        crossval2 = CrossValidator(estimator=nested_pipeline,
+                                   estimatorParamMaps=paramGrid,
+                                   evaluator=MulticlassClassificationEvaluator(),
+                                   numFolds=2)  # use 3+ folds in practice
+        cv2Path = temp_path + "/cv2"
+        crossval2.save(cv2Path)
+        loadedCV2 = CrossValidator.load(cv2Path)
+        self.assert_param_maps_equal(loadedCV2.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedCV2.getEstimator().uid, crossval2.getEstimator().uid)
+
+        # Run cross-validation, and choose the best set of parameters.
+        cvModel2 = crossval2.fit(training)
+        # test save/load of CrossValidatorModel
+        cvModelPath2 = temp_path + "/cvModel2"
+        cvModel2.save(cvModelPath2)
+        loadedModel2 = CrossValidatorModel.load(cvModelPath2)
+        self.assertEqual(loadedModel2.bestModel.uid, cvModel2.bestModel.uid)
+        loaded_nested_pipeline_model = loadedModel2.bestModel.stages[1]
+        original_nested_pipeline_model = cvModel2.bestModel.stages[1]
+        self.assertEqual(loaded_nested_pipeline_model.uid, original_nested_pipeline_model.uid)
+        self.assertEqual(len(loaded_nested_pipeline_model.stages),
+                         len(original_nested_pipeline_model.stages))
+        for loadedStage, originalStage in zip(loaded_nested_pipeline_model.stages,
+                                              original_nested_pipeline_model.stages):
+            self.assertEqual(loadedStage.uid, originalStage.uid)
+
+
+class TrainValidationSplitTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
 
     def test_fit_minimize_metric(self):
         dataset = self.spark.createDataFrame([
@@ -381,16 +532,35 @@ def test_save_load_trained_model(self):
         lr = LogisticRegression()
         grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
         evaluator = BinaryClassificationEvaluator()
-        tvs = TrainValidationSplit(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
+        tvs = TrainValidationSplit(
+            estimator=lr,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True,
+            seed=42
+        )
         tvsModel = tvs.fit(dataset)
         lrModel = tvsModel.bestModel
 
-        tvsModelPath = temp_path + "/tvsModel"
-        lrModel.save(tvsModelPath)
-        loadedLrModel = LogisticRegressionModel.load(tvsModelPath)
+        lrModelPath = temp_path + "/lrModel"
+        lrModel.save(lrModelPath)
+        loadedLrModel = LogisticRegressionModel.load(lrModelPath)
         self.assertEqual(loadedLrModel.uid, lrModel.uid)
         self.assertEqual(loadedLrModel.intercept, lrModel.intercept)
 
+        tvsModelPath = temp_path + "/tvsModel"
+        tvsModel.save(tvsModelPath)
+        loadedTvsModel = TrainValidationSplitModel.load(tvsModelPath)
+        for param in [
+            lambda x: x.getSeed(),
+            lambda x: x.getTrainRatio(),
+        ]:
+            self.assertEqual(param(tvsModel), param(loadedTvsModel))
+
+        self.assertTrue(all(
+            loadedTvsModel.isSet(param) for param in loadedTvsModel.params
+        ))
+
     def test_save_load_simple_estimator(self):
         # This tests saving and loading the trained model only.
         # Save/load for TrainValidationSplit will be added later: SPARK-13786
@@ -413,7 +583,8 @@ def test_save_load_simple_estimator(self):
         loadedTvs = TrainValidationSplit.load(tvsPath)
         self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
         self.assertEqual(loadedTvs.getEvaluator().uid, tvs.getEvaluator().uid)
-        self.assertEqual(loadedTvs.getEstimatorParamMaps(), tvs.getEstimatorParamMaps())
+        self.assert_param_maps_equal(
+            loadedTvs.getEstimatorParamMaps(), tvs.getEstimatorParamMaps())
 
         tvsModelPath = temp_path + "/tvsModel"
         tvsModel.save(tvsModelPath)
@@ -494,6 +665,7 @@ def test_save_load_nested_estimator(self):
         tvsPath = temp_path + "/tvs"
         tvs.save(tvsPath)
         loadedTvs = TrainValidationSplit.load(tvsPath)
+        self.assert_param_maps_equal(loadedTvs.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
         self.assertEqual(loadedTvs.getEvaluator().uid, tvs.getEvaluator().uid)
 
@@ -509,8 +681,86 @@ def test_save_load_nested_estimator(self):
         tvsModelPath = temp_path + "/tvsModel"
         tvsModel.save(tvsModelPath)
         loadedModel = TrainValidationSplitModel.load(tvsModelPath)
+        self.assert_param_maps_equal(loadedModel.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedModel.bestModel.uid, tvsModel.bestModel.uid)
 
+    def test_save_load_pipeline_estimator(self):
+        temp_path = tempfile.mkdtemp()
+        training = self.spark.createDataFrame([
+            (0, "a b c d e spark", 1.0),
+            (1, "b d", 0.0),
+            (2, "spark f g h", 1.0),
+            (3, "hadoop mapreduce", 0.0),
+            (4, "b spark who", 1.0),
+            (5, "g d a y", 0.0),
+            (6, "spark fly", 1.0),
+            (7, "was mapreduce", 0.0),
+        ], ["id", "text", "label"])
+
+        # Configure an ML pipeline, which consists of tree stages: tokenizer, hashingTF, and lr.
+        tokenizer = Tokenizer(inputCol="text", outputCol="words")
+        hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
+
+        ova = OneVsRest(classifier=LogisticRegression())
+        lr1 = LogisticRegression().setMaxIter(5)
+        lr2 = LogisticRegression().setMaxIter(10)
+
+        pipeline = Pipeline(stages=[tokenizer, hashingTF, ova])
+
+        paramGrid = ParamGridBuilder() \
+            .addGrid(hashingTF.numFeatures, [10, 100]) \
+            .addGrid(ova.classifier, [lr1, lr2]) \
+            .build()
+
+        tvs = TrainValidationSplit(estimator=pipeline,
+                                   estimatorParamMaps=paramGrid,
+                                   evaluator=MulticlassClassificationEvaluator())
+        tvsPath = temp_path + "/tvs"
+        tvs.save(tvsPath)
+        loadedTvs = TrainValidationSplit.load(tvsPath)
+        self.assert_param_maps_equal(loadedTvs.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
+
+        # Run train validation split, and choose the best set of parameters.
+        tvsModel = tvs.fit(training)
+
+        # test save/load of CrossValidatorModel
+        tvsModelPath = temp_path + "/tvsModel"
+        tvsModel.save(tvsModelPath)
+        loadedModel = TrainValidationSplitModel.load(tvsModelPath)
+        self.assertEqual(loadedModel.bestModel.uid, tvsModel.bestModel.uid)
+        self.assertEqual(len(loadedModel.bestModel.stages), len(tvsModel.bestModel.stages))
+        for loadedStage, originalStage in zip(loadedModel.bestModel.stages,
+                                              tvsModel.bestModel.stages):
+            self.assertEqual(loadedStage.uid, originalStage.uid)
+
+        # Test nested pipeline
+        nested_pipeline = Pipeline(stages=[tokenizer, Pipeline(stages=[hashingTF, ova])])
+        tvs2 = TrainValidationSplit(estimator=nested_pipeline,
+                                    estimatorParamMaps=paramGrid,
+                                    evaluator=MulticlassClassificationEvaluator())
+        tvs2Path = temp_path + "/tvs2"
+        tvs2.save(tvs2Path)
+        loadedTvs2 = TrainValidationSplit.load(tvs2Path)
+        self.assert_param_maps_equal(loadedTvs2.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedTvs2.getEstimator().uid, tvs2.getEstimator().uid)
+
+        # Run train validation split, and choose the best set of parameters.
+        tvsModel2 = tvs2.fit(training)
+        # test save/load of CrossValidatorModel
+        tvsModelPath2 = temp_path + "/tvsModel2"
+        tvsModel2.save(tvsModelPath2)
+        loadedModel2 = TrainValidationSplitModel.load(tvsModelPath2)
+        self.assertEqual(loadedModel2.bestModel.uid, tvsModel2.bestModel.uid)
+        loaded_nested_pipeline_model = loadedModel2.bestModel.stages[1]
+        original_nested_pipeline_model = tvsModel2.bestModel.stages[1]
+        self.assertEqual(loaded_nested_pipeline_model.uid, original_nested_pipeline_model.uid)
+        self.assertEqual(len(loaded_nested_pipeline_model.stages),
+                         len(original_nested_pipeline_model.stages))
+        for loadedStage, originalStage in zip(loaded_nested_pipeline_model.stages,
+                                              original_nested_pipeline_model.stages):
+            self.assertEqual(loadedStage.uid, originalStage.uid)
+
     def test_copy(self):
         dataset = self.spark.createDataFrame([
             (10, 10.0),
@@ -525,11 +775,30 @@ def test_copy(self):
         grid = ParamGridBuilder() \
             .addGrid(iee.inducedError, [100.0, 0.0, 10000.0]) \
             .build()
-        tvs = TrainValidationSplit(estimator=iee, estimatorParamMaps=grid, evaluator=evaluator)
+        tvs = TrainValidationSplit(
+            estimator=iee,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True
+        )
         tvsModel = tvs.fit(dataset)
         tvsCopied = tvs.copy()
         tvsModelCopied = tvsModel.copy()
 
+        for param in [
+            lambda x: x.getCollectSubModels(),
+            lambda x: x.getParallelism(),
+            lambda x: x.getSeed(),
+            lambda x: x.getTrainRatio(),
+        ]:
+            self.assertEqual(param(tvs), param(tvsCopied))
+
+        for param in [
+            lambda x: x.getSeed(),
+            lambda x: x.getTrainRatio(),
+        ]:
+            self.assertEqual(param(tvsModel), param(tvsModelCopied))
+
         self.assertEqual(tvs.getEstimator().uid, tvsCopied.getEstimator().uid,
                          "Copied TrainValidationSplit has the same uid of Estimator")
 
@@ -541,6 +810,19 @@ def test_copy(self):
             self.assertEqual(tvsModel.validationMetrics[index],
                              tvsModelCopied.validationMetrics[index])
 
+        tvsModel.validationMetrics[0] = 'foo'
+        self.assertNotEqual(
+            tvsModelCopied.validationMetrics[0],
+            'foo',
+            "Changing the original validationMetrics should not affect the copied model"
+        )
+        tvsModel.subModels[0].getInducedError = lambda: 'foo'
+        self.assertNotEqual(
+            tvsModelCopied.subModels[0].getInducedError(),
+            'foo',
+            "Changing the original subModels should not affect the copied model"
+        )
+
 
 if __name__ == "__main__":
     from pyspark.ml.tests.test_tuning import *
diff --git a/python/pyspark/ml/tests/test_util.py b/python/pyspark/ml/tests/test_util.py
new file mode 100644
index 0000000000000..498a649e480a8
--- /dev/null
+++ b/python/pyspark/ml/tests/test_util.py
@@ -0,0 +1,84 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+
+from pyspark.ml import Pipeline
+from pyspark.ml.classification import LogisticRegression, OneVsRest
+from pyspark.ml.feature import VectorAssembler
+from pyspark.ml.linalg import Vectors
+from pyspark.ml.util import MetaAlgorithmReadWrite
+from pyspark.testing.mlutils import SparkSessionTestCase
+
+
+class MetaAlgorithmReadWriteTests(SparkSessionTestCase):
+
+    def test_getAllNestedStages(self):
+        def _check_uid_set_equal(stages, expected_stages):
+            uids = set(map(lambda x: x.uid, stages))
+            expected_uids = set(map(lambda x: x.uid, expected_stages))
+            self.assertEqual(uids, expected_uids)
+
+        df1 = self.spark.createDataFrame([
+            (Vectors.dense([1., 2.]), 1.0),
+            (Vectors.dense([-1., -2.]), 0.0),
+        ], ['features', 'label'])
+        df2 = self.spark.createDataFrame([
+            (1., 2., 1.0),
+            (1., 2., 0.0),
+        ], ['a', 'b', 'label'])
+        vs = VectorAssembler(inputCols=['a', 'b'], outputCol='features')
+        lr = LogisticRegression()
+        pipeline = Pipeline(stages=[vs, lr])
+        pipelineModel = pipeline.fit(df2)
+        ova = OneVsRest(classifier=lr)
+        ovaModel = ova.fit(df1)
+
+        ova_pipeline = Pipeline(stages=[vs, ova])
+        nested_pipeline = Pipeline(stages=[ova_pipeline])
+
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(pipeline),
+            [pipeline, vs, lr]
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(pipelineModel),
+            [pipelineModel] + pipelineModel.stages
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(ova),
+            [ova, lr]
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(ovaModel),
+            [ovaModel, lr] + ovaModel.models
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(nested_pipeline),
+            [nested_pipeline, ova_pipeline, vs, ova, lr]
+        )
+
+
+if __name__ == "__main__":
+    from pyspark.ml.tests.test_util import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py
index cb4542c5d025f..e16ea5753f1b3 100644
--- a/python/pyspark/ml/tuning.py
+++ b/python/pyspark/ml/tuning.py
@@ -26,8 +26,9 @@
 from pyspark.ml.param import Params, Param, TypeConverters
 from pyspark.ml.param.shared import HasCollectSubModels, HasParallelism, HasSeed
 from pyspark.ml.util import *
-from pyspark.ml.wrapper import JavaParams
-from pyspark.sql.functions import rand
+from pyspark.ml.wrapper import JavaParams, JavaEstimator, JavaWrapper
+from pyspark.sql.functions import col, lit, rand, UserDefinedFunction
+from pyspark.sql.types import BooleanType
 
 __all__ = ['ParamGridBuilder', 'CrossValidator', 'CrossValidatorModel', 'TrainValidationSplit',
            'TrainValidationSplitModel']
@@ -50,6 +51,10 @@ def _parallelFitTasks(est, train, eva, validation, epm, collectSubModel):
 
     def singleTask():
         index, model = next(modelIter)
+        # TODO: duplicate evaluator to take extra params from input
+        #  Note: Supporting tuning params in evaluator need update method
+        #  `MetaAlgorithmReadWrite.getAllNestedStages`, make it return
+        #  all nested stages and evaluators
         metric = eva.evaluate(model.transform(validation, epm[index]))
         return index, metric, model if collectSubModel else None
 
@@ -169,8 +174,16 @@ def _from_java_impl(cls, java_stage):
         # Load information from java_stage to the instance.
         estimator = JavaParams._from_java(java_stage.getEstimator())
         evaluator = JavaParams._from_java(java_stage.getEvaluator())
-        epms = [estimator._transfer_param_map_from_java(epm)
-                for epm in java_stage.getEstimatorParamMaps()]
+        if isinstance(estimator, JavaEstimator):
+            epms = [estimator._transfer_param_map_from_java(epm)
+                    for epm in java_stage.getEstimatorParamMaps()]
+        elif MetaAlgorithmReadWrite.isMetaEstimator(estimator):
+            # Meta estimator such as Pipeline, OneVsRest
+            epms = _ValidatorSharedReadWrite.meta_estimator_transfer_param_maps_from_java(
+                estimator, java_stage.getEstimatorParamMaps())
+        else:
+            raise ValueError('Unsupported estimator used in tuning: ' + str(estimator))
+
         return estimator, epms, evaluator
 
     def _to_java_impl(self):
@@ -181,15 +194,80 @@ def _to_java_impl(self):
         gateway = SparkContext._gateway
         cls = SparkContext._jvm.org.apache.spark.ml.param.ParamMap
 
-        java_epms = gateway.new_array(cls, len(self.getEstimatorParamMaps()))
-        for idx, epm in enumerate(self.getEstimatorParamMaps()):
-            java_epms[idx] = self.getEstimator()._transfer_param_map_to_java(epm)
+        estimator = self.getEstimator()
+        if isinstance(estimator, JavaEstimator):
+            java_epms = gateway.new_array(cls, len(self.getEstimatorParamMaps()))
+            for idx, epm in enumerate(self.getEstimatorParamMaps()):
+                java_epms[idx] = self.getEstimator()._transfer_param_map_to_java(epm)
+        elif MetaAlgorithmReadWrite.isMetaEstimator(estimator):
+            # Meta estimator such as Pipeline, OneVsRest
+            java_epms = _ValidatorSharedReadWrite.meta_estimator_transfer_param_maps_to_java(
+                estimator, self.getEstimatorParamMaps())
+        else:
+            raise ValueError('Unsupported estimator used in tuning: ' + str(estimator))
 
         java_estimator = self.getEstimator()._to_java()
         java_evaluator = self.getEvaluator()._to_java()
         return java_estimator, java_epms, java_evaluator
 
 
+class _ValidatorSharedReadWrite:
+    @staticmethod
+    def meta_estimator_transfer_param_maps_to_java(pyEstimator, pyParamMaps):
+        pyStages = MetaAlgorithmReadWrite.getAllNestedStages(pyEstimator)
+        stagePairs = list(map(lambda stage: (stage, stage._to_java()), pyStages))
+        sc = SparkContext._active_spark_context
+
+        paramMapCls = SparkContext._jvm.org.apache.spark.ml.param.ParamMap
+        javaParamMaps = SparkContext._gateway.new_array(paramMapCls, len(pyParamMaps))
+
+        for idx, pyParamMap in enumerate(pyParamMaps):
+            javaParamMap = JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
+            for pyParam, pyValue in pyParamMap.items():
+                javaParam = None
+                for pyStage, javaStage in stagePairs:
+                    if pyStage._testOwnParam(pyParam.parent, pyParam.name):
+                        javaParam = javaStage.getParam(pyParam.name)
+                        break
+                if javaParam is None:
+                    raise ValueError('Resolve param in estimatorParamMaps failed: ' + str(pyParam))
+                if isinstance(pyValue, Params) and hasattr(pyValue, "_to_java"):
+                    javaValue = pyValue._to_java()
+                else:
+                    javaValue = _py2java(sc, pyValue)
+                pair = javaParam.w(javaValue)
+                javaParamMap.put([pair])
+            javaParamMaps[idx] = javaParamMap
+        return javaParamMaps
+
+    @staticmethod
+    def meta_estimator_transfer_param_maps_from_java(pyEstimator, javaParamMaps):
+        pyStages = MetaAlgorithmReadWrite.getAllNestedStages(pyEstimator)
+        stagePairs = list(map(lambda stage: (stage, stage._to_java()), pyStages))
+        sc = SparkContext._active_spark_context
+        pyParamMaps = []
+        for javaParamMap in javaParamMaps:
+            pyParamMap = dict()
+            for javaPair in javaParamMap.toList():
+                javaParam = javaPair.param()
+                pyParam = None
+                for pyStage, javaStage in stagePairs:
+                    if pyStage._testOwnParam(javaParam.parent(), javaParam.name()):
+                        pyParam = pyStage.getParam(javaParam.name())
+                if pyParam is None:
+                    raise ValueError('Resolve param in estimatorParamMaps failed: ' +
+                                     javaParam.parent() + '.' + javaParam.name())
+                javaValue = javaPair.value()
+                if sc._jvm.Class.forName("org.apache.spark.ml.util.DefaultParamsWritable") \
+                        .isInstance(javaValue):
+                    pyValue = JavaParams._from_java(javaValue)
+                else:
+                    pyValue = _java2py(sc, javaValue)
+                pyParamMap[pyParam] = pyValue
+            pyParamMaps.append(pyParamMap)
+        return pyParamMaps
+
+
 class _CrossValidatorParams(_ValidatorParams):
     """
     Params for :py:class:`CrossValidator` and :py:class:`CrossValidatorModel`.
@@ -200,6 +278,10 @@ class _CrossValidatorParams(_ValidatorParams):
     numFolds = Param(Params._dummy(), "numFolds", "number of folds for cross validation",
                      typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_CrossValidatorParams, self).__init__(*args)
+        self._setDefault(numFolds=3)
+
     @since("1.4.0")
     def getNumFolds(self):
         """
@@ -249,6 +331,8 @@ class CrossValidator(Estimator, _CrossValidatorParams, HasParallelism, HasCollec
     [0.5, ...
     >>> evaluator.evaluate(cvModel.transform(dataset))
     0.8333...
+    >>> evaluator.evaluate(cvModelRead.transform(dataset))
+    0.8333...
 
     .. versionadded:: 1.4.0
     """
@@ -261,7 +345,7 @@ def __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, numF
                  seed=None, parallelism=1, collectSubModels=False)
         """
         super(CrossValidator, self).__init__()
-        self._setDefault(numFolds=3, parallelism=1)
+        self._setDefault(parallelism=1)
         kwargs = self._input_kwargs
         self._set(**kwargs)
 
@@ -473,9 +557,12 @@ def copy(self, extra=None):
         if extra is None:
             extra = dict()
         bestModel = self.bestModel.copy(extra)
-        avgMetrics = self.avgMetrics
-        subModels = self.subModels
-        return CrossValidatorModel(bestModel, avgMetrics, subModels)
+        avgMetrics = list(self.avgMetrics)
+        subModels = [
+            [sub_model.copy() for sub_model in fold_sub_models]
+            for fold_sub_models in self.subModels
+        ]
+        return self._copyValues(CrossValidatorModel(bestModel, avgMetrics, subModels), extra=extra)
 
     @since("2.3.0")
     def write(self):
@@ -499,8 +586,16 @@ def _from_java(cls, java_stage):
         avgMetrics = _java2py(sc, java_stage.avgMetrics())
         estimator, epms, evaluator = super(CrossValidatorModel, cls)._from_java_impl(java_stage)
 
-        py_stage = cls(bestModel=bestModel, avgMetrics=avgMetrics)._set(estimator=estimator)
-        py_stage = py_stage._set(estimatorParamMaps=epms)._set(evaluator=evaluator)
+        py_stage = cls(bestModel=bestModel, avgMetrics=avgMetrics)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "numFolds": java_stage.getNumFolds(),
+            "seed": java_stage.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            py_stage = py_stage._set(**{param_name: param_val})
 
         if java_stage.hasSubModels():
             py_stage.subModels = [[JavaParams._from_java(sub_model)
@@ -524,9 +619,17 @@ def _to_java(self):
                                              _py2java(sc, self.avgMetrics))
         estimator, epms, evaluator = super(CrossValidatorModel, self)._to_java_impl()
 
-        _java_obj.set("evaluator", evaluator)
-        _java_obj.set("estimator", estimator)
-        _java_obj.set("estimatorParamMaps", epms)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "numFolds": self.getNumFolds(),
+            "seed": self.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            java_param = _java_obj.getParam(param_name)
+            pair = java_param.w(param_val)
+            _java_obj.set(pair)
 
         if self.subModels is not None:
             java_sub_models = [[sub_model._to_java() for sub_model in fold_sub_models]
@@ -545,6 +648,10 @@ class _TrainValidationSplitParams(_ValidatorParams):
     trainRatio = Param(Params._dummy(), "trainRatio", "Param for ratio between train and\
      validation data. Must be between 0 and 1.", typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_TrainValidationSplitParams, self).__init__(*args)
+        self._setDefault(trainRatio=0.75)
+
     @since("2.0.0")
     def getTrainRatio(self):
         """
@@ -590,6 +697,8 @@ class TrainValidationSplit(Estimator, _TrainValidationSplitParams, HasParallelis
     [0.5, ...
     >>> evaluator.evaluate(tvsModel.transform(dataset))
     0.833...
+    >>> evaluator.evaluate(tvsModelRead.transform(dataset))
+    0.833...
 
     .. versionadded:: 2.0.0
     """
@@ -602,7 +711,7 @@ def __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, trai
                  parallelism=1, collectSubModels=False, seed=None)
         """
         super(TrainValidationSplit, self).__init__()
-        self._setDefault(trainRatio=0.75, parallelism=1)
+        self._setDefault(parallelism=1)
         kwargs = self._input_kwargs
         self._set(**kwargs)
 
@@ -806,8 +915,11 @@ def copy(self, extra=None):
             extra = dict()
         bestModel = self.bestModel.copy(extra)
         validationMetrics = list(self.validationMetrics)
-        subModels = self.subModels
-        return TrainValidationSplitModel(bestModel, validationMetrics, subModels)
+        subModels = [model.copy() for model in self.subModels]
+        return self._copyValues(
+            TrainValidationSplitModel(bestModel, validationMetrics, subModels),
+            extra=extra
+        )
 
     @since("2.3.0")
     def write(self):
@@ -835,8 +947,16 @@ def _from_java(cls, java_stage):
                                            cls)._from_java_impl(java_stage)
         # Create a new instance of this stage.
         py_stage = cls(bestModel=bestModel,
-                       validationMetrics=validationMetrics)._set(estimator=estimator)
-        py_stage = py_stage._set(estimatorParamMaps=epms)._set(evaluator=evaluator)
+                       validationMetrics=validationMetrics)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "trainRatio": java_stage.getTrainRatio(),
+            "seed": java_stage.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            py_stage = py_stage._set(**{param_name: param_val})
 
         if java_stage.hasSubModels():
             py_stage.subModels = [JavaParams._from_java(sub_model)
@@ -859,9 +979,17 @@ def _to_java(self):
             _py2java(sc, self.validationMetrics))
         estimator, epms, evaluator = super(TrainValidationSplitModel, self)._to_java_impl()
 
-        _java_obj.set("evaluator", evaluator)
-        _java_obj.set("estimator", estimator)
-        _java_obj.set("estimatorParamMaps", epms)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "trainRatio": self.getTrainRatio(),
+            "seed": self.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            java_param = _java_obj.getParam(param_name)
+            pair = java_param.w(param_val)
+            _java_obj.set(pair)
 
         if self.subModels is not None:
             java_sub_models = [sub_model._to_java() for sub_model in self.subModels]
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index 81b785e71f302..0f9d8e72ab2b7 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -345,8 +345,6 @@ def read(cls):
 @inherit_doc
 class DefaultParamsWritable(MLWritable):
     """
-    .. note:: DeveloperApi
-
     Helper trait for making simple :py:class:`Params` types writable.  If a :py:class:`Params`
     class stores all data as :py:class:`Param` values, then extending this trait will provide
     a default implementation of writing saved instances of the class.
@@ -370,8 +368,6 @@ def write(self):
 @inherit_doc
 class DefaultParamsWriter(MLWriter):
     """
-    .. note:: DeveloperApi
-
     Specialization of :py:class:`MLWriter` for :py:class:`Params` types
 
     Class for writing Estimators and Transformers whose parameters are JSON-serializable.
@@ -390,6 +386,7 @@ def saveImpl(self, path):
     def saveMetadata(instance, path, sc, extraMetadata=None, paramMap=None):
         """
         Saves metadata + Params to: path + "/metadata"
+
         - class
         - timestamp
         - sparkVersion
@@ -397,6 +394,7 @@ def saveMetadata(instance, path, sc, extraMetadata=None, paramMap=None):
         - paramMap
         - defaultParamMap (since 2.4.0)
         - (optionally, extra metadata)
+
         :param extraMetadata:  Extra metadata to be saved at same level as uid, paramMap, etc.
         :param paramMap:  If given, this is saved in the "paramMap" field.
         """
@@ -443,8 +441,6 @@ def _get_metadata_to_save(instance, sc, extraMetadata=None, paramMap=None):
 @inherit_doc
 class DefaultParamsReadable(MLReadable):
     """
-    .. note:: DeveloperApi
-
     Helper trait for making simple :py:class:`Params` types readable.
     If a :py:class:`Params` class stores all data as :py:class:`Param` values,
     then extending this trait will provide a default implementation of reading saved
@@ -464,8 +460,6 @@ def read(cls):
 @inherit_doc
 class DefaultParamsReader(MLReader):
     """
-    .. note:: DeveloperApi
-
     Specialization of :py:class:`MLReader` for :py:class:`Params` types
 
     Default :py:class:`MLReader` implementation for transformers and estimators that
@@ -569,6 +563,7 @@ def loadParamsInstance(path, sc):
 class HasTrainingSummary(object):
     """
     Base class for models that provides Training summary.
+
     .. versionadded:: 3.0.0
     """
 
@@ -589,3 +584,41 @@ def summary(self):
         no summary exists.
         """
         return (self._call_java("summary"))
+
+
+class MetaAlgorithmReadWrite:
+
+    @staticmethod
+    def isMetaEstimator(pyInstance):
+        from pyspark.ml import Estimator, Pipeline
+        from pyspark.ml.tuning import _ValidatorParams
+        from pyspark.ml.classification import OneVsRest
+        return isinstance(pyInstance, Pipeline) or isinstance(pyInstance, OneVsRest) or \
+            (isinstance(pyInstance, Estimator) and isinstance(pyInstance, _ValidatorParams))
+
+    @staticmethod
+    def getAllNestedStages(pyInstance):
+        from pyspark.ml import Pipeline, PipelineModel
+        from pyspark.ml.tuning import _ValidatorParams
+        from pyspark.ml.classification import OneVsRest, OneVsRestModel
+
+        # TODO: We need to handle `RFormulaModel.pipelineModel` here after Pyspark RFormulaModel
+        #  support pipelineModel property.
+        if isinstance(pyInstance, Pipeline):
+            pySubStages = pyInstance.getStages()
+        elif isinstance(pyInstance, PipelineModel):
+            pySubStages = pyInstance.stages
+        elif isinstance(pyInstance, _ValidatorParams):
+            raise ValueError('PySpark does not support nested validator.')
+        elif isinstance(pyInstance, OneVsRest):
+            pySubStages = [pyInstance.getClassifier()]
+        elif isinstance(pyInstance, OneVsRestModel):
+            pySubStages = [pyInstance.getClassifier()] + pyInstance.models
+        else:
+            pySubStages = []
+
+        nestedStages = []
+        for pySubStage in pySubStages:
+            nestedStages.extend(MetaAlgorithmReadWrite.getAllNestedStages(pySubStage))
+
+        return [pyInstance] + nestedStages
diff --git a/python/pyspark/mllib/feature.py b/python/pyspark/mllib/feature.py
index 905c4da2d880d..3efae6ff0ecc3 100644
--- a/python/pyspark/mllib/feature.py
+++ b/python/pyspark/mllib/feature.py
@@ -43,8 +43,6 @@
 
 class VectorTransformer(object):
     """
-    .. note:: DeveloperApi
-
     Base class for transformation of a vector or RDD of vector
     """
     def transform(self, vector):
diff --git a/python/pyspark/mllib/tests/test_streaming_algorithms.py b/python/pyspark/mllib/tests/test_streaming_algorithms.py
index 2077809a043f1..5818a7c088841 100644
--- a/python/pyspark/mllib/tests/test_streaming_algorithms.py
+++ b/python/pyspark/mllib/tests/test_streaming_algorithms.py
@@ -327,7 +327,7 @@ def condition():
                 return True
             return "Latest errors: " + ", ".join(map(lambda x: str(x), errors))
 
-        eventually(condition, timeout=60.0)
+        eventually(condition, timeout=180.0)
 
 
 class StreamingLinearRegressionWithTests(MLLibStreamingTestCase):
@@ -434,9 +434,9 @@ def test_train_prediction(self):
         slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
         slr.setInitialWeights([0.0])
 
-        # Create ten batches with 100 sample points in each.
+        # Create fifteen batches with 100 sample points in each.
         batches = []
-        for i in range(10):
+        for i in range(15):
             batch = LinearDataGenerator.generateLinearInput(
                 0.0, [10.0], [0.0], [1.0 / 3.0], 100, 42 + i, 0.1)
             batches.append(self.sc.parallelize(batch))
@@ -463,7 +463,7 @@ def condition():
                 return True
             return "Latest errors: " + ", ".join(map(lambda x: str(x), errors))
 
-        eventually(condition)
+        eventually(condition, timeout=180.0)
 
 
 if __name__ == "__main__":
diff --git a/python/pyspark/mllib/util.py b/python/pyspark/mllib/util.py
index 1a0ce42dc4e4f..f0f9cda4672b1 100644
--- a/python/pyspark/mllib/util.py
+++ b/python/pyspark/mllib/util.py
@@ -372,7 +372,7 @@ def save(self, sc, path):
          * human-readable (JSON) model metadata to path/metadata/
          * Parquet formatted data to path/data/
 
-        The model may be loaded using py:meth:`Loader.load`.
+        The model may be loaded using :py:meth:`Loader.load`.
 
         :param sc: Spark context used to save model data.
         :param path: Path specifying the directory in which to save
@@ -412,7 +412,7 @@ class Loader(object):
     def load(cls, sc, path):
         """
         Load a model from the given path. The model should have been
-        saved using py:meth:`Saveable.save`.
+        saved using :py:meth:`Saveable.save`.
 
         :param sc: Spark context used for loading model files.
         :param path: Path specifying the directory to which the model
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
index 52ab86c0d88ee..b5b72dade12a8 100644
--- a/python/pyspark/rdd.py
+++ b/python/pyspark/rdd.py
@@ -889,6 +889,19 @@ def collect(self):
             sock_info = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
         return list(_load_from_socket(sock_info, self._jrdd_deserializer))
 
+    def collectWithJobGroup(self, groupId, description, interruptOnCancel=False):
+        """
+        .. note:: Experimental
+
+        When collect rdd, use this method to specify job group.
+
+        .. versionadded:: 3.0.0
+        """
+        with SCCallSiteSync(self.context) as css:
+            sock_info = self.ctx._jvm.PythonRDD.collectAndServeWithJobGroup(
+                self._jrdd.rdd(), groupId, description, interruptOnCancel)
+        return list(_load_from_socket(sock_info, self._jrdd_deserializer))
+
     def reduce(self, f):
         """
         Reduces the elements of this RDD using the specified commutative and
diff --git a/python/pyspark/resourceinformation.py b/python/pyspark/resource.py
similarity index 89%
rename from python/pyspark/resourceinformation.py
rename to python/pyspark/resource.py
index aaed21374b6ee..b0e41cced85b5 100644
--- a/python/pyspark/resourceinformation.py
+++ b/python/pyspark/resource.py
@@ -26,8 +26,10 @@ class ResourceInformation(object):
 
     One example is GPUs, where the addresses would be the indices of the GPUs
 
-    @param name the name of the resource
-    @param addresses an array of strings describing the addresses of the resource
+    :param name: the name of the resource
+    :param addresses: an array of strings describing the addresses of the resource
+
+    .. versionadded:: 3.0.0
     """
 
     def __init__(self, name, addresses):
diff --git a/python/pyspark/sql/__init__.py b/python/pyspark/sql/__init__.py
index 0a8d71c12e060..c28cb8c3b9cbe 100644
--- a/python/pyspark/sql/__init__.py
+++ b/python/pyspark/sql/__init__.py
@@ -43,7 +43,7 @@
 
 
 from pyspark.sql.types import Row
-from pyspark.sql.context import SQLContext, UDFRegistration
+from pyspark.sql.context import SQLContext, HiveContext, UDFRegistration
 from pyspark.sql.session import SparkSession
 from pyspark.sql.column import Column
 from pyspark.sql.catalog import Catalog
@@ -55,7 +55,7 @@
 
 
 __all__ = [
-    'SparkSession', 'SQLContext', 'UDFRegistration',
+    'SparkSession', 'SQLContext', 'HiveContext', 'UDFRegistration',
     'DataFrame', 'GroupedData', 'Column', 'Catalog', 'Row',
     'DataFrameNaFunctions', 'DataFrameStatFunctions', 'Window', 'WindowSpec',
     'DataFrameReader', 'DataFrameWriter', 'PandasCogroupedOps'
diff --git a/python/pyspark/sql/catalog.py b/python/pyspark/sql/catalog.py
index 08cf6ee330785..974251f63b37a 100644
--- a/python/pyspark/sql/catalog.py
+++ b/python/pyspark/sql/catalog.py
@@ -138,6 +138,26 @@ def listColumns(self, tableName, dbName=None):
                 isBucket=jcolumn.isBucket()))
         return columns
 
+    @since(2.0)
+    def createExternalTable(self, tableName, path=None, source=None, schema=None, **options):
+        """Creates a table based on the dataset in a data source.
+
+        It returns the DataFrame associated with the external table.
+
+        The data source is specified by the ``source`` and a set of ``options``.
+        If ``source`` is not specified, the default data source configured by
+        ``spark.sql.sources.default`` will be used.
+
+        Optionally, a schema can be provided as the schema of the returned :class:`DataFrame` and
+        created external table.
+
+        :return: :class:`DataFrame`
+        """
+        warnings.warn(
+            "createExternalTable is deprecated since Spark 2.2, please use createTable instead.",
+            DeprecationWarning)
+        return self.createTable(tableName, path, source, schema, **options)
+
     @since(2.2)
     def createTable(self, tableName, path=None, source=None, schema=None, **options):
         """Creates a table based on the dataset in a data source.
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index 59d1408e26ad5..ef4944c9121a4 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -17,11 +17,14 @@
 
 import sys
 import json
+import warnings
 
 if sys.version >= '3':
     basestring = str
     long = int
 
+from py4j.java_gateway import is_instance_of
+
 from pyspark import copy_func, since
 from pyspark.context import SparkContext
 from pyspark.rdd import ignore_unicode_prefix
@@ -296,12 +299,14 @@ def getItem(self, key):
         +----+------+
         |   1| value|
         +----+------+
-
-        .. versionchanged:: 3.0
-           If `key` is a `Column` object, the indexing operator should be used instead.
-           For example, `map_col.getItem(col('id'))` should be replaced with `map_col[col('id')]`.
         """
-        return _bin_op("getItem")(self, key)
+        if isinstance(key, Column):
+            warnings.warn(
+                "A column as 'key' in getItem is deprecated as of Spark 3.0, and will not "
+                "be supported in the future release. Use `column[key]` or `column.key` syntax "
+                "instead.",
+                DeprecationWarning)
+        return self[key]
 
     @since(1.3)
     def getField(self, name):
@@ -323,12 +328,18 @@ def getField(self, name):
         |  1|
         +---+
         """
+        if isinstance(name, Column):
+            warnings.warn(
+                "A column as 'name' in getField is deprecated as of Spark 3.0, and will not "
+                "be supported in the future release. Use `column[name]` or `column.name` syntax "
+                "instead.",
+                DeprecationWarning)
         return self[name]
 
     def __getattr__(self, item):
         if item.startswith("__"):
             raise AttributeError(item)
-        return self.getField(item)
+        return self[item]
 
     def __getitem__(self, k):
         if isinstance(k, slice):
@@ -540,7 +551,8 @@ def alias(self, *alias, **kwargs):
 
         :param alias: strings of desired column names (collects all positional arguments passed)
         :param metadata: a dict of information to be stored in ``metadata`` attribute of the
-            corresponding :class: `StructField` (optional, keyword only argument)
+            corresponding :class:`StructField <pyspark.sql.types.StructField>` (optional, keyword
+            only argument)
 
         .. versionchanged:: 2.2
            Added optional ``metadata`` argument.
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 68d5ef4ca7b15..d9e5080cf0a9d 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -32,7 +32,7 @@
 from pyspark.sql.udf import UDFRegistration
 from pyspark.sql.utils import install_exception_handler
 
-__all__ = ["SQLContext"]
+__all__ = ["SQLContext", "HiveContext"]
 
 
 class SQLContext(object):
@@ -56,6 +56,8 @@ class SQLContext(object):
     def __init__(self, sparkContext, sparkSession=None, jsqlContext=None):
         """Creates a new SQLContext.
 
+        .. note:: Deprecated in 3.0.0. Use :func:`SparkSession.builder.getOrCreate()` instead.
+
         >>> from datetime import datetime
         >>> sqlContext = SQLContext(sc)
         >>> allTypes = sc.parallelize([Row(i=1, s="string", d=1.0, l=1,
@@ -70,6 +72,11 @@ def __init__(self, sparkContext, sparkSession=None, jsqlContext=None):
         >>> df.rdd.map(lambda x: (x.i, x.s, x.d, x.l, x.b, x.time, x.row.a, x.list)).collect()
         [(1, u'string', 1.0, 1, True, datetime.datetime(2014, 8, 1, 14, 1, 5), 1, [1, 2, 3])]
         """
+        if sparkSession is None:
+            warnings.warn(
+                "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
+                DeprecationWarning)
+
         self._sc = sparkContext
         self._jsc = self._sc._jsc
         self._jvm = self._sc._jvm
@@ -105,9 +112,16 @@ def getOrCreate(cls, sc):
         Get the existing SQLContext or create a new one with given SparkContext.
 
         :param sc: SparkContext
+
+        .. note:: Deprecated in 3.0.0. Use :func:`SparkSession.builder.getOrCreate()` instead.
         """
+        warnings.warn(
+            "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
+            DeprecationWarning)
+
         if cls._instantiatedContext is None:
-            jsqlContext = sc._jvm.SQLContext.getOrCreate(sc._jsc.sc())
+            jsqlContext = sc._jvm.SparkSession.builder().sparkContext(
+                sc._jsc.sc()).getOrCreate().sqlContext()
             sparkSession = SparkSession(sc, jsqlContext.sparkSession())
             cls(sc, sparkSession, jsqlContext)
         return cls._instantiatedContext
@@ -325,6 +339,24 @@ def dropTempTable(self, tableName):
         """
         self.sparkSession.catalog.dropTempView(tableName)
 
+    @since(1.3)
+    def createExternalTable(self, tableName, path=None, source=None, schema=None, **options):
+        """Creates an external table based on the dataset in a data source.
+
+        It returns the DataFrame associated with the external table.
+
+        The data source is specified by the ``source`` and a set of ``options``.
+        If ``source`` is not specified, the default data source configured by
+        ``spark.sql.sources.default`` will be used.
+
+        Optionally, a schema can be provided as the schema of the returned :class:`DataFrame` and
+        created external table.
+
+        :return: :class:`DataFrame`
+        """
+        return self.sparkSession.catalog.createExternalTable(
+            tableName, path, source, schema, **options)
+
     @ignore_unicode_prefix
     @since(1.0)
     def sql(self, sqlQuery):
@@ -448,6 +480,53 @@ def streams(self):
         return StreamingQueryManager(self._ssql_ctx.streams())
 
 
+class HiveContext(SQLContext):
+    """A variant of Spark SQL that integrates with data stored in Hive.
+
+    Configuration for Hive is read from ``hive-site.xml`` on the classpath.
+    It supports running both SQL and HiveQL commands.
+
+    :param sparkContext: The SparkContext to wrap.
+    :param jhiveContext: An optional JVM Scala HiveContext. If set, we do not instantiate a new
+        :class:`HiveContext` in the JVM, instead we make all calls to this object.
+
+    .. note:: Deprecated in 2.0.0. Use SparkSession.builder.enableHiveSupport().getOrCreate().
+    """
+
+    def __init__(self, sparkContext, jhiveContext=None):
+        warnings.warn(
+            "HiveContext is deprecated in Spark 2.0.0. Please use " +
+            "SparkSession.builder.enableHiveSupport().getOrCreate() instead.",
+            DeprecationWarning)
+        if jhiveContext is None:
+            sparkContext._conf.set("spark.sql.catalogImplementation", "hive")
+            sparkSession = SparkSession.builder._sparkContext(sparkContext).getOrCreate()
+        else:
+            sparkSession = SparkSession(sparkContext, jhiveContext.sparkSession())
+        SQLContext.__init__(self, sparkContext, sparkSession, jhiveContext)
+
+    @classmethod
+    def _createForTesting(cls, sparkContext):
+        """(Internal use only) Create a new HiveContext for testing.
+
+        All test code that touches HiveContext *must* go through this method. Otherwise,
+        you may end up launching multiple derby instances and encounter with incredibly
+        confusing error messages.
+        """
+        jsc = sparkContext._jsc.sc()
+        jtestHive = sparkContext._jvm.org.apache.spark.sql.hive.test.TestHiveContext(jsc, False)
+        return cls(sparkContext, jtestHive)
+
+    def refreshTable(self, tableName):
+        """Invalidate and refresh all the cached the metadata of the given
+        table. For performance reasons, Spark SQL or the external data source
+        library it uses might cache certain metadata about a table, such as the
+        location of blocks. When those change outside of Spark SQL, users should
+        call this function to invalidate the cache.
+        """
+        self._ssql_ctx.refreshTable(tableName)
+
+
 def _test():
     import os
     import doctest
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 2432b8127840b..8ba2ffab4a331 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -122,6 +122,25 @@ def toJSON(self, use_unicode=True):
         rdd = self._jdf.toJSON()
         return RDD(rdd.toJavaRDD(), self._sc, UTF8Deserializer(use_unicode))
 
+    @since(1.3)
+    def registerTempTable(self, name):
+        """Registers this DataFrame as a temporary table using the given name.
+
+        The lifetime of this temporary table is tied to the :class:`SparkSession`
+        that was used to create this :class:`DataFrame`.
+
+        >>> df.registerTempTable("people")
+        >>> df2 = spark.sql("select * from people")
+        >>> sorted(df.collect()) == sorted(df2.collect())
+        True
+        >>> spark.catalog.dropTempView("people")
+
+        .. note:: Deprecated in 2.0, use createOrReplaceTempView instead.
+        """
+        warnings.warn(
+            "Deprecated in 2.0, use createOrReplaceTempView instead.", DeprecationWarning)
+        self._jdf.createOrReplaceTempView(name)
+
     @since(2.0)
     def createTempView(self, name):
         """Creates a local temporary view with this :class:`DataFrame`.
@@ -257,6 +276,8 @@ def explain(self, extended=None, mode=None):
         """Prints the (logical and physical) plans to the console for debugging purpose.
 
         :param extended: boolean, default ``False``. If ``False``, prints only the physical plan.
+            When this is a string without specifying the ``mode``, it works as the mode is
+            specified.
         :param mode: specifies the expected output format of plans.
 
             * ``simple``: Print only a physical plan.
@@ -284,14 +305,20 @@ def explain(self, extended=None, mode=None):
         == Physical Plan ==
         * Scan ExistingRDD (1)
         (1) Scan ExistingRDD [codegen id : 1]
-        Output: [age#0, name#1]
+        Output [2]: [age#0, name#1]
+        ...
+
+        >>> df.explain("cost")
+        == Optimized Logical Plan ==
+        ...Statistics...
+        ...
 
         .. versionchanged:: 3.0.0
            Added optional argument `mode` to specify the expected output format of plans.
         """
 
         if extended is not None and mode is not None:
-            raise Exception("extended and mode can not be specified simultaneously")
+            raise Exception("extended and mode should not be set together.")
 
         # For the no argument case: df.explain()
         is_no_argument = extended is None and mode is None
@@ -299,18 +326,22 @@ def explain(self, extended=None, mode=None):
         # For the cases below:
         #   explain(True)
         #   explain(extended=False)
-        is_extended_case = extended is not None and isinstance(extended, bool)
+        is_extended_case = isinstance(extended, bool) and mode is None
+
+        # For the case when extended is mode:
+        #   df.explain("formatted")
+        is_extended_as_mode = isinstance(extended, basestring) and mode is None
 
-        # For the mode specified: df.explain(mode="formatted")
-        is_mode_case = mode is not None and isinstance(mode, basestring)
+        # For the mode specified:
+        #   df.explain(mode="formatted")
+        is_mode_case = extended is None and isinstance(mode, basestring)
 
-        if not is_no_argument and not (is_extended_case or is_mode_case):
-            if extended is not None:
-                err_msg = "extended (optional) should be provided as bool" \
-                    ", got {0}".format(type(extended))
-            else:  # For mode case
-                err_msg = "mode (optional) should be provided as str, got {0}".format(type(mode))
-            raise TypeError(err_msg)
+        if not (is_no_argument or is_extended_case or is_extended_as_mode or is_mode_case):
+            argtypes = [
+                str(type(arg)) for arg in [extended, mode] if arg is not None]
+            raise TypeError(
+                "extended (optional) and mode (optional) should be a string "
+                "and bool; however, got [%s]." % ", ".join(argtypes))
 
         # Sets an explain mode depending on a given argument
         if is_no_argument:
@@ -319,6 +350,8 @@ def explain(self, extended=None, mode=None):
             explain_mode = "extended" if extended else "simple"
         elif is_mode_case:
             explain_mode = mode
+        elif is_extended_as_mode:
+            explain_mode = extended
 
         print(self._sc._jvm.PythonSQLUtils.explainString(self._jdf.queryExecution(), explain_mode))
 
@@ -2118,7 +2151,7 @@ def drop(self, *cols):
 
     @ignore_unicode_prefix
     def toDF(self, *cols):
-        """Returns a new class:`DataFrame` that with new specified column names
+        """Returns a new :class:`DataFrame` that with new specified column names
 
         :param cols: list of new column names (string)
 
@@ -2130,9 +2163,9 @@ def toDF(self, *cols):
 
     @since(3.0)
     def transform(self, func):
-        """Returns a new class:`DataFrame`. Concise syntax for chaining custom transformations.
+        """Returns a new :class:`DataFrame`. Concise syntax for chaining custom transformations.
 
-        :param func: a function that takes and returns a class:`DataFrame`.
+        :param func: a function that takes and returns a :class:`DataFrame`.
 
         >>> from pyspark.sql.functions import col
         >>> df = spark.createDataFrame([(1, 1.0), (2, 2.0)], ["int", "float"])
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index e80d556cc89e3..041f841817657 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -236,12 +236,14 @@ def _options_to_str(options):
     'degrees': """
                Converts an angle measured in radians to an approximately equivalent angle
                measured in degrees.
+
                :param col: angle in radians
                :return: angle in degrees, as if computed by `java.lang.Math.toDegrees()`
                """,
     'radians': """
                Converts an angle measured in degrees to an approximately equivalent angle
                measured in radians.
+
                :param col: angle in degrees
                :return: angle in radians, as if computed by `java.lang.Math.toRadians()`
                """,
@@ -294,6 +296,8 @@ def _options_to_str(options):
 
 # Wraps deprecated functions (keys) with the messages (values).
 _functions_deprecated = {
+    'toDegrees': 'Deprecated in 2.1, use degrees instead.',
+    'toRadians': 'Deprecated in 2.1, use radians instead.',
 }
 
 for _name, _doc in _functions.items():
@@ -317,13 +321,22 @@ def _options_to_str(options):
 del _name, _doc
 
 
+@since(1.3)
+def approxCountDistinct(col, rsd=None):
+    """
+    .. note:: Deprecated in 2.1, use :func:`approx_count_distinct` instead.
+    """
+    warnings.warn("Deprecated in 2.1, use approx_count_distinct instead.", DeprecationWarning)
+    return approx_count_distinct(col, rsd)
+
+
 @since(2.1)
 def approx_count_distinct(col, rsd=None):
     """Aggregate function: returns a new :class:`Column` for approximate distinct count of
     column `col`.
 
-    :param rsd: maximum estimation error allowed (default = 0.05). For rsd < 0.01, it is more
-        efficient to use :func:`countDistinct`
+    :param rsd: maximum relative standard deviation allowed (default = 0.05).
+        For rsd < 0.01, it is more efficient to use :func:`countDistinct`
 
     >>> df.agg(approx_count_distinct(df.age).alias('distinct_ages')).collect()
     [Row(distinct_ages=2)]
@@ -586,7 +599,7 @@ def nanvl(col1, col2):
 @since(1.4)
 def rand(seed=None):
     """Generates a random column with independent and identically distributed (i.i.d.) samples
-    from U[0.0, 1.0].
+    uniformly distributed in [0.0, 1.0).
 
     .. note:: The function is non-deterministic in general case.
 
@@ -896,7 +909,8 @@ def ntile(n):
 @since(1.5)
 def current_date():
     """
-    Returns the current date as a :class:`DateType` column.
+    Returns the current date at the start of query evaluation as a :class:`DateType` column.
+    All calls of current_date within the same query return the same value.
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.current_date())
@@ -904,7 +918,8 @@ def current_date():
 
 def current_timestamp():
     """
-    Returns the current timestamp as a :class:`TimestampType` column.
+    Returns the current timestamp at the start of query evaluation as a :class:`TimestampType`
+    column. All calls of current_timestamp within the same query return the same value.
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.current_timestamp())
@@ -918,8 +933,9 @@ def date_format(date, format):
     format given by the second argument.
 
     A pattern could be for instance `dd.MM.yyyy` and could return a string like '18.03.1993'. All
-    pattern letters of the Java class `java.time.format.DateTimeFormatter` can be used.
+    pattern letters of `datetime pattern`_. can be used.
 
+    .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
     .. note:: Use when ever possible specialized functions like `year`. These benefit from a
         specialized implementation.
 
@@ -1136,8 +1152,7 @@ def months_between(date1, date2, roundOff=True):
 @since(2.2)
 def to_date(col, format=None):
     """Converts a :class:`Column` into :class:`pyspark.sql.types.DateType`
-    using the optionally specified format. Specify formats according to
-    `DateTimeFormatter <https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html>`_. # noqa
+    using the optionally specified format. Specify formats according to `datetime pattern`_.
     By default, it follows casting rules to :class:`pyspark.sql.types.DateType` if the format
     is omitted. Equivalent to ``col.cast("date")``.
 
@@ -1160,8 +1175,7 @@ def to_date(col, format=None):
 @since(2.2)
 def to_timestamp(col, format=None):
     """Converts a :class:`Column` into :class:`pyspark.sql.types.TimestampType`
-    using the optionally specified format. Specify formats according to
-    `DateTimeFormatter <https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html>`_. # noqa
+    using the optionally specified format. Specify formats according to `datetime pattern`_.
     By default, it follows casting rules to :class:`pyspark.sql.types.TimestampType` if the format
     is omitted. Equivalent to ``col.cast("timestamp")``.
 
@@ -1247,7 +1261,7 @@ def last_day(date):
 
 @ignore_unicode_prefix
 @since(1.5)
-def from_unixtime(timestamp, format="uuuu-MM-dd HH:mm:ss"):
+def from_unixtime(timestamp, format="yyyy-MM-dd HH:mm:ss"):
     """
     Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string
     representing the timestamp of that moment in the current system time zone in the given
@@ -1264,9 +1278,9 @@ def from_unixtime(timestamp, format="uuuu-MM-dd HH:mm:ss"):
 
 
 @since(1.5)
-def unix_timestamp(timestamp=None, format='uuuu-MM-dd HH:mm:ss'):
+def unix_timestamp(timestamp=None, format='yyyy-MM-dd HH:mm:ss'):
     """
-    Convert time string with given pattern ('uuuu-MM-dd HH:mm:ss', by default)
+    Convert time string with given pattern ('yyyy-MM-dd HH:mm:ss', by default)
     to Unix time stamp (in seconds), using the default timezone and the default
     locale, return null if fail.
 
@@ -1301,7 +1315,12 @@ def from_utc_timestamp(timestamp, tz):
     timestamp to string according to the session local timezone.
 
     :param timestamp: the column that contains timestamps
-    :param tz: a string that has the ID of timezone, e.g. "GMT", "America/Los_Angeles", etc
+    :param tz: A string detailing the time zone ID that the input should be adjusted to. It should
+               be in the format of either region-based zone IDs or zone offsets. Region IDs must
+               have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
+               the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
+               supported as aliases of '+00:00'. Other short names are not recommended to use
+               because they can be ambiguous.
 
     .. versionchanged:: 2.4
        `tz` can take a :class:`Column` containing timezone ID strings.
@@ -1335,7 +1354,12 @@ def to_utc_timestamp(timestamp, tz):
     timestamp to string according to the session local timezone.
 
     :param timestamp: the column that contains timestamps
-    :param tz: a string that has the ID of timezone, e.g. "GMT", "America/Los_Angeles", etc
+    :param tz: A string detailing the time zone ID that the input should be adjusted to. It should
+               be in the format of either region-based zone IDs or zone offsets. Region IDs must
+               have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
+               the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
+               supported as aliases of '+00:00'. Other short names are not recommended to use
+               because they can be ambiguous.
 
     .. versionchanged:: 2.4
        `tz` can take a :class:`Column` containing timezone ID strings.
@@ -2116,6 +2140,7 @@ def array_remove(col, element):
 def array_distinct(col):
     """
     Collection function: removes duplicate values from the array.
+
     :param col: name of column or expression
 
     >>> df = spark.createDataFrame([([1, 2, 3, 2],), ([4, 5, 5, 4],)], ['data'])
@@ -2766,12 +2791,12 @@ def map_concat(*cols):
     :param cols: list of column names (string) or list of :class:`Column` expressions
 
     >>> from pyspark.sql.functions import map_concat
-    >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c', 1, 'd') as map2")
+    >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c') as map2")
     >>> df.select(map_concat("map1", "map2").alias("map3")).show(truncate=False)
     +------------------------+
     |map3                    |
     +------------------------+
-    |[1 -> d, 2 -> b, 3 -> c]|
+    |[1 -> a, 2 -> b, 3 -> c]|
     +------------------------+
     """
     sc = SparkContext._active_spark_context
diff --git a/python/pyspark/sql/pandas/conversion.py b/python/pyspark/sql/pandas/conversion.py
index 8548cd222bf10..e6d8e9f24a557 100644
--- a/python/pyspark/sql/pandas/conversion.py
+++ b/python/pyspark/sql/pandas/conversion.py
@@ -21,6 +21,7 @@
     xrange = range
 else:
     from itertools import izip as zip
+from collections import Counter
 
 from pyspark import since
 from pyspark.rdd import _load_from_socket
@@ -102,13 +103,17 @@ def toPandas(self):
                 try:
                     from pyspark.sql.pandas.types import _check_series_localize_timestamps
                     import pyarrow
-                    batches = self._collect_as_arrow()
+                    # Rename columns to avoid duplicated column names.
+                    tmp_column_names = ['col_{}'.format(i) for i in range(len(self.columns))]
+                    batches = self.toDF(*tmp_column_names)._collect_as_arrow()
                     if len(batches) > 0:
                         table = pyarrow.Table.from_batches(batches)
                         # Pandas DataFrame created from PyArrow uses datetime64[ns] for date type
                         # values, but we should use datetime.date to match the behavior with when
                         # Arrow optimization is disabled.
                         pdf = table.to_pandas(date_as_object=True)
+                        # Rename back to the original column names.
+                        pdf.columns = self.columns
                         for field in self.schema:
                             if isinstance(field.dataType, TimestampType):
                                 pdf[field.name] = \
@@ -131,9 +136,16 @@ def toPandas(self):
 
         # Below is toPandas without Arrow optimization.
         pdf = pd.DataFrame.from_records(self.collect(), columns=self.columns)
+        column_counter = Counter(self.columns)
+
+        dtype = [None] * len(self.schema)
+        for fieldIdx, field in enumerate(self.schema):
+            # For duplicate column name, we use `iloc` to access it.
+            if column_counter[field.name] > 1:
+                pandas_col = pdf.iloc[:, fieldIdx]
+            else:
+                pandas_col = pdf[field.name]
 
-        dtype = {}
-        for field in self.schema:
             pandas_type = PandasConversionMixin._to_corrected_pandas_type(field.dataType)
             # SPARK-21766: if an integer field is nullable and has null values, it can be
             # inferred by pandas as float column. Once we convert the column with NaN back
@@ -141,16 +153,36 @@ def toPandas(self):
             # float type, not the corrected type from the schema in this case.
             if pandas_type is not None and \
                 not(isinstance(field.dataType, IntegralType) and field.nullable and
-                    pdf[field.name].isnull().any()):
-                dtype[field.name] = pandas_type
+                    pandas_col.isnull().any()):
+                dtype[fieldIdx] = pandas_type
             # Ensure we fall back to nullable numpy types, even when whole column is null:
-            if isinstance(field.dataType, IntegralType) and pdf[field.name].isnull().any():
-                dtype[field.name] = np.float64
-            if isinstance(field.dataType, BooleanType) and pdf[field.name].isnull().any():
-                dtype[field.name] = np.object
+            if isinstance(field.dataType, IntegralType) and pandas_col.isnull().any():
+                dtype[fieldIdx] = np.float64
+            if isinstance(field.dataType, BooleanType) and pandas_col.isnull().any():
+                dtype[fieldIdx] = np.object
+
+        df = pd.DataFrame()
+        for index, t in enumerate(dtype):
+            column_name = self.schema[index].name
+
+            # For duplicate column name, we use `iloc` to access it.
+            if column_counter[column_name] > 1:
+                series = pdf.iloc[:, index]
+            else:
+                series = pdf[column_name]
+
+            if t is not None:
+                series = series.astype(t, copy=False)
+
+            # `insert` API makes copy of data, we only do it for Series of duplicate column names.
+            # `pdf.iloc[:, index] = pdf.iloc[:, index]...` doesn't always work because `iloc` could
+            # return a view or a copy depending by context.
+            if column_counter[column_name] > 1:
+                df.insert(index, column_name, series, allow_duplicates=True)
+            else:
+                df[column_name] = series
 
-        for f, t in dtype.items():
-            pdf[f] = pdf[f].astype(t, copy=False)
+        pdf = df
 
         if timezone is None:
             return pdf
@@ -381,7 +413,7 @@ def _create_from_pandas_with_arrow(self, pdf, schema, timezone):
 
         # Slice the DataFrame to be batched
         step = -(-len(pdf) // self.sparkContext.defaultParallelism)  # round int up
-        pdf_slices = (pdf[start:start + step] for start in xrange(0, len(pdf), step))
+        pdf_slices = (pdf.iloc[start:start + step] for start in xrange(0, len(pdf), step))
 
         # Create list of Arrow (columns, type) for serializer dump_stream
         arrow_data = [[(c, t) for (_, c), t in zip(pdf_slice.iteritems(), arrow_types)]
diff --git a/python/pyspark/sql/pandas/functions.py b/python/pyspark/sql/pandas/functions.py
index 30602789a33a9..094dc357b6822 100644
--- a/python/pyspark/sql/pandas/functions.py
+++ b/python/pyspark/sql/pandas/functions.py
@@ -43,303 +43,228 @@ class PandasUDFType(object):
 @since(2.3)
 def pandas_udf(f=None, returnType=None, functionType=None):
     """
-    Creates a vectorized user defined function (UDF).
+    Creates a pandas user defined function (a.k.a. vectorized user defined function).
+
+    Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer
+    data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF
+    is defined using the `pandas_udf` as a decorator or to wrap the function, and no
+    additional configuration is required. A Pandas UDF behaves as a regular PySpark function
+    API in general.
 
     :param f: user-defined function. A python function if used as a standalone function
     :param returnType: the return type of the user-defined function. The value can be either a
         :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
     :param functionType: an enum value in :class:`pyspark.sql.functions.PandasUDFType`.
-                         Default: SCALAR.
-
-    .. seealso:: :meth:`pyspark.sql.DataFrame.mapInPandas`
-    .. seealso:: :meth:`pyspark.sql.GroupedData.applyInPandas`
-    .. seealso:: :meth:`pyspark.sql.PandasCogroupedOps.applyInPandas`
-
-    The function type of the UDF can be one of the following:
-
-    1. SCALAR
-
-       A scalar UDF defines a transformation: One or more `pandas.Series` -> A `pandas.Series`.
-       The length of the returned `pandas.Series` must be of the same as the input `pandas.Series`.
-       If the return type is :class:`StructType`, the returned value should be a `pandas.DataFrame`.
-
-       :class:`MapType`, nested :class:`StructType` are currently not supported as output types.
-
-       Scalar UDFs can be used with :meth:`pyspark.sql.DataFrame.withColumn` and
-       :meth:`pyspark.sql.DataFrame.select`.
-
-       >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-       >>> from pyspark.sql.types import IntegerType, StringType
-       >>> slen = pandas_udf(lambda s: s.str.len(), IntegerType())  # doctest: +SKIP
-       >>> @pandas_udf(StringType())  # doctest: +SKIP
-       ... def to_upper(s):
-       ...     return s.str.upper()
-       ...
-       >>> @pandas_udf("integer", PandasUDFType.SCALAR)  # doctest: +SKIP
-       ... def add_one(x):
-       ...     return x + 1
-       ...
-       >>> df = spark.createDataFrame([(1, "John Doe", 21)],
-       ...                            ("id", "name", "age"))  # doctest: +SKIP
-       >>> df.select(slen("name").alias("slen(name)"), to_upper("name"), add_one("age")) \\
-       ...     .show()  # doctest: +SKIP
-       +----------+--------------+------------+
-       |slen(name)|to_upper(name)|add_one(age)|
-       +----------+--------------+------------+
-       |         8|      JOHN DOE|          22|
-       +----------+--------------+------------+
-       >>> @pandas_udf("first string, last string")  # doctest: +SKIP
-       ... def split_expand(n):
-       ...     return n.str.split(expand=True)
-       >>> df.select(split_expand("name")).show()  # doctest: +SKIP
-       +------------------+
-       |split_expand(name)|
-       +------------------+
-       |       [John, Doe]|
-       +------------------+
-
-       .. note:: The length of `pandas.Series` within a scalar UDF is not that of the whole input
-           column, but is the length of an internal batch used for each call to the function.
-           Therefore, this can be used, for example, to ensure the length of each returned
-           `pandas.Series`, and can not be used as the column length.
-
-    2. SCALAR_ITER
-
-       A scalar iterator UDF is semantically the same as the scalar Pandas UDF above except that the
-       wrapped Python function takes an iterator of batches as input instead of a single batch and,
-       instead of returning a single output batch, it yields output batches or explicitly returns an
-       generator or an iterator of output batches.
-       It is useful when the UDF execution requires initializing some state, e.g., loading a machine
-       learning model file to apply inference to every input batch.
-
-       .. note:: It is not guaranteed that one invocation of a scalar iterator UDF will process all
-           batches from one partition, although it is currently implemented this way.
-           Your code shall not rely on this behavior because it might change in the future for
-           further optimization, e.g., one invocation processes multiple partitions.
-
-       Scalar iterator UDFs are used with :meth:`pyspark.sql.DataFrame.withColumn` and
-       :meth:`pyspark.sql.DataFrame.select`.
-
-       >>> import pandas as pd  # doctest: +SKIP
-       >>> from pyspark.sql.functions import col, pandas_udf, struct, PandasUDFType
-       >>> pdf = pd.DataFrame([1, 2, 3], columns=["x"])  # doctest: +SKIP
-       >>> df = spark.createDataFrame(pdf)  # doctest: +SKIP
-
-       When the UDF is called with a single column that is not `StructType`, the input to the
-       underlying function is an iterator of `pd.Series`.
-
-       >>> @pandas_udf("long", PandasUDFType.SCALAR_ITER)  # doctest: +SKIP
-       ... def plus_one(batch_iter):
-       ...     for x in batch_iter:
-       ...         yield x + 1
-       ...
-       >>> df.select(plus_one(col("x"))).show()  # doctest: +SKIP
-       +-----------+
-       |plus_one(x)|
-       +-----------+
-       |          2|
-       |          3|
-       |          4|
-       +-----------+
-
-       When the UDF is called with more than one columns, the input to the underlying function is an
-       iterator of `pd.Series` tuple.
-
-       >>> @pandas_udf("long", PandasUDFType.SCALAR_ITER)  # doctest: +SKIP
-       ... def multiply_two_cols(batch_iter):
-       ...     for a, b in batch_iter:
-       ...         yield a * b
-       ...
-       >>> df.select(multiply_two_cols(col("x"), col("x"))).show()  # doctest: +SKIP
-       +-----------------------+
-       |multiply_two_cols(x, x)|
-       +-----------------------+
-       |                      1|
-       |                      4|
-       |                      9|
-       +-----------------------+
-
-       When the UDF is called with a single column that is `StructType`, the input to the underlying
-       function is an iterator of `pd.DataFrame`.
-
-       >>> @pandas_udf("long", PandasUDFType.SCALAR_ITER)  # doctest: +SKIP
-       ... def multiply_two_nested_cols(pdf_iter):
-       ...    for pdf in pdf_iter:
-       ...        yield pdf["a"] * pdf["b"]
-       ...
-       >>> df.select(
-       ...     multiply_two_nested_cols(
-       ...         struct(col("x").alias("a"), col("x").alias("b"))
-       ...     ).alias("y")
-       ... ).show()  # doctest: +SKIP
-       +---+
-       |  y|
-       +---+
-       |  1|
-       |  4|
-       |  9|
-       +---+
-
-       In the UDF, you can initialize some states before processing batches, wrap your code with
-       `try ... finally ...` or use context managers to ensure the release of resources at the end
-       or in case of early termination.
-
-       >>> y_bc = spark.sparkContext.broadcast(1)  # doctest: +SKIP
-       >>> @pandas_udf("long", PandasUDFType.SCALAR_ITER)  # doctest: +SKIP
-       ... def plus_y(batch_iter):
-       ...     y = y_bc.value  # initialize some state
-       ...     try:
-       ...         for x in batch_iter:
-       ...             yield x + y
-       ...     finally:
-       ...         pass  # release resources here, if any
-       ...
-       >>> df.select(plus_y(col("x"))).show()  # doctest: +SKIP
-       +---------+
-       |plus_y(x)|
-       +---------+
-       |        2|
-       |        3|
-       |        4|
-       +---------+
-
-    3. GROUPED_MAP
-
-       A grouped map UDF defines transformation: A `pandas.DataFrame` -> A `pandas.DataFrame`
-       The returnType should be a :class:`StructType` describing the schema of the returned
-       `pandas.DataFrame`. The column labels of the returned `pandas.DataFrame` must either match
-       the field names in the defined returnType schema if specified as strings, or match the
-       field data types by position if not strings, e.g. integer indices.
-       The length of the returned `pandas.DataFrame` can be arbitrary.
-
-       Grouped map UDFs are used with :meth:`pyspark.sql.GroupedData.apply`.
-
-       >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-       >>> df = spark.createDataFrame(
-       ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-       ...     ("id", "v"))  # doctest: +SKIP
-       >>> @pandas_udf("id long, v double", PandasUDFType.GROUPED_MAP)  # doctest: +SKIP
-       ... def normalize(pdf):
-       ...     v = pdf.v
-       ...     return pdf.assign(v=(v - v.mean()) / v.std())
-       >>> df.groupby("id").apply(normalize).show()  # doctest: +SKIP
-       +---+-------------------+
-       | id|                  v|
-       +---+-------------------+
-       |  1|-0.7071067811865475|
-       |  1| 0.7071067811865475|
-       |  2|-0.8320502943378437|
-       |  2|-0.2773500981126146|
-       |  2| 1.1094003924504583|
-       +---+-------------------+
-
-       Alternatively, the user can define a function that takes two arguments.
-       In this case, the grouping key(s) will be passed as the first argument and the data will
-       be passed as the second argument. The grouping key(s) will be passed as a tuple of numpy
-       data types, e.g., `numpy.int32` and `numpy.float64`. The data will still be passed in
-       as a `pandas.DataFrame` containing all columns from the original Spark DataFrame.
-       This is useful when the user does not want to hardcode grouping key(s) in the function.
-
-       >>> import pandas as pd  # doctest: +SKIP
-       >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-       >>> df = spark.createDataFrame(
-       ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-       ...     ("id", "v"))  # doctest: +SKIP
-       >>> @pandas_udf("id long, v double", PandasUDFType.GROUPED_MAP)  # doctest: +SKIP
-       ... def mean_udf(key, pdf):
-       ...     # key is a tuple of one numpy.int64, which is the value
-       ...     # of 'id' for the current group
-       ...     return pd.DataFrame([key + (pdf.v.mean(),)])
-       >>> df.groupby('id').apply(mean_udf).show()  # doctest: +SKIP
-       +---+---+
-       | id|  v|
-       +---+---+
-       |  1|1.5|
-       |  2|6.0|
-       +---+---+
-       >>> @pandas_udf(
-       ...    "id long, `ceil(v / 2)` long, v double",
-       ...    PandasUDFType.GROUPED_MAP)  # doctest: +SKIP
-       >>> def sum_udf(key, pdf):
-       ...     # key is a tuple of two numpy.int64s, which is the values
-       ...     # of 'id' and 'ceil(df.v / 2)' for the current group
-       ...     return pd.DataFrame([key + (pdf.v.sum(),)])
-       >>> df.groupby(df.id, ceil(df.v / 2)).apply(sum_udf).show()  # doctest: +SKIP
-       +---+-----------+----+
-       | id|ceil(v / 2)|   v|
-       +---+-----------+----+
-       |  2|          5|10.0|
-       |  1|          1| 3.0|
-       |  2|          3| 5.0|
-       |  2|          2| 3.0|
-       +---+-----------+----+
-
-       .. note:: If returning a new `pandas.DataFrame` constructed with a dictionary, it is
-           recommended to explicitly index the columns by name to ensure the positions are correct,
-           or alternatively use an `OrderedDict`.
-           For example, `pd.DataFrame({'id': ids, 'a': data}, columns=['id', 'a'])` or
-           `pd.DataFrame(OrderedDict([('id', ids), ('a', data)]))`.
-
-       .. seealso:: :meth:`pyspark.sql.GroupedData.apply`
-
-    4. GROUPED_AGG
-
-       A grouped aggregate UDF defines a transformation: One or more `pandas.Series` -> A scalar
-       The `returnType` should be a primitive data type, e.g., :class:`DoubleType`.
-       The returned scalar can be either a python primitive type, e.g., `int` or `float`
-       or a numpy data type, e.g., `numpy.int64` or `numpy.float64`.
-
-       :class:`MapType` and :class:`StructType` are currently not supported as output types.
-
-       Group aggregate UDFs are used with :meth:`pyspark.sql.GroupedData.agg` and
-       :class:`pyspark.sql.Window`
-
-       This example shows using grouped aggregated UDFs with groupby:
-
-       >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-       >>> df = spark.createDataFrame(
-       ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-       ...     ("id", "v"))
-       >>> @pandas_udf("double", PandasUDFType.GROUPED_AGG)  # doctest: +SKIP
-       ... def mean_udf(v):
-       ...     return v.mean()
-       >>> df.groupby("id").agg(mean_udf(df['v'])).show()  # doctest: +SKIP
-       +---+-----------+
-       | id|mean_udf(v)|
-       +---+-----------+
-       |  1|        1.5|
-       |  2|        6.0|
-       +---+-----------+
-
-       This example shows using grouped aggregated UDFs as window functions.
-
-       >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-       >>> from pyspark.sql import Window
-       >>> df = spark.createDataFrame(
-       ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-       ...     ("id", "v"))
-       >>> @pandas_udf("double", PandasUDFType.GROUPED_AGG)  # doctest: +SKIP
-       ... def mean_udf(v):
-       ...     return v.mean()
-       >>> w = (Window.partitionBy('id')
-       ...            .orderBy('v')
-       ...            .rowsBetween(-1, 0))
-       >>> df.withColumn('mean_v', mean_udf(df['v']).over(w)).show()  # doctest: +SKIP
-       +---+----+------+
-       | id|   v|mean_v|
-       +---+----+------+
-       |  1| 1.0|   1.0|
-       |  1| 2.0|   1.5|
-       |  2| 3.0|   3.0|
-       |  2| 5.0|   4.0|
-       |  2|10.0|   7.5|
-       +---+----+------+
-
-       .. note:: For performance reasons, the input series to window functions are not copied.
+        Default: SCALAR.
+
+        .. note:: This parameter exists for compatibility. Using Python type hints is encouraged.
+
+    In order to use this API, customarily the below are imported:
+
+    >>> import pandas as pd
+    >>> from pyspark.sql.functions import pandas_udf
+
+    From Spark 3.0 with Python 3.6+, `Python type hints <https://www.python.org/dev/peps/pep-0484>`_
+    detect the function types as below:
+
+    >>> @pandas_udf(IntegerType())
+    ... def slen(s: pd.Series) -> pd.Series:
+    ...     return s.str.len()
+
+    Prior to Spark 3.0, the pandas UDF used `functionType` to decide the execution type as below:
+
+    >>> from pyspark.sql.functions import PandasUDFType
+    >>> from pyspark.sql.types import IntegerType
+    >>> @pandas_udf(IntegerType(), PandasUDFType.SCALAR)
+    ... def slen(s):
+    ...     return s.str.len()
+
+    It is preferred to specify type hints for the pandas UDF instead of specifying pandas UDF
+    type via `functionType` which will be deprecated in the future releases.
+
+    Note that the type hint should use `pandas.Series` in all cases but there is one variant
+    that `pandas.DataFrame` should be used for its input or output type hint instead when the input
+    or output column is of :class:`pyspark.sql.types.StructType`. The following example shows
+    a Pandas UDF which takes long column, string column and struct column, and outputs a struct
+    column. It requires the function to specify the type hints of `pandas.Series` and
+    `pandas.DataFrame` as below:
+
+    >>> @pandas_udf("col1 string, col2 long")
+    >>> def func(s1: pd.Series, s2: pd.Series, s3: pd.DataFrame) -> pd.DataFrame:
+    ...     s3['col2'] = s1 + s2.str.len()
+    ...     return s3
+    ...
+    >>> # Create a Spark DataFrame that has three columns including a sturct column.
+    ... df = spark.createDataFrame(
+    ...     [[1, "a string", ("a nested string",)]],
+    ...     "long_col long, string_col string, struct_col struct<col1:string>")
+    >>> df.printSchema()
+    root
+    |-- long_column: long (nullable = true)
+    |-- string_column: string (nullable = true)
+    |-- struct_column: struct (nullable = true)
+    |    |-- col1: string (nullable = true)
+    >>> df.select(func("long_col", "string_col", "struct_col")).printSchema()
+    |-- func(long_col, string_col, struct_col): struct (nullable = true)
+    |    |-- col1: string (nullable = true)
+    |    |-- col2: long (nullable = true)
+
+    In the following sections, it describes the cominations of the supported type hints. For
+    simplicity, `pandas.DataFrame` variant is omitted.
+
+    * Series to Series
+        `pandas.Series`, ... -> `pandas.Series`
+
+        The function takes one or more `pandas.Series` and outputs one `pandas.Series`.
+        The output of the function should always be of the same length as the input.
+
+        >>> @pandas_udf("string")
+        ... def to_upper(s: pd.Series) -> pd.Series:
+        ...     return s.str.upper()
+        ...
+        >>> df = spark.createDataFrame([("John Doe",)], ("name",))
+        >>> df.select(to_upper("name")).show()
+        +--------------+
+        |to_upper(name)|
+        +--------------+
+        |      JOHN DOE|
+        +--------------+
+
+        >>> @pandas_udf("first string, last string")
+        ... def split_expand(s: pd.Series) -> pd.DataFrame:
+        ...     return s.str.split(expand=True)
+        ...
+        >>> df = spark.createDataFrame([("John Doe",)], ("name",))
+        >>> df.select(split_expand("name")).show()
+        +------------------+
+        |split_expand(name)|
+        +------------------+
+        |       [John, Doe]|
+        +------------------+
+
+        .. note:: The length of the input is not that of the whole input column, but is the
+            length of an internal batch used for each call to the function.
+
+    * Iterator of Series to Iterator of Series
+        `Iterator[pandas.Series]` -> `Iterator[pandas.Series]`
+
+        The function takes an iterator of `pandas.Series` and outputs an iterator of
+        `pandas.Series`. In this case, the created pandas UDF instance requires one input
+        column when this is called as a PySpark column. The length of the entire output from
+        the function should be the same length of the entire input; therefore, it can
+        prefetch the data from the input iterator as long as the lengths are the same.
+
+        It is also useful when the UDF execution
+        requires initializing some states although internally it works identically as
+        Series to Series case. The pseudocode below illustrates the example.
+
+        .. highlight:: python
+        .. code-block:: python
+
+            @pandas_udf("long")
+            def calculate(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
+                # Do some expensive initialization with a state
+                state = very_expensive_initialization()
+                for x in iterator:
+                    # Use that state for whole iterator.
+                    yield calculate_with_state(x, state)
+
+            df.select(calculate("value")).show()
+
+        >>> from typing import Iterator
+        >>> @pandas_udf("long")
+        ... def plus_one(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
+        ...     for s in iterator:
+        ...         yield s + 1
+        ...
+        >>> df = spark.createDataFrame(pd.DataFrame([1, 2, 3], columns=["v"]))
+        >>> df.select(plus_one(df.v)).show()
+        +-----------+
+        |plus_one(v)|
+        +-----------+
+        |          2|
+        |          3|
+        |          4|
+        +-----------+
+
+        .. note:: The length of each series is the length of a batch internally used.
+
+    * Iterator of Multiple Series to Iterator of Series
+        `Iterator[Tuple[pandas.Series, ...]]` -> `Iterator[pandas.Series]`
+
+        The function takes an iterator of a tuple of multiple `pandas.Series` and outputs an
+        iterator of `pandas.Series`. In this case, the created pandas UDF instance requires
+        input columns as many as the series when this is called as a PySpark column.
+        Otherwise, it has the same characteristics and restrictions as Iterator of Series
+        to Iterator of Series case.
+
+        >>> from typing import Iterator, Tuple
+        >>> from pyspark.sql.functions import struct, col
+        >>> @pandas_udf("long")
+        ... def multiply(iterator: Iterator[Tuple[pd.Series, pd.DataFrame]]) -> Iterator[pd.Series]:
+        ...     for s1, df in iterator:
+        ...         yield s1 * df.v
+        ...
+        >>> df = spark.createDataFrame(pd.DataFrame([1, 2, 3], columns=["v"]))
+        >>> df.withColumn('output', multiply(col("v"), struct(col("v")))).show()
+        +---+------+
+        |  v|output|
+        +---+------+
+        |  1|     1|
+        |  2|     4|
+        |  3|     9|
+        +---+------+
+
+        .. note:: The length of each series is the length of a batch internally used.
+
+    * Series to Scalar
+        `pandas.Series`, ... -> `Any`
+
+        The function takes `pandas.Series` and returns a scalar value. The `returnType`
+        should be a primitive data type, and the returned scalar can be either a python primitive
+        type, e.g., int or float or a numpy data type, e.g., numpy.int64 or numpy.float64.
+        `Any` should ideally be a specific scalar type accordingly.
+
+        >>> @pandas_udf("double")
+        ... def mean_udf(v: pd.Series) -> float:
+        ...     return v.mean()
+        ...
+        >>> df = spark.createDataFrame(
+        ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], ("id", "v"))
+        >>> df.groupby("id").agg(mean_udf(df['v'])).show()
+        +---+-----------+
+        | id|mean_udf(v)|
+        +---+-----------+
+        |  1|        1.5|
+        |  2|        6.0|
+        +---+-----------+
+
+        This UDF can also be used as window functions as below:
+
+        >>> from pyspark.sql import Window
+        >>> @pandas_udf("double")
+        ... def mean_udf(v: pd.Series) -> float:
+        ...     return v.mean()
+        ...
+        >>> df = spark.createDataFrame(
+        ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], ("id", "v"))
+        >>> w = Window.partitionBy('id').orderBy('v').rowsBetween(-1, 0)
+        >>> df.withColumn('mean_v', mean_udf("v").over(w)).show()
+        +---+----+------+
+        | id|   v|mean_v|
+        +---+----+------+
+        |  1| 1.0|   1.0|
+        |  1| 2.0|   1.5|
+        |  2| 3.0|   3.0|
+        |  2| 5.0|   4.0|
+        |  2|10.0|   7.5|
+        +---+----+------+
+
+        .. note:: For performance reasons, the input series to window functions are not copied.
             Therefore, mutating the input series is not allowed and will cause incorrect results.
             For the same reason, users should also not rely on the index of the input series.
 
-       .. seealso:: :meth:`pyspark.sql.GroupedData.agg` and :class:`pyspark.sql.Window`
+        .. seealso:: :meth:`pyspark.sql.GroupedData.agg` and :class:`pyspark.sql.Window`
 
     .. note:: The user-defined functions do not support conditional expressions or short circuiting
         in boolean expressions and it ends up with being executed all internally. If the functions
@@ -348,10 +273,21 @@ def pandas_udf(f=None, returnType=None, functionType=None):
     .. note:: The user-defined functions do not take keyword arguments on the calling side.
 
     .. note:: The data type of returned `pandas.Series` from the user-defined functions should be
-        matched with defined returnType (see :meth:`types.to_arrow_type` and
+        matched with defined `returnType` (see :meth:`types.to_arrow_type` and
         :meth:`types.from_arrow_type`). When there is mismatch between them, Spark might do
         conversion on returned data. The conversion is not guaranteed to be correct and results
         should be checked for accuracy by users.
+
+    .. note:: Currently,
+        :class:`pyspark.sql.types.MapType`,
+        :class:`pyspark.sql.types.ArrayType` of :class:`pyspark.sql.types.TimestampType` and
+        nested :class:`pyspark.sql.types.StructType`
+        are currently not supported as output types.
+
+    .. seealso:: :meth:`pyspark.sql.DataFrame.mapInPandas`
+    .. seealso:: :meth:`pyspark.sql.GroupedData.applyInPandas`
+    .. seealso:: :meth:`pyspark.sql.PandasCogroupedOps.applyInPandas`
+    .. seealso:: :meth:`pyspark.sql.UDFRegistration.register`
     """
 
     # The following table shows most of Pandas data and SQL type conversions in Pandas UDFs that
@@ -448,6 +384,14 @@ def _create_pandas_udf(f, returnType, evalType):
                 "In Python 3.6+ and Spark 3.0+, it is preferred to specify type hints for "
                 "pandas UDF instead of specifying pandas UDF type which will be deprecated "
                 "in the future releases. See SPARK-28264 for more details.", UserWarning)
+        elif evalType in [PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF,
+                          PythonEvalType.SQL_MAP_PANDAS_ITER_UDF,
+                          PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF]:
+            # In case of 'SQL_GROUPED_MAP_PANDAS_UDF',  deprecation warning is being triggered
+            # at `apply` instead.
+            # In case of 'SQL_MAP_PANDAS_ITER_UDF' and 'SQL_COGROUPED_MAP_PANDAS_UDF', the
+            # evaluation type will always be set.
+            pass
         elif len(argspec.annotations) > 0:
             evalType = infer_eval_type(signature(f))
             assert evalType is not None
@@ -480,25 +424,3 @@ def _create_pandas_udf(f, returnType, evalType):
             "or three arguments (key, left, right).")
 
     return _create_udf(f, returnType, evalType)
-
-
-def _test():
-    import doctest
-    from pyspark.sql import SparkSession
-    import pyspark.sql.pandas.functions
-    globs = pyspark.sql.pandas.functions.__dict__.copy()
-    spark = SparkSession.builder\
-        .master("local[4]")\
-        .appName("sql.pandas.functions tests")\
-        .getOrCreate()
-    globs['spark'] = spark
-    (failure_count, test_count) = doctest.testmod(
-        pyspark.sql.pandas.functions, globs=globs,
-        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF)
-    spark.stop()
-    if failure_count:
-        sys.exit(-1)
-
-
-if __name__ == "__main__":
-    _test()
diff --git a/python/pyspark/sql/pandas/group_ops.py b/python/pyspark/sql/pandas/group_ops.py
index 3152271ba9df8..b93f0516cadb1 100644
--- a/python/pyspark/sql/pandas/group_ops.py
+++ b/python/pyspark/sql/pandas/group_ops.py
@@ -88,29 +88,27 @@ def applyInPandas(self, func, schema):
         to the user-function and the returned `pandas.DataFrame` are combined as a
         :class:`DataFrame`.
 
-        The returned `pandas.DataFrame` can be of arbitrary length and its schema must match the
-        returnType of the pandas udf.
-
-        .. note:: This function requires a full shuffle. All the data of a group will be loaded
-            into memory, so the user should be aware of the potential OOM risk if data is skewed
-            and certain groups are too large to fit in memory.
+        The `schema` should be a :class:`StructType` describing the schema of the returned
+        `pandas.DataFrame`. The column labels of the returned `pandas.DataFrame` must either match
+        the field names in the defined schema if specified as strings, or match the
+        field data types by position if not strings, e.g. integer indices.
+        The length of the returned `pandas.DataFrame` can be arbitrary.
 
         :param func: a Python native function that takes a `pandas.DataFrame`, and outputs a
             `pandas.DataFrame`.
         :param schema: the return type of the `func` in PySpark. The value can be either a
             :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
-        .. note:: Experimental
-
-        >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
+        >>> import pandas as pd  # doctest: +SKIP
+        >>> from pyspark.sql.functions import pandas_udf, ceil
         >>> df = spark.createDataFrame(
         ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-        ...     ("id", "v"))
+        ...     ("id", "v"))  # doctest: +SKIP
         >>> def normalize(pdf):
         ...     v = pdf.v
         ...     return pdf.assign(v=(v - v.mean()) / v.std())
-        >>> df.groupby("id").applyInPandas(normalize, schema="id long, v double").show()
-        ... # doctest: +SKIP
+        >>> df.groupby("id").applyInPandas(
+        ...     normalize, schema="id long, v double").show()  # doctest: +SKIP
         +---+-------------------+
         | id|                  v|
         +---+-------------------+
@@ -121,8 +119,56 @@ def applyInPandas(self, func, schema):
         |  2| 1.1094003924504583|
         +---+-------------------+
 
-        .. seealso:: :meth:`pyspark.sql.functions.pandas_udf`
+        Alternatively, the user can pass a function that takes two arguments.
+        In this case, the grouping key(s) will be passed as the first argument and the data will
+        be passed as the second argument. The grouping key(s) will be passed as a tuple of numpy
+        data types, e.g., `numpy.int32` and `numpy.float64`. The data will still be passed in
+        as a `pandas.DataFrame` containing all columns from the original Spark DataFrame.
+        This is useful when the user does not want to hardcode grouping key(s) in the function.
+
+        >>> df = spark.createDataFrame(
+        ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
+        ...     ("id", "v"))  # doctest: +SKIP
+        >>> def mean_func(key, pdf):
+        ...     # key is a tuple of one numpy.int64, which is the value
+        ...     # of 'id' for the current group
+        ...     return pd.DataFrame([key + (pdf.v.mean(),)])
+        >>> df.groupby('id').applyInPandas(
+        ...     mean_func, schema="id long, v double").show()  # doctest: +SKIP
+        +---+---+
+        | id|  v|
+        +---+---+
+        |  1|1.5|
+        |  2|6.0|
+        +---+---+
+        >>> def sum_func(key, pdf):
+        ...     # key is a tuple of two numpy.int64s, which is the values
+        ...     # of 'id' and 'ceil(df.v / 2)' for the current group
+        ...     return pd.DataFrame([key + (pdf.v.sum(),)])
+        >>> df.groupby(df.id, ceil(df.v / 2)).applyInPandas(
+        ...     sum_func, schema="id long, `ceil(v / 2)` long, v double").show()  # doctest: +SKIP
+        +---+-----------+----+
+        | id|ceil(v / 2)|   v|
+        +---+-----------+----+
+        |  2|          5|10.0|
+        |  1|          1| 3.0|
+        |  2|          3| 5.0|
+        |  2|          2| 3.0|
+        +---+-----------+----+
+
+        .. note:: This function requires a full shuffle. All the data of a group will be loaded
+            into memory, so the user should be aware of the potential OOM risk if data is skewed
+            and certain groups are too large to fit in memory.
+
+        .. note:: If returning a new `pandas.DataFrame` constructed with a dictionary, it is
+            recommended to explicitly index the columns by name to ensure the positions are correct,
+            or alternatively use an `OrderedDict`.
+            For example, `pd.DataFrame({'id': ids, 'a': data}, columns=['id', 'a'])` or
+            `pd.DataFrame(OrderedDict([('id', ids), ('a', data)]))`.
 
+        .. note:: Experimental
+
+        .. seealso:: :meth:`pyspark.sql.functions.pandas_udf`
         """
         from pyspark.sql import GroupedData
         from pyspark.sql.functions import pandas_udf, PandasUDFType
@@ -176,14 +222,11 @@ def applyInPandas(self, func, schema):
         `pandas.DataFrame` to the user-function and the returned `pandas.DataFrame` are combined as
         a :class:`DataFrame`.
 
-        The returned `pandas.DataFrame` can be of arbitrary length and its schema must match the
-        returnType of the pandas udf.
-
-        .. note:: This function requires a full shuffle. All the data of a cogroup will be loaded
-            into memory, so the user should be aware of the potential OOM risk if data is skewed
-            and certain groups are too large to fit in memory.
-
-        .. note:: Experimental
+        The `schema` should be a :class:`StructType` describing the schema of the returned
+        `pandas.DataFrame`. The column labels of the returned `pandas.DataFrame` must either match
+        the field names in the defined schema if specified as strings, or match the
+        field data types by position if not strings, e.g. integer indices.
+        The length of the returned `pandas.DataFrame` can be arbitrary.
 
         :param func: a Python native function that takes two `pandas.DataFrame`\\s, and
             outputs a `pandas.DataFrame`, or that takes one tuple (grouping keys) and two
@@ -191,7 +234,7 @@ def applyInPandas(self, func, schema):
         :param schema: the return type of the `func` in PySpark. The value can be either a
             :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
-        >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
+        >>> from pyspark.sql.functions import pandas_udf
         >>> df1 = spark.createDataFrame(
         ...     [(20000101, 1, 1.0), (20000101, 2, 2.0), (20000102, 1, 3.0), (20000102, 2, 4.0)],
         ...     ("time", "id", "v1"))
@@ -232,6 +275,18 @@ def applyInPandas(self, func, schema):
         |20000102|  1|3.0|  x|
         +--------+---+---+---+
 
+        .. note:: This function requires a full shuffle. All the data of a cogroup will be loaded
+            into memory, so the user should be aware of the potential OOM risk if data is skewed
+            and certain groups are too large to fit in memory.
+
+        .. note:: If returning a new `pandas.DataFrame` constructed with a dictionary, it is
+            recommended to explicitly index the columns by name to ensure the positions are correct,
+            or alternatively use an `OrderedDict`.
+            For example, `pd.DataFrame({'id': ids, 'a': data}, columns=['id', 'a'])` or
+            `pd.DataFrame(OrderedDict([('id', ids), ('a', data)]))`.
+
+        .. note:: Experimental
+
         .. seealso:: :meth:`pyspark.sql.functions.pandas_udf`
 
         """
diff --git a/python/pyspark/sql/pandas/map_ops.py b/python/pyspark/sql/pandas/map_ops.py
index 75cacd797f9dd..9835e88c6ac21 100644
--- a/python/pyspark/sql/pandas/map_ops.py
+++ b/python/pyspark/sql/pandas/map_ops.py
@@ -45,10 +45,10 @@ def mapInPandas(self, func, schema):
         :param schema: the return type of the `func` in PySpark. The value can be either a
             :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
-        >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
+        >>> from pyspark.sql.functions import pandas_udf
         >>> df = spark.createDataFrame([(1, 21), (2, 30)], ("id", "age"))
-        >>> def filter_func(batch_iter):
-        ...     for pdf in batch_iter:
+        >>> def filter_func(iterator):
+        ...     for pdf in iterator:
         ...         yield pdf[pdf.id == 1]
         >>> df.mapInPandas(filter_func, df.schema).show()  # doctest: +SKIP
         +---+---+
diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py
index 6f46e92f5a2a4..106684eb9b795 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -156,13 +156,16 @@ def create_array(s, t):
                 s = _check_series_convert_timestamps_internal(s, self._timezone)
             try:
                 array = pa.Array.from_pandas(s, mask=mask, type=t, safe=self._safecheck)
-            except pa.ArrowException as e:
-                error_msg = "Exception thrown when converting pandas.Series (%s) to Arrow " + \
-                            "Array (%s). It can be caused by overflows or other unsafe " + \
-                            "conversions warned by Arrow. Arrow safe type check can be " + \
-                            "disabled by using SQL config " + \
-                            "`spark.sql.execution.pandas.arrowSafeTypeConversion`."
-                raise RuntimeError(error_msg % (s.dtype, t), e)
+            except ValueError as e:
+                if self._safecheck:
+                    error_msg = "Exception thrown when converting pandas.Series (%s) to " + \
+                                "Arrow Array (%s). It can be caused by overflows or other " + \
+                                "unsafe conversions warned by Arrow. Arrow safe type check " + \
+                                "can be disabled by using SQL config " + \
+                                "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+                    raise ValueError(error_msg % (s.dtype, t), e)
+                else:
+                    raise e
             return array
 
         arrs = []
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 3d3280dbd9943..0c54abbb7df21 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -48,7 +48,7 @@ def _set_opts(self, schema=None, **options):
 class DataFrameReader(OptionUtils):
     """
     Interface used to load a :class:`DataFrame` from external storage systems
-    (e.g. file systems, key-value stores, etc). Use :func:`spark.read`
+    (e.g. file systems, key-value stores, etc). Use :attr:`SparkSession.read`
     to access this.
 
     .. versionadded:: 1.4
@@ -105,9 +105,18 @@ def option(self, key, value):
         """Adds an input option for the underlying data source.
 
         You can set the following option(s) for reading files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to parse timestamps
-                in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to parse
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
             * ``pathGlobFilter``: an optional glob pattern to only include files with paths matching
                 the pattern. The syntax follows org.apache.hadoop.fs.GlobFilter.
                 It does not change the behavior of partition discovery.
@@ -120,9 +129,18 @@ def options(self, **options):
         """Adds input options for the underlying data source.
 
         You can set the following option(s) for reading files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to parse timestamps
-                in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to parse
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
             * ``pathGlobFilter``: an optional glob pattern to only include files with paths matching
                 the pattern. The syntax follows org.apache.hadoop.fs.GlobFilter.
                 It does not change the behavior of partition discovery.
@@ -133,7 +151,7 @@ def options(self, **options):
 
     @since(1.4)
     def load(self, path=None, format=None, schema=None, **options):
-        """Loads data from a data source and returns it as a :class`DataFrame`.
+        """Loads data from a data source and returns it as a :class:`DataFrame`.
 
         :param path: optional string or a list of string for file-system backed data sources.
         :param format: optional string for format of the data source. Default to 'parquet'.
@@ -205,15 +223,15 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param mode: allows a mode for dealing with corrupt records during parsing. If None is
                      set, it uses the default value, ``PERMISSIVE``.
 
-                * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
                   into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
                   fields to ``null``. To keep corrupt records, an user can set a string type \
                   field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
                   schema does not have the field, it drops corrupt records during parsing. \
                   When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \
                   field in an output schema.
-                *  ``DROPMALFORMED`` : ignores the whole corrupted records.
-                *  ``FAILFAST`` : throws an exception when it meets corrupted records.
+                *  ``DROPMALFORMED``: ignores the whole corrupted records.
+                *  ``FAILFAST``: throws an exception when it meets corrupted records.
 
         :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
                                           created by ``PERMISSIVE`` mode. This overrides
@@ -221,14 +239,13 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
                                           it uses the value specified in
                                           ``spark.sql.columnNameOfCorruptRecord``.
         :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           follow the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param multiLine: parse one record, which may span multiple lines, per file. If None is
                           set, it uses the default value, ``false``.
         :param allowUnquotedControlChars: allows JSON Strings to contain unquoted control
@@ -254,7 +271,9 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param recursiveFileLookup: recursively scan a directory for files. Using this option
                                     disables `partition discovery`_.
 
-        .. _partition discovery: /sql-data-sources-parquet.html#partition-discovery
+        .. _partition discovery:
+          https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery
+        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
         >>> df1 = spark.read.json('python/test_support/sql/people.json')
         >>> df1.dtypes
@@ -402,6 +421,9 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
                         character. By default (None), it is disabled.
         :param header: uses the first line as names of columns. If None is set, it uses the
                        default value, ``false``.
+                       .. note:: if the given path is a RDD of Strings, this header
+                       option will remove all lines same with the header if exists.
+
         :param inferSchema: infers the input schema automatically from data. It requires one extra
                        pass over the data. If None is set, it uses the default value, ``false``.
         :param enforceSchema: If it is set to ``true``, the specified or inferred schema will be
@@ -430,14 +452,13 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param negativeInf: sets the string representation of a negative infinity value. If None
                             is set, it uses the default value, ``Inf``.
         :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           follow the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param maxColumns: defines a hard limit of how many columns a record can have. If None is
                            set, it uses the default value, ``20480``.
         :param maxCharsPerColumn: defines the maximum number of characters allowed for any given
@@ -452,7 +473,7 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
                      be controlled by ``spark.sql.csv.parser.columnPruning.enabled``
                      (enabled by default).
 
-                * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
                   into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
                   fields to ``null``. To keep corrupt records, an user can set a string type \
                   field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
@@ -461,8 +482,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
                   When it meets a record having fewer tokens than the length of the schema, \
                   sets ``null`` to extra fields. When the record has more tokens than the \
                   length of the schema, it drops extra tokens.
-                * ``DROPMALFORMED`` : ignores the whole corrupted records.
-                * ``FAILFAST`` : throws an exception when it meets corrupted records.
+                * ``DROPMALFORMED``: ignores the whole corrupted records.
+                * ``FAILFAST``: throws an exception when it meets corrupted records.
 
         :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
                                           created by ``PERMISSIVE`` mode. This overrides
@@ -616,7 +637,7 @@ def jdbc(self, url, table, column=None, lowerBound=None, upperBound=None, numPar
 class DataFrameWriter(OptionUtils):
     """
     Interface used to write a :class:`DataFrame` to external storage systems
-    (e.g. file systems, key-value stores, etc). Use :func:`DataFrame.write`
+    (e.g. file systems, key-value stores, etc). Use :attr:`DataFrame.write`
     to access this.
 
     .. versionadded:: 1.4
@@ -665,9 +686,18 @@ def option(self, key, value):
         """Adds an output option for the underlying data source.
 
         You can set the following option(s) for writing files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to format
-                timestamps in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to format
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
         """
         self._jwrite = self._jwrite.option(key, to_str(value))
         return self
@@ -677,9 +707,18 @@ def options(self, **options):
         """Adds output options for the underlying data source.
 
         You can set the following option(s) for writing files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to format
-                timestamps in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to format
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
         """
         for k in options:
             self._jwrite = self._jwrite.option(k, to_str(options[k]))
@@ -794,7 +833,7 @@ def save(self, path=None, format=None, mode=None, partitionBy=None, **options):
     def insertInto(self, tableName, overwrite=None):
         """Inserts the content of the :class:`DataFrame` to the specified table.
 
-        It requires that the schema of the class:`DataFrame` is the same as the
+        It requires that the schema of the :class:`DataFrame` is the same as the
         schema of the table.
 
         Optionally overwriting any existing data.
@@ -850,14 +889,13 @@ def json(self, path, mode=None, compression=None, dateFormat=None, timestampForm
                             known case-insensitive shorten names (none, bzip2, gzip, lz4,
                             snappy and deflate).
         :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           follow the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param encoding: specifies encoding (charset) of saved json files. If None is set,
                         the default UTF-8 charset will be used.
         :param lineSep: defines the line separator that should be used for writing. If None is
@@ -954,15 +992,14 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
                        the default value, ``false``.
         :param nullValue: sets the string representation of a null value. If None is set, it uses
                           the default value, empty string.
-        :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+        :param dateFormat: sets the string that indicates a date format. Custom date formats follow
+                           the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param ignoreLeadingWhiteSpace: a flag indicating whether or not leading whitespaces from
                                         values being written should be skipped. If None is set, it
                                         uses the default value, ``true``.
diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index 233f4927389d2..e9486a3d3238a 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -265,7 +265,10 @@ def newSession(self):
     @since(3.0)
     def getActiveSession(cls):
         """
-        Returns the active SparkSession for the current thread, returned by the builder.
+        Returns the active SparkSession for the current thread, returned by the builder
+
+        :return: :class:`SparkSession` if an active session exists for the current thread
+
         >>> s = SparkSession.getActiveSession()
         >>> l = [('Alice', 1)]
         >>> rdd = s.sparkContext.parallelize(l)
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index f17a52f6b3dc8..2450a4c93c460 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -125,6 +125,7 @@ def lastProgress(self):
         """
         Returns the most recent :class:`StreamingQueryProgress` update of this streaming query or
         None if there were no progress updates
+
         :return: a map
         """
         lastProgress = self._jsq.lastProgress()
@@ -276,9 +277,9 @@ def resetTerminated(self):
 
 class DataStreamReader(OptionUtils):
     """
-    Interface used to load a streaming :class:`DataFrame` from external storage systems
-    (e.g. file systems, key-value stores, etc). Use :func:`spark.readStream`
-    to access this.
+    Interface used to load a streaming :class:`DataFrame <pyspark.sql.DataFrame>` from external
+    storage systems (e.g. file systems, key-value stores, etc).
+    Use :attr:`SparkSession.readStream <pyspark.sql.SparkSession.readStream>` to access this.
 
     .. note:: Evolving.
 
@@ -338,9 +339,18 @@ def option(self, key, value):
         """Adds an input option for the underlying data source.
 
         You can set the following option(s) for reading files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to parse timestamps
-                in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to parse
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
 
         .. note:: Evolving.
 
@@ -354,9 +364,18 @@ def options(self, **options):
         """Adds input options for the underlying data source.
 
         You can set the following option(s) for reading files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to parse timestamps
-                in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to parse
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
 
         .. note:: Evolving.
 
@@ -368,7 +387,8 @@ def options(self, **options):
 
     @since(2.0)
     def load(self, path=None, format=None, schema=None, **options):
-        """Loads a data stream from a data source and returns it as a :class`DataFrame`.
+        """Loads a data stream from a data source and returns it as a
+        :class:`DataFrame <pyspark.sql.DataFrame>`.
 
         .. note:: Evolving.
 
@@ -441,15 +461,15 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param mode: allows a mode for dealing with corrupt records during parsing. If None is
                      set, it uses the default value, ``PERMISSIVE``.
 
-                * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
                   into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
                   fields to ``null``. To keep corrupt records, an user can set a string type \
                   field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
                   schema does not have the field, it drops corrupt records during parsing. \
                   When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \
                   field in an output schema.
-                *  ``DROPMALFORMED`` : ignores the whole corrupted records.
-                *  ``FAILFAST`` : throws an exception when it meets corrupted records.
+                *  ``DROPMALFORMED``: ignores the whole corrupted records.
+                *  ``FAILFAST``: throws an exception when it meets corrupted records.
 
         :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
                                           created by ``PERMISSIVE`` mode. This overrides
@@ -457,14 +477,13 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
                                           it uses the value specified in
                                           ``spark.sql.columnNameOfCorruptRecord``.
         :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           follow the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param multiLine: parse one record, which may span multiple lines, per file. If None is
                           set, it uses the default value, ``false``.
         :param allowUnquotedControlChars: allows JSON Strings to contain unquoted control
@@ -488,7 +507,9 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param recursiveFileLookup: recursively scan a directory for files. Using this option
                                     disables `partition discovery`_.
 
-        .. _partition discovery: /sql-data-sources-parquet.html#partition-discovery
+        .. _partition discovery:
+          https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery
+        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
         >>> json_sdf = spark.readStream.json(tempfile.mkdtemp(), schema = sdf_schema)
         >>> json_sdf.isStreaming
@@ -669,14 +690,13 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param negativeInf: sets the string representation of a negative infinity value. If None
                             is set, it uses the default value, ``Inf``.
         :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           follow the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param maxColumns: defines a hard limit of how many columns a record can have. If None is
                            set, it uses the default value, ``20480``.
         :param maxCharsPerColumn: defines the maximum number of characters allowed for any given
@@ -687,7 +707,7 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param mode: allows a mode for dealing with corrupt records during parsing. If None is
                      set, it uses the default value, ``PERMISSIVE``.
 
-                * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
                   into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
                   fields to ``null``. To keep corrupt records, an user can set a string type \
                   field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
@@ -696,8 +716,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
                   When it meets a record having fewer tokens than the length of the schema, \
                   sets ``null`` to extra fields. When the record has more tokens than the \
                   length of the schema, it drops extra tokens.
-                * ``DROPMALFORMED`` : ignores the whole corrupted records.
-                * ``FAILFAST`` : throws an exception when it meets corrupted records.
+                * ``DROPMALFORMED``: ignores the whole corrupted records.
+                * ``FAILFAST``: throws an exception when it meets corrupted records.
 
         :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
                                           created by ``PERMISSIVE`` mode. This overrides
@@ -750,8 +770,9 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
 
 class DataStreamWriter(object):
     """
-    Interface used to write a streaming :class:`DataFrame` to external storage systems
-    (e.g. file systems, key-value stores, etc). Use :func:`DataFrame.writeStream`
+    Interface used to write a streaming :class:`DataFrame <pyspark.sql.DataFrame>` to external
+    storage systems (e.g. file systems, key-value stores, etc).
+    Use :attr:`DataFrame.writeStream <pyspark.sql.DataFrame.writeStream>`
     to access this.
 
     .. note:: Evolving.
@@ -774,11 +795,11 @@ def outputMode(self, outputMode):
 
         Options include:
 
-        * `append`:Only the new rows in the streaming DataFrame/Dataset will be written to
+        * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to
            the sink
-        * `complete`:All the rows in the streaming DataFrame/Dataset will be written to the sink
+        * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the sink
            every time these is some updates
-        * `update`:only the rows that were updated in the streaming DataFrame/Dataset will be
+        * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be
            written to the sink every time there are some updates. If the query doesn't contain
            aggregations, it will be equivalent to `append` mode.
 
@@ -809,9 +830,18 @@ def option(self, key, value):
         """Adds an output option for the underlying data source.
 
         You can set the following option(s) for writing files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to format
-                timestamps in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to format
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
 
         .. note:: Evolving.
         """
@@ -823,9 +853,18 @@ def options(self, **options):
         """Adds output options for the underlying data source.
 
         You can set the following option(s) for writing files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to format
-                timestamps in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to format
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
 
        .. note:: Evolving.
         """
@@ -1131,11 +1170,11 @@ def start(self, path=None, format=None, outputMode=None, partitionBy=None, query
         :param outputMode: specifies how data of a streaming DataFrame/Dataset is written to a
                            streaming sink.
 
-            * `append`:Only the new rows in the streaming DataFrame/Dataset will be written to the
+            * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to the
               sink
-            * `complete`:All the rows in the streaming DataFrame/Dataset will be written to the sink
-               every time these is some updates
-            * `update`:only the rows that were updated in the streaming DataFrame/Dataset will be
+            * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the
+              sink every time these is some updates
+            * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be
               written to the sink every time there are some updates. If the query doesn't contain
               aggregations, it will be equivalent to `append` mode.
         :param partitionBy: names of partitioning columns
diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 98f44dfd29da5..2c6231da3c161 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -21,6 +21,9 @@
 import time
 import unittest
 import warnings
+import sys
+if sys.version >= '3':
+    basestring = unicode = str
 
 from pyspark import SparkContext, SparkConf
 from pyspark.sql import Row, SparkSession
@@ -263,11 +266,12 @@ def test_createDataFrame_with_schema(self):
     def test_createDataFrame_with_incorrect_schema(self):
         pdf = self.create_pandas_data_frame()
         fields = list(self.schema)
-        fields[0], fields[1] = fields[1], fields[0]  # swap str with int
+        fields[5], fields[6] = fields[6], fields[5]  # swap decimal with date
         wrong_schema = StructType(fields)
-        with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, "integer.*required"):
-                self.spark.createDataFrame(pdf, schema=wrong_schema)
+        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegexp(Exception, "[D|d]ecimal.*got.*date"):
+                    self.spark.createDataFrame(pdf, schema=wrong_schema)
 
     def test_createDataFrame_with_names(self):
         pdf = self.create_pandas_data_frame()
@@ -297,9 +301,9 @@ def test_createDataFrame_does_not_modify_input(self):
         # Some series get converted for Spark to consume, this makes sure input is unchanged
         pdf = self.create_pandas_data_frame()
         # Use a nanosecond value to make sure it is not truncated
-        pdf.ix[0, '8_timestamp_t'] = pd.Timestamp(1)
+        pdf.iloc[0, 7] = pd.Timestamp(1)
         # Integers with nulls will get NaNs filled with 0 and will be casted
-        pdf.ix[1, '2_int_t'] = None
+        pdf.iloc[1, 1] = None
         pdf_copy = pdf.copy(deep=True)
         self.spark.createDataFrame(pdf, schema=self.schema)
         self.assertTrue(pdf.equals(pdf_copy))
@@ -415,6 +419,25 @@ def run_test(num_records, num_parts, max_records, use_delay=False):
         for case in cases:
             run_test(*case)
 
+    def test_createDataFrame_with_float_index(self):
+        # SPARK-32098: float index should not produce duplicated or truncated Spark DataFrame
+        self.assertEqual(
+            self.spark.createDataFrame(
+                pd.DataFrame({'a': [1, 2, 3]}, index=[2., 3., 4.])).distinct().count(), 3)
+
+    def test_no_partition_toPandas(self):
+        # SPARK-32301: toPandas should work from a Spark DataFrame with no partitions
+        # Forward-ported from SPARK-32300.
+        pdf = self.spark.sparkContext.emptyRDD().toDF("col1 int").toPandas()
+        self.assertEqual(len(pdf), 0)
+        self.assertEqual(list(pdf.columns), ["col1"])
+
+    def test_createDataFrame_empty_partition(self):
+        pdf = pd.DataFrame({"c1": [1], "c2": ["string"]})
+        df = self.spark.createDataFrame(pdf)
+        self.assertEqual([Row(c1=1, c2='string')], df.collect())
+        self.assertGreater(self.spark.sparkContext.defaultParallelism, len(pdf))
+
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
diff --git a/python/pyspark/sql/tests/test_column.py b/python/pyspark/sql/tests/test_column.py
index d9d933110dab5..58bf896a10c2a 100644
--- a/python/pyspark/sql/tests/test_column.py
+++ b/python/pyspark/sql/tests/test_column.py
@@ -18,8 +18,6 @@
 
 import sys
 
-from py4j.protocol import Py4JJavaError
-
 from pyspark.sql import Column, Row
 from pyspark.sql.types import *
 from pyspark.sql.utils import AnalysisException
@@ -87,7 +85,7 @@ def test_column_operators(self):
                                 "Cannot apply 'in' operator against a column",
                                 lambda: 1 in cs)
 
-    def test_column_apply(self):
+    def test_column_accessor(self):
         from pyspark.sql.functions import col
 
         self.assertIsInstance(col("foo")[1:3], Column)
@@ -95,16 +93,6 @@ def test_column_apply(self):
         self.assertIsInstance(col("foo")["bar"], Column)
         self.assertRaises(ValueError, lambda: col("foo")[0:10:2])
 
-    def test_column_getitem(self):
-        from pyspark.sql.functions import col, create_map, lit
-
-        map_col = create_map(lit(0), lit(100), lit(1), lit(200))
-        self.assertRaisesRegexp(
-            Py4JJavaError,
-            "Unsupported literal type class org.apache.spark.sql.Column id",
-            lambda: map_col.getItem(col('id'))
-        )
-
     def test_column_select(self):
         df = self.df
         self.assertEqual(self.testData, df.select("*").collect())
diff --git a/python/pyspark/sql/tests/test_context.py b/python/pyspark/sql/tests/test_context.py
index d57ebc48cf5e6..3a0c7bb913519 100644
--- a/python/pyspark/sql/tests/test_context.py
+++ b/python/pyspark/sql/tests/test_context.py
@@ -27,9 +27,9 @@
 
 import py4j
 
+from pyspark import SparkContext, SQLContext
 from pyspark.sql import Row, SparkSession
 from pyspark.sql.types import *
-from pyspark.sql.window import Window
 from pyspark.testing.utils import ReusedPySparkTestCase
 
 
@@ -111,99 +111,6 @@ def test_save_and_load_table(self):
 
         shutil.rmtree(tmpPath)
 
-    def test_window_functions(self):
-        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
-        w = Window.partitionBy("value").orderBy("key")
-        from pyspark.sql import functions as F
-        sel = df.select(df.value, df.key,
-                        F.max("key").over(w.rowsBetween(0, 1)),
-                        F.min("key").over(w.rowsBetween(0, 1)),
-                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
-                        F.row_number().over(w),
-                        F.rank().over(w),
-                        F.dense_rank().over(w),
-                        F.ntile(2).over(w))
-        rs = sorted(sel.collect())
-        expected = [
-            ("1", 1, 1, 1, 1, 1, 1, 1, 1),
-            ("2", 1, 1, 1, 3, 1, 1, 1, 1),
-            ("2", 1, 2, 1, 3, 2, 1, 1, 1),
-            ("2", 2, 2, 2, 3, 3, 3, 2, 2)
-        ]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-    def test_window_functions_without_partitionBy(self):
-        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
-        w = Window.orderBy("key", df.value)
-        from pyspark.sql import functions as F
-        sel = df.select(df.value, df.key,
-                        F.max("key").over(w.rowsBetween(0, 1)),
-                        F.min("key").over(w.rowsBetween(0, 1)),
-                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
-                        F.row_number().over(w),
-                        F.rank().over(w),
-                        F.dense_rank().over(w),
-                        F.ntile(2).over(w))
-        rs = sorted(sel.collect())
-        expected = [
-            ("1", 1, 1, 1, 4, 1, 1, 1, 1),
-            ("2", 1, 1, 1, 4, 2, 2, 2, 1),
-            ("2", 1, 2, 1, 4, 3, 2, 2, 2),
-            ("2", 2, 2, 2, 4, 4, 4, 3, 2)
-        ]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-    def test_window_functions_cumulative_sum(self):
-        df = self.spark.createDataFrame([("one", 1), ("two", 2)], ["key", "value"])
-        from pyspark.sql import functions as F
-
-        # Test cumulative sum
-        sel = df.select(
-            df.key,
-            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding, 0)))
-        rs = sorted(sel.collect())
-        expected = [("one", 1), ("two", 3)]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-        # Test boundary values less than JVM's Long.MinValue and make sure we don't overflow
-        sel = df.select(
-            df.key,
-            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding - 1, 0)))
-        rs = sorted(sel.collect())
-        expected = [("one", 1), ("two", 3)]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-        # Test boundary values greater than JVM's Long.MaxValue and make sure we don't overflow
-        frame_end = Window.unboundedFollowing + 1
-        sel = df.select(
-            df.key,
-            F.sum(df.value).over(Window.rowsBetween(Window.currentRow, frame_end)))
-        rs = sorted(sel.collect())
-        expected = [("one", 3), ("two", 2)]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-    def test_collect_functions(self):
-        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
-        from pyspark.sql import functions
-
-        self.assertEqual(
-            sorted(df.select(functions.collect_set(df.key).alias('r')).collect()[0].r),
-            [1, 2])
-        self.assertEqual(
-            sorted(df.select(functions.collect_list(df.key).alias('r')).collect()[0].r),
-            [1, 1, 1, 2])
-        self.assertEqual(
-            sorted(df.select(functions.collect_set(df.value).alias('r')).collect()[0].r),
-            ["1", "2"])
-        self.assertEqual(
-            sorted(df.select(functions.collect_list(df.value).alias('r')).collect()[0].r),
-            ["1", "2", "2", "2"])
-
     def test_limit_and_take(self):
         df = self.spark.range(1, 1000, numPartitions=10)
 
@@ -222,13 +129,6 @@ def assert_runs_only_one_job_stage_and_task(job_group_name, f):
         # Regression test for SPARK-17514: limit(n).collect() should the perform same as take(n)
         assert_runs_only_one_job_stage_and_task("collect_limit", lambda: df.limit(1).collect())
 
-    def test_datetime_functions(self):
-        from pyspark.sql import functions
-        from datetime import date
-        df = self.spark.range(1).selectExpr("'2017-01-22' as dateCol")
-        parse_result = df.select(functions.to_date(functions.col("dateCol"))).first()
-        self.assertEquals(date(2017, 1, 22), parse_result['to_date(`dateCol`)'])
-
     def test_unbounded_frames(self):
         from pyspark.sql import functions as F
         from pyspark.sql import window
@@ -259,6 +159,23 @@ def range_frame_match():
         reload(window)
 
 
+class SQLContextTests(unittest.TestCase):
+
+    def test_get_or_create(self):
+        sc = None
+        sql_context = None
+        try:
+            sc = SparkContext('local[4]', "SQLContextTests")
+            sql_context = SQLContext.getOrCreate(sc)
+            assert(isinstance(sql_context, SQLContext))
+        finally:
+            SQLContext._instantiatedContext = None
+            if sql_context is not None:
+                sql_context.sparkSession.stop()
+            if sc is not None:
+                sc.stop()
+
+
 if __name__ == "__main__":
     from pyspark.sql.tests.test_context import *
 
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index d738449799bda..738c9846169d1 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -529,6 +529,39 @@ def test_to_pandas(self):
         self.assertEquals(types[4], np.object)  # datetime.date
         self.assertEquals(types[5], 'datetime64[ns]')
 
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    def test_to_pandas_with_duplicated_column_names(self):
+        import numpy as np
+
+        sql = "select 1 v, 1 v"
+        for arrowEnabled in [False, True]:
+            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrowEnabled}):
+                df = self.spark.sql(sql)
+                pdf = df.toPandas()
+                types = pdf.dtypes
+                self.assertEquals(types.iloc[0], np.int32)
+                self.assertEquals(types.iloc[1], np.int32)
+
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    def test_to_pandas_on_cross_join(self):
+        import numpy as np
+
+        sql = """
+        select t1.*, t2.* from (
+          select explode(sequence(1, 3)) v
+        ) t1 left join (
+          select explode(sequence(1, 3)) v
+        ) t2
+        """
+        for arrowEnabled in [False, True]:
+            with self.sql_conf({"spark.sql.crossJoin.enabled": True,
+                                "spark.sql.execution.arrow.pyspark.enabled": arrowEnabled}):
+                df = self.spark.sql(sql)
+                pdf = df.toPandas()
+                types = pdf.dtypes
+                self.assertEquals(types.iloc[0], np.int32)
+                self.assertEquals(types.iloc[1], np.int32)
+
     @unittest.skipIf(have_pandas, "Required Pandas was found.")
     def test_to_pandas_required_pandas_not_found(self):
         with QuietTest(self.sc):
diff --git a/python/pyspark/sql/tests/test_functions.py b/python/pyspark/sql/tests/test_functions.py
index fa9ee57ff5f90..fd2ad22fa7b78 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -18,7 +18,7 @@
 import datetime
 import sys
 
-from pyspark.sql import Row
+from pyspark.sql import Row, Window
 from pyspark.sql.functions import udf, input_file_name
 from pyspark.testing.sqlutils import ReusedSQLTestCase
 
@@ -337,6 +337,106 @@ def test_overlay(self):
 
         self.assertListEqual(actual, expected)
 
+    def test_window_functions(self):
+        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
+        w = Window.partitionBy("value").orderBy("key")
+        from pyspark.sql import functions as F
+        sel = df.select(df.value, df.key,
+                        F.max("key").over(w.rowsBetween(0, 1)),
+                        F.min("key").over(w.rowsBetween(0, 1)),
+                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
+                        F.row_number().over(w),
+                        F.rank().over(w),
+                        F.dense_rank().over(w),
+                        F.ntile(2).over(w))
+        rs = sorted(sel.collect())
+        expected = [
+            ("1", 1, 1, 1, 1, 1, 1, 1, 1),
+            ("2", 1, 1, 1, 3, 1, 1, 1, 1),
+            ("2", 1, 2, 1, 3, 2, 1, 1, 1),
+            ("2", 2, 2, 2, 3, 3, 3, 2, 2)
+        ]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+    def test_window_functions_without_partitionBy(self):
+        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
+        w = Window.orderBy("key", df.value)
+        from pyspark.sql import functions as F
+        sel = df.select(df.value, df.key,
+                        F.max("key").over(w.rowsBetween(0, 1)),
+                        F.min("key").over(w.rowsBetween(0, 1)),
+                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
+                        F.row_number().over(w),
+                        F.rank().over(w),
+                        F.dense_rank().over(w),
+                        F.ntile(2).over(w))
+        rs = sorted(sel.collect())
+        expected = [
+            ("1", 1, 1, 1, 4, 1, 1, 1, 1),
+            ("2", 1, 1, 1, 4, 2, 2, 2, 1),
+            ("2", 1, 2, 1, 4, 3, 2, 2, 2),
+            ("2", 2, 2, 2, 4, 4, 4, 3, 2)
+        ]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+    def test_window_functions_cumulative_sum(self):
+        df = self.spark.createDataFrame([("one", 1), ("two", 2)], ["key", "value"])
+        from pyspark.sql import functions as F
+
+        # Test cumulative sum
+        sel = df.select(
+            df.key,
+            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding, 0)))
+        rs = sorted(sel.collect())
+        expected = [("one", 1), ("two", 3)]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+        # Test boundary values less than JVM's Long.MinValue and make sure we don't overflow
+        sel = df.select(
+            df.key,
+            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding - 1, 0)))
+        rs = sorted(sel.collect())
+        expected = [("one", 1), ("two", 3)]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+        # Test boundary values greater than JVM's Long.MaxValue and make sure we don't overflow
+        frame_end = Window.unboundedFollowing + 1
+        sel = df.select(
+            df.key,
+            F.sum(df.value).over(Window.rowsBetween(Window.currentRow, frame_end)))
+        rs = sorted(sel.collect())
+        expected = [("one", 3), ("two", 2)]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+    def test_collect_functions(self):
+        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
+        from pyspark.sql import functions
+
+        self.assertEqual(
+            sorted(df.select(functions.collect_set(df.key).alias('r')).collect()[0].r),
+            [1, 2])
+        self.assertEqual(
+            sorted(df.select(functions.collect_list(df.key).alias('r')).collect()[0].r),
+            [1, 1, 1, 2])
+        self.assertEqual(
+            sorted(df.select(functions.collect_set(df.value).alias('r')).collect()[0].r),
+            ["1", "2"])
+        self.assertEqual(
+            sorted(df.select(functions.collect_list(df.value).alias('r')).collect()[0].r),
+            ["1", "2", "2", "2"])
+
+    def test_datetime_functions(self):
+        from pyspark.sql import functions
+        from datetime import date
+        df = self.spark.range(1).selectExpr("'2017-01-22' as dateCol")
+        parse_result = df.select(functions.to_date(functions.col("dateCol"))).first()
+        self.assertEquals(date(2017, 1, 22), parse_result['to_date(`dateCol`)'])
+
 
 if __name__ == "__main__":
     import unittest
diff --git a/python/pyspark/sql/tests/test_pandas_cogrouped_map.py b/python/pyspark/sql/tests/test_pandas_cogrouped_map.py
index 3ed9d2ac62fd3..c1cb30c3caa91 100644
--- a/python/pyspark/sql/tests/test_pandas_cogrouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_cogrouped_map.py
@@ -19,7 +19,7 @@
 import sys
 
 from pyspark.sql.functions import array, explode, col, lit, udf, sum, pandas_udf, PandasUDFType
-from pyspark.sql.types import DoubleType, StructType, StructField
+from pyspark.sql.types import DoubleType, StructType, StructField, Row
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
 from pyspark.testing.utils import QuietTest
@@ -193,6 +193,22 @@ def test_wrong_args(self):
             left.groupby('id').cogroup(right.groupby('id')) \
                 .applyInPandas(lambda: 1, StructType([StructField("d", DoubleType())]))
 
+    def test_case_insensitive_grouping_column(self):
+        # SPARK-31915: case-insensitive grouping column should work.
+        df1 = self.spark.createDataFrame([(1, 1)], ("column", "value"))
+
+        row = df1.groupby("ColUmn").cogroup(
+            df1.groupby("COLUMN")
+        ).applyInPandas(lambda r, l: r + l, "column long, value long").first()
+        self.assertEquals(row.asDict(), Row(column=2, value=2).asDict())
+
+        df2 = self.spark.createDataFrame([(1, 1)], ("column", "value"))
+
+        row = df1.groupby("ColUmn").cogroup(
+            df2.groupby("COLUMN")
+        ).applyInPandas(lambda r, l: r + l, "column long, value long").first()
+        self.assertEquals(row.asDict(), Row(column=2, value=2).asDict())
+
     @staticmethod
     def _test_with_key(left, right, isLeft):
 
diff --git a/python/pyspark/sql/tests/test_pandas_grouped_map.py b/python/pyspark/sql/tests/test_pandas_grouped_map.py
index 51dd07fd7d70c..8e02b291cddea 100644
--- a/python/pyspark/sql/tests/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_grouped_map.py
@@ -390,11 +390,11 @@ def rename_pdf(pdf, names):
         # Function returns a pdf with required column names, but order could be arbitrary using dict
         def change_col_order(pdf):
             # Constructing a DataFrame from a dict should result in the same order,
-            # but use from_items to ensure the pdf column order is different than schema
-            return pd.DataFrame.from_items([
+            # but use OrderedDict to ensure the pdf column order is different than schema
+            return pd.DataFrame.from_dict(OrderedDict([
                 ('id', pdf.id),
                 ('u', pdf.v * 2),
-                ('v', pdf.v)])
+                ('v', pdf.v)]))
 
         ordered_udf = pandas_udf(
             change_col_order,
@@ -450,15 +450,16 @@ def int_index(pdf):
         def column_name_typo(pdf):
             return pd.DataFrame({'iid': pdf.id, 'v': pdf.v})
 
-        @pandas_udf('id long, v int', PandasUDFType.GROUPED_MAP)
+        @pandas_udf('id long, v decimal', PandasUDFType.GROUPED_MAP)
         def invalid_positional_types(pdf):
-            return pd.DataFrame([(u'a', 1.2)])
+            return pd.DataFrame([(1, datetime.date(2020, 10, 5))])
 
-        with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
-                grouped_df.apply(column_name_typo).collect()
-            with self.assertRaisesRegexp(Exception, "an integer is required"):
-                grouped_df.apply(invalid_positional_types).collect()
+        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
+                    grouped_df.apply(column_name_typo).collect()
+                with self.assertRaisesRegexp(Exception, "[D|d]ecimal.*got.*date"):
+                    grouped_df.apply(invalid_positional_types).collect()
 
     def test_positional_assignment_conf(self):
         with self.sql_conf({
@@ -545,13 +546,13 @@ def f(pdf):
 
     def test_grouped_over_window_with_key(self):
 
-        data = [(0, 1, "2018-03-10T00:00:00+00:00", False),
-                (1, 2, "2018-03-11T00:00:00+00:00", False),
-                (2, 2, "2018-03-12T00:00:00+00:00", False),
-                (3, 3, "2018-03-15T00:00:00+00:00", False),
-                (4, 3, "2018-03-16T00:00:00+00:00", False),
-                (5, 3, "2018-03-17T00:00:00+00:00", False),
-                (6, 3, "2018-03-21T00:00:00+00:00", False)]
+        data = [(0, 1, "2018-03-10T00:00:00+00:00", [0]),
+                (1, 2, "2018-03-11T00:00:00+00:00", [0]),
+                (2, 2, "2018-03-12T00:00:00+00:00", [0]),
+                (3, 3, "2018-03-15T00:00:00+00:00", [0]),
+                (4, 3, "2018-03-16T00:00:00+00:00", [0]),
+                (5, 3, "2018-03-17T00:00:00+00:00", [0]),
+                (6, 3, "2018-03-21T00:00:00+00:00", [0])]
 
         expected_window = [
             {'start': datetime.datetime(2018, 3, 10, 0, 0),
@@ -562,30 +563,53 @@ def test_grouped_over_window_with_key(self):
              'end': datetime.datetime(2018, 3, 25, 0, 0)},
         ]
 
-        expected = {0: (1, expected_window[0]),
-                    1: (2, expected_window[0]),
-                    2: (2, expected_window[0]),
-                    3: (3, expected_window[1]),
-                    4: (3, expected_window[1]),
-                    5: (3, expected_window[1]),
-                    6: (3, expected_window[2])}
+        expected_key = {0: (1, expected_window[0]),
+                        1: (2, expected_window[0]),
+                        2: (2, expected_window[0]),
+                        3: (3, expected_window[1]),
+                        4: (3, expected_window[1]),
+                        5: (3, expected_window[1]),
+                        6: (3, expected_window[2])}
+
+        # id -> array of group with len of num records in window
+        expected = {0: [1],
+                    1: [2, 2],
+                    2: [2, 2],
+                    3: [3, 3, 3],
+                    4: [3, 3, 3],
+                    5: [3, 3, 3],
+                    6: [3]}
 
         df = self.spark.createDataFrame(data, ['id', 'group', 'ts', 'result'])
         df = df.select(col('id'), col('group'), col('ts').cast('timestamp'), col('result'))
 
-        @pandas_udf(df.schema, PandasUDFType.GROUPED_MAP)
         def f(key, pdf):
             group = key[0]
             window_range = key[1]
-            # Result will be True if group and window range equal to expected
-            is_expected = pdf.id.apply(lambda id: (expected[id][0] == group and
-                                                   expected[id][1] == window_range))
-            return pdf.assign(result=is_expected)
 
-        result = df.groupby('group', window('ts', '5 days')).apply(f).select('result').collect()
+            # Make sure the key with group and window values are correct
+            for _, i in pdf.id.iteritems():
+                assert expected_key[i][0] == group, "{} != {}".format(expected_key[i][0], group)
+                assert expected_key[i][1] == window_range, \
+                    "{} != {}".format(expected_key[i][1], window_range)
+
+            return pdf.assign(result=[[group] * len(pdf)] * len(pdf))
+
+        result = df.groupby('group', window('ts', '5 days')).applyInPandas(f, df.schema)\
+            .select('id', 'result').collect()
+
+        for r in result:
+            self.assertListEqual(expected[r[0]], r[1])
+
+    def test_case_insensitive_grouping_column(self):
+        # SPARK-31915: case-insensitive grouping column should work.
+        def my_pandas_udf(pdf):
+            return pdf.assign(score=0.5)
 
-        # Check that all group and window_range values from udf matched expected
-        self.assertTrue(all([r[0] for r in result]))
+        df = self.spark.createDataFrame([[1, 1]], ["column", "score"])
+        row = df.groupby('COLUMN').applyInPandas(
+            my_pandas_udf, schema="column integer, score float").first()
+        self.assertEquals(row.asDict(), Row(column=1, score=0.5).asDict())
 
 
 if __name__ == "__main__":
diff --git a/python/pyspark/sql/tests/test_pandas_udf.py b/python/pyspark/sql/tests/test_pandas_udf.py
index 94801a16ce7cd..7fa65f0e792b9 100644
--- a/python/pyspark/sql/tests/test_pandas_udf.py
+++ b/python/pyspark/sql/tests/test_pandas_udf.py
@@ -19,14 +19,12 @@
 
 from pyspark.sql.functions import udf, pandas_udf, PandasUDFType
 from pyspark.sql.types import *
-from pyspark.sql.utils import ParseException
+from pyspark.sql.utils import ParseException, PythonException
 from pyspark.rdd import PythonEvalType
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
 from pyspark.testing.utils import QuietTest
 
-from py4j.protocol import Py4JJavaError
-
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
@@ -157,14 +155,14 @@ def foofoo(x, y):
 
         # plain udf (test for SPARK-23754)
         self.assertRaisesRegexp(
-            Py4JJavaError,
+            PythonException,
             exc_message,
             df.withColumn('v', udf(foo)('id')).collect
         )
 
         # pandas scalar udf
         self.assertRaisesRegexp(
-            Py4JJavaError,
+            PythonException,
             exc_message,
             df.withColumn(
                 'v', pandas_udf(foo, 'double', PandasUDFType.SCALAR)('id')
@@ -173,7 +171,7 @@ def foofoo(x, y):
 
         # pandas grouped map
         self.assertRaisesRegexp(
-            Py4JJavaError,
+            PythonException,
             exc_message,
             df.groupBy('id').apply(
                 pandas_udf(foo, df.schema, PandasUDFType.GROUPED_MAP)
@@ -181,7 +179,7 @@ def foofoo(x, y):
         )
 
         self.assertRaisesRegexp(
-            Py4JJavaError,
+            PythonException,
             exc_message,
             df.groupBy('id').apply(
                 pandas_udf(foofoo, df.schema, PandasUDFType.GROUPED_MAP)
@@ -190,7 +188,7 @@ def foofoo(x, y):
 
         # pandas grouped agg
         self.assertRaisesRegexp(
-            Py4JJavaError,
+            PythonException,
             exc_message,
             df.groupBy('id').agg(
                 pandas_udf(foo, 'double', PandasUDFType.GROUPED_AGG)('id')
@@ -211,14 +209,14 @@ def udf(column):
 
         # Since 0.11.0, PyArrow supports the feature to raise an error for unsafe cast.
         with self.sql_conf({
-                "spark.sql.execution.pandas.arrowSafeTypeConversion": True}):
+                "spark.sql.execution.pandas.convertToArrowArraySafely": True}):
             with self.assertRaisesRegexp(Exception,
                                          "Exception thrown when converting pandas.Series"):
                 df.select(['A']).withColumn('udf', udf('A')).collect()
 
         # Disabling Arrow safe type check.
         with self.sql_conf({
-                "spark.sql.execution.pandas.arrowSafeTypeConversion": False}):
+                "spark.sql.execution.pandas.convertToArrowArraySafely": False}):
             df.select(['A']).withColumn('udf', udf('A')).collect()
 
     def test_pandas_udf_arrow_overflow(self):
@@ -232,13 +230,13 @@ def udf(column):
 
         # When enabling safe type check, Arrow 0.11.0+ disallows overflow cast.
         with self.sql_conf({
-                "spark.sql.execution.pandas.arrowSafeTypeConversion": True}):
+                "spark.sql.execution.pandas.convertToArrowArraySafely": True}):
             with self.assertRaisesRegexp(Exception,
                                          "Exception thrown when converting pandas.Series"):
                 df.withColumn('udf', udf('id')).collect()
 
         # Disabling safe type check, let Arrow do the cast anyway.
-        with self.sql_conf({"spark.sql.execution.pandas.arrowSafeTypeConversion": False}):
+        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
             df.withColumn('udf', udf('id')).collect()
 
 
diff --git a/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py b/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
index 974ad560daebf..224c8ce97f559 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
@@ -357,7 +357,7 @@ def test_complex_expressions(self):
                         plus_one(sum_udf(col('v1'))),
                         sum_udf(plus_one(col('v2'))))
                    .sort(['id', '(v % 2)'])
-                   .toPandas().sort_index(by=['id', '(v % 2)']))
+                   .toPandas().sort_values(by=['id', '(v % 2)']))
 
         expected1 = (df.withColumn('v1', df.v + 1)
                      .withColumn('v2', df.v + 2)
@@ -368,7 +368,7 @@ def test_complex_expressions(self):
                           plus_one(sum(col('v1'))),
                           sum(plus_one(col('v2'))))
                      .sort(['id', '(v % 2)'])
-                     .toPandas().sort_index(by=['id', '(v % 2)']))
+                     .toPandas().sort_values(by=['id', '(v % 2)']))
 
         # Test complex expressions with sql expression, scala pandas UDF and
         # group aggregate pandas UDF
@@ -381,7 +381,7 @@ def test_complex_expressions(self):
                         plus_two(sum_udf(col('v1'))),
                         sum_udf(plus_two(col('v2'))))
                    .sort(['id', '(v % 2)'])
-                   .toPandas().sort_index(by=['id', '(v % 2)']))
+                   .toPandas().sort_values(by=['id', '(v % 2)']))
 
         expected2 = (df.withColumn('v1', df.v + 1)
                      .withColumn('v2', df.v + 2)
@@ -392,7 +392,7 @@ def test_complex_expressions(self):
                           plus_two(sum(col('v1'))),
                           sum(plus_two(col('v2'))))
                      .sort(['id', '(v % 2)'])
-                     .toPandas().sort_index(by=['id', '(v % 2)']))
+                     .toPandas().sort_values(by=['id', '(v % 2)']))
 
         # Test sequential groupby aggregate
         result3 = (df.groupby('id')
@@ -491,6 +491,23 @@ def max_udf(v):
             agg2 = self.spark.sql("select max_udf(id) from table")
             assert_frame_equal(agg1.toPandas(), agg2.toPandas())
 
+    def test_no_predicate_pushdown_through(self):
+        # SPARK-30921: We should not pushdown predicates of PythonUDFs through Aggregate.
+        import numpy as np
+
+        @pandas_udf('float', PandasUDFType.GROUPED_AGG)
+        def mean(x):
+            return np.mean(x)
+
+        df = self.spark.createDataFrame([
+            Row(id=1, foo=42), Row(id=2, foo=1), Row(id=2, foo=2)
+        ])
+
+        agg = df.groupBy('id').agg(mean('foo').alias("mean"))
+        filtered = agg.filter(agg['mean'] > 40.0)
+
+        assert(filtered.collect()[0]["mean"] == 42.0)
+
 
 if __name__ == "__main__":
     from pyspark.sql.tests.test_pandas_udf_grouped_agg import *
diff --git a/python/pyspark/sql/tests/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
index 8e7e85ffe6b73..7260e80e2cfca 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
@@ -445,8 +445,8 @@ def iter_udf_wong_output_size(it):
         with QuietTest(self.sc):
             with self.assertRaisesRegexp(
                     Exception,
-                    "The number of output rows of pandas iterator UDF should be "
-                    "the same with input rows"):
+                    "The length of output in Scalar iterator.*"
+                    "the length of output was 1"):
                 df.select(iter_udf_wong_output_size(col('id'))).collect()
 
         @pandas_udf(LongType(), PandasUDFType.SCALAR_ITER)
@@ -461,7 +461,7 @@ def iter_udf_not_reading_all_input(it):
             with QuietTest(self.sc):
                 with self.assertRaisesRegexp(
                         Exception,
-                        "SQL_SCALAR_PANDAS_ITER_UDF should exhaust the input iterator"):
+                        "pandas iterator UDF should exhaust"):
                     df1.select(iter_udf_not_reading_all_input(col('id'))).collect()
 
     def test_vectorized_udf_chained(self):
@@ -868,7 +868,7 @@ def test_close(batch_iter):
 
             with QuietTest(self.sc):
                 with self.sql_conf({"spark.sql.execution.arrow.maxRecordsPerBatch": 1,
-                                    "spark.sql.pandas.udf.buffer.size": 4}):
+                                    "spark.sql.execution.pandas.udf.buffer.size": 4}):
                     self.spark.range(10).repartition(1) \
                         .select(test_close(col("id"))).limit(2).collect()
                     # wait here because python udf worker will take some time to detect
diff --git a/python/pyspark/sql/tests/test_pandas_udf_typehints.py b/python/pyspark/sql/tests/test_pandas_udf_typehints.py
index 7c83c78f108d9..2582080056864 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_typehints.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_typehints.py
@@ -261,6 +261,48 @@ def test_group_agg_udf_type_hint(self):
         expected = df.groupby('id').agg(mean(df.v).alias('weighted_mean(v, 1.0)')).sort('id')
         assert_frame_equal(expected.toPandas(), actual.toPandas())
 
+    def test_ignore_type_hint_in_group_apply_in_pandas(self):
+        df = self.spark.range(10)
+        exec(
+            "def pandas_plus_one(v: pd.DataFrame) -> pd.DataFrame:\n"
+            "    return v + 1",
+            self.local)
+
+        pandas_plus_one = self.local["pandas_plus_one"]
+
+        actual = df.groupby('id').applyInPandas(pandas_plus_one, schema=df.schema).sort('id')
+        expected = df.selectExpr("id + 1 as id")
+        assert_frame_equal(expected.toPandas(), actual.toPandas())
+
+    def test_ignore_type_hint_in_cogroup_apply_in_pandas(self):
+        df = self.spark.range(10)
+        exec(
+            "def pandas_plus_one(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame:\n"
+            "    return left + 1",
+            self.local)
+
+        pandas_plus_one = self.local["pandas_plus_one"]
+
+        actual = df.groupby('id').cogroup(
+            self.spark.range(10).groupby("id")
+        ).applyInPandas(pandas_plus_one, schema=df.schema).sort('id')
+        expected = df.selectExpr("id + 1 as id")
+        assert_frame_equal(expected.toPandas(), actual.toPandas())
+
+    def test_ignore_type_hint_in_map_in_pandas(self):
+        df = self.spark.range(10)
+        exec(
+            "from typing import Iterator\n"
+            "def pandas_plus_one(iter: Iterator[pd.DataFrame]) -> Iterator[pd.DataFrame]:\n"
+            "    return map(lambda v: v + 1, iter)",
+            self.local)
+
+        pandas_plus_one = self.local["pandas_plus_one"]
+
+        actual = df.mapInPandas(pandas_plus_one, schema=df.schema)
+        expected = df.selectExpr("id + 1 as id")
+        assert_frame_equal(expected.toPandas(), actual.toPandas())
+
 
 if __name__ == "__main__":
     from pyspark.sql.tests.test_pandas_udf_typehints import *
diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py
index 37ce62b7f7159..786564a59936e 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -26,6 +26,7 @@
 
 from pyspark.sql import Row
 from pyspark.sql.functions import col, UserDefinedFunction
+from pyspark.sql.utils import AnalysisException
 from pyspark.sql.types import *
 from pyspark.sql.types import _array_signed_int_typecode_ctype_mappings, _array_type_mappings, \
     _array_unsigned_int_typecode_ctype_mappings, _infer_type, _make_type_verifier, _merge_type
@@ -205,13 +206,13 @@ def test_create_dataframe_from_dict_respects_schema(self):
 
     def test_negative_decimal(self):
         try:
-            self.spark.sql("set spark.sql.legacy.allowNegativeScaleOfDecimal.enabled=true")
+            self.spark.sql("set spark.sql.legacy.allowNegativeScaleOfDecimal=true")
             df = self.spark.createDataFrame([(1, ), (11, )], ["value"])
             ret = df.select(col("value").cast(DecimalType(1, -1))).collect()
             actual = list(map(lambda r: int(r.value), ret))
             self.assertEqual(actual, [0, 10])
         finally:
-            self.spark.sql("set spark.sql.legacy.allowNegativeScaleOfDecimal.enabled=false")
+            self.spark.sql("set spark.sql.legacy.allowNegativeScaleOfDecimal=false")
 
     def test_create_dataframe_from_objects(self):
         data = [MyObject(1, "1"), MyObject(2, "2")]
@@ -437,6 +438,14 @@ def test_cast_to_string_with_udt(self):
         result = df.select(col('point').cast('string'), col('pypoint').cast('string')).head()
         self.assertEqual(result, Row(point=u'(1.0, 2.0)', pypoint=u'[3.0, 4.0]'))
 
+    def test_cast_to_udt_with_udt(self):
+        from pyspark.sql.functions import col
+        row = Row(point=ExamplePoint(1.0, 2.0), python_only_point=PythonOnlyPoint(1.0, 2.0))
+        df = self.spark.createDataFrame([row])
+        self.assertRaises(AnalysisException, lambda: df.select(col("point").cast(PythonOnlyUDT())))
+        self.assertRaises(AnalysisException,
+                          lambda: df.select(col("python_only_point").cast(ExamplePointUDT())))
+
     def test_struct_type(self):
         struct1 = StructType().add("f1", StringType(), True).add("f2", StringType(), True, None)
         struct2 = StructType([StructField("f1", StringType(), True),
@@ -725,7 +734,8 @@ def assertCollectSuccess(typecode, value):
         if sys.version_info[0] < 3:
             all_types = set(['c', 'b', 'B', 'u', 'h', 'H', 'i', 'I', 'l', 'L', 'f', 'd'])
         else:
-            all_types = set(array.typecodes)
+            # PyPy seems not having array.typecodes.
+            all_types = set(['b', 'B', 'u', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd'])
         unsupported_types = all_types - set(supported_types)
         # test unsupported types
         for t in unsupported_types:
diff --git a/python/pyspark/sql/tests/test_udf.py b/python/pyspark/sql/tests/test_udf.py
index 061d3f5e1f7ac..ea7ec9f1e7c0c 100644
--- a/python/pyspark/sql/tests/test_udf.py
+++ b/python/pyspark/sql/tests/test_udf.py
@@ -21,6 +21,8 @@
 import tempfile
 import unittest
 
+import py4j
+
 from pyspark import SparkContext
 from pyspark.sql import SparkSession, Column, Row
 from pyspark.sql.functions import UserDefinedFunction, udf
@@ -357,6 +359,32 @@ def test_udf_registration_returns_udf(self):
             df.select(add_four("id").alias("plus_four")).collect()
         )
 
+    @unittest.skipIf(not test_compiled, test_not_compiled_message)
+    def test_register_java_function(self):
+        self.spark.udf.registerJavaFunction(
+            "javaStringLength", "test.org.apache.spark.sql.JavaStringLength", IntegerType())
+        [value] = self.spark.sql("SELECT javaStringLength('test')").first()
+        self.assertEqual(value, 4)
+
+        self.spark.udf.registerJavaFunction(
+            "javaStringLength2", "test.org.apache.spark.sql.JavaStringLength")
+        [value] = self.spark.sql("SELECT javaStringLength2('test')").first()
+        self.assertEqual(value, 4)
+
+        self.spark.udf.registerJavaFunction(
+            "javaStringLength3", "test.org.apache.spark.sql.JavaStringLength", "integer")
+        [value] = self.spark.sql("SELECT javaStringLength3('test')").first()
+        self.assertEqual(value, 4)
+
+    @unittest.skipIf(not test_compiled, test_not_compiled_message)
+    def test_register_java_udaf(self):
+        self.spark.udf.registerJavaUDAF("javaUDAF", "test.org.apache.spark.sql.MyDoubleAvg")
+        df = self.spark.createDataFrame([(1, "a"), (2, "b"), (3, "a")], ["id", "name"])
+        df.createOrReplaceTempView("df")
+        row = self.spark.sql(
+            "SELECT name, javaUDAF(id) as avg from df group by name order by name desc").first()
+        self.assertEqual(row.asDict(), Row(name='b', avg=102.0).asDict())
+
     def test_non_existed_udf(self):
         spark = self.spark
         self.assertRaisesRegexp(AnalysisException, "Can not load class non_existed_udf",
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 8afff77b723a8..320a68dffe7a3 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -76,7 +76,7 @@ def json(self):
 
     def needConversion(self):
         """
-        Does this type need to conversion between Python object and internal SQL object.
+        Does this type needs conversion between Python object and internal SQL object.
 
         This is used to avoid the unnecessary conversion for ArrayType/MapType/StructType.
         """
@@ -210,17 +210,17 @@ class DecimalType(FractionalType):
 
     The precision can be up to 38, the scale must be less or equal to precision.
 
-    When create a DecimalType, the default precision and scale is (10, 0). When infer
+    When creating a DecimalType, the default precision and scale is (10, 0). When inferring
     schema from decimal.Decimal objects, it will be DecimalType(38, 18).
 
-    :param precision: the maximum total number of digits (default: 10)
+    :param precision: the maximum (i.e. total) number of digits (default: 10)
     :param scale: the number of digits on right side of dot. (default: 0)
     """
 
     def __init__(self, precision=10, scale=0):
         self.precision = precision
         self.scale = scale
-        self.hasPrecisionInfo = True  # this is public API
+        self.hasPrecisionInfo = True  # this is a public API
 
     def simpleString(self):
         return "decimal(%d,%d)" % (self.precision, self.scale)
@@ -457,8 +457,8 @@ class StructType(DataType):
 
     This is the data type representing a :class:`Row`.
 
-    Iterating a :class:`StructType` will iterate its :class:`StructField`\\s.
-    A contained :class:`StructField` can be accessed by name or position.
+    Iterating a :class:`StructType` will iterate over its :class:`StructField`\\s.
+    A contained :class:`StructField` can be accessed by its name or position.
 
     >>> struct1 = StructType([StructField("f1", StringType(), True)])
     >>> struct1["f1"]
@@ -492,8 +492,8 @@ def __init__(self, fields=None):
 
     def add(self, field, data_type=None, nullable=True, metadata=None):
         """
-        Construct a StructType by adding new elements to it to define the schema. The method accepts
-        either:
+        Construct a StructType by adding new elements to it, to define the schema.
+        The method accepts either:
 
             a) A single parameter which is a StructField object.
             b) Between 2 and 4 parameters as (name, data_type, nullable (optional),
@@ -676,7 +676,7 @@ def needConversion(self):
     @classmethod
     def _cachedSqlType(cls):
         """
-        Cache the sqlType() into class, because it's heavy used in `toInternal`.
+        Cache the sqlType() into class, because it's heavily used in `toInternal`.
         """
         if not hasattr(cls, "_cached_sql_type"):
             cls._cached_sql_type = cls.sqlType()
@@ -693,7 +693,7 @@ def fromInternal(self, obj):
 
     def serialize(self, obj):
         """
-        Converts the a user-type object into a SQL datum.
+        Converts a user-type object into a SQL datum.
         """
         raise NotImplementedError("UDT must implement toInternal().")
 
@@ -760,7 +760,7 @@ def __eq__(self, other):
 def _parse_datatype_string(s):
     """
     Parses the given data type string to a :class:`DataType`. The data type string format equals
-    to :class:`DataType.simpleString`, except that top level struct type can omit
+    :class:`DataType.simpleString`, except that the top level struct type can omit
     the ``struct<>`` and atomic types use ``typeName()`` as their format, e.g. use ``byte`` instead
     of ``tinyint`` for :class:`ByteType`. We can also use ``int`` as a short name
     for :class:`IntegerType`. Since Spark 2.3, this also supports a schema in a DDL-formatted
@@ -921,7 +921,7 @@ def _parse_datatype_json_value(json_value):
 # We should be careful here. The size of these types in python depends on C
 # implementation. We need to make sure that this conversion does not lose any
 # precision. Also, JVM only support signed types, when converting unsigned types,
-# keep in mind that it required 1 more bit when stored as singed types.
+# keep in mind that it require 1 more bit when stored as signed types.
 #
 # Reference for C integer size, see:
 # ISO/IEC 9899:201x specification, chapter 5.2.4.2.1 Sizes of integer types <limits.h>.
@@ -959,7 +959,7 @@ def _int_size_to_type(size):
     if size <= 64:
         return LongType
 
-# The list of all supported array typecodes is stored here
+# The list of all supported array typecodes, is stored here
 _array_type_mappings = {
     # Warning: Actual properties for float and double in C is not specified in C.
     # On almost every system supported by both python and JVM, they are IEEE 754
@@ -995,9 +995,9 @@ def _int_size_to_type(size):
     _array_type_mappings['c'] = StringType
 
 # SPARK-21465:
-# In python2, array of 'L' happened to be mistakenly partially supported. To
+# In python2, array of 'L' happened to be mistakenly, just partially supported. To
 # avoid breaking user's code, we should keep this partial support. Below is a
-# dirty hacking to keep this partial support and make the unit test passes
+# dirty hacking to keep this partial support and pass the unit test.
 import platform
 if sys.version_info[0] < 3 and platform.python_implementation() != 'PyPy':
     if 'L' not in _array_type_mappings.keys():
@@ -1071,7 +1071,7 @@ def _infer_schema(row, names=None):
 
 
 def _has_nulltype(dt):
-    """ Return whether there is NullType in `dt` or not """
+    """ Return whether there is a NullType in `dt` or not """
     if isinstance(dt, StructType):
         return any(_has_nulltype(f.dataType) for f in dt.fields)
     elif isinstance(dt, ArrayType):
@@ -1211,7 +1211,7 @@ def _make_type_verifier(dataType, nullable=True, name=None):
 
     This verifier also checks the value of obj against datatype and raises a ValueError if it's not
     within the allowed range, e.g. using 128 as ByteType will overflow. Note that, Python float is
-    not checked, so it will become infinity when cast to Java float if it overflows.
+    not checked, so it will become infinity when cast to Java float, if it overflows.
 
     >>> _make_type_verifier(StructType([]))(None)
     >>> _make_type_verifier(StringType())("")
@@ -1433,7 +1433,7 @@ class Row(tuple):
     ``key in row`` will search through row keys.
 
     Row can be used to create a row object by using named arguments.
-    It is not allowed to omit a named argument to represent the value is
+    It is not allowed to omit a named argument to represent that the value is
     None or missing. This should be explicitly set to None in this case.
 
     NOTE: As of Spark 3.0.0, Rows created from named arguments no longer have
@@ -1524,9 +1524,15 @@ def __new__(cls, *args, **kwargs):
 
     def asDict(self, recursive=False):
         """
-        Return as an dict
+        Return as a dict
 
-        :param recursive: turns the nested Row as dict (default: False).
+        :param recursive: turns the nested Rows to dict (default: False).
+
+        .. note:: If a row contains duplicate field names, e.g., the rows of a join
+            between two :class:`DataFrame` that both have the fields of same names,
+            one of the duplicate fields will be selected by ``asDict``. ``__getitem__``
+            will also return one of the duplicate fields, however returned value might
+            be different to ``asDict``.
 
         >>> Row(name="Alice", age=11).asDict() == {'name': 'Alice', 'age': 11}
         True
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 433c5fc845c59..da68583b04e1c 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -297,17 +297,18 @@ def register(self, name, f, returnType=None):
             >>> spark.sql("SELECT random_udf()").collect()  # doctest: +SKIP
             [Row(random_udf()=82)]
 
-            >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-            >>> @pandas_udf("integer", PandasUDFType.SCALAR)  # doctest: +SKIP
-            ... def add_one(x):
-            ...     return x + 1
+            >>> import pandas as pd  # doctest: +SKIP
+            >>> from pyspark.sql.functions import pandas_udf
+            >>> @pandas_udf("integer")  # doctest: +SKIP
+            ... def add_one(s: pd.Series) -> pd.Series:
+            ...     return s + 1
             ...
             >>> _ = spark.udf.register("add_one", add_one)  # doctest: +SKIP
             >>> spark.sql("SELECT add_one(id) FROM range(3)").collect()  # doctest: +SKIP
             [Row(add_one(id)=1), Row(add_one(id)=2), Row(add_one(id)=3)]
 
-            >>> @pandas_udf("integer", PandasUDFType.GROUPED_AGG)  # doctest: +SKIP
-            ... def sum_udf(v):
+            >>> @pandas_udf("integer")  # doctest: +SKIP
+            ... def sum_udf(v: pd.Series) -> int:
             ...     return v.sum()
             ...
             >>> _ = spark.udf.register("sum_udf", sum_udf)  # doctest: +SKIP
@@ -364,17 +365,20 @@ def registerJavaFunction(self, name, javaClassName, returnType=None):
         >>> from pyspark.sql.types import IntegerType
         >>> spark.udf.registerJavaFunction(
         ...     "javaStringLength", "test.org.apache.spark.sql.JavaStringLength", IntegerType())
-        >>> spark.sql("SELECT javaStringLength('test')").collect()
+        ... # doctest: +SKIP
+        >>> spark.sql("SELECT javaStringLength('test')").collect()  # doctest: +SKIP
         [Row(javaStringLength(test)=4)]
 
         >>> spark.udf.registerJavaFunction(
         ...     "javaStringLength2", "test.org.apache.spark.sql.JavaStringLength")
-        >>> spark.sql("SELECT javaStringLength2('test')").collect()
+        ... # doctest: +SKIP
+        >>> spark.sql("SELECT javaStringLength2('test')").collect()  # doctest: +SKIP
         [Row(javaStringLength2(test)=4)]
 
         >>> spark.udf.registerJavaFunction(
         ...     "javaStringLength3", "test.org.apache.spark.sql.JavaStringLength", "integer")
-        >>> spark.sql("SELECT javaStringLength3('test')").collect()
+        ... # doctest: +SKIP
+        >>> spark.sql("SELECT javaStringLength3('test')").collect()  # doctest: +SKIP
         [Row(javaStringLength3(test)=4)]
         """
 
@@ -394,10 +398,11 @@ def registerJavaUDAF(self, name, javaClassName):
         :param javaClassName: fully qualified name of java class
 
         >>> spark.udf.registerJavaUDAF("javaUDAF", "test.org.apache.spark.sql.MyDoubleAvg")
+        ... # doctest: +SKIP
         >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "a")],["id", "name"])
         >>> df.createOrReplaceTempView("df")
-        >>> spark.sql("SELECT name, javaUDAF(id) as avg from df group by name order by name desc") \
-                .collect()
+        >>> q = "SELECT name, javaUDAF(id) as avg from df group by name order by name desc"
+        >>> spark.sql(q).collect()  # doctest: +SKIP
         [Row(name=u'b', avg=102.0), Row(name=u'a', avg=102.0)]
         """
 
@@ -414,6 +419,9 @@ def _test():
         .appName("sql.udf tests")\
         .getOrCreate()
     globs['spark'] = spark
+    # Hack to skip the unit tests in register. These are currently being tested in proper tests.
+    # We should reenable this test once we completely drop Python 2.
+    del pyspark.sql.udf.UDFRegistration.register
     (failure_count, test_count) = doctest.testmod(
         pyspark.sql.udf, globs=globs,
         optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE)
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index 147ac3325efd9..1d5bc49d252e2 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -18,8 +18,19 @@
 import py4j
 import sys
 
+from pyspark import SparkContext
+
 if sys.version_info.major >= 3:
     unicode = str
+    # Disable exception chaining (PEP 3134) in captured exceptions
+    # in order to hide JVM stacktace.
+    exec("""
+def raise_from(e):
+    raise e from None
+""")
+else:
+    def raise_from(e):
+        raise e
 
 
 class CapturedException(Exception):
@@ -29,7 +40,11 @@ def __init__(self, desc, stackTrace, cause=None):
         self.cause = convert_exception(cause) if cause is not None else None
 
     def __str__(self):
+        sql_conf = SparkContext._jvm.org.apache.spark.sql.internal.SQLConf.get()
+        debug_enabled = sql_conf.pysparkJVMStacktraceEnabled()
         desc = self.desc
+        if debug_enabled:
+            desc = desc + "\n\nJVM stacktrace:\n%s" % self.stackTrace
         # encode unicode instance for python2 for human readable description
         if sys.version_info.major < 3 and isinstance(desc, unicode):
             return str(desc.encode('utf-8'))
@@ -67,6 +82,12 @@ class QueryExecutionException(CapturedException):
     """
 
 
+class PythonException(CapturedException):
+    """
+    Exceptions thrown from Python workers.
+    """
+
+
 class UnknownException(CapturedException):
     """
     None of the above exceptions.
@@ -75,21 +96,30 @@ class UnknownException(CapturedException):
 
 def convert_exception(e):
     s = e.toString()
-    stackTrace = '\n\t at '.join(map(lambda x: x.toString(), e.getStackTrace()))
     c = e.getCause()
+    stacktrace = SparkContext._jvm.org.apache.spark.util.Utils.exceptionString(e)
+
     if s.startswith('org.apache.spark.sql.AnalysisException: '):
-        return AnalysisException(s.split(': ', 1)[1], stackTrace, c)
+        return AnalysisException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.catalyst.analysis'):
-        return AnalysisException(s.split(': ', 1)[1], stackTrace, c)
+        return AnalysisException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.catalyst.parser.ParseException: '):
-        return ParseException(s.split(': ', 1)[1], stackTrace, c)
+        return ParseException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.streaming.StreamingQueryException: '):
-        return StreamingQueryException(s.split(': ', 1)[1], stackTrace, c)
+        return StreamingQueryException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.execution.QueryExecutionException: '):
-        return QueryExecutionException(s.split(': ', 1)[1], stackTrace, c)
+        return QueryExecutionException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('java.lang.IllegalArgumentException: '):
-        return IllegalArgumentException(s.split(': ', 1)[1], stackTrace, c)
-    return UnknownException(s, stackTrace, c)
+        return IllegalArgumentException(s.split(': ', 1)[1], stacktrace, c)
+    if c is not None and (
+            c.toString().startswith('org.apache.spark.api.python.PythonException: ')
+            # To make sure this only catches Python UDFs.
+            and any(map(lambda v: "org.apache.spark.sql.execution.python" in v.toString(),
+                        c.getStackTrace()))):
+        msg = ("\n  An exception was thrown from the Python worker. "
+               "Please see the stack trace below.\n%s" % c.getMessage())
+        return PythonException(msg, stacktrace)
+    return UnknownException(s, stacktrace, c)
 
 
 def capture_sql_exception(f):
@@ -99,7 +129,9 @@ def deco(*a, **kw):
         except py4j.protocol.Py4JJavaError as e:
             converted = convert_exception(e.java_exception)
             if not isinstance(converted, UnknownException):
-                raise converted
+                # Hide where the exception came from that shows a non-Pythonic
+                # JVM exception message.
+                raise_from(converted)
             else:
                 raise
     return deco
diff --git a/python/pyspark/streaming/context.py b/python/pyspark/streaming/context.py
index 769121c19ff4d..6199611940dc9 100644
--- a/python/pyspark/streaming/context.py
+++ b/python/pyspark/streaming/context.py
@@ -17,7 +17,7 @@
 
 from __future__ import print_function
 
-from py4j.java_gateway import java_import
+from py4j.java_gateway import java_import, is_instance_of
 
 from pyspark import RDD, SparkConf
 from pyspark.serializers import NoOpSerializer, UTF8Deserializer, CloudPickleSerializer
@@ -341,8 +341,17 @@ def union(self, *dstreams):
             raise ValueError("All DStreams should have same serializer")
         if len(set(s._slideDuration for s in dstreams)) > 1:
             raise ValueError("All DStreams should have same slide duration")
-        cls = SparkContext._jvm.org.apache.spark.streaming.api.java.JavaDStream
-        jdstreams = SparkContext._gateway.new_array(cls, len(dstreams))
+        jdstream_cls = SparkContext._jvm.org.apache.spark.streaming.api.java.JavaDStream
+        jpair_dstream_cls = SparkContext._jvm.org.apache.spark.streaming.api.java.JavaPairDStream
+        gw = SparkContext._gateway
+        if is_instance_of(gw, dstreams[0]._jdstream, jdstream_cls):
+            cls = jdstream_cls
+        elif is_instance_of(gw, dstreams[0]._jdstream, jpair_dstream_cls):
+            cls = jpair_dstream_cls
+        else:
+            cls_name = dstreams[0]._jdstream.getClass().getCanonicalName()
+            raise TypeError("Unsupported Java DStream class %s" % cls_name)
+        jdstreams = gw.new_array(cls, len(dstreams))
         for i in range(0, len(dstreams)):
             jdstreams[i] = dstreams[i]._jdstream
         return DStream(self._jssc.union(jdstreams), self, dstreams[0]._jrdd_deserializer)
diff --git a/python/pyspark/streaming/tests/test_dstream.py b/python/pyspark/streaming/tests/test_dstream.py
index 7ecdf6b0b12db..89edb23070c69 100644
--- a/python/pyspark/streaming/tests/test_dstream.py
+++ b/python/pyspark/streaming/tests/test_dstream.py
@@ -30,8 +30,9 @@
 
 
 @unittest.skipIf(
-    "pypy" in platform.python_implementation().lower() and "COVERAGE_PROCESS_START" in os.environ,
-    "PyPy implementation causes to hang DStream tests forever when Coverage report is used.")
+    "pypy" in platform.python_implementation().lower(),
+    "The tests fail in PyPy3 implementation for an unknown reason. "
+    "With PyPy, it causes to hang DStream tests forever when Coverage report is used.")
 class BasicOperationTests(PySparkStreamingTestCase):
 
     def test_map(self):
@@ -394,8 +395,9 @@ def failed_func(i):
 
 
 @unittest.skipIf(
-    "pypy" in platform.python_implementation().lower() and "COVERAGE_PROCESS_START" in os.environ,
-    "PyPy implementation causes to hang DStream tests forever when Coverage report is used.")
+    "pypy" in platform.python_implementation().lower(),
+    "The tests fail in PyPy3 implementation for an unknown reason. "
+    "With PyPy, it causes to hang DStream tests forever when Coverage report is used.")
 class WindowFunctionTests(PySparkStreamingTestCase):
 
     timeout = 15
@@ -474,8 +476,9 @@ def func(dstream):
 
 
 @unittest.skipIf(
-    "pypy" in platform.python_implementation().lower() and "COVERAGE_PROCESS_START" in os.environ,
-    "PyPy implementation causes to hang DStream tests forever when Coverage report is used.")
+    "pypy" in platform.python_implementation().lower(),
+    "The tests fail in PyPy3 implementation for an unknown reason. "
+    "With PyPy, it causes to hang DStream tests forever when Coverage report is used.")
 class CheckpointTests(unittest.TestCase):
 
     setupCalled = False
diff --git a/python/pyspark/taskcontext.py b/python/pyspark/taskcontext.py
index d648f63338514..8f419a5e8446a 100644
--- a/python/pyspark/taskcontext.py
+++ b/python/pyspark/taskcontext.py
@@ -16,9 +16,10 @@
 #
 
 from __future__ import print_function
+import json
 
 from pyspark.java_gateway import local_connect_and_auth
-from pyspark.serializers import write_int, UTF8Deserializer
+from pyspark.serializers import read_int, write_int, write_with_length, UTF8Deserializer
 
 
 class TaskContext(object):
@@ -107,22 +108,35 @@ def resources(self):
 
 
 BARRIER_FUNCTION = 1
+ALL_GATHER_FUNCTION = 2
 
 
-def _load_from_socket(port, auth_secret):
+def _load_from_socket(port, auth_secret, function, all_gather_message=None):
     """
     Load data from a given socket, this is a blocking method thus only return when the socket
     connection has been closed.
     """
     (sockfile, sock) = local_connect_and_auth(port, auth_secret)
-    # The barrier() call may block forever, so no timeout
+
+    # The call may block forever, so no timeout
     sock.settimeout(None)
-    # Make a barrier() function call.
-    write_int(BARRIER_FUNCTION, sockfile)
+
+    if function == BARRIER_FUNCTION:
+        # Make a barrier() function call.
+        write_int(function, sockfile)
+    elif function == ALL_GATHER_FUNCTION:
+        # Make a all_gather() function call.
+        write_int(function, sockfile)
+        write_with_length(all_gather_message.encode("utf-8"), sockfile)
+    else:
+        raise ValueError("Unrecognized function type")
     sockfile.flush()
 
     # Collect result.
-    res = UTF8Deserializer().loads(sockfile)
+    len = read_int(sockfile)
+    res = []
+    for i in range(len):
+        res.append(UTF8Deserializer().loads(sockfile))
 
     # Release resources.
     sockfile.close()
@@ -199,7 +213,29 @@ def barrier(self):
             raise Exception("Not supported to call barrier() before initialize " +
                             "BarrierTaskContext.")
         else:
-            _load_from_socket(self._port, self._secret)
+            _load_from_socket(self._port, self._secret, BARRIER_FUNCTION)
+
+    def allGather(self, message=""):
+        """
+        .. note:: Experimental
+
+        This function blocks until all tasks in the same stage have reached this routine.
+        Each task passes in a message and returns with a list of all the messages passed in
+        by each of those tasks.
+
+        .. warning:: In a barrier stage, each task much have the same number of `allGather()`
+            calls, in all possible code branches.
+            Otherwise, you may get the job hanging or a SparkException after timeout.
+
+        .. versionadded:: 3.0.0
+        """
+        if not isinstance(message, str):
+            raise ValueError("Argument `message` must be of type `str`")
+        elif self._port is None or self._secret is None:
+            raise Exception("Not supported to call barrier() before initialize " +
+                            "BarrierTaskContext.")
+        else:
+            return _load_from_socket(self._port, self._secret, ALL_GATHER_FUNCTION, message)
 
     def getTaskInfos(self):
         """
diff --git a/python/pyspark/tests/test_context.py b/python/pyspark/tests/test_context.py
index c7f435a582210..29b49fd4417e1 100644
--- a/python/pyspark/tests/test_context.py
+++ b/python/pyspark/tests/test_context.py
@@ -267,6 +267,30 @@ def test_resources(self):
             resources = sc.resources
             self.assertEqual(len(resources), 0)
 
+    def test_disallow_to_create_spark_context_in_executors(self):
+        # SPARK-32160: SparkContext should not created in executors if the config is set.
+
+        def create_spark_context():
+            conf = SparkConf().set("spark.executor.allowSparkContext", "false")
+            with SparkContext(conf=conf):
+                pass
+
+        with SparkContext("local-cluster[3, 1, 1024]") as sc:
+            with self.assertRaises(Exception) as context:
+                sc.range(2).foreach(lambda _: create_spark_context())
+            self.assertIn("SparkContext should only be created and accessed on the driver.",
+                          str(context.exception))
+
+    def test_allow_to_create_spark_context_in_executors(self):
+        # SPARK-32160: SparkContext can be created in executors.
+
+        def create_spark_context():
+            with SparkContext():
+                pass
+
+        with SparkContext("local-cluster[3, 1, 1024]") as sc:
+            sc.range(2).foreach(lambda _: create_spark_context())
+
 
 class ContextTestsWithResources(unittest.TestCase):
 
diff --git a/python/pyspark/tests/test_rdd.py b/python/pyspark/tests/test_rdd.py
index 15cc48ae2153d..cf582201893aa 100644
--- a/python/pyspark/tests/test_rdd.py
+++ b/python/pyspark/tests/test_rdd.py
@@ -166,6 +166,17 @@ def test_zip_chaining(self):
             set([(x, (x, x)) for x in 'abc'])
         )
 
+    def test_union_pair_rdd(self):
+        # SPARK-31788: test if pair RDDs can be combined by union.
+        rdd = self.sc.parallelize([1, 2])
+        pair_rdd = rdd.zip(rdd)
+        unionRDD = self.sc.union([pair_rdd, pair_rdd])
+        self.assertEqual(
+            set(unionRDD.collect()),
+            set([(1, 1), (2, 2), (1, 1), (2, 2)])
+        )
+        self.assertEqual(unionRDD.count(), 4)
+
     def test_deleting_input_files(self):
         # Regression test for SPARK-1025
         tempFile = tempfile.NamedTemporaryFile(delete=False)
@@ -783,6 +794,67 @@ def fail_last(x):
         for i in range(4):
             self.assertEqual(i, next(it))
 
+    def test_multiple_group_jobs(self):
+        import threading
+        group_a = "job_ids_to_cancel"
+        group_b = "job_ids_to_run"
+
+        threads = []
+        thread_ids = range(4)
+        thread_ids_to_cancel = [i for i in thread_ids if i % 2 == 0]
+        thread_ids_to_run = [i for i in thread_ids if i % 2 != 0]
+
+        # A list which records whether job is cancelled.
+        # The index of the array is the thread index which job run in.
+        is_job_cancelled = [False for _ in thread_ids]
+
+        def run_job(job_group, index):
+            """
+            Executes a job with the group ``job_group``. Each job waits for 3 seconds
+            and then exits.
+            """
+            try:
+                self.sc.parallelize([15]).map(lambda x: time.sleep(x)) \
+                    .collectWithJobGroup(job_group, "test rdd collect with setting job group")
+                is_job_cancelled[index] = False
+            except Exception:
+                # Assume that exception means job cancellation.
+                is_job_cancelled[index] = True
+
+        # Test if job succeeded when not cancelled.
+        run_job(group_a, 0)
+        self.assertFalse(is_job_cancelled[0])
+
+        # Run jobs
+        for i in thread_ids_to_cancel:
+            t = threading.Thread(target=run_job, args=(group_a, i))
+            t.start()
+            threads.append(t)
+
+        for i in thread_ids_to_run:
+            t = threading.Thread(target=run_job, args=(group_b, i))
+            t.start()
+            threads.append(t)
+
+        # Wait to make sure all jobs are executed.
+        time.sleep(3)
+        # And then, cancel one job group.
+        self.sc.cancelJobGroup(group_a)
+
+        # Wait until all threads launching jobs are finished.
+        for t in threads:
+            t.join()
+
+        for i in thread_ids_to_cancel:
+            self.assertTrue(
+                is_job_cancelled[i],
+                "Thread {i}: Job in group A was not cancelled.".format(i=i))
+
+        for i in thread_ids_to_run:
+            self.assertFalse(
+                is_job_cancelled[i],
+                "Thread {i}: Job in group B did not succeeded.".format(i=i))
+
 
 if __name__ == "__main__":
     import unittest
diff --git a/python/pyspark/tests/test_taskcontext.py b/python/pyspark/tests/test_taskcontext.py
index 6095a384679af..b2fa9919c65da 100644
--- a/python/pyspark/tests/test_taskcontext.py
+++ b/python/pyspark/tests/test_taskcontext.py
@@ -127,12 +127,32 @@ def f(iterator):
 
         def context_barrier(x):
             tc = BarrierTaskContext.get()
-            time.sleep(random.randint(1, 10))
+            time.sleep(random.randint(1, 5) * 2)
             tc.barrier()
             return time.time()
 
         times = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
-        self.assertTrue(max(times) - min(times) < 1)
+        self.assertTrue(max(times) - min(times) < 2)
+
+    def test_all_gather(self):
+        """
+        Verify that BarrierTaskContext.allGather() performs global sync among all barrier tasks
+        within a stage and passes messages properly.
+        """
+        rdd = self.sc.parallelize(range(10), 4)
+
+        def f(iterator):
+            yield sum(iterator)
+
+        def context_barrier(x):
+            tc = BarrierTaskContext.get()
+            time.sleep(random.randint(1, 10))
+            out = tc.allGather(str(tc.partitionId()))
+            pids = [int(e) for e in out]
+            return pids
+
+        pids = rdd.barrier().mapPartitions(f).map(context_barrier).collect()[0]
+        self.assertEqual(pids, [0, 1, 2, 3])
 
     def test_barrier_infos(self):
         """
@@ -215,7 +235,7 @@ def f(iterator):
 
         def context_barrier(x):
             tc = BarrierTaskContext.get()
-            time.sleep(random.randint(1, 10))
+            time.sleep(random.randint(1, 5) * 2)
             tc.barrier()
             return (time.time(), os.getpid())
 
@@ -223,7 +243,7 @@ def context_barrier(x):
         times = list(map(lambda x: x[0], result))
         pids = list(map(lambda x: x[1], result))
         # check both barrier and worker reuse effect
-        self.assertTrue(max(times) - min(times) < 1)
+        self.assertTrue(max(times) - min(times) < 2)
         for pid in pids:
             self.assertTrue(pid in worker_pids)
 
diff --git a/python/pyspark/tests/test_worker.py b/python/pyspark/tests/test_worker.py
index 9d7deb23da604..f51d4b24f11b9 100644
--- a/python/pyspark/tests/test_worker.py
+++ b/python/pyspark/tests/test_worker.py
@@ -98,6 +98,15 @@ def raise_exception(_):
             self.assertRaises(Exception, lambda: rdd.foreach(raise_exception))
         self.assertEqual(100, rdd.map(str).count())
 
+    def test_after_non_exception_error(self):
+        # SPARK-33339: Pyspark application will hang due to non Exception
+        def raise_system_exit(_):
+            raise SystemExit()
+        rdd = self.sc.parallelize(range(100), 1)
+        with QuietTest(self.sc):
+            self.assertRaises(Exception, lambda: rdd.foreach(raise_system_exit))
+        self.assertEqual(100, rdd.map(str).count())
+
     def test_after_jvm_exception(self):
         tempFile = tempfile.NamedTemporaryFile(delete=False)
         tempFile.write(b"Hello World!")
@@ -137,7 +146,7 @@ def count():
         t.daemon = True
         t.start()
         t.join(5)
-        self.assertTrue(not t.isAlive())
+        self.assertTrue(not t.is_alive())
         self.assertEqual(100000, rdd.count())
 
     def test_with_different_versions_of_python(self):
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index 93137560de25e..6e53e57f156e6 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -114,33 +114,6 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
-def _warn_pin_thread(name):
-    if os.environ.get("PYSPARK_PIN_THREAD", "false").lower() == "true":
-        msg = (
-            "PYSPARK_PIN_THREAD feature is enabled. "
-            "However, note that it cannot inherit the local properties from the parent thread "
-            "although it isolates each thread on PVM and JVM with its own local properties. "
-            "\n"
-            "To work around this, you should manually copy and set the local properties from "
-            "the parent thread to the child thread when you create another thread.")
-    else:
-        msg = (
-            "Currently, '%s' (set to local properties) with multiple threads does "
-            "not properly work. "
-            "\n"
-            "Internally threads on PVM and JVM are not synced, and JVM thread can be reused "
-            "for multiple threads on PVM, which fails to isolate local properties for each "
-            "thread on PVM. "
-            "\n"
-            "To work around this, you can set PYSPARK_PIN_THREAD to true (see SPARK-22340). "
-            "However, note that it cannot inherit the local properties from the parent thread "
-            "although it isolates each thread on PVM and JVM with its own local properties. "
-            "\n"
-            "To work around this, you should manually copy and set the local properties from "
-            "the parent thread to the child thread when you create another thread." % name)
-    warnings.warn(msg, UserWarning)
-
-
 def _print_missing_jar(lib_name, pkg_name, jar_name, spark_version):
     print("""
 ________________________________________________________________________________________________
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index 1abc41279ebe8..e6410c11cf86d 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.0.dev0"
+__version__ = "3.0.2.dev0"
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index 5d498421e259d..0bce87d5c9f90 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -19,6 +19,7 @@
 Worker that receives input from Piped RDD.
 """
 from __future__ import print_function
+from __future__ import absolute_import
 import os
 import sys
 import time
@@ -35,7 +36,7 @@
 from pyspark.java_gateway import local_connect_and_auth
 from pyspark.taskcontext import BarrierTaskContext, TaskContext
 from pyspark.files import SparkFiles
-from pyspark.resourceinformation import ResourceInformation
+from pyspark.resource import ResourceInformation
 from pyspark.rdd import PythonEvalType
 from pyspark.serializers import write_with_length, write_int, read_long, read_bool, \
     write_long, read_int, SpecialLengths, UTF8Deserializer, PickleSerializer, \
@@ -306,7 +307,7 @@ def read_udfs(pickleSer, infile, eval_type):
 
         # NOTE: if timezone is set here, that implies respectSessionTimeZone is True
         timezone = runner_conf.get("spark.sql.session.timeZone", None)
-        safecheck = runner_conf.get("spark.sql.execution.pandas.arrowSafeTypeConversion",
+        safecheck = runner_conf.get("spark.sql.execution.pandas.convertToArrowArraySafely",
                                     "false").lower() == 'true'
         # Used by SQL_GROUPED_MAP_PANDAS_UDF and SQL_SCALAR_PANDAS_UDF when returning StructType
         assign_cols_by_name = runner_conf.get(
@@ -341,7 +342,7 @@ def read_udfs(pickleSer, infile, eval_type):
             pickleSer, infile, eval_type, runner_conf, udf_index=0)
 
         def func(_, iterator):
-            num_input_rows = [0]
+            num_input_rows = [0]  # TODO(SPARK-29909): Use nonlocal after we drop Python 2.
 
             def map_batch(batch):
                 udf_args = [batch[offset] for offset in arg_offsets]
@@ -357,8 +358,13 @@ def map_batch(batch):
             num_output_rows = 0
             for result_batch, result_type in result_iter:
                 num_output_rows += len(result_batch)
+                # This assert is for Scalar Iterator UDF to fail fast.
+                # The length of the entire input can only be explicitly known
+                # by consuming the input iterator in user side. Therefore,
+                # it's very unlikely the output length is higher than
+                # input length.
                 assert is_map_iter or num_output_rows <= num_input_rows[0], \
-                    "Pandas MAP_ITER UDF outputted more rows than input rows."
+                    "Pandas SCALAR_ITER UDF outputted more rows than input rows."
                 yield (result_batch, result_type)
 
             if is_scalar_iter:
@@ -367,14 +373,14 @@ def map_batch(batch):
                 except StopIteration:
                     pass
                 else:
-                    raise RuntimeError("SQL_SCALAR_PANDAS_ITER_UDF should exhaust the input "
+                    raise RuntimeError("pandas iterator UDF should exhaust the input "
                                        "iterator.")
 
-            if is_scalar_iter and num_output_rows != num_input_rows[0]:
-                raise RuntimeError("The number of output rows of pandas iterator UDF should be "
-                                   "the same with input rows. The input rows number is %d but the "
-                                   "output rows number is %d." %
-                                   (num_input_rows[0], num_output_rows))
+                if num_output_rows != num_input_rows[0]:
+                    raise RuntimeError(
+                        "The length of output in Scalar iterator pandas UDF should be "
+                        "the same with the input's; however, the length of output was %d and the "
+                        "length of input was %d." % (num_output_rows, num_input_rows[0]))
 
         # profiling is not supported for UDF
         return func, None, ser, ser
@@ -465,7 +471,7 @@ def main(infile, outfile):
         version = utf8_deserializer.loads(infile)
         if version != "%d.%d" % sys.version_info[:2]:
             raise Exception(("Python in worker has different version %s than that in " +
-                             "driver %s, PySpark cannot run with different minor versions." +
+                             "driver %s, PySpark cannot run with different minor versions. " +
                              "Please check environment variables PYSPARK_PYTHON and " +
                              "PYSPARK_DRIVER_PYTHON are correctly set.") %
                             ("%d.%d" % sys.version_info[:2], version))
@@ -602,7 +608,7 @@ def process():
         # reuse.
         TaskContext._setTaskContext(None)
         BarrierTaskContext._setTaskContext(None)
-    except Exception:
+    except BaseException:
         try:
             exc_info = traceback.format_exc()
             if isinstance(exc_info, bytes):
@@ -616,7 +622,7 @@ def process():
         except IOError:
             # JVM close the socket
             pass
-        except Exception:
+        except BaseException:
             # Write the error to stderr if it happened while serializing
             print("PySpark worker failed with exception:", file=sys.stderr)
             print(traceback.format_exc(), file=sys.stderr)
diff --git a/python/run-tests.py b/python/run-tests.py
index b677a5134ec93..f269478b2a2b8 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -51,7 +51,7 @@ def print_red(text):
     print('\033[31m' + text + '\033[0m')
 
 
-SKIPPED_TESTS = Manager().dict()
+SKIPPED_TESTS = None
 LOG_FILE = os.path.join(SPARK_HOME, "python/unit-tests.log")
 FAILURE_REPORTING_LOCK = Lock()
 LOGGER = logging.getLogger()
@@ -75,7 +75,9 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
         'SPARK_PREPEND_CLASSES': '1',
         'PYSPARK_PYTHON': which(pyspark_python),
         'PYSPARK_DRIVER_PYTHON': which(pyspark_python),
-        'PYSPARK_ROW_FIELD_SORTING_ENABLED': 'true'
+        'PYSPARK_ROW_FIELD_SORTING_ENABLED': 'true',
+        # Preserve legacy nested timezone behavior for pyarrow>=2, remove after SPARK-32285
+        'PYARROW_IGNORE_TIMEZONE': '1'
     })
 
     # Create a unique temp directory under 'target/' for each run. The TMPDIR variable is
@@ -85,12 +87,17 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
         tmp_dir = os.path.join(target_dir, str(uuid.uuid4()))
     os.mkdir(tmp_dir)
     env["TMPDIR"] = tmp_dir
+    metastore_dir = os.path.join(tmp_dir, str(uuid.uuid4()))
+    while os.path.isdir(metastore_dir):
+        metastore_dir = os.path.join(metastore_dir, str(uuid.uuid4()))
+    os.mkdir(metastore_dir)
 
     # Also override the JVM's temp directory by setting driver and executor options.
     java_options = "-Djava.io.tmpdir={0} -Dio.netty.tryReflectionSetAccessible=true".format(tmp_dir)
     spark_args = [
         "--conf", "spark.driver.extraJavaOptions='{0}'".format(java_options),
         "--conf", "spark.executor.extraJavaOptions='{0}'".format(java_options),
+        "--conf", "spark.sql.warehouse.dir='{0}'".format(metastore_dir),
         "pyspark-shell"
     ]
     env["PYSPARK_SUBMIT_ARGS"] = " ".join(spark_args)
@@ -142,6 +149,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
             skipped_counts = len(skipped_tests)
             if skipped_counts > 0:
                 key = (pyspark_python, test_name)
+                assert SKIPPED_TESTS is not None
                 SKIPPED_TESTS[key] = skipped_tests
             per_test_output.close()
         except:
@@ -161,7 +169,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
 
 
 def get_default_python_executables():
-    python_execs = [x for x in ["python3.6", "python2.7", "pypy"] if which(x)]
+    python_execs = [x for x in ["python3.6", "python2.7", "pypy3", "pypy"] if which(x)]
 
     if "python3.6" not in python_execs:
         p = which("python3")
@@ -322,4 +330,5 @@ def process_queue(task_queue):
 
 
 if __name__ == "__main__":
+    SKIPPED_TESTS = Manager().dict()
     main()
diff --git a/python/setup.py b/python/setup.py
index 40b49aaeeb27c..78b5a691a7ce3 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -16,14 +16,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import glob
 import os
 import sys
 from setuptools import setup
 from shutil import copyfile, copytree, rmtree
 
-if sys.version_info < (3, 6):
-    print("Python versions prior to 3.6 are not supported for pip installed PySpark.",
+if sys.version_info < (2, 7):
+    print("Python versions prior to 2.7 are not supported for pip installed PySpark.",
           file=sys.stderr)
     sys.exit(-1)
 
@@ -206,7 +207,7 @@ def _supports_symlinks():
             'pyspark.examples.src.main.python': ['*.py', '*/*.py']},
         scripts=scripts,
         license='http://www.apache.org/licenses/LICENSE-2.0',
-        install_requires=['py4j==0.10.8.1'],
+        install_requires=['py4j==0.10.9'],
         extras_require={
             'ml': ['numpy>=1.7'],
             'mllib': ['numpy>=1.7'],
@@ -225,6 +226,7 @@ def _supports_symlinks():
             'Programming Language :: Python :: 3.6',
             'Programming Language :: Python :: 3.7',
             'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3.9',
             'Programming Language :: Python :: Implementation :: CPython',
             'Programming Language :: Python :: Implementation :: PyPy']
     )
diff --git a/repl/pom.xml b/repl/pom.xml
index 3aee53cc9599a..92911ee094b4d 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
index 5428fa4ee9df7..818b9900d0836 100644
--- a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
@@ -233,6 +233,7 @@ class ExecutorClassLoaderSuite
       .setMaster("local")
       .setAppName("executor-class-loader-test")
       .set("spark.network.timeout", "11s")
+      .set("spark.network.timeoutInterval", "11s")
       .set("spark.repl.class.outputDir", tempDir1.getAbsolutePath)
     val sc = new SparkContext(conf)
     try {
diff --git a/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala
index 4795306692f7a..e11a54bc88070 100644
--- a/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala
@@ -380,6 +380,67 @@ class SingletonReplSuite extends SparkFunSuite {
     assertDoesNotContain("Exception", output)
   }
 
+  test("SPARK-31399: should clone+clean line object w/ non-serializable state in ClosureCleaner") {
+    // Test ClosureCleaner when a closure captures the enclosing `this` REPL line object, and that
+    // object contains an unused non-serializable field.
+    // Specifically, the closure in this test case contains a directly nested closure, and the
+    // capture is triggered by the inner closure.
+    // `ns` should be nulled out, but `topLevelValue` should stay intact.
+
+    // Can't use :paste mode because PipedOutputStream/PipedInputStream doesn't work well with the
+    // EOT control character (i.e. Ctrl+D).
+    // Just write things on a single line to emulate :paste mode.
+
+    // NOTE: in order for this test case to trigger the intended scenario, the following three
+    //       variables need to be in the same "input", which will make the REPL pack them into the
+    //       same REPL line object:
+    //         - ns: a non-serializable state, not accessed by the closure;
+    //         - topLevelValue: a serializable state, accessed by the closure;
+    //         - closure: the starting closure, captures the enclosing REPL line object.
+    val output = runInterpreter(
+      """
+        |class NotSerializableClass(val x: Int)
+        |val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
+        |(j: Int) => {
+        |  (1 to j).flatMap { x =>
+        |    (1 to x).map { y => y + topLevelValue }
+        |  }
+        |}
+        |val r = sc.parallelize(0 to 2).map(closure).collect
+      """.stripMargin)
+    assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = " +
+      "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+    assertDoesNotContain("Exception", output)
+  }
+
+  test("SPARK-31399: ClosureCleaner should discover indirectly nested closure in inner class") {
+    // Similar to the previous test case, but with indirect closure nesting instead.
+    // There's still nested closures involved, but the inner closure is indirectly nested in the
+    // outer closure, with a level of inner class in between them.
+    // This changes how the inner closure references/captures the outer closure/enclosing `this`
+    // REPL line object, and covers a different code path in inner closure discovery.
+
+    // `ns` should be nulled out, but `topLevelValue` should stay intact.
+
+    val output = runInterpreter(
+      """
+        |class NotSerializableClass(val x: Int)
+        |val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
+        |(j: Int) => {
+        |  class InnerFoo {
+        |    val innerClosure = (x: Int) => (1 to x).map { y => y + topLevelValue }
+        |  }
+        |  val innerFoo = new InnerFoo
+        |  (1 to j).flatMap(innerFoo.innerClosure)
+        |}
+        |val r = sc.parallelize(0 to 2).map(closure).collect
+      """.stripMargin)
+    assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = " +
+       "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+    assertDoesNotContain("Array(Vector(), Vector(1null), Vector(1null, 1null, 2null)", output)
+    assertDoesNotContain("Exception", output)
+  }
+
   test("newProductSeqEncoder with REPL defined class") {
     val output = runInterpreter(
       """
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 8c46738b259bc..ef6c38a64ec8c 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
@@ -29,7 +29,8 @@
   <name>Spark Project Kubernetes</name>
   <properties>
     <sbt.project.name>kubernetes</sbt.project.name>
-    <kubernetes.client.version>4.6.4</kubernetes.client.version>
+    <!-- Note: Please update the kubernetes client version in kubernetes/integration-tests/pom.xml -->
+    <kubernetes.client.version>4.9.2</kubernetes.client.version>
   </properties>
 
   <dependencies>
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index b326650476add..8a415058cc690 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -33,6 +33,7 @@ private[spark] object Config extends Logging {
         "to .kube/config under your home directory.  If not specified then your current " +
         "context is used.  You can always override specific aspects of the config file " +
         "provided configuration using other Spark on K8S configuration options.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -40,12 +41,14 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.driver.master")
       .doc("The internal Kubernetes master (API server) address " +
         "to be used for driver to request executors.")
+      .version("3.0.0")
       .stringConf
       .createWithDefault(KUBERNETES_MASTER_INTERNAL_URL)
 
   val KUBERNETES_NAMESPACE =
     ConfigBuilder("spark.kubernetes.namespace")
       .doc("The namespace that will be used for running the driver and executor pods.")
+      .version("2.3.0")
       .stringConf
       .createWithDefault("default")
 
@@ -54,22 +57,26 @@ private[spark] object Config extends Logging {
       .doc("Container image to use for Spark containers. Individual container types " +
         "(e.g. driver or executor) can also be configured to use different images if desired, " +
         "by setting the container type-specific image name.")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
   val DRIVER_CONTAINER_IMAGE =
     ConfigBuilder("spark.kubernetes.driver.container.image")
       .doc("Container image to use for the driver.")
+      .version("2.3.0")
       .fallbackConf(CONTAINER_IMAGE)
 
   val EXECUTOR_CONTAINER_IMAGE =
     ConfigBuilder("spark.kubernetes.executor.container.image")
       .doc("Container image to use for the executors.")
+      .version("2.3.0")
       .fallbackConf(CONTAINER_IMAGE)
 
   val CONTAINER_IMAGE_PULL_POLICY =
     ConfigBuilder("spark.kubernetes.container.image.pullPolicy")
       .doc("Kubernetes image pull policy. Valid values are Always, Never, and IfNotPresent.")
+      .version("2.3.0")
       .stringConf
       .checkValues(Set("Always", "Never", "IfNotPresent"))
       .createWithDefault("IfNotPresent")
@@ -78,6 +85,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.container.image.pullSecrets")
       .doc("Comma separated list of the Kubernetes secrets used " +
         "to access private image registries.")
+      .version("2.4.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
@@ -95,24 +103,28 @@ private[spark] object Config extends Logging {
   val SUBMISSION_CLIENT_REQUEST_TIMEOUT =
     ConfigBuilder("spark.kubernetes.submission.requestTimeout")
       .doc("request timeout to be used in milliseconds for starting the driver")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
   val SUBMISSION_CLIENT_CONNECTION_TIMEOUT =
     ConfigBuilder("spark.kubernetes.submission.connectionTimeout")
       .doc("connection timeout to be used in milliseconds for starting the driver")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
   val DRIVER_CLIENT_REQUEST_TIMEOUT =
     ConfigBuilder("spark.kubernetes.driver.requestTimeout")
       .doc("request timeout to be used in milliseconds for driver to request executors")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
   val DRIVER_CLIENT_CONNECTION_TIMEOUT =
     ConfigBuilder("spark.kubernetes.driver.connectionTimeout")
       .doc("connection timeout to be used in milliseconds for driver to request executors")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
@@ -122,48 +134,56 @@ private[spark] object Config extends Logging {
         "this service account when requesting executor pods from the API server. If specific " +
         "credentials are given for the driver pod to use, the driver will favor " +
         "using those credentials instead.")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_DRIVER_LIMIT_CORES =
     ConfigBuilder("spark.kubernetes.driver.limit.cores")
       .doc("Specify the hard cpu limit for the driver pod")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_DRIVER_REQUEST_CORES =
     ConfigBuilder("spark.kubernetes.driver.request.cores")
       .doc("Specify the cpu request for the driver pod")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_DRIVER_SUBMIT_CHECK =
     ConfigBuilder("spark.kubernetes.submitInDriver")
     .internal()
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(false)
 
   val KUBERNETES_EXECUTOR_LIMIT_CORES =
     ConfigBuilder("spark.kubernetes.executor.limit.cores")
       .doc("Specify the hard cpu limit for each executor pod")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_EXECUTOR_SCHEDULER_NAME =
     ConfigBuilder("spark.kubernetes.executor.scheduler.name")
       .doc("Specify the scheduler name for each executor pod")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_EXECUTOR_REQUEST_CORES =
     ConfigBuilder("spark.kubernetes.executor.request.cores")
       .doc("Specify the cpu request for each executor pod")
+      .version("2.4.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_DRIVER_POD_NAME =
     ConfigBuilder("spark.kubernetes.driver.pod.name")
       .doc("Name of the driver pod.")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
@@ -171,12 +191,14 @@ private[spark] object Config extends Logging {
   val KUBERNETES_DRIVER_POD_NAME_PREFIX =
     ConfigBuilder("spark.kubernetes.driver.resourceNamePrefix")
       .internal()
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_EXECUTOR_POD_NAME_PREFIX =
     ConfigBuilder("spark.kubernetes.executor.podNamePrefix")
       .doc("Prefix to use in front of the executor pod names.")
+      .version("2.3.0")
       .internal()
       .stringConf
       .createOptional
@@ -184,6 +206,7 @@ private[spark] object Config extends Logging {
   val KUBERNETES_ALLOCATION_BATCH_SIZE =
     ConfigBuilder("spark.kubernetes.allocation.batch.size")
       .doc("Number of pods to launch at once in each round of executor allocation.")
+      .version("2.3.0")
       .intConf
       .checkValue(value => value > 0, "Allocation batch size should be a positive integer")
       .createWithDefault(5)
@@ -191,6 +214,7 @@ private[spark] object Config extends Logging {
   val KUBERNETES_ALLOCATION_BATCH_DELAY =
     ConfigBuilder("spark.kubernetes.allocation.batch.delay")
       .doc("Time to wait between each round of executor allocation.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(value => value > 0, "Allocation batch delay must be a positive time value.")
       .createWithDefaultString("1s")
@@ -199,6 +223,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.executor.lostCheck.maxAttempts")
       .doc("Maximum number of attempts allowed for checking the reason of an executor loss " +
         "before it is assumed that the executor failed.")
+      .version("2.3.0")
       .intConf
       .checkValue(value => value > 0, "Maximum attempts of checks of executor lost reason " +
         "must be a positive integer")
@@ -208,12 +233,14 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.submission.waitAppCompletion")
       .doc("In cluster mode, whether to wait for the application to finish before exiting the " +
         "launcher process.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
   val REPORT_INTERVAL =
     ConfigBuilder("spark.kubernetes.report.interval")
       .doc("Interval between reports of the current app status in cluster mode.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(interval => interval > 0, s"Logging interval must be a positive time value.")
       .createWithDefaultString("1s")
@@ -222,6 +249,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.executor.apiPollingInterval")
       .doc("Interval between polls against the Kubernetes API server to inspect the " +
         "state of executors.")
+      .version("2.4.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(interval => interval > 0, s"API server polling interval must be a" +
         " positive time value.")
@@ -231,6 +259,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.executor.eventProcessingInterval")
       .doc("Interval between successive inspection of executor events sent from the" +
         " Kubernetes API.")
+      .version("2.4.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(interval => interval > 0, s"Event processing interval must be a positive" +
         " time value.")
@@ -240,6 +269,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.memoryOverheadFactor")
       .doc("This sets the Memory Overhead Factor that will allocate memory to non-JVM jobs " +
         "which in the case of JVM tasks will default to 0.10 and 0.40 for non-JVM jobs")
+      .version("2.4.0")
       .doubleConf
       .checkValue(mem_overhead => mem_overhead >= 0 && mem_overhead < 1,
         "Ensure that memory overhead is a double between 0 --> 1.0")
@@ -248,6 +278,7 @@ private[spark] object Config extends Logging {
   val PYSPARK_MAJOR_PYTHON_VERSION =
     ConfigBuilder("spark.kubernetes.pyspark.pythonVersion")
       .doc("This sets the major Python version. Either 2 or 3. (Python2 or Python3)")
+      .version("2.4.0")
       .stringConf
       .checkValue(pv => List("2", "3").contains(pv),
         "Ensure that major Python version is either Python2 or Python3")
@@ -258,6 +289,7 @@ private[spark] object Config extends Logging {
       .doc("Specify the local location of the krb5.conf file to be mounted on the driver " +
         "and executors for Kerberos. Note: The KDC defined needs to be " +
         "visible from inside the containers ")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -266,6 +298,7 @@ private[spark] object Config extends Logging {
       .doc("Specify the name of the ConfigMap, containing the krb5.conf file, to be mounted " +
         "on the driver and executors for Kerberos. Note: The KDC defined" +
         "needs to be visible from inside the containers ")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -273,6 +306,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.hadoop.configMapName")
       .doc("Specify the name of the ConfigMap, containing the HADOOP_CONF_DIR files, " +
         "to be mounted on the driver and executors for custom Hadoop configuration.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -280,6 +314,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.kerberos.tokenSecret.name")
       .doc("Specify the name of the secret where your existing delegation tokens are stored. " +
         "This removes the need for the job user to provide any keytab for launching a job")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -287,13 +322,15 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.kerberos.tokenSecret.itemKey")
       .doc("Specify the item key of the data where your existing delegation tokens are stored. " +
         "This removes the need for the job user to provide any keytab for launching a job")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val APP_RESOURCE_TYPE =
     ConfigBuilder("spark.kubernetes.resource.type")
-      .doc("This sets the resource type internally")
       .internal()
+      .doc("This sets the resource type internally")
+      .version("2.4.1")
       .stringConf
       .checkValues(Set(APP_RESOURCE_TYPE_JAVA, APP_RESOURCE_TYPE_PYTHON, APP_RESOURCE_TYPE_R))
       .createOptional
@@ -304,30 +341,35 @@ private[spark] object Config extends Logging {
         "their medium set to Memory so that they will be created as tmpfs (i.e. RAM) backed " +
         "volumes. This may improve performance but scratch space usage will count towards " +
         "your pods memory limit so you may wish to request more memory.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val KUBERNETES_DRIVER_PODTEMPLATE_FILE =
     ConfigBuilder("spark.kubernetes.driver.podTemplateFile")
       .doc("File containing a template pod spec for the driver")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_EXECUTOR_PODTEMPLATE_FILE =
     ConfigBuilder("spark.kubernetes.executor.podTemplateFile")
       .doc("File containing a template pod spec for executors")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_DRIVER_PODTEMPLATE_CONTAINER_NAME =
     ConfigBuilder("spark.kubernetes.driver.podTemplateContainerName")
       .doc("container name to be used as a basis for the driver in the given pod template")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME =
     ConfigBuilder("spark.kubernetes.executor.podTemplateContainerName")
       .doc("container name to be used as a basis for executors in the given pod template")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -340,12 +382,14 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.executor.deleteOnTermination")
       .doc("If set to false then executor pods will not be deleted in case " +
         "of failure or normal termination.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val KUBERNETES_DYN_ALLOC_KILL_GRACE_PERIOD =
     ConfigBuilder("spark.kubernetes.dynamicAllocation.deleteGracePeriod")
       .doc("How long to wait for executors to shut down gracefully before a forceful kill.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("5s")
 
@@ -353,6 +397,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.appKillPodDeletionGracePeriod")
       .doc("Time to wait for graceful deletion of Spark pods when spark-submit" +
         " is used for killing an application.")
+      .version("3.0.0")
       .timeConf(TimeUnit.SECONDS)
       .createOptional
 
@@ -360,11 +405,32 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.file.upload.path")
       .doc("Hadoop compatible file system path where files from the local file system " +
         "will be uploded to in cluster mode.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
+  val KUBERNETES_EXECUTOR_CHECK_ALL_CONTAINERS =
+    ConfigBuilder("spark.kubernetes.executor.checkAllContainers")
+      .doc("If set to true, all containers in the executor pod will be checked when reporting" +
+        "executor status.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA =
+    ConfigBuilder("spark.kubernetes.executor.missingPodDetectDelta")
+      .doc("When a registered executor's POD is missing from the Kubernetes API server's polled " +
+        "list of PODs then this delta time is taken as the accepted time difference between the " +
+        "registration time and the time of the polling. After this time the POD is considered " +
+        "missing from the cluster and the executor will be removed.")
+      .version("3.1.1")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .checkValue(delay => delay > 0, "delay must be a positive time value")
+      .createWithDefaultString("30s")
+
   val KUBERNETES_DRIVER_LABEL_PREFIX = "spark.kubernetes.driver.label."
   val KUBERNETES_DRIVER_ANNOTATION_PREFIX = "spark.kubernetes.driver.annotation."
+  val KUBERNETES_DRIVER_SERVICE_ANNOTATION_PREFIX = "spark.kubernetes.driver.service.annotation."
   val KUBERNETES_DRIVER_SECRETS_PREFIX = "spark.kubernetes.driver.secrets."
   val KUBERNETES_DRIVER_SECRET_KEY_REF_PREFIX = "spark.kubernetes.driver.secretKeyRef."
   val KUBERNETES_DRIVER_VOLUMES_PREFIX = "spark.kubernetes.driver.volumes."
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
index a3c74ff7b2885..e885e1ff2665e 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
@@ -75,7 +75,7 @@ private[spark] object Constants {
   val EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME = "pod-spec-template.yml"
   val EXECUTOR_POD_SPEC_TEMPLATE_MOUNTPATH = "/opt/spark/pod-template"
   val POD_TEMPLATE_VOLUME = "pod-template-volume"
-  val POD_TEMPLATE_CONFIGMAP = "podspec-configmap"
+  val POD_TEMPLATE_CONFIGMAP = "driver-podspec-conf-map"
   val POD_TEMPLATE_KEY = "podspec-configmap-key"
 
   // Miscellaneous
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
index 09943b7974ed9..6dcae3e8b0e39 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
@@ -106,6 +106,11 @@ private[spark] class KubernetesDriverConf(
     KubernetesUtils.parsePrefixedKeyValuePairs(sparkConf, KUBERNETES_DRIVER_ANNOTATION_PREFIX)
   }
 
+  def serviceAnnotations: Map[String, String] = {
+    KubernetesUtils.parsePrefixedKeyValuePairs(sparkConf,
+      KUBERNETES_DRIVER_SERVICE_ANNOTATION_PREFIX)
+  }
+
   override def secretNamesToMountPaths: Map[String, String] = {
     KubernetesUtils.parsePrefixedKeyValuePairs(sparkConf, KUBERNETES_DRIVER_SECRETS_PREFIX)
   }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
index e234b1780a7d0..c49f4a15de974 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
@@ -23,7 +23,7 @@ import java.util.UUID
 
 import scala.collection.JavaConverters._
 
-import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, PodBuilder, Quantity, QuantityBuilder}
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, PodBuilder, Quantity}
 import io.fabric8.kubernetes.client.KubernetesClient
 import org.apache.commons.codec.binary.Hex
 import org.apache.hadoop.fs.{FileSystem, Path}
@@ -234,9 +234,7 @@ private[spark] object KubernetesUtils extends Logging {
         throw new SparkException(s"Resource: ${request.id.resourceName} was requested, " +
           "but vendor was not specified.")
       }
-      val quantity = new QuantityBuilder(false)
-        .withAmount(request.amount.toString)
-        .build()
+      val quantity = new Quantity(request.amount.toString)
       (KubernetesConf.buildKubernetesResourceName(vendorDomain, request.id.resourceName), quantity)
     }.toMap
   }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
index e2d70d7692ef3..63f18129328b0 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
@@ -80,14 +80,10 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf)
           .build()
       }
 
-    val driverCpuQuantity = new QuantityBuilder(false)
-      .withAmount(driverCoresRequest)
-      .build()
-    val driverMemoryQuantity = new QuantityBuilder(false)
-      .withAmount(s"${driverMemoryWithOverheadMiB}Mi")
-      .build()
+    val driverCpuQuantity = new Quantity(driverCoresRequest)
+    val driverMemoryQuantity = new Quantity(s"${driverMemoryWithOverheadMiB}Mi")
     val maybeCpuLimitQuantity = driverLimitCores.map { limitCores =>
-      ("cpu", new QuantityBuilder(false).withAmount(limitCores).build())
+      ("cpu", new Quantity(limitCores))
     }
 
     val driverResourceQuantities =
@@ -158,7 +154,6 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf)
     val additionalProps = mutable.Map(
       KUBERNETES_DRIVER_POD_NAME.key -> driverPodName,
       "spark.app.id" -> conf.appId,
-      KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> conf.resourceNamePrefix,
       KUBERNETES_DRIVER_SUBMIT_CHECK.key -> "true",
       MEMORY_OVERHEAD_FACTOR.key -> overheadFactor.toString)
     // try upload local, resolvable files to a hadoop compatible file system
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
index d88bd5858bc94..6a26df2997fd2 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
@@ -88,12 +88,8 @@ private[spark] class BasicExecutorFeatureStep(
       // Replace dangerous characters in the remaining string with a safe alternative.
       .replaceAll("[^\\w-]+", "_")
 
-    val executorMemoryQuantity = new QuantityBuilder(false)
-      .withAmount(s"${executorMemoryTotal}Mi")
-      .build()
-    val executorCpuQuantity = new QuantityBuilder(false)
-      .withAmount(executorCoresRequest)
-      .build()
+    val executorMemoryQuantity = new Quantity(s"${executorMemoryTotal}Mi")
+    val executorCpuQuantity = new Quantity(executorCoresRequest)
 
     val executorResourceQuantities =
       KubernetesUtils.buildResourcesQuantities(SPARK_EXECUTOR_PREFIX,
@@ -183,9 +179,7 @@ private[spark] class BasicExecutorFeatureStep(
       .addToArgs("executor")
       .build()
     val containerWithLimitCores = executorLimitCores.map { limitCores =>
-      val executorCpuLimitQuantity = new QuantityBuilder(false)
-        .withAmount(limitCores)
-        .build()
+      val executorCpuLimitQuantity = new Quantity(limitCores)
       new ContainerBuilder(executorContainer)
         .editResources()
           .addToLimits("cpu", executorCpuLimitQuantity)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala
index 925bcdf3e637f..1e9c60c871479 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala
@@ -69,6 +69,7 @@ private[spark] class DriverServiceFeatureStep(
     val driverService = new ServiceBuilder()
       .withNewMetadata()
         .withName(resolvedServiceName)
+        .addToAnnotations(kubernetesConf.serviceAnnotations.asJava)
         .endMetadata()
       .withNewSpec()
         .withClusterIP("None")
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
index 8548e7057cdf0..4599df99b3c61 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
@@ -65,7 +65,7 @@ private[spark] class MountVolumesFeatureStep(conf: KubernetesConf)
           new VolumeBuilder()
             .withEmptyDir(
               new EmptyDirVolumeSource(medium.getOrElse(""),
-              new Quantity(sizeLimit.orNull)))
+                sizeLimit.map(new Quantity(_)).orNull))
       }
 
       val volume = volumeBuilder.withName(spec.volumeName).build()
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala
index 7f41ca43589b6..1040419a4a6e9 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala
@@ -31,6 +31,8 @@ private[spark] class PodTemplateConfigMapStep(conf: KubernetesConf)
 
   private val hasTemplate = conf.contains(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE)
 
+  private val configmapName = s"${conf.resourceNamePrefix}-$POD_TEMPLATE_CONFIGMAP"
+
   def configurePod(pod: SparkPod): SparkPod = {
     if (hasTemplate) {
       val podWithVolume = new PodBuilder(pod.pod)
@@ -38,7 +40,7 @@ private[spark] class PodTemplateConfigMapStep(conf: KubernetesConf)
             .addNewVolume()
               .withName(POD_TEMPLATE_VOLUME)
               .withNewConfigMap()
-                .withName(POD_TEMPLATE_CONFIGMAP)
+                .withName(configmapName)
                 .addNewItem()
                   .withKey(POD_TEMPLATE_KEY)
                   .withPath(EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)
@@ -76,7 +78,7 @@ private[spark] class PodTemplateConfigMapStep(conf: KubernetesConf)
       val podTemplateString = Files.toString(new File(podTemplateFile), StandardCharsets.UTF_8)
       Seq(new ConfigMapBuilder()
           .withNewMetadata()
-            .withName(POD_TEMPLATE_CONFIGMAP)
+            .withName(configmapName)
           .endMetadata()
           .addToData(POD_TEMPLATE_KEY, podTemplateString)
         .build())
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
index 8e5532d70b5da..b5ad444867864 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
@@ -21,8 +21,10 @@ import java.util.{Collections, UUID}
 import java.util.Properties
 
 import io.fabric8.kubernetes.api.model._
-import io.fabric8.kubernetes.client.KubernetesClient
+import io.fabric8.kubernetes.client.{KubernetesClient, Watch}
+import io.fabric8.kubernetes.client.Watcher.Action
 import scala.collection.mutable
+import scala.util.control.Breaks._
 import scala.util.control.NonFatal
 
 import org.apache.spark.SparkConf
@@ -122,25 +124,37 @@ private[spark] class Client(
         .endSpec()
       .build()
     val driverPodName = resolvedDriverPod.getMetadata.getName
-    Utils.tryWithResource(
-      kubernetesClient
-        .pods()
-        .withName(driverPodName)
-        .watch(watcher)) { _ =>
-      val createdDriverPod = kubernetesClient.pods().create(resolvedDriverPod)
-      try {
-        val otherKubernetesResources =
-          resolvedDriverSpec.driverKubernetesResources ++ Seq(configMap)
-        addDriverOwnerReference(createdDriverPod, otherKubernetesResources)
-        kubernetesClient.resourceList(otherKubernetesResources: _*).createOrReplace()
-      } catch {
-        case NonFatal(e) =>
-          kubernetesClient.pods().delete(createdDriverPod)
-          throw e
-      }
 
-      val sId = Seq(conf.namespace, driverPodName).mkString(":")
-      watcher.watchOrStop(sId)
+    var watch: Watch = null
+    val createdDriverPod = kubernetesClient.pods().create(resolvedDriverPod)
+    try {
+      val otherKubernetesResources = resolvedDriverSpec.driverKubernetesResources ++ Seq(configMap)
+      addDriverOwnerReference(createdDriverPod, otherKubernetesResources)
+      kubernetesClient.resourceList(otherKubernetesResources: _*).createOrReplace()
+    } catch {
+      case NonFatal(e) =>
+        kubernetesClient.pods().delete(createdDriverPod)
+        throw e
+    }
+    val sId = Seq(conf.namespace, driverPodName).mkString(":")
+    breakable {
+      while (true) {
+        val podWithName = kubernetesClient
+          .pods()
+          .withName(driverPodName)
+        // Reset resource to old before we start the watch, this is important for race conditions
+        watcher.reset()
+        watch = podWithName.watch(watcher)
+
+        // Send the latest pod state we know to the watcher to make sure we didn't miss anything
+        watcher.eventReceived(Action.MODIFIED, podWithName.get())
+
+        // Break the while loop if the pod is completed or we don't want to wait
+        if(watcher.watchOrStop(sId)) {
+          watch.close()
+          break
+        }
+      }
     }
   }
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
index ce3c80c0f85b1..aa27a9ef508ca 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
@@ -19,6 +19,7 @@ package org.apache.spark.deploy.k8s.submit
 import io.fabric8.kubernetes.api.model.Pod
 import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher}
 import io.fabric8.kubernetes.client.Watcher.Action
+import java.net.HttpURLConnection.HTTP_GONE
 
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.KubernetesDriverConf
@@ -26,7 +27,8 @@ import org.apache.spark.deploy.k8s.KubernetesUtils._
 import org.apache.spark.internal.Logging
 
 private[k8s] trait LoggingPodStatusWatcher extends Watcher[Pod] {
-  def watchOrStop(submissionId: String): Unit
+  def watchOrStop(submissionId: String): Boolean
+  def reset(): Unit
 }
 
 /**
@@ -42,10 +44,16 @@ private[k8s] class LoggingPodStatusWatcherImpl(conf: KubernetesDriverConf)
 
   private var podCompleted = false
 
+  private var resourceTooOldReceived = false
+
   private var pod = Option.empty[Pod]
 
   private def phase: String = pod.map(_.getStatus.getPhase).getOrElse("unknown")
 
+  override def reset(): Unit = {
+    resourceTooOldReceived = false
+  }
+
   override def eventReceived(action: Action, pod: Pod): Unit = {
     this.pod = Option(pod)
     action match {
@@ -62,7 +70,12 @@ private[k8s] class LoggingPodStatusWatcherImpl(conf: KubernetesDriverConf)
 
   override def onClose(e: KubernetesClientException): Unit = {
     logDebug(s"Stopping watching application $appId with last-observed phase $phase")
-    closeWatch()
+    if(e != null && e.getCode == HTTP_GONE) {
+      resourceTooOldReceived = true
+      logDebug(s"Got HTTP Gone code, resource version changed in k8s api: $e")
+    } else {
+      closeWatch()
+    }
   }
 
   private def logLongStatus(): Unit = {
@@ -78,20 +91,26 @@ private[k8s] class LoggingPodStatusWatcherImpl(conf: KubernetesDriverConf)
     this.notifyAll()
   }
 
-  override def watchOrStop(sId: String): Unit = if (conf.get(WAIT_FOR_APP_COMPLETION)) {
+  override def watchOrStop(sId: String): Boolean = if (conf.get(WAIT_FOR_APP_COMPLETION)) {
     logInfo(s"Waiting for application ${conf.appName} with submission ID $sId to finish...")
     val interval = conf.get(REPORT_INTERVAL)
     synchronized {
-      while (!podCompleted) {
+      while (!podCompleted && !resourceTooOldReceived) {
         wait(interval)
         logInfo(s"Application status for $appId (phase: $phase)")
       }
     }
-    logInfo(
-      pod.map { p => s"Container final statuses:\n\n${containersDescription(p)}" }
-        .getOrElse("No containers were found in the driver pod."))
-    logInfo(s"Application ${conf.appName} with submission ID $sId finished")
+
+    if(podCompleted) {
+      logInfo(
+        pod.map { p => s"Container final statuses:\n\n${containersDescription(p)}" }
+          .getOrElse("No containers were found in the driver pod."))
+      logInfo(s"Application ${conf.appName} with submission ID $sId finished")
+    }
+    podCompleted
   } else {
     logInfo(s"Deployed Spark application ${conf.appName} with submission ID $sId into Kubernetes")
+    // Always act like the application has completed since we don't want to wait for app completion
+    true
   }
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
index b394f35b15111..3869ef90bfe5a 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
@@ -64,13 +64,13 @@ private[spark] class ExecutorPodsAllocator(
 
   // Executor IDs that have been requested from Kubernetes but have not been detected in any
   // snapshot yet. Mapped to the timestamp when they were created.
-  private val newlyCreatedExecutors = mutable.Map.empty[Long, Long]
+  private val newlyCreatedExecutors = mutable.LinkedHashMap.empty[Long, Long]
 
   private val dynamicAllocationEnabled = Utils.isDynamicAllocationEnabled(conf)
 
   private val hasPendingPods = new AtomicBoolean()
 
-  private var lastSnapshot = ExecutorPodsSnapshot(Nil)
+  private var lastSnapshot = ExecutorPodsSnapshot()
 
   // Executors that have been deleted by this allocator but not yet detected as deleted in
   // a snapshot from the API server. This is used to deny registration from these executors
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
index d6b75824d05d8..b1a6694b9fa3a 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
@@ -44,6 +44,9 @@ private[spark] class ExecutorPodsLifecycleManager(
   private val eventProcessingInterval = conf.get(KUBERNETES_EXECUTOR_EVENT_PROCESSING_INTERVAL)
 
   private lazy val shouldDeleteExecutors = conf.get(KUBERNETES_DELETE_EXECUTORS)
+  private lazy val missingPodDetectDelta = conf.get(KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA)
+
+  private var lastFullSnapshotTs: Long = 0
 
   def start(schedulerBackend: KubernetesClusterSchedulerBackend): Unit = {
     snapshotsStore.addSubscriber(eventProcessingInterval) {
@@ -83,33 +86,34 @@ private[spark] class ExecutorPodsLifecycleManager(
 
     // Reconcile the case where Spark claims to know about an executor but the corresponding pod
     // is missing from the cluster. This would occur if we miss a deletion event and the pod
-    // transitions immediately from running io absent. We only need to check against the latest
-    // snapshot for this, and we don't do this for executors in the deleted executors cache or
-    // that we just removed in this round.
-    if (snapshots.nonEmpty) {
-      val latestSnapshot = snapshots.last
-      (schedulerBackend.getExecutorIds().map(_.toLong).toSet
-        -- latestSnapshot.executorPods.keySet
-        -- execIdsRemovedInThisRound).foreach { missingExecutorId =>
-        if (removedExecutorsCache.getIfPresent(missingExecutorId) == null) {
-          val exitReasonMessage = s"The executor with ID $missingExecutorId was not found in the" +
-            s" cluster but we didn't get a reason why. Marking the executor as failed. The" +
-            s" executor may have been deleted but the driver missed the deletion event."
+    // transitions immediately from running to absent. We only need to check against the latest
+    // fresh full snapshot (coming from ExecutorPodsPollingSnapshotSource) for this, and we don't
+    // do this for executors in the deleted executors cache or that we just removed in this round.
+    if (snapshots.nonEmpty && lastFullSnapshotTs != snapshots.last.fullSnapshotTs) {
+      lastFullSnapshotTs = snapshots.last.fullSnapshotTs
+      val lostExecutorsWithRegistrationTs =
+        schedulerBackend.getExecutorsWithRegistrationTs().map(t => (t._1.toLong, t._2)) --
+        snapshots.last.executorPods.keySet -- execIdsRemovedInThisRound
+
+      lostExecutorsWithRegistrationTs.foreach { case (lostExecId, lostExecRegistrationTs) =>
+        if (removedExecutorsCache.getIfPresent(lostExecId) == null &&
+            lastFullSnapshotTs - lostExecRegistrationTs > missingPodDetectDelta) {
+          val exitReasonMessage = s"The executor with ID $lostExecId (registered at " +
+            s"$lostExecRegistrationTs ms) was not found in the cluster at the polling time " +
+            s"($lastFullSnapshotTs ms) which is after the accepted detect delta time " +
+            s"($missingPodDetectDelta ms) configured by " +
+            s"`${KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA.key}`. " +
+            "The executor may have been deleted but the driver missed the deletion event. " +
+            "Marking this executor as failed."
           logDebug(exitReasonMessage)
           val exitReason = ExecutorExited(
             UNKNOWN_EXIT_CODE,
             exitCausedByApp = false,
             exitReasonMessage)
-          schedulerBackend.doRemoveExecutor(missingExecutorId.toString, exitReason)
-          execIdsRemovedInThisRound += missingExecutorId
+          schedulerBackend.doRemoveExecutor(lostExecId.toString, exitReason)
         }
       }
     }
-
-    if (execIdsRemovedInThisRound.nonEmpty) {
-      logDebug(s"Removed executors with ids ${execIdsRemovedInThisRound.mkString(",")}" +
-        s" from Spark that were either found to be deleted or non-existent in the cluster.")
-    }
   }
 
   private def onFinalNonDeletedState(
@@ -186,4 +190,3 @@ private[spark] class ExecutorPodsLifecycleManager(
 private object ExecutorPodsLifecycleManager {
   val UNKNOWN_EXIT_CODE = -1
 }
-
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
index 435a5f1461c92..16064dc66544a 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
@@ -18,6 +18,7 @@ package org.apache.spark.scheduler.cluster.k8s
 
 import java.util.Locale
 
+import io.fabric8.kubernetes.api.model.ContainerStateTerminated
 import io.fabric8.kubernetes.api.model.Pod
 
 import org.apache.spark.deploy.k8s.Constants._
@@ -26,23 +27,30 @@ import org.apache.spark.internal.Logging
 /**
  * An immutable view of the current executor pods that are running in the cluster.
  */
-private[spark] case class ExecutorPodsSnapshot(executorPods: Map[Long, ExecutorPodState]) {
+private[spark] case class ExecutorPodsSnapshot(
+    executorPods: Map[Long, ExecutorPodState],
+    fullSnapshotTs: Long) {
 
   import ExecutorPodsSnapshot._
 
   def withUpdate(updatedPod: Pod): ExecutorPodsSnapshot = {
     val newExecutorPods = executorPods ++ toStatesByExecutorId(Seq(updatedPod))
-    new ExecutorPodsSnapshot(newExecutorPods)
+    new ExecutorPodsSnapshot(newExecutorPods, fullSnapshotTs)
   }
 }
 
 object ExecutorPodsSnapshot extends Logging {
+  private var shouldCheckAllContainers: Boolean = _
 
-  def apply(executorPods: Seq[Pod]): ExecutorPodsSnapshot = {
-    ExecutorPodsSnapshot(toStatesByExecutorId(executorPods))
+  def apply(executorPods: Seq[Pod], fullSnapshotTs: Long): ExecutorPodsSnapshot = {
+    ExecutorPodsSnapshot(toStatesByExecutorId(executorPods), fullSnapshotTs)
   }
 
-  def apply(): ExecutorPodsSnapshot = ExecutorPodsSnapshot(Map.empty[Long, ExecutorPodState])
+  def apply(): ExecutorPodsSnapshot = ExecutorPodsSnapshot(Map.empty[Long, ExecutorPodState], 0)
+
+  def setShouldCheckAllContainers(watchAllContainers: Boolean): Unit = {
+    shouldCheckAllContainers = watchAllContainers
+  }
 
   private def toStatesByExecutorId(executorPods: Seq[Pod]): Map[Long, ExecutorPodState] = {
     executorPods.map { pod =>
@@ -59,7 +67,15 @@ object ExecutorPodsSnapshot extends Logging {
         case "pending" =>
           PodPending(pod)
         case "running" =>
-          PodRunning(pod)
+          if (shouldCheckAllContainers &&
+            "Never" == pod.getSpec.getRestartPolicy &&
+            pod.getStatus.getContainerStatuses.stream
+              .map[ContainerStateTerminated](cs => cs.getState.getTerminated)
+              .anyMatch(t => t != null && t.getExitCode != 0)) {
+            PodFailed(pod)
+          } else {
+            PodRunning(pod)
+          }
         case "failed" =>
           PodFailed(pod)
         case "succeeded" =>
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala
index 8aa20bfbc82db..502892314d667 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala
@@ -23,7 +23,11 @@ import javax.annotation.concurrent.GuardedBy
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
-import org.apache.spark.util.{ThreadUtils, Utils}
+import org.apache.spark.internal.Logging
+import org.apache.spark.util.Clock
+import org.apache.spark.util.SystemClock
+import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.Utils
 
 /**
  * Controls the propagation of the Spark application's executor pods state to subscribers that
@@ -47,8 +51,10 @@ import org.apache.spark.util.{ThreadUtils, Utils}
  * time-windowed chunks. Each subscriber can choose to receive their snapshot chunks at different
  * time intervals.
  */
-private[spark] class ExecutorPodsSnapshotsStoreImpl(subscribersExecutor: ScheduledExecutorService)
-  extends ExecutorPodsSnapshotsStore {
+private[spark] class ExecutorPodsSnapshotsStoreImpl(
+    subscribersExecutor: ScheduledExecutorService,
+    clock: Clock = new SystemClock)
+  extends ExecutorPodsSnapshotsStore with Logging {
 
   private val SNAPSHOT_LOCK = new Object()
 
@@ -93,7 +99,7 @@ private[spark] class ExecutorPodsSnapshotsStoreImpl(subscribersExecutor: Schedul
   }
 
   override def replaceSnapshot(newSnapshot: Seq[Pod]): Unit = SNAPSHOT_LOCK.synchronized {
-    currentSnapshot = ExecutorPodsSnapshot(newSnapshot)
+    currentSnapshot = ExecutorPodsSnapshot(newSnapshot, clock.getTimeMillis())
     addCurrentSnapshotToSubscribers()
   }
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
index b9d7a7083f41a..f2638129cd786 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
@@ -91,10 +91,13 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit
     val schedulerExecutorService = ThreadUtils.newDaemonSingleThreadScheduledExecutor(
       "kubernetes-executor-maintenance")
 
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(
+      sc.conf.get(KUBERNETES_EXECUTOR_CHECK_ALL_CONTAINERS))
     val subscribersExecutor = ThreadUtils
       .newDaemonThreadPoolScheduledExecutor(
         "kubernetes-executor-snapshots-subscribers", 2)
     val snapshotsStore = new ExecutorPodsSnapshotsStoreImpl(subscribersExecutor)
+
     val removedExecutorsCache = CacheBuilder.newBuilder()
       .expireAfterWrite(3, TimeUnit.MINUTES)
       .build[java.lang.Long, java.lang.Long]()
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala
index ee830a91f3272..7763b116f98ab 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala
@@ -48,6 +48,7 @@ object KubernetesTestConf {
       labels: Map[String, String] = Map.empty,
       environment: Map[String, String] = Map.empty,
       annotations: Map[String, String] = Map.empty,
+      serviceAnnotations: Map[String, String] = Map.empty,
       secretEnvNamesToKeyRefs: Map[String, String] = Map.empty,
       secretNamesToMountPaths: Map[String, String] = Map.empty,
       volumes: Seq[KubernetesVolumeSpec] = Seq.empty): KubernetesDriverConf = {
@@ -59,6 +60,7 @@ object KubernetesTestConf {
     setPrefixedConfigs(conf, KUBERNETES_DRIVER_LABEL_PREFIX, labels)
     setPrefixedConfigs(conf, KUBERNETES_DRIVER_ENV_PREFIX, environment)
     setPrefixedConfigs(conf, KUBERNETES_DRIVER_ANNOTATION_PREFIX, annotations)
+    setPrefixedConfigs(conf, KUBERNETES_DRIVER_SERVICE_ANNOTATION_PREFIX, serviceAnnotations)
     setPrefixedConfigs(conf, KUBERNETES_DRIVER_SECRETS_PREFIX, secretNamesToMountPaths)
     setPrefixedConfigs(conf, KUBERNETES_DRIVER_SECRET_KEY_REF_PREFIX, secretEnvNamesToKeyRefs)
     setVolumeSpecs(conf, KUBERNETES_DRIVER_VOLUMES_PREFIX, volumes)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala
index 707c823d69cf0..26bd317de8ec6 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala
@@ -101,8 +101,7 @@ abstract class PodBuilderSuite extends SparkFunSuite {
     assert(container.getArgs.contains("arg"))
     assert(container.getCommand.equals(List("command").asJava))
     assert(container.getEnv.asScala.exists(_.getName == "env-key"))
-    assert(container.getResources.getLimits.get("gpu") ===
-      new QuantityBuilder().withAmount("1").build())
+    assert(container.getResources.getLimits.get("gpu") === new Quantity("1"))
     assert(container.getSecurityContext.getRunAsNonRoot)
     assert(container.getStdin)
     assert(container.getTerminationMessagePath === "termination-message-path")
@@ -156,7 +155,7 @@ abstract class PodBuilderSuite extends SparkFunSuite {
           .withImagePullPolicy("Always")
           .withName("executor-container")
           .withNewResources()
-            .withLimits(Map("gpu" -> new QuantityBuilder().withAmount("1").build()).asJava)
+            .withLimits(Map("gpu" -> new Quantity("1")).asJava)
             .endResources()
           .withNewSecurityContext()
             .withRunAsNonRoot(true)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
index ce66afd9448a9..c8c934b436ffa 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
@@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s.features
 
 import scala.collection.JavaConverters._
 
-import io.fabric8.kubernetes.api.model.{ContainerPort, ContainerPortBuilder, LocalObjectReferenceBuilder}
+import io.fabric8.kubernetes.api.model.{ContainerPort, ContainerPortBuilder, LocalObjectReferenceBuilder, Quantity}
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s.{KubernetesTestConf, SparkPod}
@@ -105,13 +105,13 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
 
     val resourceRequirements = configuredPod.container.getResources
     val requests = resourceRequirements.getRequests.asScala
-    assert(requests("cpu").getAmount === "2")
-    assert(requests("memory").getAmount === "456Mi")
+    assert(amountAndFormat(requests("cpu")) === "2")
+    assert(amountAndFormat(requests("memory")) === "456Mi")
     val limits = resourceRequirements.getLimits.asScala
-    assert(limits("memory").getAmount === "456Mi")
-    assert(limits("cpu").getAmount === "4")
+    assert(amountAndFormat(limits("memory")) === "456Mi")
+    assert(amountAndFormat(limits("cpu")) === "4")
     resources.foreach { case (k8sName, testRInfo) =>
-      assert(limits(k8sName).getAmount === testRInfo.count)
+      assert(amountAndFormat(limits(k8sName)) === testRInfo.count)
     }
 
     val driverPodMetadata = configuredPod.pod.getMetadata
@@ -124,7 +124,6 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
     val expectedSparkConf = Map(
       KUBERNETES_DRIVER_POD_NAME.key -> "spark-driver-pod",
       "spark.app.id" -> KubernetesTestConf.APP_ID,
-      KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> kubernetesConf.resourceNamePrefix,
       "spark.kubernetes.submitInDriver" -> "true",
       MEMORY_OVERHEAD_FACTOR.key -> MEMORY_OVERHEAD_FACTOR.defaultValue.get.toString)
     assert(featureStep.getAdditionalPodSystemProperties() === expectedSparkConf)
@@ -141,7 +140,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
       .configurePod(basePod)
       .container.getResources
       .getRequests.asScala
-    assert(requests1("cpu").getAmount === "1")
+    assert(amountAndFormat(requests1("cpu")) === "1")
 
     // if spark.driver.cores is set it should be used
     sparkConf.set(DRIVER_CORES, 10)
@@ -149,7 +148,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
       .configurePod(basePod)
       .container.getResources
       .getRequests.asScala
-    assert(requests2("cpu").getAmount === "10")
+    assert(amountAndFormat(requests2("cpu")) === "10")
 
     // spark.kubernetes.driver.request.cores should be preferred over spark.driver.cores
     Seq("0.1", "100m").foreach { value =>
@@ -158,7 +157,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
         .configurePod(basePod)
         .container.getResources
         .getRequests.asScala
-      assert(requests3("cpu").getAmount === value)
+      assert(amountAndFormat(requests3("cpu")) === value)
     }
   }
 
@@ -204,7 +203,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
         mainAppResource = resource)
       val step = new BasicDriverFeatureStep(conf)
       val pod = step.configurePod(SparkPod.initialPod())
-      val mem = pod.container.getResources.getRequests.get("memory").getAmount()
+      val mem = amountAndFormat(pod.container.getResources.getRequests.get("memory"))
       val expected = (driverMem + driverMem * expectedFactor).toInt
       assert(mem === s"${expected}Mi")
 
@@ -219,4 +218,6 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
       .withContainerPort(portNumber)
       .withProtocol("TCP")
       .build()
+
+  private def amountAndFormat(quantity: Quantity): String = quantity.getAmount + quantity.getFormat
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
index f375b1fe6a5cd..da50372d04c73 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
@@ -128,10 +128,11 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     val executor = step.configurePod(SparkPod.initialPod())
 
     assert(executor.container.getResources.getLimits.size() === 3)
-    assert(executor.container.getResources
-      .getLimits.get("memory").getAmount === "1408Mi")
+    assert(amountAndFormat(executor.container.getResources
+      .getLimits.get("memory")) === "1408Mi")
     gpuResources.foreach { case (k8sName, testRInfo) =>
-      assert(executor.container.getResources.getLimits.get(k8sName).getAmount === testRInfo.count)
+      assert(amountAndFormat(
+        executor.container.getResources.getLimits.get(k8sName)) === testRInfo.count)
     }
   }
 
@@ -151,8 +152,8 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     assert(executor.container.getImage === EXECUTOR_IMAGE)
     assert(executor.container.getVolumeMounts.isEmpty)
     assert(executor.container.getResources.getLimits.size() === 1)
-    assert(executor.container.getResources
-      .getLimits.get("memory").getAmount === "1408Mi")
+    assert(amountAndFormat(executor.container.getResources
+      .getLimits.get("memory")) === "1408Mi")
 
     // The pod has no node selector, volumes.
     assert(executor.pod.getSpec.getNodeSelector.isEmpty)
@@ -201,7 +202,7 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf))
     val executor = step.configurePod(SparkPod.initialPod())
     // This is checking that basic executor + executorMemory = 1408 + 42 = 1450
-    assert(executor.container.getResources.getRequests.get("memory").getAmount === "1450Mi")
+    assert(amountAndFormat(executor.container.getResources.getRequests.get("memory")) === "1450Mi")
   }
 
   test("auth secret propagation") {
@@ -273,4 +274,6 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     val expectedEnvs = defaultEnvs ++ additionalEnvVars ++ extraJavaOptsEnvs
     assert(containerEnvs === expectedEnvs)
   }
+
+  private def amountAndFormat(quantity: Quantity): String = quantity.getAmount + quantity.getFormat
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala
index 9068289bab581..18afd10395566 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala
@@ -38,6 +38,9 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
   private val DRIVER_LABELS = Map(
     "label1key" -> "label1value",
     "label2key" -> "label2value")
+  private val DRIVER_SERVICE_ANNOTATIONS = Map(
+    "annotation1key" -> "annotation1value",
+    "annotation2key" -> "annotation2value")
 
   test("Headless service has a port for the driver RPC, the block manager and driver ui.") {
     val sparkConf = new SparkConf(false)
@@ -46,7 +49,8 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
       .set(UI_PORT, 4080)
     val kconf = KubernetesTestConf.createDriverConf(
       sparkConf = sparkConf,
-      labels = DRIVER_LABELS)
+      labels = DRIVER_LABELS,
+      serviceAnnotations = DRIVER_SERVICE_ANNOTATIONS)
     val configurationStep = new DriverServiceFeatureStep(kconf)
     assert(configurationStep.configurePod(SparkPod.initialPod()) === SparkPod.initialPod())
     assert(configurationStep.getAdditionalKubernetesResources().size === 1)
@@ -79,7 +83,9 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
   }
 
   test("Ports should resolve to defaults in SparkConf and in the service.") {
-    val kconf = KubernetesTestConf.createDriverConf(labels = DRIVER_LABELS)
+    val kconf = KubernetesTestConf.createDriverConf(
+      labels = DRIVER_LABELS,
+      serviceAnnotations = DRIVER_SERVICE_ANNOTATIONS)
     val configurationStep = new DriverServiceFeatureStep(kconf)
     val resolvedService = configurationStep
       .getAdditionalKubernetesResources()
@@ -164,6 +170,9 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
     DRIVER_LABELS.foreach { case (k, v) =>
       assert(service.getSpec.getSelector.get(k) === v)
     }
+    DRIVER_SERVICE_ANNOTATIONS.foreach { case (k, v) =>
+      assert(service.getMetadata.getAnnotations.get(k) === v)
+    }
     assert(service.getSpec.getPorts.size() === 3)
     val driverServicePorts = service.getSpec.getPorts.asScala
     assert(driverServicePorts.head.getName === DRIVER_PORT_NAME)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
index 8c430eeb3fa71..3888062785324 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
@@ -79,7 +79,8 @@ class MountVolumesFeatureStepSuite extends SparkFunSuite {
     assert(configuredPod.pod.getSpec.getVolumes.size() === 1)
     val emptyDir = configuredPod.pod.getSpec.getVolumes.get(0).getEmptyDir
     assert(emptyDir.getMedium === "Memory")
-    assert(emptyDir.getSizeLimit.getAmount === "6G")
+    assert(emptyDir.getSizeLimit.getAmount ===  "6")
+    assert(emptyDir.getSizeLimit.getFormat === "G")
     assert(configuredPod.container.getVolumeMounts.size() === 1)
     assert(configuredPod.container.getVolumeMounts.get(0).getMountPath === "/tmp")
     assert(configuredPod.container.getVolumeMounts.get(0).getName === "testVolume")
@@ -101,7 +102,7 @@ class MountVolumesFeatureStepSuite extends SparkFunSuite {
     assert(configuredPod.pod.getSpec.getVolumes.size() === 1)
     val emptyDir = configuredPod.pod.getSpec.getVolumes.get(0).getEmptyDir
     assert(emptyDir.getMedium === "")
-    assert(emptyDir.getSizeLimit.getAmount === null)
+    assert(emptyDir.getSizeLimit === null)
     assert(configuredPod.container.getVolumeMounts.size() === 1)
     assert(configuredPod.container.getVolumeMounts.get(0).getMountPath === "/tmp")
     assert(configuredPod.container.getVolumeMounts.get(0).getName === "testVolume")
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala
index 051320fa44c5e..1b38fd6a0d2ab 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala
@@ -16,13 +16,14 @@
  */
 package org.apache.spark.deploy.k8s.features
 
-import java.io.{File, PrintWriter}
+import java.io.PrintWriter
 import java.nio.file.Files
 
 import io.fabric8.kubernetes.api.model.ConfigMap
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s._
+import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.util.Utils
 
 class PodTemplateConfigMapStepSuite extends SparkFunSuite {
@@ -56,8 +57,9 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite {
 
     assert(configuredPod.pod.getSpec.getVolumes.size() === 1)
     val volume = configuredPod.pod.getSpec.getVolumes.get(0)
+    val generatedResourceName = s"${kubernetesConf.resourceNamePrefix}-$POD_TEMPLATE_CONFIGMAP"
     assert(volume.getName === Constants.POD_TEMPLATE_VOLUME)
-    assert(volume.getConfigMap.getName === Constants.POD_TEMPLATE_CONFIGMAP)
+    assert(volume.getConfigMap.getName === generatedResourceName)
     assert(volume.getConfigMap.getItems.size() === 1)
     assert(volume.getConfigMap.getItems.get(0).getKey === Constants.POD_TEMPLATE_KEY)
     assert(volume.getConfigMap.getItems.get(0).getPath ===
@@ -70,7 +72,7 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite {
 
     val resources = step.getAdditionalKubernetesResources()
     assert(resources.size === 1)
-    assert(resources.head.getMetadata.getName === Constants.POD_TEMPLATE_CONFIGMAP)
+    assert(resources.head.getMetadata.getName === generatedResourceName)
     assert(resources.head.isInstanceOf[ConfigMap])
     val configMap = resources.head.asInstanceOf[ConfigMap]
     assert(configMap.getData.size() === 1)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala
index 5d49ac0bbaafa..d9ec3feb526ee 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala
@@ -136,6 +136,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
     createdResourcesArgumentCaptor = ArgumentCaptor.forClass(classOf[HasMetadata])
     when(podOperations.create(FULL_EXPECTED_POD)).thenReturn(POD_WITH_OWNER_REFERENCE)
     when(namedPods.watch(loggingPodStatusWatcher)).thenReturn(mock[Watch])
+    when(loggingPodStatusWatcher.watchOrStop(kconf.namespace + ":" + POD_NAME)).thenReturn(true)
     doReturn(resourceList)
       .when(kubernetesClient)
       .resourceList(createdResourcesArgumentCaptor.capture())
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
index 9ac7e0222054a..2a727cf19981a 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
@@ -19,8 +19,14 @@ package org.apache.spark.scheduler.cluster.k8s
 import io.fabric8.kubernetes.api.model.Pod
 import scala.collection.mutable
 
+import org.apache.spark.util.ManualClock
+
 class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore {
 
+  ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
+
+  val clock = new ManualClock()
+
   private val snapshotsBuffer = mutable.Buffer.empty[ExecutorPodsSnapshot]
   private val subscribers = mutable.Buffer.empty[Seq[ExecutorPodsSnapshot] => Unit]
 
@@ -45,7 +51,7 @@ class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore
   }
 
   override def replaceSnapshot(newSnapshot: Seq[Pod]): Unit = {
-    currentSnapshot = ExecutorPodsSnapshot(newSnapshot)
+    currentSnapshot = ExecutorPodsSnapshot(newSnapshot, clock.getTimeMillis())
     snapshotsBuffer += currentSnapshot
   }
 
@@ -54,7 +60,7 @@ class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore
       case (_, PodDeleted(_)) => false
       case _ => true
     }
-    currentSnapshot = ExecutorPodsSnapshot(nonDeleted)
+    currentSnapshot = ExecutorPodsSnapshot(nonDeleted, clock.getTimeMillis())
     snapshotsBuffer += currentSnapshot
   }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
index 2e883623a4b1c..30312d27b59ed 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
@@ -71,6 +71,33 @@ object ExecutorLifecycleTestUtils {
       .build()
   }
 
+  /**
+   * [SPARK-30821]
+   * This creates a pod with one container in running state and one container in failed
+   * state (terminated with non-zero exit code). This pod is used for unit-testing the
+   * spark.kubernetes.executor.checkAllContainers Spark Conf.
+   */
+  def runningExecutorWithFailedContainer(executorId: Long): Pod = {
+    new PodBuilder(podWithAttachedContainerForId(executorId))
+      .editOrNewStatus()
+        .withPhase("running")
+        .addNewContainerStatus()
+          .withNewState()
+            .withNewTerminated()
+              .withExitCode(1)
+            .endTerminated()
+          .endState()
+        .endContainerStatus()
+        .addNewContainerStatus()
+          .withNewState()
+            .withNewRunning()
+            .endRunning()
+          .endState()
+        .endContainerStatus()
+      .endStatus()
+      .build()
+  }
+
   def succeededExecutor(executorId: Long): Pod = {
     new PodBuilder(podWithAttachedContainerForId(executorId))
       .editOrNewStatus()
@@ -112,7 +139,10 @@ object ExecutorLifecycleTestUtils {
         .addToLabels(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID)
         .addToLabels(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
         .addToLabels(SPARK_EXECUTOR_ID_LABEL, executorId.toString)
-        .endMetadata()
+      .endMetadata()
+      .editOrNewSpec()
+        .withRestartPolicy("Never")
+      .endSpec()
       .build()
     val container = new ContainerBuilder()
       .withName("spark-executor")
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala
index 9920f4d3ea737..7021cb78da0fa 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala
@@ -56,7 +56,7 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte
     val removedExecutorsCache = CacheBuilder.newBuilder().build[java.lang.Long, java.lang.Long]
     snapshotsStore = new DeterministicExecutorPodsSnapshotsStore()
     namedExecutorPods = mutable.Map.empty[String, PodResource[Pod, DoneablePod]]
-    when(schedulerBackend.getExecutorIds()).thenReturn(Seq.empty[String])
+    when(schedulerBackend.getExecutorsWithRegistrationTs()).thenReturn(Map.empty[String, Long])
     when(kubernetesClient.pods()).thenReturn(podOperations)
     when(podOperations.withName(any(classOf[String]))).thenAnswer(namedPodsAnswer())
     eventHandlerUnderTest = new ExecutorPodsLifecycleManager(
@@ -90,13 +90,23 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte
 
   test("When the scheduler backend lists executor ids that aren't present in the cluster," +
     " remove those executors from Spark.") {
-    when(schedulerBackend.getExecutorIds()).thenReturn(Seq("1"))
-    val msg = s"The executor with ID 1 was not found in the cluster but we didn't" +
-      s" get a reason why. Marking the executor as failed. The executor may have been" +
-      s" deleted but the driver missed the deletion event."
-    val expectedLossReason = ExecutorExited(-1, exitCausedByApp = false, msg)
+      when(schedulerBackend.getExecutorsWithRegistrationTs()).thenReturn(Map("1" -> 7L))
+    val missingPodDelta =
+      eventHandlerUnderTest.conf.get(Config.KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA)
+    snapshotsStore.clock.advance(missingPodDelta + 7)
     snapshotsStore.replaceSnapshot(Seq.empty[Pod])
     snapshotsStore.notifySubscribers()
+    verify(schedulerBackend, never()).doRemoveExecutor(any(), any())
+
+    // 1 more millisecond and the accepted delta is over so the missing POD will be detected
+    snapshotsStore.clock.advance(1)
+    snapshotsStore.replaceSnapshot(Seq.empty[Pod])
+    snapshotsStore.notifySubscribers()
+    val msg = "The executor with ID 1 (registered at 7 ms) was not found in the cluster at " +
+      "the polling time (30008 ms) which is after the accepted detect delta time (30000 ms) " +
+      "configured by `spark.kubernetes.executor.missingPodDetectDelta`. The executor may have " +
+      "been deleted but the driver missed the deletion event. Marking this executor as failed."
+    val expectedLossReason = ExecutorExited(-1, exitCausedByApp = false, msg)
     verify(schedulerBackend).doRemoveExecutor("1", expectedLossReason)
   }
 
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
index 70e19c904eddb..7d25450f2edf4 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
@@ -16,35 +16,59 @@
  */
 package org.apache.spark.scheduler.cluster.k8s
 
+import io.fabric8.kubernetes.api.model.Pod
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.scheduler.cluster.k8s.ExecutorLifecycleTestUtils._
 
 class ExecutorPodsSnapshotSuite extends SparkFunSuite {
 
+  def testCase(pod: Pod, state: Pod => ExecutorPodState): (Pod, ExecutorPodState) =
+    (pod, state(pod))
+
+  def doTest(testCases: Seq[(Pod, ExecutorPodState)]): Unit = {
+    val snapshot = ExecutorPodsSnapshot(testCases.map(_._1), 0)
+    for (((_, state), i) <- testCases.zipWithIndex) {
+      assertResult(state.getClass.getName, s"executor ID $i") {
+        snapshot.executorPods(i).getClass.getName
+      }
+    }
+  }
+
   test("States are interpreted correctly from pod metadata.") {
-    val pods = Seq(
-      pendingExecutor(0),
-      runningExecutor(1),
-      succeededExecutor(2),
-      failedExecutorWithoutDeletion(3),
-      deletedExecutor(4),
-      unknownExecutor(5))
-    val snapshot = ExecutorPodsSnapshot(pods)
-    assert(snapshot.executorPods ===
-      Map(
-        0L -> PodPending(pods(0)),
-        1L -> PodRunning(pods(1)),
-        2L -> PodSucceeded(pods(2)),
-        3L -> PodFailed(pods(3)),
-        4L -> PodDeleted(pods(4)),
-        5L -> PodUnknown(pods(5))))
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
+    val testCases = Seq(
+      testCase(pendingExecutor(0), PodPending),
+      testCase(runningExecutor(1), PodRunning),
+      testCase(succeededExecutor(2), PodSucceeded),
+      testCase(failedExecutorWithoutDeletion(3), PodFailed),
+      testCase(deletedExecutor(4), PodDeleted),
+      testCase(unknownExecutor(5), PodUnknown)
+    )
+    doTest(testCases)
+  }
+
+  test("SPARK-30821: States are interpreted correctly from pod metadata"
+    + " when configured to check all containers.") {
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(true)
+    val testCases = Seq(
+      testCase(pendingExecutor(0), PodPending),
+      testCase(runningExecutor(1), PodRunning),
+      testCase(runningExecutorWithFailedContainer(2), PodFailed),
+      testCase(succeededExecutor(3), PodSucceeded),
+      testCase(failedExecutorWithoutDeletion(4), PodFailed),
+      testCase(deletedExecutor(5), PodDeleted),
+      testCase(unknownExecutor(6), PodUnknown)
+    )
+    doTest(testCases)
   }
 
   test("Updates add new pods for non-matching ids and edit existing pods for matching ids") {
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
     val originalPods = Seq(
       pendingExecutor(0),
       runningExecutor(1))
-    val originalSnapshot = ExecutorPodsSnapshot(originalPods)
+    val originalSnapshot = ExecutorPodsSnapshot(originalPods, 0)
     val snapshotWithUpdatedPod = originalSnapshot.withUpdate(succeededExecutor(1))
     assert(snapshotWithUpdatedPod.executorPods ===
       Map(
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
index cf54b3c4eb329..b4240bbacf106 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
@@ -26,15 +26,19 @@ import scala.collection.mutable
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.util.ManualClock
 
 class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter {
 
   private var eventBufferScheduler: DeterministicScheduler = _
   private var eventQueueUnderTest: ExecutorPodsSnapshotsStoreImpl = _
+  private var clock: ManualClock = _
 
   before {
     eventBufferScheduler = new DeterministicScheduler()
-    eventQueueUnderTest = new ExecutorPodsSnapshotsStoreImpl(eventBufferScheduler)
+    clock = new ManualClock()
+    eventQueueUnderTest = new ExecutorPodsSnapshotsStoreImpl(eventBufferScheduler, clock)
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
   }
 
   test("Subscribers get notified of events periodically.") {
@@ -51,6 +55,7 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
     assert(receivedSnapshots1 === Seq(ExecutorPodsSnapshot()))
     assert(receivedSnapshots2 === Seq(ExecutorPodsSnapshot()))
 
+    clock.advance(100)
     pushPodWithIndex(1)
     // Force time to move forward so that the buffer is emitted, scheduling the
     // processing task on the subscription executor...
@@ -59,7 +64,7 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
 
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
     assert(receivedSnapshots2 === Seq(ExecutorPodsSnapshot()))
 
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
@@ -67,29 +72,29 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
     // Don't repeat snapshots
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
     assert(receivedSnapshots2 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
     pushPodWithIndex(2)
     pushPodWithIndex(3)
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
 
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)), 0)))
     assert(receivedSnapshots2 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
 
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)), 0)))
     assert(receivedSnapshots1 === receivedSnapshots2)
   }
 
@@ -112,13 +117,14 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
     assert(receivedSnapshots === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
+    clock.advance(100)
     eventQueueUnderTest.replaceSnapshot(Seq(podWithIndex(2)))
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
     assert(receivedSnapshots === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(2)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(2)), 100)))
   }
 
   private def pushPodWithIndex(index: Int): Unit =
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
index b6eeff1cd18a9..23d2b035e9e80 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
@@ -14,8 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+ARG java_image_tag=11-jre-slim
 
-FROM openjdk:8-jdk-slim
+FROM openjdk:${java_image_tag}
 
 ARG spark_uid=185
 
@@ -27,6 +28,7 @@ ARG spark_uid=185
 # docker build -t spark:latest -f kubernetes/dockerfiles/spark/Dockerfile .
 
 RUN set -ex && \
+    sed -i 's/http:\/\/deb.\(.*\)/https:\/\/deb.\1/g' /etc/apt/sources.list && \
     apt-get update && \
     ln -s /lib /lib64 && \
     apt install -y bash tini libc6 libpam-modules krb5-user libnss3 && \
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
index 59f375b707ca7..f63f2d0d58e22 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
@@ -25,7 +25,14 @@ USER 0
 
 RUN mkdir ${SPARK_HOME}/R
 
-RUN apt-get update && apt install -y r-base r-base-dev && rm -rf /var/cache/apt/*
+# Install R 3.6.3 (http://cloud.r-project.org/bin/linux/debian/)
+RUN \
+  echo "deb http://cloud.r-project.org/bin/linux/debian buster-cran35/" >> /etc/apt/sources.list && \
+  apt install -y gnupg && \
+  (apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF' || apt-key adv --keyserver keys.openpgp.org --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF') && \
+  apt-get update && \
+  apt install -y -t buster-cran35 r-base r-base-dev && \
+  rm -rf /var/cache/apt/*
 
 COPY R ${SPARK_HOME}/R
 ENV R_HOME /usr/lib/R
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
index 6ee3523c8edab..8218c2982c6f8 100755
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
@@ -58,8 +58,8 @@ fi
 
 # If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
 # It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n ${HADOOP_HOME}  ] && [ -z ${SPARK_DIST_CLASSPATH}  ]; then
-  export SPARK_DIST_CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath)  
+if [ -n "${HADOOP_HOME}"  ] && [ -z "${SPARK_DIST_CLASSPATH}"  ]; then
+  export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
 fi
 
 if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md
index d7ad35a175a61..6409c227ec287 100644
--- a/resource-managers/kubernetes/integration-tests/README.md
+++ b/resource-managers/kubernetes/integration-tests/README.md
@@ -6,13 +6,21 @@ title: Spark on Kubernetes Integration Tests
 # Running the Kubernetes Integration Tests
 
 Note that the integration test framework is currently being heavily revised and
-is subject to change. Note that currently the integration tests only run with Java 8.
+is subject to change.
 
 The simplest way to run the integration tests is to install and run Minikube, then run the following from this
 directory:
 
     ./dev/dev-run-integration-tests.sh
 
+To run tests with Java 11 instead of Java 8, use `--java-image-tag` to specify the base image.
+
+    ./dev/dev-run-integration-tests.sh --java-image-tag 11-jre-slim
+
+To run tests with Hadoop 3.2 instead of Hadoop 2.7, use `--hadoop-profile`.
+
+    ./dev/dev-run-integration-tests.sh --hadoop-profile hadoop-3.2
+
 The minimum tested version of Minikube is 0.23.0. The kube-dns addon must be enabled. Minikube should
 run with a minimum of 4 CPUs and 6G of memory:
 
@@ -183,7 +191,14 @@ to the wrapper scripts and using the wrapper scripts will simply set these appro
       A specific image tag to use, when set assumes images with those tags are already built and available in the 
       specified image repository.  When set to <code>N/A</code> (the default) fresh images will be built.
     </td>
-    <td><code>N/A</code>
+    <td><code>N/A</code></td>
+  </tr>
+  <tr>
+    <td><code>spark.kubernetes.test.javaImageTag</code></td>
+    <td>
+      A specific OpenJDK base image tag to use, when set uses it instead of 8-jre-slim.
+    </td>
+    <td><code>8-jre-slim</code></td>
   </tr>
   <tr>
     <td><code>spark.kubernetes.test.imageTagFile</code></td>
diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
index 1f0a8035cea7b..75e1ca8d577ca 100755
--- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
+++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
@@ -23,6 +23,7 @@ DEPLOY_MODE="minikube"
 IMAGE_REPO="docker.io/kubespark"
 SPARK_TGZ="N/A"
 IMAGE_TAG="N/A"
+JAVA_IMAGE_TAG=
 BASE_IMAGE_NAME=
 JVM_IMAGE_NAME=
 PYTHON_IMAGE_NAME=
@@ -34,6 +35,7 @@ CONTEXT=
 INCLUDE_TAGS="k8s"
 EXCLUDE_TAGS=
 JAVA_VERSION="8"
+HADOOP_PROFILE="hadoop-2.7"
 MVN="$TEST_ROOT_DIR/build/mvn"
 
 SCALA_VERSION=$("$MVN" help:evaluate -Dexpression=scala.binary.version 2>/dev/null\
@@ -52,6 +54,10 @@ while (( "$#" )); do
       IMAGE_TAG="$2"
       shift
       ;;
+    --java-image-tag)
+      JAVA_IMAGE_TAG="$2"
+      shift
+      ;;
     --deploy-mode)
       DEPLOY_MODE="$2"
       shift
@@ -104,6 +110,10 @@ while (( "$#" )); do
       JAVA_VERSION="$2"
       shift
       ;;
+    --hadoop-profile)
+      HADOOP_PROFILE="$2"
+      shift
+      ;;
     *)
       break
       ;;
@@ -120,27 +130,32 @@ properties=(
   -Dtest.include.tags=$INCLUDE_TAGS
 )
 
-if [ -n $NAMESPACE ];
+if [ -n "$JAVA_IMAGE_TAG" ];
+then
+  properties=( ${properties[@]} -Dspark.kubernetes.test.javaImageTag=$JAVA_IMAGE_TAG )
+fi
+
+if [ -n "$NAMESPACE" ];
 then
   properties=( ${properties[@]} -Dspark.kubernetes.test.namespace=$NAMESPACE )
 fi
 
-if [ -n $SERVICE_ACCOUNT ];
+if [ -n "$SERVICE_ACCOUNT" ];
 then
   properties=( ${properties[@]} -Dspark.kubernetes.test.serviceAccountName=$SERVICE_ACCOUNT )
 fi
 
-if [ -n $CONTEXT ];
+if [ -n "$CONTEXT" ];
 then
   properties=( ${properties[@]} -Dspark.kubernetes.test.kubeConfigContext=$CONTEXT )
 fi
 
-if [ -n $SPARK_MASTER ];
+if [ -n "$SPARK_MASTER" ];
 then
   properties=( ${properties[@]} -Dspark.kubernetes.test.master=$SPARK_MASTER )
 fi
 
-if [ -n $EXCLUDE_TAGS ];
+if [ -n "$EXCLUDE_TAGS" ];
 then
   properties=( ${properties[@]} -Dtest.exclude.tags=$EXCLUDE_TAGS )
 fi
@@ -156,4 +171,4 @@ properties+=(
   -Dspark.kubernetes.test.rImage=$R_IMAGE_NAME
 )
 
-$TEST_ROOT_DIR/build/mvn integration-test -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests -am -Pscala-$SCALA_VERSION -Pkubernetes -Pkubernetes-integration-tests ${properties[@]}
+$TEST_ROOT_DIR/build/mvn integration-test -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests -am -Pscala-$SCALA_VERSION -P$HADOOP_PROFILE -Pkubernetes -Pkubernetes-integration-tests ${properties[@]}
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 8e1043f77db6d..693e720df1aa3 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,18 +20,15 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
   <artifactId>spark-kubernetes-integration-tests_2.12</artifactId>
   <properties>
     <download-maven-plugin.version>1.3.0</download-maven-plugin.version>
-    <exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
     <extraScalaTestArgs></extraScalaTestArgs>
-    <kubernetes-client.version>4.6.4</kubernetes-client.version>
-    <scala-maven-plugin.version>3.2.2</scala-maven-plugin.version>
-    <scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
+    <kubernetes-client.version>4.9.2</kubernetes-client.version>
     <sbt.project.name>kubernetes-integration-tests</sbt.project.name>
 
     <!-- Integration Test Configuration Properties -->
@@ -39,6 +36,7 @@
     <spark.kubernetes.test.sparkTgz></spark.kubernetes.test.sparkTgz>
     <spark.kubernetes.test.unpackSparkDir>${project.build.directory}/spark-dist-unpacked</spark.kubernetes.test.unpackSparkDir>
     <spark.kubernetes.test.imageTag>N/A</spark.kubernetes.test.imageTag>
+    <spark.kubernetes.test.javaImageTag>8-jre-slim</spark.kubernetes.test.javaImageTag>
     <spark.kubernetes.test.imageTagFile>${project.build.directory}/imageTag.txt</spark.kubernetes.test.imageTagFile>
     <spark.kubernetes.test.deployMode>minikube</spark.kubernetes.test.deployMode>
     <spark.kubernetes.test.imageRepo>docker.io/kubespark</spark.kubernetes.test.imageRepo>
@@ -76,12 +74,6 @@
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
       <type>test-jar</type>
     </dependency>
-    <dependency>
-      <groupId>com.amazonaws</groupId>
-      <artifactId>aws-java-sdk</artifactId>
-      <version>1.7.4</version>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
   <build>
@@ -109,6 +101,9 @@
                 <argument>--image-tag</argument>
                 <argument>${spark.kubernetes.test.imageTag}</argument>
 
+                <argument>--java-image-tag</argument>
+                <argument>${spark.kubernetes.test.javaImageTag}</argument>
+
                 <argument>--image-tag-output-file</argument>
                 <argument>${spark.kubernetes.test.imageTagFile}</argument>
 
@@ -117,6 +112,9 @@
 
                 <argument>--spark-tgz</argument>
                 <argument>${spark.kubernetes.test.sparkTgz}</argument>
+
+                <argument>--test-exclude-tags</argument>
+                <argument>"${test.exclude.tags}"</argument>
               </arguments>
             </configuration>
           </execution>
@@ -182,4 +180,31 @@
 
   </build>
 
+  <profiles>
+    <profile>
+      <id>hadoop-2.7</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-java-sdk</artifactId>
+          <version>1.7.4</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+    <profile>
+      <id>hadoop-3.2</id>
+      <dependencies>
+        <dependency>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-java-sdk-bundle</artifactId>
+          <version>1.11.375</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
 </project>
diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
index 9e04b963fc40e..beda56cf37c94 100755
--- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
+++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
@@ -23,7 +23,10 @@ IMAGE_TAG_OUTPUT_FILE="$TEST_ROOT_DIR/target/image-tag.txt"
 DEPLOY_MODE="minikube"
 IMAGE_REPO="docker.io/kubespark"
 IMAGE_TAG="N/A"
+JAVA_IMAGE_TAG="8-jre-slim"
 SPARK_TGZ="N/A"
+MVN="$TEST_ROOT_DIR/build/mvn"
+EXCLUDE_TAGS=""
 
 # Parse arguments
 while (( "$#" )); do
@@ -40,6 +43,10 @@ while (( "$#" )); do
       IMAGE_TAG="$2"
       shift
       ;;
+    --java-image-tag)
+      JAVA_IMAGE_TAG="$2"
+      shift
+      ;;
     --image-tag-output-file)
       IMAGE_TAG_OUTPUT_FILE="$2"
       shift
@@ -52,6 +59,10 @@ while (( "$#" )); do
       SPARK_TGZ="$2"
       shift
       ;;
+    --test-exclude-tags)
+      EXCLUDE_TAGS="$2"
+      shift
+      ;;
     *)
       break
       ;;
@@ -79,14 +90,24 @@ fi
 # If there is a specific Spark image skip building and extraction/copy
 if [[ $IMAGE_TAG == "N/A" ]];
 then
-  IMAGE_TAG=$(uuidgen);
+  VERSION=$("$MVN" help:evaluate -Dexpression=project.version \
+    | grep -v "INFO"\
+    | grep -v "WARNING"\
+    | tail -n 1)
+  IMAGE_TAG=${VERSION}_$(uuidgen)
   cd $SPARK_INPUT_DIR
 
+  # OpenJDK base-image tag (e.g. 8-jre-slim, 11-jre-slim)
+  JAVA_IMAGE_TAG_BUILD_ARG="-b java_image_tag=$JAVA_IMAGE_TAG"
+
   # Build PySpark image
   LANGUAGE_BINDING_BUILD_ARGS="-p $DOCKER_FILE_BASE_PATH/bindings/python/Dockerfile"
 
   # Build SparkR image
-  LANGUAGE_BINDING_BUILD_ARGS="$LANGUAGE_BINDING_BUILD_ARGS -R $DOCKER_FILE_BASE_PATH/bindings/R/Dockerfile"
+  tags=(${EXCLUDE_TAGS//,/ })
+  if [[ ! ${tags[@]} =~ "r" ]]; then
+    LANGUAGE_BINDING_BUILD_ARGS="$LANGUAGE_BINDING_BUILD_ARGS -R $DOCKER_FILE_BASE_PATH/bindings/R/Dockerfile"
+  fi
 
   # Unset SPARK_HOME to let the docker-image-tool script detect SPARK_HOME. Otherwise, it cannot
   # indicate the unpacked directory as its home. See SPARK-28550.
@@ -95,7 +116,7 @@ then
   case $DEPLOY_MODE in
     cloud)
       # Build images
-      $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+      $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $JAVA_IMAGE_TAG_BUILD_ARG $LANGUAGE_BINDING_BUILD_ARGS build
 
       # Push images appropriately
       if [[ $IMAGE_REPO == gcr.io* ]] ;
@@ -109,13 +130,13 @@ then
     docker-for-desktop)
        # Only need to build as this will place it in our local Docker repo which is all
        # we need for Docker for Desktop to work so no need to also push
-       $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+       $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $JAVA_IMAGE_TAG_BUILD_ARG $LANGUAGE_BINDING_BUILD_ARGS build
        ;;
 
     minikube)
        # Only need to build and if we do this with the -m option for minikube we will
        # build the images directly using the minikube Docker daemon so no need to push
-       $SPARK_INPUT_DIR/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+       $SPARK_INPUT_DIR/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG $JAVA_IMAGE_TAG_BUILD_ARG $LANGUAGE_BINDING_BUILD_ARGS build
        ;;
     *)
        echo "Unrecognized deploy mode $DEPLOY_MODE" && exit 1
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
index 6b340f2558cca..76221e46d1cf4 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
@@ -111,6 +111,6 @@ private[spark] object BasicTestsSuite {
   val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE =
      s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt"
   val REMOTE_PAGE_RANK_DATA_FILE =
-    "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt"
+    "https://raw.githubusercontent.com/apache/spark/master/data/mllib/pagerank_data.txt"
   val REMOTE_PAGE_RANK_FILE_NAME = "pagerank_data.txt"
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
index 7181774b9f17e..e712b95cdbcea 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
@@ -35,17 +35,16 @@ import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube
 private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
   import KubernetesSuite.k8sTestTag
 
-  val cName = "ceph-nano"
+  val cName = "minio"
   val svcName = s"$cName-s3"
-  val bucket = "spark"
-
-  private def getCephContainer(): Container = {
-    val envVars = Map ( "NETWORK_AUTO_DETECT" -> "4",
-      "RGW_FRONTEND_PORT" -> "8000",
-      "SREE_PORT" -> "5001",
-      "CEPH_DEMO_UID" -> "nano",
-      "CEPH_DAEMON" -> "demo",
-      "DEBUG" -> "verbose"
+  val BUCKET = "spark"
+  val ACCESS_KEY = "minio"
+  val SECRET_KEY = "miniostorage"
+
+  private def getMinioContainer(): Container = {
+    val envVars = Map (
+      "MINIO_ACCESS_KEY" -> ACCESS_KEY,
+      "MINIO_SECRET_KEY" -> SECRET_KEY
     ).map( envV =>
       new EnvVarBuilder()
         .withName(envV._1)
@@ -54,22 +53,19 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
     ).toArray
 
     val resources = Map(
-      "cpu" -> new QuantityBuilder()
-        .withAmount("1")
-        .build(),
-      "memory" -> new QuantityBuilder()
-        .withAmount("512M")
-        .build()
+      "cpu" -> new Quantity("1"),
+      "memory" -> new Quantity("512M")
     ).asJava
 
     new ContainerBuilder()
-      .withImage("ceph/daemon:latest")
+      .withImage("minio/minio:latest")
       .withImagePullPolicy("Always")
       .withName(cName)
+      .withArgs("server", "/data")
       .withPorts(new ContainerPortBuilder()
           .withName(svcName)
           .withProtocol("TCP")
-          .withContainerPort(8000)
+          .withContainerPort(9000)
         .build()
       )
       .withResources(new ResourceRequirementsBuilder()
@@ -81,10 +77,9 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
       .build()
   }
 
-  // Based on https://github.com/ceph/cn
-  private def setupCephStorage(): Unit = {
-    val labels = Map("app" -> "ceph", "daemon" -> "nano").asJava
-    val cephService = new ServiceBuilder()
+  private def setupMinioStorage(): Unit = {
+    val labels = Map("app" -> "minio").asJava
+    val minioService = new ServiceBuilder()
       .withNewMetadata()
         .withName(svcName)
       .withLabels(labels)
@@ -92,9 +87,9 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
       .withNewSpec()
         .withPorts(new ServicePortBuilder()
           .withName("https")
-          .withPort(8000)
+          .withPort(9000)
           .withProtocol("TCP")
-          .withTargetPort(new IntOrString(8000))
+          .withTargetPort(new IntOrString(9000))
           .build()
         )
         .withType("NodePort")
@@ -102,7 +97,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
       .endSpec()
       .build()
 
-    val cephStatefulSet = new StatefulSetBuilder()
+    val minioStatefulSet = new StatefulSetBuilder()
       .withNewMetadata()
         .withName(cName)
         .withLabels(labels)
@@ -110,7 +105,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
       .withNewSpec()
         .withReplicas(1)
         .withNewSelector()
-          .withMatchLabels(Map("app" -> "ceph").asJava)
+          .withMatchLabels(Map("app" -> "minio").asJava)
         .endSelector()
         .withServiceName(cName)
         .withNewTemplate()
@@ -119,7 +114,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
             .withLabels(labels)
            .endMetadata()
           .withNewSpec()
-            .withContainers(getCephContainer())
+            .withContainers(getMinioContainer())
           .endSpec()
         .endTemplate()
       .endSpec()
@@ -128,16 +123,16 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
     kubernetesTestComponents
       .kubernetesClient
       .services()
-      .create(cephService)
+      .create(minioService)
 
     kubernetesTestComponents
       .kubernetesClient
       .apps()
       .statefulSets()
-      .create(cephStatefulSet)
+      .create(minioStatefulSet)
   }
 
- private def deleteCephStorage(): Unit = {
+ private def deleteMinioStorage(): Unit = {
     kubernetesTestComponents
       .kubernetesClient
       .apps()
@@ -153,49 +148,36 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
   }
 
   test("Launcher client dependencies", k8sTestTag, MinikubeTag) {
+    val packages = if (Utils.isHadoop3) {
+      "org.apache.hadoop:hadoop-aws:3.2.0"
+    } else {
+      "com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.6"
+    }
     val fileName = Utils.createTempFile(FILE_CONTENTS, HOST_PATH)
     try {
-      setupCephStorage()
-      val cephUrlStr = getServiceUrl(svcName)
-      val cephUrl = new URL(cephUrlStr)
-      val cephHost = cephUrl.getHost
-      val cephPort = cephUrl.getPort
+      setupMinioStorage()
+      val minioUrlStr = getServiceUrl(svcName)
+      val minioUrl = new URL(minioUrlStr)
+      val minioHost = minioUrl.getHost
+      val minioPort = minioUrl.getPort
       val examplesJar = Utils.getExamplesJarAbsolutePath(sparkHomeDir)
-      val (accessKey, secretKey) = getCephCredentials()
       sparkAppConf
-        .set("spark.hadoop.fs.s3a.access.key", accessKey)
-        .set("spark.hadoop.fs.s3a.secret.key", secretKey)
+        .set("spark.hadoop.fs.s3a.access.key", ACCESS_KEY)
+        .set("spark.hadoop.fs.s3a.secret.key", SECRET_KEY)
         .set("spark.hadoop.fs.s3a.connection.ssl.enabled", "false")
-        .set("spark.hadoop.fs.s3a.endpoint", s"$cephHost:$cephPort")
-        .set("spark.kubernetes.file.upload.path", s"s3a://$bucket")
+        .set("spark.hadoop.fs.s3a.endpoint", s"$minioHost:$minioPort")
+        .set("spark.kubernetes.file.upload.path", s"s3a://$BUCKET")
         .set("spark.files", s"$HOST_PATH/$fileName")
         .set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
-        .set("spark.jars.packages", "com.amazonaws:aws-java-sdk:" +
-          "1.7.4,org.apache.hadoop:hadoop-aws:2.7.6")
+        .set("spark.jars.packages", packages)
         .set("spark.driver.extraJavaOptions", "-Divy.cache.dir=/tmp -Divy.home=/tmp")
-      createS3Bucket(accessKey, secretKey, cephUrlStr)
+      createS3Bucket(ACCESS_KEY, SECRET_KEY, minioUrlStr)
       runSparkRemoteCheckAndVerifyCompletion(appResource = examplesJar,
         appArgs = Array(fileName),
         timeout = Option(DEPS_TIMEOUT))
     } finally {
       // make sure this always runs
-      deleteCephStorage()
-    }
-  }
-
-  // There isn't a cleaner way to get the credentials
-  // when ceph-nano runs on k8s
-  private def getCephCredentials(): (String, String) = {
-    Eventually.eventually(TIMEOUT, INTERVAL) {
-      val cephPod = kubernetesTestComponents
-        .kubernetesClient
-        .pods()
-        .withName(s"$cName-0")
-        .get()
-      implicit val podName: String = cephPod.getMetadata.getName
-      implicit val components: KubernetesTestComponents = kubernetesTestComponents
-      val contents = Utils.executeCommand("cat", "/nano_user_details")
-    (extractS3Key(contents, "access_key"), extractS3Key(contents, "secret_key"))
+      deleteMinioStorage()
     }
   }
 
@@ -215,10 +197,10 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
         val credentials = new BasicAWSCredentials(accessKey, secretKey)
         val s3client = new AmazonS3Client(credentials)
         s3client.setEndpoint(endPoint)
-        s3client.createBucket(bucket)
+        s3client.createBucket(BUCKET)
       } catch {
         case e: Exception =>
-          throw new SparkException(s"Failed to create bucket $bucket.", e)
+          throw new SparkException(s"Failed to create bucket $BUCKET.", e)
       }
     }
   }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 0d4fcccc35cf9..e6fb4e1bdffd4 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -59,15 +59,15 @@ class KubernetesSuite extends SparkFunSuite
   protected var appLocator: String = _
 
   // Default memory limit is 1024M + 384M (minimum overhead constant)
-  private val baseMemory = s"${1024 + 384}Mi"
+  private val baseMemory = s"${1024 + 384}"
   protected val memOverheadConstant = 0.8
-  private val standardNonJVMMemory = s"${(1024 + 0.4*1024).toInt}Mi"
+  private val standardNonJVMMemory = s"${(1024 + 0.4*1024).toInt}"
   protected val additionalMemory = 200
   // 209715200 is 200Mi
   protected val additionalMemoryInBytes = 209715200
-  private val extraDriverTotalMemory = s"${(1024 + memOverheadConstant*1024).toInt}Mi"
+  private val extraDriverTotalMemory = s"${(1024 + memOverheadConstant*1024).toInt}"
   private val extraExecTotalMemory =
-    s"${(1024 + memOverheadConstant*1024 + additionalMemory).toInt}Mi"
+    s"${(1024 + memOverheadConstant*1024 + additionalMemory).toInt}"
 
   /**
    * Build the image ref for the given image name, taking the repo and tag from the
@@ -419,11 +419,12 @@ class KubernetesSuite extends SparkFunSuite
 
 private[spark] object KubernetesSuite {
   val k8sTestTag = Tag("k8s")
+  val rTestTag = Tag("r")
   val MinikubeTag = Tag("minikube")
   val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi"
   val SPARK_DFS_READ_WRITE_TEST = "org.apache.spark.examples.DFSReadWriteTest"
   val SPARK_REMOTE_MAIN_CLASS: String = "org.apache.spark.examples.SparkRemoteFileTest"
   val SPARK_DRIVER_MAIN_CLASS: String = "org.apache.spark.examples.DriverSubmissionTest"
-  val TIMEOUT = PatienceConfiguration.Timeout(Span(2, Minutes))
+  val TIMEOUT = PatienceConfiguration.Timeout(Span(3, Minutes))
   val INTERVAL = PatienceConfiguration.Interval(Span(2, Seconds))
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
index 4cfda8ab9ba77..af980f0494369 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
@@ -69,7 +69,7 @@ private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesCl
       .set("spark.master", s"k8s://${kubernetesClient.getMasterUrl}")
       .set("spark.kubernetes.namespace", namespace)
       .set("spark.executor.cores", "1")
-      .set("spark.executors.instances", "1")
+      .set("spark.executor.instances", "1")
       .set("spark.app.name", "spark-test-app")
       .set(IS_TESTING.key, "false")
       .set(UI_ENABLED.key, "true")
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
index f0218217e6afb..86f8cddcef7da 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
@@ -45,7 +45,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite =>
         .withName("test-local-pv")
       .endMetadata()
       .withNewSpec()
-        .withCapacity(Map("storage" -> new QuantityBuilder().withAmount("1Gi").build()).asJava)
+        .withCapacity(Map("storage" -> new Quantity("1Gi")).asJava)
         .withAccessModes("ReadWriteOnce")
         .withPersistentVolumeReclaimPolicy("Retain")
         .withStorageClassName("test-local-storage")
@@ -56,7 +56,8 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite =>
                 .withMatchExpressions(new NodeSelectorRequirementBuilder()
                   .withKey("kubernetes.io/hostname")
                   .withOperator("In")
-                  .withValues("minikube", "docker-for-desktop", "docker-desktop").build()).build())
+                  .withValues("minikube", "m01", "docker-for-desktop", "docker-desktop")
+                  .build()).build())
             .endRequired()
           .endNodeAffinity()
       .endSpec()
@@ -71,8 +72,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite =>
         .withAccessModes("ReadWriteOnce")
         .withStorageClassName("test-local-storage")
         .withResources(new ResourceRequirementsBuilder()
-        .withRequests(Map("storage" -> new QuantityBuilder()
-          .withAmount("1Gi").build()).asJava).build())
+          .withRequests(Map("storage" -> new Quantity("1Gi")).asJava).build())
       .endSpec()
 
     kubernetesTestComponents
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala
index e81562a923228..b7c8886a15ae7 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala
@@ -19,9 +19,9 @@ package org.apache.spark.deploy.k8s.integrationtest
 private[spark] trait RTestsSuite { k8sSuite: KubernetesSuite =>
 
   import RTestsSuite._
-  import KubernetesSuite.k8sTestTag
+  import KubernetesSuite.{k8sTestTag, rTestTag}
 
-  test("Run SparkR on simple dataframe.R example", k8sTestTag) {
+  test("Run SparkR on simple dataframe.R example", k8sTestTag, rTestTag) {
     sparkAppConf.set("spark.kubernetes.container.image", rImage)
     runSparkApplicationAndVerifyCompletion(
       appResource = SPARK_R_DATAFRAME_TEST,
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
index 9f85805b9d315..0000a94725763 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
@@ -21,13 +21,16 @@ import java.nio.file.{Files, Path}
 import java.util.concurrent.CountDownLatch
 
 import scala.collection.JavaConverters._
+import scala.util.Try
 
 import io.fabric8.kubernetes.client.dsl.ExecListener
 import okhttp3.Response
 import org.apache.commons.io.output.ByteArrayOutputStream
+import org.apache.hadoop.util.VersionInfo
 
 import org.apache.spark.{SPARK_VERSION, SparkException}
 import org.apache.spark.internal.Logging
+import org.apache.spark.util.{Utils => SparkUtils}
 
 object Utils extends Logging {
 
@@ -131,4 +134,8 @@ object Utils extends Logging {
         s"under spark home test dir ${sparkHomeDir.toAbsolutePath}!")
     }
   }
+
+  def isHadoop3(): Boolean = {
+    VersionInfo.getVersion.startsWith("3")
+  }
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
index a7d35b67d1b92..7753b0977aa57 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.spark.deploy.k8s.integrationtest.backend.minikube
 
-import java.nio.file.Paths
+import java.nio.file.{Files, Paths}
 
 import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient}
 
@@ -68,15 +68,23 @@ private[spark] object Minikube extends Logging {
   def getKubernetesClient: DefaultKubernetesClient = {
     val kubernetesMaster = s"https://${getMinikubeIp}:8443"
     val userHome = System.getProperty("user.home")
+    val minikubeBasePath = Paths.get(userHome, MINIKUBE_PATH).toString
+    val profileDir = if (Files.exists(Paths.get(minikubeBasePath, "apiserver.crt"))) {
+      // For Minikube <1.9
+      ""
+    } else {
+      // For Minikube >=1.9
+      Paths.get("profiles", executeMinikube("profile")(0)).toString
+    }
+    val apiServerCertPath = Paths.get(minikubeBasePath, profileDir, "apiserver.crt")
+    val apiServerKeyPath = Paths.get(minikubeBasePath, profileDir, "apiserver.key")
     val kubernetesConf = new ConfigBuilder()
       .withApiVersion("v1")
       .withMasterUrl(kubernetesMaster)
       .withCaCertFile(
         Paths.get(userHome, MINIKUBE_PATH, "ca.crt").toFile.getAbsolutePath)
-      .withClientCertFile(
-        Paths.get(userHome, MINIKUBE_PATH, "apiserver.crt").toFile.getAbsolutePath)
-      .withClientKeyFile(
-        Paths.get(userHome, MINIKUBE_PATH, "apiserver.key").toFile.getAbsolutePath)
+      .withClientCertFile(apiServerCertPath.toFile.getAbsolutePath)
+      .withClientKeyFile(apiServerKeyPath.toFile.getAbsolutePath)
       .build()
     new DefaultKubernetesClient(kubernetesConf)
   }
@@ -120,8 +128,11 @@ private[spark] object Minikube extends Logging {
 
   private def executeMinikube(action: String, args: String*): Seq[String] = {
     ProcessUtils.executeProcess(
-      Array("bash", "-c", s"minikube $action ${args.mkString(" ")}"),
-      MINIKUBE_STARTUP_TIMEOUT_SECONDS)
+      Array("bash", "-c", s"MINIKUBE_IN_STYLE=true minikube $action ${args.mkString(" ")}"),
+      MINIKUBE_STARTUP_TIMEOUT_SECONDS).filter{x =>
+      !x.contains("There is a newer version of minikube") &&
+      !x.contains("https://github.com/kubernetes")
+    }
   }
 
   def minikubeServiceAction(args: String*): String = {
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index 107ba365aa775..ec93eefa1d8f7 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
index 79a113792de91..e1c0d18b73a2b 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
@@ -28,6 +28,7 @@ package object config {
       ConfigBuilder(s"spark.mesos.$taskType.secret.names")
         .doc("A comma-separated list of secret reference names. Consult the Mesos Secret " +
           "protobuf for more information.")
+        .version("2.3.0")
         .stringConf
         .toSequence
         .createOptional
@@ -35,6 +36,7 @@ package object config {
     private[spark] val SECRET_VALUES =
       ConfigBuilder(s"spark.mesos.$taskType.secret.values")
         .doc("A comma-separated list of secret values.")
+        .version("2.3.0")
         .stringConf
         .toSequence
         .createOptional
@@ -43,6 +45,7 @@ package object config {
       ConfigBuilder(s"spark.mesos.$taskType.secret.envkeys")
         .doc("A comma-separated list of the environment variables to contain the secrets." +
           "The environment variable will be set on the driver.")
+        .version("2.3.0")
         .stringConf
         .toSequence
         .createOptional
@@ -51,6 +54,7 @@ package object config {
       ConfigBuilder(s"spark.mesos.$taskType.secret.filenames")
         .doc("A comma-separated list of file paths secret will be written to.  Consult the Mesos " +
           "Secret protobuf for more information.")
+        .version("2.3.0")
         .stringConf
         .toSequence
         .createOptional
@@ -59,6 +63,7 @@ package object config {
   private[spark] val CREDENTIAL_PRINCIPAL =
     ConfigBuilder("spark.mesos.principal")
       .doc("Name of the Kerberos principal to authenticate Spark to Mesos.")
+      .version("1.5.0")
       .stringConf
       .createOptional
 
@@ -66,18 +71,21 @@ package object config {
     ConfigBuilder("spark.mesos.principal.file")
       .doc("The path of file which contains the name of the Kerberos principal " +
         "to authenticate Spark to Mesos.")
+      .version("2.4.0")
       .stringConf
       .createOptional
 
   private[spark] val CREDENTIAL_SECRET =
     ConfigBuilder("spark.mesos.secret")
       .doc("The secret value to authenticate Spark to Mesos.")
+      .version("1.5.0")
       .stringConf
       .createOptional
 
   private[spark] val CREDENTIAL_SECRET_FILE =
     ConfigBuilder("spark.mesos.secret.file")
       .doc("The path of file which contains the secret value to authenticate Spark to Mesos.")
+      .version("2.4.0")
       .stringConf
       .createOptional
 
@@ -85,6 +93,7 @@ package object config {
 
   private[spark] val SHUFFLE_CLEANER_INTERVAL_S =
     ConfigBuilder("spark.shuffle.cleaner.interval")
+      .version("2.0.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("30s")
 
@@ -92,6 +101,7 @@ package object config {
     ConfigBuilder("spark.mesos.dispatcher.webui.url")
       .doc("Set the Spark Mesos dispatcher webui_url for interacting with the " +
         "framework. If unset it will point to Spark's internal web UI.")
+      .version("2.0.0")
       .stringConf
       .createOptional
 
@@ -99,6 +109,7 @@ package object config {
     ConfigBuilder("spark.mesos.dispatcher.historyServer.url")
       .doc("Set the URL of the history server. The dispatcher will then " +
         "link each driver to its entry in the history server.")
+      .version("2.1.0")
       .stringConf
       .createOptional
 
@@ -107,6 +118,7 @@ package object config {
       .doc("Mesos labels to add to the driver.  Labels are free-form key-value pairs. Key-value " +
         "pairs should be separated by a colon, and commas used to list more than one." +
         "Ex. key:value,key2:value2")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
@@ -114,6 +126,7 @@ package object config {
     ConfigBuilder("spark.mesos.driver.webui.url")
       .doc("Set the Spark Mesos driver webui_url for interacting with the framework. " +
         "If unset it will point to Spark's internal web UI.")
+      .version("2.0.0")
       .stringConf
       .createOptional
 
@@ -125,6 +138,7 @@ package object config {
     ConfigBuilder("spark.mesos.driver.failoverTimeout")
       .doc("Amount of time in seconds that the master will wait to hear from the driver, " +
           "during a temporary disconnection, before tearing down all the executors.")
+      .version("2.3.0")
       .doubleConf
       .createWithDefault(0.0)
 
@@ -132,6 +146,7 @@ package object config {
     ConfigBuilder("spark.mesos.network.name")
       .doc("Attach containers to the given named network. If this job is launched " +
         "in cluster mode, also launch the driver in the given named network.")
+      .version("2.1.0")
       .stringConf
       .createOptional
 
@@ -140,6 +155,7 @@ package object config {
       .doc("Network labels to pass to CNI plugins.  This is a comma-separated list " +
         "of key-value pairs, where each key-value pair has the format key:value. " +
         "Example: key1:val1,key2:val2")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
@@ -147,19 +163,21 @@ package object config {
     ConfigBuilder("spark.mesos.driver.constraints")
       .doc("Attribute based constraints on mesos resource offers. Applied by the dispatcher " +
         "when launching drivers. Default is to accept all offers with sufficient resources.")
+      .version("2.2.1")
       .stringConf
       .createWithDefault("")
 
   private[spark] val DRIVER_FRAMEWORK_ID =
     ConfigBuilder("spark.mesos.driver.frameworkId")
+      .version("2.1.0")
       .stringConf
       .createOptional
 
   private[spark] val EXECUTOR_URI =
-    ConfigBuilder("spark.executor.uri").stringConf.createOptional
+    ConfigBuilder("spark.executor.uri").version("0.8.0").stringConf.createOptional
 
   private[spark] val PROXY_BASE_URL =
-    ConfigBuilder("spark.mesos.proxy.baseURL").stringConf.createOptional
+    ConfigBuilder("spark.mesos.proxy.baseURL").version("2.3.0").stringConf.createOptional
 
   private[spark] val COARSE_MODE =
     ConfigBuilder("spark.mesos.coarse")
@@ -167,22 +185,26 @@ package object config {
         "Spark acquires one long-lived Mesos task on each machine. If set to false, runs over " +
         "Mesos cluster in \"fine-grained\" sharing mode, where one Mesos task is created per " +
         "Spark task.")
-      .booleanConf.createWithDefault(true)
+      .version("0.6.0")
+      .booleanConf
+      .createWithDefault(true)
 
   private[spark] val COARSE_SHUTDOWN_TIMEOUT =
     ConfigBuilder("spark.mesos.coarse.shutdownTimeout")
+      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(_ >= 0, s"spark.mesos.coarse.shutdownTimeout must be >= 0")
       .createWithDefaultString("10s")
 
   private[spark] val MAX_DRIVERS =
-    ConfigBuilder("spark.mesos.maxDrivers").intConf.createWithDefault(200)
+    ConfigBuilder("spark.mesos.maxDrivers").version("1.4.0").intConf.createWithDefault(200)
 
   private[spark] val RETAINED_DRIVERS =
-    ConfigBuilder("spark.mesos.retainedDrivers").intConf.createWithDefault(200)
+    ConfigBuilder("spark.mesos.retainedDrivers").version("1.4.0").intConf.createWithDefault(200)
 
   private[spark] val CLUSTER_RETRY_WAIT_MAX_SECONDS =
     ConfigBuilder("spark.mesos.cluster.retry.wait.max")
+      .version("1.4.0")
       .intConf
       .createWithDefault(60) // 1 minute
 
@@ -190,6 +212,7 @@ package object config {
     ConfigBuilder("spark.mesos.fetcherCache.enable")
       .doc("If set to true, all URIs (example: `spark.executor.uri`, `spark.mesos.uris`) will be " +
         "cached by the Mesos Fetcher Cache.")
+      .version("2.1.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -201,6 +224,7 @@ package object config {
         "resource from the host's file system. If the value is unknown it prints a warning msg " +
         "in the dispatcher logs and defaults to `host`. If the value is `container` then spark " +
         "submit in the container will use the jar in the container's path: `/path/to/jar`.")
+      .version("2.4.0")
       .stringConf
       .checkValues(Set("host", "container"))
       .createWithDefault("host")
@@ -210,12 +234,14 @@ package object config {
       .doc("Time to consider unused resources refused, serves as a fallback of " +
         "`spark.mesos.rejectOfferDurationForUnmetConstraints`, " +
         "`spark.mesos.rejectOfferDurationForReachedMaxCores`.")
+      .version("2.2.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("120s")
 
   private[spark] val REJECT_OFFER_DURATION_FOR_UNMET_CONSTRAINTS =
     ConfigBuilder("spark.mesos.rejectOfferDurationForUnmetConstraints")
       .doc("Time to consider unused resources refused with unmet constraints.")
+      .version("1.6.0")
       .timeConf(TimeUnit.SECONDS)
       .createOptional
 
@@ -223,6 +249,7 @@ package object config {
     ConfigBuilder("spark.mesos.rejectOfferDurationForReachedMaxCores")
       .doc("Time to consider unused resources refused when maximum number of cores " +
         "`spark.cores.max` is reached.")
+      .version("2.0.0")
       .timeConf(TimeUnit.SECONDS)
       .createOptional
 
@@ -231,6 +258,7 @@ package object config {
       .doc("A comma-separated list of URIs to be downloaded to the sandbox when driver or " +
         "executor is launched by Mesos. This applies to both coarse-grained and fine-grained " +
         "mode.")
+      .version("1.5.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
@@ -241,6 +269,7 @@ package object config {
         "By default, the executors will simply use the driver's Spark home directory, which may " +
         "not be visible to them. Note that this is only relevant if a Spark binary package is " +
         "not specified through `spark.executor.uri`.")
+      .version("1.1.1")
       .stringConf
       .createOptional
 
@@ -250,6 +279,7 @@ package object config {
         "include the cores used to run the Spark tasks. In other words, even if no Spark task " +
         "is being run, each Mesos executor will occupy the number of cores configured here. " +
         "The value can be a floating point number.")
+      .version("1.4.0")
       .doubleConf
       .createWithDefault(1.0)
 
@@ -259,6 +289,7 @@ package object config {
         "more cores allocated. It instead means that an executor will \"pretend\" it has more " +
         "cores, so that the driver will send it more tasks. Use this to increase parallelism. " +
         "This setting is only used for Mesos coarse-grained mode.")
+      .version("0.6.0")
       .intConf
       .createWithDefault(0)
 
@@ -267,6 +298,7 @@ package object config {
       .doc("The amount of additional memory, specified in MiB, to be allocated per executor. " +
         "By default, the overhead will be larger of either 384 or 10% of " +
         "`spark.executor.memory`. If set, the final overhead will be this value.")
+      .version("1.1.1")
       .intConf
       .createOptional
 
@@ -277,6 +309,7 @@ package object config {
         "The installed path of Spark in the image can be specified with " +
         "`spark.mesos.executor.home`; the installed path of the Mesos library can be specified " +
         "with `spark.executorEnv.MESOS_NATIVE_JAVA_LIBRARY`.")
+      .version("1.4.0")
       .stringConf
       .createOptional
 
@@ -285,11 +318,13 @@ package object config {
       .doc("Force Mesos agents to pull the image specified in " +
         "`spark.mesos.executor.docker.image`. By default Mesos agents will not pull images they " +
         "already have cached.")
+      .version("2.1.0")
       .booleanConf
       .createOptional
 
   private[spark] val EXECUTOR_DOCKER_PORT_MAPS =
     ConfigBuilder("spark.mesos.executor.docker.portmaps")
+      .version("1.4.0")
       .stringConf
       .toSequence
       .createOptional
@@ -299,6 +334,7 @@ package object config {
       .doc("Set the list of custom parameters which will be passed into the `docker run` " +
         "command when launching the Spark executor on Mesos using the docker containerizer. " +
         "The format of this property is a list of key/value pairs which pair looks key1=value1.")
+      .version("2.2.0")
       .stringConf
       .toSequence
       .createOptional
@@ -309,6 +345,7 @@ package object config {
         "using `spark.mesos.executor.docker.image`. The format of this property is a list of " +
         "mappings following the form passed to `docker run -v`. That is they take the form:  " +
         "`[host_path:]container_path[:ro|:rw]`")
+      .version("1.4.0")
       .stringConf
       .toSequence
       .createOptional
@@ -318,6 +355,7 @@ package object config {
       .doc("Set the maximum number GPU resources to acquire for this job. Note that executors " +
         "will still launch when no GPU resources are found since this configuration is just an " +
         "upper limit and not a guaranteed amount.")
+      .version("2.1.0")
       .intConf
       .createWithDefault(0)
 
@@ -327,6 +365,7 @@ package object config {
         "Key-value pairs should be separated by a colon, and commas used to list more than one. " +
         "If your label includes a colon or comma, you can escape it with a backslash. " +
         "Ex. key:value,key2:a\\:b.")
+      .version("2.2.0")
       .stringConf
       .createWithDefault("")
 
@@ -335,6 +374,7 @@ package object config {
       .doc("Attribute-based constraints on mesos resource offers. By default, all resource " +
         "offers will be accepted. This setting applies only to executors. Refer to Mesos " +
         "Attributes & Resources doc for more information on attributes.")
+      .version("1.5.0")
       .stringConf
       .createWithDefault("")
 
@@ -344,6 +384,7 @@ package object config {
         "Mesos supports two types of containerizers for docker: the \"docker\" containerizer, " +
         "and the preferred \"mesos\" containerizer. " +
         "Read more here: http://mesos.apache.org/documentation/latest/container-image/")
+      .version("2.1.0")
       .stringConf
       .checkValues(Set("docker", "mesos"))
       .createWithDefault("docker")
@@ -352,6 +393,7 @@ package object config {
     ConfigBuilder("spark.mesos.role")
       .doc("Set the role of this Spark framework for Mesos. Roles are used in Mesos for " +
         "reservations and resource weight sharing.")
+      .version("1.5.0")
       .stringConf
       .createOptional
 
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
index e916125ffdb67..dff0b0622cd87 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -34,6 +34,7 @@ import org.apache.spark.{SecurityManager, SparkConf, SparkContext, SparkExceptio
 import org.apache.spark.deploy.mesos.config._
 import org.apache.spark.deploy.security.HadoopDelegationTokenManager
 import org.apache.spark.internal.config
+import org.apache.spark.internal.config.Network
 import org.apache.spark.internal.config.Tests.IS_TESTING
 import org.apache.spark.launcher.{LauncherBackend, SparkAppHandle}
 import org.apache.spark.network.netty.SparkTransportConf
@@ -643,7 +644,9 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
           .registerDriverWithShuffleService(
             slave.hostname,
             externalShufflePort,
-            sc.conf.get(config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT),
+            sc.conf.get(
+              config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT
+            ).getOrElse(Utils.timeStringAsMs(s"${sc.conf.get(Network.NETWORK_TIMEOUT)}s")),
             sc.conf.get(config.EXECUTOR_HEARTBEAT_INTERVAL))
         slave.shuffleRegistered = true
       }
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
index 7b2f6a2535eda..402d80f6dd2fb 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
@@ -137,7 +137,7 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging {
     val containerInfo = ContainerInfo.newBuilder()
       .setType(containerType)
 
-    conf.get(EXECUTOR_DOCKER_IMAGE).map { image =>
+    conf.get(EXECUTOR_DOCKER_IMAGE).foreach { image =>
       val forcePullImage = conf
         .get(EXECUTOR_DOCKER_FORCE_PULL_IMAGE).contains(true)
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index 0e5df14e060da..f0031cf2466b5 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 1e8f4084ef9c7..99efa5c497abc 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
 import org.apache.spark._
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.deploy.history.HistoryServer
+import org.apache.spark.deploy.security.HadoopDelegationTokenManager
 import org.apache.spark.deploy.yarn.config._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
@@ -774,6 +775,11 @@ private[spark] class ApplicationMaster(
       driver.send(RegisterClusterManager(self))
     }
 
+    override def receive: PartialFunction[Any, Unit] = {
+      case UpdateDelegationTokens(tokens) =>
+        SparkHadoopUtil.get.addDelegationTokens(tokens, sparkConf)
+    }
+
     override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
       case r: RequestExecutors =>
         Option(allocator) match {
@@ -805,9 +811,6 @@ private[spark] class ApplicationMaster(
           case None =>
             logWarning("Container allocator is not ready to find executor loss reasons yet.")
         }
-
-      case UpdateDelegationTokens(tokens) =>
-        SparkHadoopUtil.get.addDelegationTokens(tokens, sparkConf)
     }
 
     override def onDisconnected(remoteAddress: RpcAddress): Unit = {
@@ -860,10 +863,22 @@ object ApplicationMaster extends Logging {
     val ugi = sparkConf.get(PRINCIPAL) match {
       // We only need to log in with the keytab in cluster mode. In client mode, the driver
       // handles the user keytab.
-      case Some(principal) if amArgs.userClass != null =>
+      case Some(principal) if master.isClusterMode =>
         val originalCreds = UserGroupInformation.getCurrentUser().getCredentials()
         SparkHadoopUtil.get.loginUserFromKeytab(principal, sparkConf.get(KEYTAB).orNull)
         val newUGI = UserGroupInformation.getCurrentUser()
+
+        if (master.appAttemptId == null || master.appAttemptId.getAttemptId > 1) {
+          // Re-obtain delegation tokens if this is not a first attempt, as they might be outdated
+          // as of now. Add the fresh tokens on top of the original user's credentials (overwrite).
+          // Set the context class loader so that the token manager has access to jars
+          // distributed by the user.
+          Utils.withContextClassLoader(master.userClassLoader) {
+            val credentialManager = new HadoopDelegationTokenManager(sparkConf, yarnConf, null)
+            credentialManager.obtainDelegationTokens(originalCreds)
+          }
+        }
+
         // Transfer the original user's tokens to the new user, since it may contain needed tokens
         // (such as those user to connect to YARN).
         newUGI.addCredentials(originalCreds)
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 696afaacb0e79..6da6a8daad2b0 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -1353,7 +1353,10 @@ private object Client extends Logging {
       }
     }
 
-    populateHadoopClasspath(conf, env)
+    if (sparkConf.get(POPULATE_HADOOP_CLASSPATH)) {
+      populateHadoopClasspath(conf, env)
+    }
+
     sys.env.get(ENV_DIST_CLASSPATH).foreach { cp =>
       addClasspathEntry(getClusterPath(sparkConf, cp), env)
     }
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala
index 2288bb55d8b47..3e33382e0a395 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala
@@ -40,7 +40,7 @@ private[yarn] case class ContainerLocalityPreferences(nodes: Array[String], rack
  * and cpus per task is 1, so the required container number is 15,
  * and host ratio is (host1: 30, host2: 30, host3: 20, host4: 10).
  *
- * 1. If requested container number (18) is more than the required container number (15):
+ * 1. If the requested container number (18) is more than the required container number (15):
  *
  * requests for 5 containers with nodes: (host1, host2, host3, host4)
  * requests for 5 containers with nodes: (host1, host2, host3)
@@ -63,16 +63,16 @@ private[yarn] case class ContainerLocalityPreferences(nodes: Array[String], rack
  * follow the method of 1 and 2.
  *
  * 4. If containers exist and some of them can match the requested localities.
- * For example if we have 1 containers on each node (host1: 1, host2: 1: host3: 1, host4: 1),
+ * For example if we have 1 container on each node (host1: 1, host2: 1: host3: 1, host4: 1),
  * and the expected containers on each node would be (host1: 5, host2: 5, host3: 4, host4: 2),
  * so the newly requested containers on each node would be updated to (host1: 4, host2: 4,
  * host3: 3, host4: 1), 12 containers by total.
  *
  *   4.1 If requested container number (18) is more than newly required containers (12). Follow
- *   method 1 with updated ratio 4 : 4 : 3 : 1.
+ *   method 1 with an updated ratio 4 : 4 : 3 : 1.
  *
- *   4.2 If request container number (10) is more than newly required containers (12). Follow
- *   method 2 with updated ratio 4 : 4 : 3 : 1.
+ *   4.2 If request container number (10) is less than newly required containers (12). Follow
+ *   method 2 with an updated ratio 4 : 4 : 3 : 1.
  *
  * 5. If containers exist and existing localities can fully cover the requested localities.
  * For example if we have 5 containers on each node (host1: 5, host2: 5, host3: 5, host4: 5),
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
index 8dfbef27be2fb..3797491bb2e92 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
@@ -29,6 +29,7 @@ package object config {
   private[spark] val APPLICATION_TAGS = ConfigBuilder("spark.yarn.tags")
     .doc("Comma-separated list of strings to pass through as YARN application tags appearing " +
       "in YARN Application Reports, which can be used for filtering when querying YARN.")
+    .version("1.5.0")
     .stringConf
     .toSequence
     .createOptional
@@ -37,6 +38,7 @@ package object config {
     .doc("Application priority for YARN to define pending applications ordering policy, those" +
       " with higher value have a better opportunity to be activated. Currently, YARN only" +
       " supports application priority when using FIFO ordering policy.")
+    .version("3.0.0")
     .intConf
     .createOptional
 
@@ -44,6 +46,7 @@ package object config {
     ConfigBuilder("spark.yarn.am.attemptFailuresValidityInterval")
       .doc("Interval after which AM failures will be considered independent and " +
         "not accumulate towards the attempt count.")
+      .version("1.6.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
@@ -51,36 +54,52 @@ package object config {
     ConfigBuilder("spark.yarn.executor.failuresValidityInterval")
       .doc("Interval after which Executor failures will be considered independent and not " +
         "accumulate towards the attempt count.")
+      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val MAX_APP_ATTEMPTS = ConfigBuilder("spark.yarn.maxAppAttempts")
     .doc("Maximum number of AM attempts before failing the app.")
+    .version("1.3.0")
     .intConf
     .createOptional
 
   private[spark] val USER_CLASS_PATH_FIRST = ConfigBuilder("spark.yarn.user.classpath.first")
     .doc("Whether to place user jars in front of Spark's classpath.")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(false)
 
+  private[spark] val POPULATE_HADOOP_CLASSPATH = ConfigBuilder("spark.yarn.populateHadoopClasspath")
+    .doc("Whether to populate Hadoop classpath from `yarn.application.classpath` and " +
+      "`mapreduce.application.classpath` Note that if this is set to `false`, it requires " +
+      "a `with-Hadoop` Spark distribution that bundles Hadoop runtime or user has to provide " +
+      "a Hadoop installation separately.")
+    .version("2.4.6")
+    .booleanConf
+    .createWithDefault(true)
+
   private[spark] val GATEWAY_ROOT_PATH = ConfigBuilder("spark.yarn.config.gatewayPath")
     .doc("Root of configuration paths that is present on gateway nodes, and will be replaced " +
       "with the corresponding path in cluster machines.")
+    .version("1.5.0")
     .stringConf
     .createWithDefault(null)
 
   private[spark] val REPLACEMENT_ROOT_PATH = ConfigBuilder("spark.yarn.config.replacementPath")
     .doc(s"Path to use as a replacement for ${GATEWAY_ROOT_PATH.key} when launching processes " +
       "in the YARN cluster.")
+    .version("1.5.0")
     .stringConf
     .createWithDefault(null)
 
   private[spark] val QUEUE_NAME = ConfigBuilder("spark.yarn.queue")
+    .version("1.0.0")
     .stringConf
     .createWithDefault("default")
 
   private[spark] val HISTORY_SERVER_ADDRESS = ConfigBuilder("spark.yarn.historyServer.address")
+    .version("1.0.0")
     .stringConf
     .createOptional
 
@@ -88,6 +107,7 @@ package object config {
     ConfigBuilder("spark.yarn.historyServer.allowTracking")
       .doc("Allow using the History Server URL for the application as the tracking URL for the " +
         "application when the Web UI is not enabled.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -95,37 +115,44 @@ package object config {
 
   private[spark] val SPARK_ARCHIVE = ConfigBuilder("spark.yarn.archive")
     .doc("Location of archive containing jars files with Spark classes.")
+    .version("2.0.0")
     .stringConf
     .createOptional
 
   private[spark] val SPARK_JARS = ConfigBuilder("spark.yarn.jars")
     .doc("Location of jars containing Spark classes.")
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createOptional
 
   private[spark] val ARCHIVES_TO_DISTRIBUTE = ConfigBuilder("spark.yarn.dist.archives")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val FILES_TO_DISTRIBUTE = ConfigBuilder("spark.yarn.dist.files")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val JARS_TO_DISTRIBUTE = ConfigBuilder("spark.yarn.dist.jars")
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val PRESERVE_STAGING_FILES = ConfigBuilder("spark.yarn.preserve.staging.files")
     .doc("Whether to preserve temporary files created by the job in HDFS.")
+    .version("1.1.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val STAGING_FILE_REPLICATION = ConfigBuilder("spark.yarn.submit.file.replication")
     .doc("Replication factor for files uploaded by Spark to HDFS.")
+    .version("0.8.1")
     .intConf
     .createOptional
 
@@ -134,93 +161,111 @@ package object config {
   private[spark] val WAIT_FOR_APP_COMPLETION = ConfigBuilder("spark.yarn.submit.waitAppCompletion")
     .doc("In cluster mode, whether to wait for the application to finish before exiting the " +
       "launcher process.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(true)
 
   private[spark] val REPORT_INTERVAL = ConfigBuilder("spark.yarn.report.interval")
     .doc("Interval between reports of the current app status.")
+    .version("0.9.0")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("1s")
 
   private[spark] val CLIENT_LAUNCH_MONITOR_INTERVAL =
     ConfigBuilder("spark.yarn.clientLaunchMonitorInterval")
       .doc("Interval between requests for status the client mode AM when starting the app.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1s")
 
   /* Shared Client-mode AM / Driver configuration. */
 
   private[spark] val AM_MAX_WAIT_TIME = ConfigBuilder("spark.yarn.am.waitTime")
+    .version("1.3.0")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("100s")
 
   private[spark] val YARN_METRICS_NAMESPACE = ConfigBuilder("spark.yarn.metrics.namespace")
     .doc("The root namespace for AM metrics reporting.")
+    .version("2.4.0")
     .stringConf
     .createOptional
 
   private[spark] val AM_NODE_LABEL_EXPRESSION = ConfigBuilder("spark.yarn.am.nodeLabelExpression")
     .doc("Node label expression for the AM.")
+    .version("1.6.0")
     .stringConf
     .createOptional
 
   private[spark] val CONTAINER_LAUNCH_MAX_THREADS =
     ConfigBuilder("spark.yarn.containerLauncherMaxThreads")
+      .version("1.2.0")
       .intConf
       .createWithDefault(25)
 
   private[spark] val MAX_EXECUTOR_FAILURES = ConfigBuilder("spark.yarn.max.executor.failures")
+    .version("1.0.0")
     .intConf
     .createOptional
 
   private[spark] val MAX_REPORTER_THREAD_FAILURES =
     ConfigBuilder("spark.yarn.scheduler.reporterThread.maxFailures")
+      .version("1.2.0")
       .intConf
       .createWithDefault(5)
 
   private[spark] val RM_HEARTBEAT_INTERVAL =
     ConfigBuilder("spark.yarn.scheduler.heartbeat.interval-ms")
+      .version("0.8.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("3s")
 
   private[spark] val INITIAL_HEARTBEAT_INTERVAL =
     ConfigBuilder("spark.yarn.scheduler.initial-allocation.interval")
+      .version("1.4.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("200ms")
 
   private[spark] val AM_FINAL_MSG_LIMIT = ConfigBuilder("spark.yarn.am.finalMessageLimit")
     .doc("The limit size of final diagnostic message for our ApplicationMaster to unregister from" +
       " the ResourceManager.")
+    .version("2.4.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("1m")
 
   /* Client-mode AM configuration. */
 
   private[spark] val AM_CORES = ConfigBuilder("spark.yarn.am.cores")
+    .version("1.3.0")
     .intConf
     .createWithDefault(1)
 
   private[spark] val AM_JAVA_OPTIONS = ConfigBuilder("spark.yarn.am.extraJavaOptions")
     .doc("Extra Java options for the client-mode AM.")
+    .version("1.3.0")
     .stringConf
     .createOptional
 
   private[spark] val AM_LIBRARY_PATH = ConfigBuilder("spark.yarn.am.extraLibraryPath")
     .doc("Extra native library path for the client-mode AM.")
+    .version("1.4.0")
     .stringConf
     .createOptional
 
   private[spark] val AM_MEMORY_OVERHEAD = ConfigBuilder("spark.yarn.am.memoryOverhead")
+    .version("1.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 
   private[spark] val AM_MEMORY = ConfigBuilder("spark.yarn.am.memory")
+    .version("1.3.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("512m")
 
   /* Driver configuration. */
 
   private[spark] val DRIVER_APP_UI_ADDRESS = ConfigBuilder("spark.driver.appUIAddress")
+    .version("1.1.0")
     .stringConf
     .createOptional
 
@@ -229,6 +274,7 @@ package object config {
   private[spark] val EXECUTOR_NODE_LABEL_EXPRESSION =
     ConfigBuilder("spark.yarn.executor.nodeLabelExpression")
       .doc("Node label expression for executors.")
+      .version("1.4.0")
       .stringConf
       .createOptional
 
@@ -237,6 +283,7 @@ package object config {
   private[spark] val YARN_UNMANAGED_AM = ConfigBuilder("spark.yarn.unmanagedAM.enabled")
     .doc("In client mode, whether to launch the Application Master service as part of the client " +
       "using unmanaged am.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -246,6 +293,7 @@ package object config {
     ConfigBuilder("spark.yarn.rolledLog.includePattern")
       .doc("Java Regex to filter the log files which match the defined include pattern and those " +
         "log files will be aggregated in a rolling fashion.")
+      .version("2.0.0")
       .stringConf
       .createOptional
 
@@ -253,6 +301,7 @@ package object config {
     ConfigBuilder("spark.yarn.rolledLog.excludePattern")
       .doc("Java Regex to filter the log files which match the defined exclude pattern and those " +
         "log files will not be aggregated in a rolling fashion.")
+      .version("2.0.0")
       .stringConf
       .createOptional
 
@@ -261,6 +310,7 @@ package object config {
   // Internal config to propagate the location of the user's jar to the driver/executors
   private[spark] val APP_JAR = ConfigBuilder("spark.yarn.user.jar")
     .internal()
+    .version("1.1.0")
     .stringConf
     .createOptional
 
@@ -268,6 +318,7 @@ package object config {
   // of the executors
   private[spark] val SECONDARY_JARS = ConfigBuilder("spark.yarn.secondary.jars")
     .internal()
+    .version("0.9.2")
     .stringConf
     .toSequence
     .createOptional
@@ -276,24 +327,28 @@ package object config {
 
   private[spark] val CACHED_FILES = ConfigBuilder("spark.yarn.cache.filenames")
     .internal()
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val CACHED_FILES_SIZES = ConfigBuilder("spark.yarn.cache.sizes")
     .internal()
+    .version("2.0.0")
     .longConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val CACHED_FILES_TIMESTAMPS = ConfigBuilder("spark.yarn.cache.timestamps")
     .internal()
+    .version("2.0.0")
     .longConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val CACHED_FILES_VISIBILITIES = ConfigBuilder("spark.yarn.cache.visibilities")
     .internal()
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
@@ -301,6 +356,7 @@ package object config {
   // Either "file" or "archive", for each file.
   private[spark] val CACHED_FILES_TYPES = ConfigBuilder("spark.yarn.cache.types")
     .internal()
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
@@ -308,20 +364,23 @@ package object config {
   // The location of the conf archive in HDFS.
   private[spark] val CACHED_CONF_ARCHIVE = ConfigBuilder("spark.yarn.cache.confArchive")
     .internal()
+    .version("2.0.0")
     .stringConf
     .createOptional
 
   /* YARN allocator-level blacklisting related config entries. */
   private[spark] val YARN_EXECUTOR_LAUNCH_BLACKLIST_ENABLED =
     ConfigBuilder("spark.yarn.blacklist.executor.launch.blacklisting.enabled")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(false)
 
   /* Initially blacklisted YARN nodes. */
   private[spark] val YARN_EXCLUDE_NODES = ConfigBuilder("spark.yarn.exclude.nodes")
-      .stringConf
-      .toSequence
-      .createWithDefault(Nil)
+    .version("3.0.0")
+    .stringConf
+    .toSequence
+    .createWithDefault(Nil)
 
   private[yarn] val YARN_EXECUTOR_RESOURCE_TYPES_PREFIX = "spark.yarn.executor.resource."
   private[yarn] val YARN_DRIVER_RESOURCE_TYPES_PREFIX = "spark.yarn.driver.resource."
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
index b42c8b933dda0..680ff9991f294 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
@@ -485,6 +485,25 @@ class ClientSuite extends SparkFunSuite with Matchers {
     }
   }
 
+  test("SPARK-31582 Being able to not populate Hadoop classpath") {
+    Seq(true, false).foreach { populateHadoopClassPath =>
+      withAppConf(Fixtures.mapAppConf) { conf =>
+        val sparkConf = new SparkConf()
+          .set(POPULATE_HADOOP_CLASSPATH, populateHadoopClassPath)
+        val env = new MutableHashMap[String, String]()
+        val args = new ClientArguments(Array("--jar", USER))
+        populateClasspath(args, conf, sparkConf, env)
+        if (populateHadoopClassPath) {
+          classpath(env) should
+            (contain (Fixtures.knownYARNAppCP) and contain (Fixtures.knownMRAppCP))
+        } else {
+          classpath(env) should
+            (not contain (Fixtures.knownYARNAppCP) and not contain (Fixtures.knownMRAppCP))
+        }
+      }
+    }
+  }
+
   private val matching = Seq(
     ("files URI match test1", "file:///file1", "file:///file2"),
     ("files URI match test2", "file:///c:file1", "file://c:file2"),
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
index b7f25656e49ac..cf2c3840a6e9b 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.deploy.yarn
 
-import scala.collection.JavaConverters._
+import java.io.{PrintWriter, StringWriter}
+
 import scala.collection.mutable.{HashMap, HashSet, Set}
 
 import org.apache.hadoop.yarn.api.records._
@@ -41,11 +42,15 @@ class LocalityPlacementStrategySuite extends SparkFunSuite {
       }
     }
 
-    val thread = new Thread(new ThreadGroup("test"), runnable, "test-thread", 32 * 1024)
+    val thread = new Thread(new ThreadGroup("test"), runnable, "test-thread", 256 * 1024)
     thread.start()
     thread.join()
 
-    assert(error === null)
+    if (error != null) {
+      val errors = new StringWriter()
+      error.printStackTrace(new PrintWriter(errors))
+      fail(s"StackOverflowError should not be thrown; however, got:\n\n$errors")
+    }
   }
 
   private def runTest(): Unit = {
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala
index e260c0b60b87c..632c66d77b707 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala
@@ -39,7 +39,7 @@ class YarnShuffleServiceMetricsSuite extends SparkFunSuite with Matchers {
     val allMetrics = Set(
       "openBlockRequestLatencyMillis", "registerExecutorRequestLatencyMillis",
       "blockTransferRateBytes", "registeredExecutorsSize", "numActiveConnections",
-      "numRegisteredConnections", "numCaughtExceptions")
+      "numCaughtExceptions")
 
     metrics.getMetrics.keySet().asScala should be (allMetrics)
   }
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
index 381a93580f961..d46cfa85c8d7b 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
@@ -23,9 +23,13 @@ import java.nio.file.attribute.PosixFilePermission._
 import java.util.EnumSet
 
 import scala.annotation.tailrec
+import scala.collection.JavaConverters._
 import scala.concurrent.duration._
 
+import com.codahale.metrics.MetricSet
 import org.apache.hadoop.fs.Path
+import org.apache.hadoop.metrics2.impl.MetricsSystemImpl
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem
 import org.apache.hadoop.service.ServiceStateException
 import org.apache.hadoop.yarn.api.records.ApplicationId
 import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -381,4 +385,25 @@ class YarnShuffleServiceSuite extends SparkFunSuite with Matchers with BeforeAnd
     s1.secretsFile should be (null)
   }
 
+  test("SPARK-31646: metrics should be registered into Node Manager's metrics system") {
+    s1 = new YarnShuffleService
+    s1.init(yarnConfig)
+
+    val metricsSource = DefaultMetricsSystem.instance.asInstanceOf[MetricsSystemImpl]
+      .getSource("sparkShuffleService").asInstanceOf[YarnShuffleServiceMetrics]
+    val metricSetRef = classOf[YarnShuffleServiceMetrics].getDeclaredField("metricSet")
+    metricSetRef.setAccessible(true)
+    val metrics = metricSetRef.get(metricsSource).asInstanceOf[MetricSet].getMetrics
+
+    assert(metrics.keySet().asScala == Set(
+      "blockTransferRateBytes",
+      "numActiveConnections",
+      "numCaughtExceptions",
+      "numRegisteredConnections",
+      "openBlockRequestLatencyMillis",
+      "registeredExecutorsSize",
+      "registerExecutorRequestLatencyMillis"
+    ))
+  }
+
 }
diff --git a/sbin/spark-config.sh b/sbin/spark-config.sh
index 0771e2a044757..b53442ec096a1 100755
--- a/sbin/spark-config.sh
+++ b/sbin/spark-config.sh
@@ -28,6 +28,6 @@ export SPARK_CONF_DIR="${SPARK_CONF_DIR:-"${SPARK_HOME}/conf"}"
 # Add the PySpark classes to the PYTHONPATH:
 if [ -z "${PYSPARK_PYTHONPATH_SET}" ]; then
   export PYTHONPATH="${SPARK_HOME}/python:${PYTHONPATH}"
-  export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.8.1-src.zip:${PYTHONPATH}"
+  export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9-src.zip:${PYTHONPATH}"
   export PYSPARK_PYTHONPATH_SET=1
 fi
diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh
index 2cb17a04f67ef..9b3b26b07842b 100755
--- a/sbin/start-slave.sh
+++ b/sbin/start-slave.sh
@@ -22,7 +22,7 @@
 # Environment Variables
 #
 #   SPARK_WORKER_INSTANCES  The number of worker instances to run on this
-#                           slave.  Default is 1.
+#                           slave.  Default is 1. Note it has been deprecate since Spark 3.0.
 #   SPARK_WORKER_PORT       The base port number for the first worker. If set,
 #                           subsequent workers will increment this number.  If
 #                           unset, Spark will find a valid port number, but
diff --git a/sql/README.md b/sql/README.md
index 67e3225e2c275..ae5ebd1d75370 100644
--- a/sql/README.md
+++ b/sql/README.md
@@ -9,4 +9,4 @@ Spark SQL is broken up into four subprojects:
  - Hive Support (sql/hive) - Includes extensions that allow users to write queries using a subset of HiveQL and access data from a Hive Metastore using Hive SerDes. There are also wrappers that allow users to run queries that include Hive UDFs, UDAFs, and UDTFs.
  - HiveServer and CLI support (sql/hive-thriftserver) - Includes support for the SQL CLI (bin/spark-sql) and a HiveServer2 (for JDBC/ODBC) compatible server.
 
-Running `./sql/create-docs.sh` generates SQL documentation for built-in functions under `sql/site`.
+Running `./sql/create-docs.sh` generates SQL documentation for built-in functions under `sql/site`, and SQL configuration documentation that gets included as part of `configuration.md` in the main `docs` directory.
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 304e3a7457fcc..e4d5ee614cb04 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 6f2bb7a9a7536..922ff10b5d5f6 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -29,12 +29,6 @@ grammar SqlBase;
    */
   public boolean legacy_exponent_literal_as_decimal_enabled = false;
 
-  /**
-   * When false, CREATE TABLE syntax without a provider will use
-   * the value of spark.sql.sources.default as its provider.
-   */
-  public boolean legacy_create_hive_table_by_default_enabled = false;
-
   /**
    * Verify whether current token is a valid decimal token (which contains dot).
    * Returns true if the character that follows the token is not a digit or letter or underscore.
@@ -64,7 +58,7 @@ grammar SqlBase;
 }
 
 singleStatement
-    : statement EOF
+    : statement ';'* EOF
     ;
 
 singleExpression
@@ -107,12 +101,7 @@ statement
         (RESTRICT | CASCADE)?                                          #dropNamespace
     | SHOW (DATABASES | NAMESPACES) ((FROM | IN) multipartIdentifier)?
         (LIKE? pattern=STRING)?                                        #showNamespaces
-    | {!legacy_create_hive_table_by_default_enabled}?
-        createTableHeader ('(' colTypeList ')')? tableProvider?
-        createTableClauses
-        (AS? query)?                                                   #createTable
-    | {legacy_create_hive_table_by_default_enabled}?
-        createTableHeader ('(' colTypeList ')')? tableProvider
+    | createTableHeader ('(' colTypeList ')')? tableProvider
         createTableClauses
         (AS? query)?                                                   #createTable
     | createTableHeader ('(' columns=colTypeList ')')?
@@ -160,10 +149,7 @@ statement
         UNSET TBLPROPERTIES (IF EXISTS)? tablePropertyList             #unsetTableProperties
     | ALTER TABLE table=multipartIdentifier
         (ALTER | CHANGE) COLUMN? column=multipartIdentifier
-        (TYPE dataType)? commentSpec? colPosition?                     #alterTableColumn
-    | ALTER TABLE table=multipartIdentifier
-        ALTER COLUMN? column=multipartIdentifier
-        setOrDrop=(SET | DROP) NOT NULL                                #alterColumnNullability
+        alterColumnAction?                                             #alterTableAlterColumn
     | ALTER TABLE table=multipartIdentifier partitionSpec?
         CHANGE COLUMN?
         colName=multipartIdentifier colType colPosition?               #hiveChangeColumn
@@ -207,6 +193,8 @@ statement
         ('(' key=tablePropertyKey ')')?                                #showTblProperties
     | SHOW COLUMNS (FROM | IN) table=multipartIdentifier
         ((FROM | IN) ns=multipartIdentifier)?                          #showColumns
+    | SHOW VIEWS ((FROM | IN) multipartIdentifier)?
+        (LIKE? pattern=STRING)?                                        #showViews
     | SHOW PARTITIONS multipartIdentifier partitionSpec?               #showPartitions
     | SHOW identifier? FUNCTIONS
         (LIKE? (multipartIdentifier | pattern=STRING))?                #showFunctions
@@ -771,8 +759,8 @@ primaryExpression
     | CASE value=expression whenClause+ (ELSE elseExpression=expression)? END                  #simpleCase
     | CAST '(' expression AS dataType ')'                                                      #cast
     | STRUCT '(' (argument+=namedExpression (',' argument+=namedExpression)*)? ')'             #struct
-    | (FIRST | FIRST_VALUE) '(' expression ((IGNORE | RESPECT) NULLS)? ')'                     #first
-    | (LAST | LAST_VALUE) '(' expression ((IGNORE | RESPECT) NULLS)? ')'                       #last
+    | FIRST '(' expression (IGNORE NULLS)? ')'                                                 #first
+    | LAST '(' expression (IGNORE NULLS)? ')'                                                  #last
     | POSITION '(' substr=valueExpression IN str=valueExpression ')'                           #position
     | constant                                                                                 #constantDefault
     | ASTERISK                                                                                 #star
@@ -830,7 +818,7 @@ errorCapturingMultiUnitsInterval
     ;
 
 multiUnitsInterval
-    : (intervalValue intervalUnit)+
+    : (intervalValue unit+=identifier)+
     ;
 
 errorCapturingUnitToUnitInterval
@@ -838,7 +826,7 @@ errorCapturingUnitToUnitInterval
     ;
 
 unitToUnitInterval
-    : value=intervalValue from=intervalUnit TO to=intervalUnit
+    : value=intervalValue from=identifier TO to=identifier
     ;
 
 intervalValue
@@ -846,16 +834,6 @@ intervalValue
     | STRING
     ;
 
-intervalUnit
-    : DAY
-    | HOUR
-    | MINUTE
-    | MONTH
-    | SECOND
-    | YEAR
-    | identifier
-    ;
-
 colPosition
     : position=FIRST | position=AFTER afterCol=errorCapturingIdentifier
     ;
@@ -983,6 +961,13 @@ number
     | MINUS? BIGDECIMAL_LITERAL       #bigDecimalLiteral
     ;
 
+alterColumnAction
+    : TYPE dataType
+    | commentSpec
+    | colPosition
+    | setOrDrop=(SET | DROP) NOT NULL
+    ;
+
 // When `SQL_standard_keyword_behavior=true`, there are 2 kinds of keywords in Spark SQL.
 // - Reserved keywords:
 //     Keywords that are reserved and can't be used as identifiers for table, view, column,
@@ -994,10 +979,12 @@ number
 // You can find the full keywords list by searching "Start of the keywords list" in this file.
 // The non-reserved keywords are listed below. Keywords not in this list are reserved keywords.
 ansiNonReserved
+//--ANSI-NON-RESERVED-START
     : ADD
     | AFTER
     | ALTER
     | ANALYZE
+    | ANTI
     | ARCHIVE
     | ARRAY
     | ASC
@@ -1120,7 +1107,6 @@ ansiNonReserved
     | REPAIR
     | REPLACE
     | RESET
-    | RESPECT
     | RESTRICT
     | REVOKE
     | RLIKE
@@ -1131,10 +1117,12 @@ ansiNonReserved
     | ROW
     | ROWS
     | SCHEMA
+    | SEMI
     | SEPARATED
     | SERDE
     | SERDEPROPERTIES
     | SET
+    | SETMINUS
     | SETS
     | SHOW
     | SKEWED
@@ -1159,6 +1147,7 @@ ansiNonReserved
     | TRIM
     | TRUE
     | TRUNCATE
+    | TYPE
     | UNARCHIVE
     | UNBOUNDED
     | UNCACHE
@@ -1168,7 +1157,9 @@ ansiNonReserved
     | USE
     | VALUES
     | VIEW
+    | VIEWS
     | WINDOW
+//--ANSI-NON-RESERVED-END
     ;
 
 // When `SQL_standard_keyword_behavior=false`, there are 2 kinds of keywords in Spark SQL.
@@ -1199,6 +1190,7 @@ strictNonReserved
     ;
 
 nonReserved
+//--DEFAULT-NON-RESERVED-START
     : ADD
     | AFTER
     | ALL
@@ -1249,7 +1241,6 @@ nonReserved
     | DATA
     | DATABASE
     | DATABASES
-    | DAY
     | DBPROPERTIES
     | DEFINED
     | DELETE
@@ -1280,7 +1271,6 @@ nonReserved
     | FIELDS
     | FILEFORMAT
     | FIRST
-    | FIRST_VALUE
     | FOLLOWING
     | FOR
     | FOREIGN
@@ -1294,7 +1284,6 @@ nonReserved
     | GROUP
     | GROUPING
     | HAVING
-    | HOUR
     | IF
     | IGNORE
     | IMPORT
@@ -1310,7 +1299,6 @@ nonReserved
     | ITEMS
     | KEYS
     | LAST
-    | LAST_VALUE
     | LATERAL
     | LAZY
     | LEADING
@@ -1328,8 +1316,6 @@ nonReserved
     | MAP
     | MATCHED
     | MERGE
-    | MINUTE
-    | MONTH
     | MSCK
     | NAMESPACE
     | NAMESPACES
@@ -1374,7 +1360,6 @@ nonReserved
     | REPAIR
     | REPLACE
     | RESET
-    | RESPECT
     | RESTRICT
     | REVOKE
     | RLIKE
@@ -1385,7 +1370,6 @@ nonReserved
     | ROW
     | ROWS
     | SCHEMA
-    | SECOND
     | SELECT
     | SEPARATED
     | SERDE
@@ -1434,20 +1418,21 @@ nonReserved
     | USER
     | VALUES
     | VIEW
+    | VIEWS
     | WHEN
     | WHERE
     | WINDOW
     | WITH
-    | YEAR
+//--DEFAULT-NON-RESERVED-END
     ;
 
 // NOTE: If you add a new token in the list below, you should update the list of keywords
-// in `docs/sql-keywords.md`. If the token is a non-reserved keyword,
-// please update `ansiNonReserved` and `nonReserved` as well.
+// and reserved tag in `docs/sql-ref-ansi-compliance.md#sql-keywords`.
 
 //============================
 // Start of the keywords list
 //============================
+//--SPARK-KEYWORD-LIST-START
 ADD: 'ADD';
 AFTER: 'AFTER';
 ALL: 'ALL';
@@ -1500,7 +1485,6 @@ CURRENT_USER: 'CURRENT_USER';
 DATA: 'DATA';
 DATABASE: 'DATABASE';
 DATABASES: 'DATABASES' | 'SCHEMAS';
-DAY: 'DAY';
 DBPROPERTIES: 'DBPROPERTIES';
 DEFINED: 'DEFINED';
 DELETE: 'DELETE';
@@ -1512,6 +1496,7 @@ DIRECTORIES: 'DIRECTORIES';
 DIRECTORY: 'DIRECTORY';
 DISTINCT: 'DISTINCT';
 DISTRIBUTE: 'DISTRIBUTE';
+DIV: 'DIV';
 DROP: 'DROP';
 ELSE: 'ELSE';
 END: 'END';
@@ -1531,7 +1516,6 @@ FIELDS: 'FIELDS';
 FILTER: 'FILTER';
 FILEFORMAT: 'FILEFORMAT';
 FIRST: 'FIRST';
-FIRST_VALUE: 'FIRST_VALUE';
 FOLLOWING: 'FOLLOWING';
 FOR: 'FOR';
 FOREIGN: 'FOREIGN';
@@ -1546,7 +1530,6 @@ GRANT: 'GRANT';
 GROUP: 'GROUP';
 GROUPING: 'GROUPING';
 HAVING: 'HAVING';
-HOUR: 'HOUR';
 IF: 'IF';
 IGNORE: 'IGNORE';
 IMPORT: 'IMPORT';
@@ -1565,7 +1548,6 @@ ITEMS: 'ITEMS';
 JOIN: 'JOIN';
 KEYS: 'KEYS';
 LAST: 'LAST';
-LAST_VALUE: 'LAST_VALUE';
 LATERAL: 'LATERAL';
 LAZY: 'LAZY';
 LEADING: 'LEADING';
@@ -1584,8 +1566,6 @@ MACRO: 'MACRO';
 MAP: 'MAP';
 MATCHED: 'MATCHED';
 MERGE: 'MERGE';
-MINUTE: 'MINUTE';
-MONTH: 'MONTH';
 MSCK: 'MSCK';
 NAMESPACE: 'NAMESPACE';
 NAMESPACES: 'NAMESPACES';
@@ -1632,7 +1612,6 @@ RENAME: 'RENAME';
 REPAIR: 'REPAIR';
 REPLACE: 'REPLACE';
 RESET: 'RESET';
-RESPECT: 'RESPECT';
 RESTRICT: 'RESTRICT';
 REVOKE: 'REVOKE';
 RIGHT: 'RIGHT';
@@ -1644,7 +1623,6 @@ ROLLUP: 'ROLLUP';
 ROW: 'ROW';
 ROWS: 'ROWS';
 SCHEMA: 'SCHEMA';
-SECOND: 'SECOND';
 SELECT: 'SELECT';
 SEMI: 'SEMI';
 SEPARATED: 'SEPARATED';
@@ -1697,11 +1675,12 @@ USER: 'USER';
 USING: 'USING';
 VALUES: 'VALUES';
 VIEW: 'VIEW';
+VIEWS: 'VIEWS';
 WHEN: 'WHEN';
 WHERE: 'WHERE';
 WINDOW: 'WINDOW';
 WITH: 'WITH';
-YEAR: 'YEAR';
+//--SPARK-KEYWORD-LIST-END
 //============================
 // End of the keywords list
 //============================
@@ -1720,7 +1699,6 @@ MINUS: '-';
 ASTERISK: '*';
 SLASH: '/';
 PERCENT: '%';
-DIV: 'DIV';
 TILDE: '~';
 AMPERSAND: '&';
 PIPE: '|';
@@ -1793,7 +1771,7 @@ fragment LETTER
     ;
 
 SIMPLE_COMMENT
-    : '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN)
+    : '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN)
     ;
 
 BRACKETED_EMPTY_COMMENT
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
index acdf6afe10ce5..579f4b387b3f9 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
@@ -31,7 +31,7 @@
  * `usage()` will be used for the function usage in brief way.
  *
  * These below are concatenated and used for the function usage in verbose way, suppose arguments,
- * examples, note, since and deprecated will be provided.
+ * examples, note, group, since and deprecated will be provided.
  *
  * `arguments()` describes arguments for the expression.
  *
@@ -39,13 +39,16 @@
  *
  * `note()` contains some notes for the expression optionally.
  *
+ * `group()` describes the category that the expression belongs to. The valid value is
+ * "agg_funcs", "array_funcs", "datetime_funcs", "json_funcs", "map_funcs" and "window_funcs".
+ *
  * `since()` contains version information for the expression. Version is specified by,
  * for example, "2.2.0".
  *
  * `deprecated()` contains deprecation information for the expression optionally, for example,
  * "Deprecated since 2.2.0. Use something else instead".
  *
- * The format, in particular for `arguments()`, `examples()`,`note()`, `since()` and
+ * The format, in particular for `arguments()`, `examples()`,`note()`, `group()`, `since()` and
  * `deprecated()`, should strictly be as follows.
  *
  * <pre>
@@ -68,6 +71,7 @@
  *   note = """
  *     ...
  *   """,
+ *   group = "agg_funcs",
  *   since = "3.0.0",
  *   deprecated = """
  *     ...
@@ -75,11 +79,12 @@
  * </code>
  * </pre>
  *
- *  We can refer the function name by `_FUNC_`, in `usage()`, `arguments()` and `examples()` as
- *  it is registered in `FunctionRegistry`.
+ *  We can refer the function name by `_FUNC_`, in `usage()`, `arguments()`, `examples()` and
+ *  `note()` as it is registered in `FunctionRegistry`.
  *
- *  Note that, if `extended()` is defined, `arguments()`, `examples()`, `note()`, `since()` and
- *  `deprecated()` should be not defined together. `extended()` exists for backward compatibility.
+ *  Note that, if `extended()` is defined, `arguments()`, `examples()`, `note()`, `group()`,
+ *  `since()` and `deprecated()` should be not defined together. `extended()` exists
+ *  for backward compatibility.
  *
  *  Note this contents are used in the SparkSQL documentation for built-in functions. The contents
  *  here are considered as a Markdown text and then rendered.
@@ -98,6 +103,13 @@
     String arguments() default "";
     String examples() default "";
     String note() default "";
+    /**
+     * Valid group names are almost the same with one defined as `groupname` in
+     * `sql/functions.scala`. But, `collection_funcs` is split into fine-grained three groups:
+     * `array_funcs`, `map_funcs`, and `json_funcs`. See `ExpressionInfo` for the
+     * detailed group names.
+     */
+    String group() default "";
     String since() default "";
     String deprecated() default "";
 }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
index 8ee90ed6f4c3b..a500822b21f02 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
@@ -19,6 +19,10 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
 /**
  * Expression information, will be used to describe a expression.
  */
@@ -31,9 +35,14 @@ public class ExpressionInfo {
     private String arguments;
     private String examples;
     private String note;
+    private String group;
     private String since;
     private String deprecated;
 
+    private static final Set<String> validGroups =
+        new HashSet<>(Arrays.asList("agg_funcs", "array_funcs", "datetime_funcs",
+            "json_funcs", "map_funcs", "window_funcs"));
+
     public String getClassName() {
         return className;
     }
@@ -75,6 +84,10 @@ public String getDeprecated() {
         return deprecated;
     }
 
+    public String getGroup() {
+        return group;
+    }
+
     public String getDb() {
         return db;
     }
@@ -87,6 +100,7 @@ public ExpressionInfo(
             String arguments,
             String examples,
             String note,
+            String group,
             String since,
             String deprecated) {
         assert name != null;
@@ -94,6 +108,7 @@ public ExpressionInfo(
         assert examples != null;
         assert examples.isEmpty() || examples.contains("    Examples:");
         assert note != null;
+        assert group != null;
         assert since != null;
         assert deprecated != null;
 
@@ -104,6 +119,7 @@ public ExpressionInfo(
         this.arguments = arguments;
         this.examples = examples;
         this.note = note;
+        this.group = group;
         this.since = since;
         this.deprecated = deprecated;
 
@@ -120,6 +136,11 @@ public ExpressionInfo(
             }
             this.extended += "\n    Note:\n      " + note.trim() + "\n";
         }
+        if (!group.isEmpty() && !validGroups.contains(group)) {
+            throw new IllegalArgumentException("'group' is malformed in the expression [" +
+                this.name + "]. It should be a value in " + validGroups + "; however, " +
+                "got [" + group + "].");
+        }
         if (!since.isEmpty()) {
             if (Integer.parseInt(since.split("\\.")[0]) < 0) {
                 throw new IllegalArgumentException("'since' is malformed in the expression [" +
@@ -140,11 +161,11 @@ public ExpressionInfo(
     }
 
     public ExpressionInfo(String className, String name) {
-        this(className, null, name, null, "", "", "", "", "");
+        this(className, null, name, null, "", "", "", "", "", "");
     }
 
     public ExpressionInfo(String className, String db, String name) {
-        this(className, db, name, null, "", "", "", "", "");
+        this(className, db, name, null, "", "", "", "", "", "");
     }
 
     /**
@@ -155,7 +176,7 @@ public ExpressionInfo(String className, String db, String name) {
     public ExpressionInfo(String className, String db, String name, String usage, String extended) {
         // `arguments` and `examples` are concatenated for the extended description. So, here
         // simply pass the `extended` as `arguments` and an empty string for `examples`.
-        this(className, db, name, usage, extended, "", "", "", "");
+        this(className, db, name, usage, extended, "", "", "", "", "");
     }
 
     private String replaceFunctionName(String usage) {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
index 23e7d1f07e4a3..034894bd86085 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
@@ -95,6 +95,10 @@ public static int calculateBitSetWidthInBytes(int numFields) {
   }
 
   public static boolean isFixedLength(DataType dt) {
+    if (dt instanceof UserDefinedType) {
+      return isFixedLength(((UserDefinedType) dt).sqlType());
+    }
+
     if (dt instanceof DecimalType) {
       return ((DecimalType) dt).precision() <= Decimal.MAX_LONG_DIGITS();
     } else {
@@ -103,6 +107,10 @@ public static boolean isFixedLength(DataType dt) {
   }
 
   public static boolean isMutable(DataType dt) {
+    if (dt instanceof UserDefinedType) {
+      return isMutable(((UserDefinedType) dt).sqlType());
+    }
+
     return mutableFieldTypes.contains(dt) || dt instanceof DecimalType ||
       dt instanceof CalendarIntervalType;
   }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VariableLengthRowBasedKeyValueBatch.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VariableLengthRowBasedKeyValueBatch.java
index c823de4810f2b..4ee913c9bf02d 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VariableLengthRowBasedKeyValueBatch.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VariableLengthRowBasedKeyValueBatch.java
@@ -19,11 +19,12 @@
 import org.apache.spark.memory.TaskMemoryManager;
 import org.apache.spark.sql.types.*;
 import org.apache.spark.unsafe.Platform;
+import org.apache.spark.unsafe.UnsafeAlignedOffset;
 
 /**
  * An implementation of `RowBasedKeyValueBatch` in which key-value records have variable lengths.
  *
- *  The format for each record looks like this:
+ *  The format for each record looks like this (in case of uaoSize = 4):
  * [4 bytes total size = (klen + vlen + 4)] [4 bytes key size = klen]
  * [UnsafeRow for key of length klen] [UnsafeRow for Value of length vlen]
  * [8 bytes pointer to next]
@@ -41,7 +42,8 @@ public final class VariableLengthRowBasedKeyValueBatch extends RowBasedKeyValueB
   @Override
   public UnsafeRow appendRow(Object kbase, long koff, int klen,
                              Object vbase, long voff, int vlen) {
-    final long recordLength = 8L + klen + vlen + 8;
+    int uaoSize = UnsafeAlignedOffset.getUaoSize();
+    final long recordLength = 2 * uaoSize + klen + vlen + 8L;
     // if run out of max supported rows or page size, return null
     if (numRows >= capacity || page == null || page.size() - pageCursor < recordLength) {
       return null;
@@ -49,10 +51,10 @@ public UnsafeRow appendRow(Object kbase, long koff, int klen,
 
     long offset = page.getBaseOffset() + pageCursor;
     final long recordOffset = offset;
-    Platform.putInt(base, offset, klen + vlen + 4);
-    Platform.putInt(base, offset + 4, klen);
+    UnsafeAlignedOffset.putSize(base, offset, klen + vlen + uaoSize);
+    UnsafeAlignedOffset.putSize(base, offset + uaoSize, klen);
 
-    offset += 8;
+    offset += 2 * uaoSize;
     Platform.copyMemory(kbase, koff, base, offset, klen);
     offset += klen;
     Platform.copyMemory(vbase, voff, base, offset, vlen);
@@ -61,11 +63,11 @@ public UnsafeRow appendRow(Object kbase, long koff, int klen,
 
     pageCursor += recordLength;
 
-    keyOffsets[numRows] = recordOffset + 8;
+    keyOffsets[numRows] = recordOffset + 2 * uaoSize;
 
     keyRowId = numRows;
-    keyRow.pointTo(base, recordOffset + 8, klen);
-    valueRow.pointTo(base, recordOffset + 8 + klen, vlen);
+    keyRow.pointTo(base, recordOffset + 2 * uaoSize, klen);
+    valueRow.pointTo(base, recordOffset + 2 * uaoSize + klen, vlen);
     numRows++;
     return valueRow;
   }
@@ -79,7 +81,7 @@ public UnsafeRow getKeyRow(int rowId) {
     assert(rowId < numRows);
     if (keyRowId != rowId) { // if keyRowId == rowId, desired keyRow is already cached
       long offset = keyOffsets[rowId];
-      int klen = Platform.getInt(base, offset - 4);
+      int klen = UnsafeAlignedOffset.getSize(base, offset - UnsafeAlignedOffset.getUaoSize());
       keyRow.pointTo(base, offset, klen);
       // set keyRowId so we can check if desired row is cached
       keyRowId = rowId;
@@ -99,9 +101,10 @@ public UnsafeRow getValueFromKey(int rowId) {
       getKeyRow(rowId);
     }
     assert(rowId >= 0);
+    int uaoSize = UnsafeAlignedOffset.getUaoSize();
     long offset = keyRow.getBaseOffset();
     int klen = keyRow.getSizeInBytes();
-    int vlen = Platform.getInt(base, offset - 8) - klen - 4;
+    int vlen = UnsafeAlignedOffset.getSize(base, offset - uaoSize * 2) - klen - uaoSize;
     valueRow.pointTo(base, offset + klen, vlen);
     return valueRow;
   }
@@ -141,14 +144,15 @@ public boolean next() {
           return false;
         }
 
-        totalLength = Platform.getInt(base, offsetInPage) - 4;
-        currentklen = Platform.getInt(base, offsetInPage + 4);
+        int uaoSize = UnsafeAlignedOffset.getUaoSize();
+        totalLength = UnsafeAlignedOffset.getSize(base, offsetInPage) - uaoSize;
+        currentklen = UnsafeAlignedOffset.getSize(base, offsetInPage + uaoSize);
         currentvlen = totalLength - currentklen;
 
-        key.pointTo(base, offsetInPage + 8, currentklen);
-        value.pointTo(base, offsetInPage + 8 + currentklen, currentvlen);
+        key.pointTo(base, offsetInPage + 2 * uaoSize, currentklen);
+        value.pointTo(base, offsetInPage + 2 * uaoSize + currentklen, currentvlen);
 
-        offsetInPage += 8 + totalLength + 8;
+        offsetInPage += 2 * uaoSize + totalLength + 8;
         recordsInPage -= 1;
         return true;
       }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/ExternalCommandRunner.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/ExternalCommandRunner.java
index 37534be31b6f6..a06483d8fa08b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/ExternalCommandRunner.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/ExternalCommandRunner.java
@@ -27,6 +27,8 @@
  * cores for Solr and so on.
  * <p>
  * This interface will be instantiated when end users call `SparkSession#executeCommand`.
+ *
+ * @since 3.0.0
  */
 @Unstable
 public interface ExternalCommandRunner {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java
index 65e0b6be00ef3..155dca5809c37 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.util.CaseInsensitiveStringMap;
 
 /**
@@ -26,8 +26,10 @@
  * some custom logic and call the built-in session catalog at the end. For example, they can
  * implement {@code createTable}, do something else before calling {@code createTable} of the
  * built-in session catalog.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface CatalogExtension extends TableCatalog, SupportsNamespaces {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java
index 34fdc5dd4d0fb..8ca4f568b9f18 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.internal.SQLConf;
 import org.apache.spark.sql.util.CaseInsensitiveStringMap;
 
@@ -38,8 +38,10 @@
  * {@code spark.sql.catalog.catalog-name.(key)=(value)} will be passed in the case insensitive
  * string map of options in initialization with the prefix removed.
  * {@code name}, is also passed and is the catalog's name; in this case, "catalog-name".
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface CatalogPlugin {
   /**
    * Called to initialize configuration.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Catalogs.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Catalogs.java
deleted file mode 100644
index b6a896c52d535..0000000000000
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Catalogs.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.connector.catalog;
-
-import org.apache.spark.SparkException;
-import org.apache.spark.annotation.Private;
-import org.apache.spark.sql.internal.SQLConf;
-import org.apache.spark.sql.util.CaseInsensitiveStringMap;
-import org.apache.spark.util.Utils;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import static scala.collection.JavaConverters.mapAsJavaMapConverter;
-
-@Private
-public class Catalogs {
-  private Catalogs() {
-  }
-
-  /**
-   * Load and configure a catalog by name.
-   * <p>
-   * This loads, instantiates, and initializes the catalog plugin for each call; it does not cache
-   * or reuse instances.
-   *
-   * @param name a String catalog name
-   * @param conf a SQLConf
-   * @return an initialized CatalogPlugin
-   * @throws CatalogNotFoundException if the plugin class cannot be found
-   * @throws SparkException if the plugin class cannot be instantiated
-   */
-  public static CatalogPlugin load(String name, SQLConf conf)
-      throws CatalogNotFoundException, SparkException {
-    String pluginClassName;
-    try {
-      pluginClassName = conf.getConfString("spark.sql.catalog." + name);
-    } catch (NoSuchElementException e){
-      throw new CatalogNotFoundException(String.format(
-          "Catalog '%s' plugin class not found: spark.sql.catalog.%s is not defined", name, name));
-    }
-
-    ClassLoader loader = Utils.getContextOrSparkClassLoader();
-
-    try {
-      Class<?> pluginClass = loader.loadClass(pluginClassName);
-
-      if (!CatalogPlugin.class.isAssignableFrom(pluginClass)) {
-        throw new SparkException(String.format(
-            "Plugin class for catalog '%s' does not implement CatalogPlugin: %s",
-            name, pluginClassName));
-      }
-
-      CatalogPlugin plugin =
-        CatalogPlugin.class.cast(pluginClass.getDeclaredConstructor().newInstance());
-
-      plugin.initialize(name, catalogOptions(name, conf));
-
-      return plugin;
-
-    } catch (ClassNotFoundException e) {
-      throw new SparkException(String.format(
-          "Cannot find catalog plugin class for catalog '%s': %s", name, pluginClassName));
-
-    } catch (NoSuchMethodException e) {
-      throw new SparkException(String.format(
-          "Failed to find public no-arg constructor for catalog '%s': %s", name, pluginClassName),
-          e);
-
-    } catch (IllegalAccessException e) {
-      throw new SparkException(String.format(
-          "Failed to call public no-arg constructor for catalog '%s': %s", name, pluginClassName),
-          e);
-
-    } catch (InstantiationException e) {
-      throw new SparkException(String.format(
-          "Cannot instantiate abstract catalog plugin class for catalog '%s': %s", name,
-          pluginClassName), e.getCause());
-
-    } catch (InvocationTargetException e) {
-      throw new SparkException(String.format(
-          "Failed during instantiating constructor for catalog '%s': %s", name, pluginClassName),
-          e.getCause());
-    }
-  }
-
-  /**
-   * Extracts a named catalog's configuration from a SQLConf.
-   *
-   * @param name a catalog name
-   * @param conf a SQLConf
-   * @return a case insensitive string map of options starting with spark.sql.catalog.(name).
-   */
-  private static CaseInsensitiveStringMap catalogOptions(String name, SQLConf conf) {
-    Map<String, String> allConfs = mapAsJavaMapConverter(conf.getAllConfs()).asJava();
-    Pattern prefix = Pattern.compile("^spark\\.sql\\.catalog\\." + name + "\\.(.+)");
-
-    HashMap<String, String> options = new HashMap<>();
-    for (Map.Entry<String, String> entry : allConfs.entrySet()) {
-      Matcher matcher = prefix.matcher(entry.getKey());
-      if (matcher.matches() && matcher.groupCount() > 0) {
-        options.put(matcher.group(1), entry.getValue());
-      }
-    }
-
-    return new CaseInsensitiveStringMap(options);
-  }
-}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
index cf0eef96d5a9c..d07d299d65a58 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
@@ -19,7 +19,7 @@
 
 import java.util.Map;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException;
 import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
@@ -33,8 +33,10 @@
  * by calling the built-in session catalog directly. This is created for convenience, so that users
  * only need to override some methods where they want to apply custom logic. For example, they can
  * override {@code createTable}, do something else before calling {@code super.createTable}.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public abstract class DelegatingCatalogExtension implements CatalogExtension {
 
   private CatalogPlugin delegate;
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java
index c3ac9c20db3b0..88c51d6c43ddf 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java
@@ -17,12 +17,14 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * Identifies an object in a catalog.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface Identifier {
 
   static Identifier of(String[] namespace, String name) {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
index a56007b2a5ab8..b6c84cdeb0d3e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
@@ -24,12 +24,12 @@
 
 import com.google.common.base.Preconditions;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  *  An {@link Identifier} implementation.
  */
-@Experimental
+@Evolving
 class IdentifierImpl implements Identifier {
 
   private String[] namespace;
@@ -55,7 +55,7 @@ public String name() {
   @Override
   public String toString() {
     return Stream.concat(Stream.of(namespace), Stream.of(name))
-      .map(CatalogV2Implicits::quote)
+      .map(CatalogV2Implicits::quoteIfNeeded)
       .collect(Collectors.joining("."));
   }
 
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java
index 1db6877c8d0be..dae29c16199df 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * NamespaceChange subclasses represent requested changes to a namespace. These are passed to
@@ -30,8 +30,10 @@
  *       removeProperty("other_prop")
  *     )
  * </pre>
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface NamespaceChange {
   /**
    * Create a NamespaceChange for setting a namespace property.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SessionConfigSupport.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SessionConfigSupport.java
index fc076ee00a623..116efc1ada16a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SessionConfigSupport.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SessionConfigSupport.java
@@ -23,6 +23,8 @@
  * A mix-in interface for {@link TableProvider}. Data sources can implement this interface to
  * propagate session configs with the specified key-prefix to all data source operations in this
  * session.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SessionConfigSupport extends TableProvider {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java
index 84b24f204bf9c..60b250adb41ef 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java
@@ -19,7 +19,7 @@
 
 import java.util.Map;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.expressions.Transform;
 import org.apache.spark.sql.connector.write.LogicalWriteInfo;
 import org.apache.spark.sql.types.StructType;
@@ -36,8 +36,10 @@
  * write will be committed. The job concludes with a call to {@link #commitStagedChanges()}, at
  * which point implementations are expected to commit the table's metadata into the metastore along
  * with the data that was written by the writes from the write builder this table created.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface StagedTable extends Table {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java
index 1c8e9c5024376..35455a0ed9975 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java
@@ -19,7 +19,7 @@
 
 import java.util.Map;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.expressions.Transform;
 import org.apache.spark.sql.connector.write.LogicalWriteInfo;
 import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
@@ -48,8 +48,10 @@
  * {@link BatchWrite#commit(WriterCommitMessage[])} is called,
  * {@link StagedTable#commitStagedChanges()} is called, at which point the staged table can
  * complete both the data write and the metadata swap operation atomically.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface StagingTableCatalog extends TableCatalog {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsCatalogOptions.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsCatalogOptions.java
index 5225b12788c49..e779d5234cfb7 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsCatalogOptions.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsCatalogOptions.java
@@ -26,6 +26,8 @@
  * sources use the `DataFrameWriter.save(path)` method, the option `path` can translate to a
  * PathIdentifier. A catalog can then use this PathIdentifier to check the existence of a table, or
  * whether a table can be created at a given directory.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SupportsCatalogOptions extends TableProvider {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
index 80aa57ca18775..106f3283a62c8 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
@@ -17,14 +17,16 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.sources.Filter;
 
 /**
  * A mix-in interface for {@link Table} delete support. Data sources can implement this
  * interface to provide the ability to delete data from tables that matches filter expressions.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface SupportsDelete {
   /**
    * Delete data from a data source table that matches filter expressions.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
index 190f1a14d7129..f70746b612e92 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException;
 import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
 
@@ -35,8 +35,10 @@
  * and uses Java packages as namespaces is not required to support the methods to create, alter, or
  * drop a namespace. Implementations are allowed to discover the existence of objects or namespaces
  * without throwing {@link NoSuchNamespaceException} when no namespace is found.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface SupportsNamespaces extends CatalogPlugin {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java
index cdff1a38ff05f..e0ed9772dcedf 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.read.Scan;
 import org.apache.spark.sql.connector.read.ScanBuilder;
 import org.apache.spark.sql.util.CaseInsensitiveStringMap;
@@ -26,8 +26,10 @@
  * A mix-in interface of {@link Table}, to indicate that it's readable. This adds
  * {@link #newScanBuilder(CaseInsensitiveStringMap)} that is used to create a scan for batch,
  * micro-batch, or continuous processing.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface SupportsRead extends Table {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java
index 90d79ed492ef3..00aa180cf030b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.write.BatchWrite;
 import org.apache.spark.sql.connector.write.LogicalWriteInfo;
 import org.apache.spark.sql.connector.write.WriteBuilder;
@@ -26,8 +26,10 @@
  * A mix-in interface of {@link Table}, to indicate that it's writable. This adds
  * {@link #newWriteBuilder(LogicalWriteInfo)} that is used to create a
  * write for batch or streaming.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface SupportsWrite extends Table {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java
index a493736c77277..8f7a87404837c 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java
@@ -36,6 +36,8 @@
  * The default implementation of {@link #partitioning()} returns an empty array of partitions, and
  * the default implementation of {@link #properties()} returns an empty map. These should be
  * overridden by implementations that support partitioning and table properties.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Table {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
index 9765118a8dbf7..68161d7225fcf 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * Capabilities that can be provided by a {@link Table} implementation.
@@ -25,8 +25,10 @@
  * Tables use {@link Table#capabilities()} to return a set of capabilities. Each capability signals
  * to Spark that the table supports a feature identified by the capability. For example, returning
  * {@link #BATCH_READ} allows Spark to read from the table using a batch scan.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public enum TableCapability {
   /**
    * Signals that the table supports reads in batch execution mode.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
index a69b23bf84d0c..b818515adf9c0 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.expressions.Transform;
 import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
@@ -34,8 +34,10 @@
  * {@link #alterTable(Identifier, TableChange...)} will be normalized to match the case used in the
  * table schema when updating, renaming, or dropping existing columns when catalyst analysis is case
  * insensitive.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface TableCatalog extends CatalogPlugin {
 
   /**
@@ -134,6 +136,8 @@ Table createTable(
    * Implementations may reject the requested changes. If any change is rejected, none of the
    * changes should be applied to the table.
    * <p>
+   * The requested changes must be applied in the order given.
+   * <p>
    * If the catalog supports views and contains a view for the identifier and not a table, this
    * must throw {@link NoSuchTableException}.
    *
@@ -172,7 +176,7 @@ Table alterTable(
    * @param newIdent the new table identifier of the table
    * @throws NoSuchTableException If the table to rename doesn't exist or is a view
    * @throws TableAlreadyExistsException If the new table name already exists or is a view
-   * @throws UnsupportedOperationException If the namespaces of old and new identiers do not
+   * @throws UnsupportedOperationException If the namespaces of old and new identifiers do not
    *                                       match (optional)
    */
   void renameTable(Identifier oldIdent, Identifier newIdent)
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
index 5ce020912f213..3ed185a82452b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
@@ -21,7 +21,7 @@
 import java.util.Objects;
 import javax.annotation.Nullable;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.types.DataType;
 
 /**
@@ -36,8 +36,10 @@
  *       deleteColumn("c")
  *     )
  * </pre>
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface TableChange {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java
index 732c5352a15ac..82731e2c8e1e8 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java
@@ -34,6 +34,8 @@
  * <p>
  * The major responsibility of this interface is to return a {@link Table} for read/write.
  * </p>
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface TableProvider {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java
index 807731ac22948..6540c91597582 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java
@@ -17,12 +17,14 @@
 
 package org.apache.spark.sql.connector.expressions;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * Base class of the public logical expression API.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface Expression {
   /**
    * Format the expression as a human readable SQL-like string.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
index a06ac9f77c5b5..791dc969ab008 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
@@ -19,15 +19,17 @@
 
 import java.util.Arrays;
 
+import org.apache.spark.annotation.Evolving;
 import scala.collection.JavaConverters;
 
-import org.apache.spark.annotation.Experimental;
 import org.apache.spark.sql.types.DataType;
 
 /**
  * Helper methods to create logical transforms to pass into Spark.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public class Expressions {
   private Expressions() {
   }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java
index 942744bf6a5d3..df9e58fa319fd 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.expressions;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.types.DataType;
 
 /**
@@ -27,8 +27,9 @@
  * the literal's {@link DataType SQL data type}.
  *
  * @param <T> the JVM type of a value held by the literal
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface Literal<T> extends Expression {
   /**
    * Returns the literal value.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java
index e6e5fa59c2499..167432fa0e86a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java
@@ -17,12 +17,14 @@
 
 package org.apache.spark.sql.connector.expressions;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * Represents a field or column reference in the public logical expression API.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface NamedReference extends Expression {
   /**
    * Returns the referenced field name as an array of String parts.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java
index edac6df8a4d3e..297205825c6a4 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java
@@ -17,15 +17,17 @@
 
 package org.apache.spark.sql.connector.expressions;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * Represents a transform function in the public logical expression API.
  * <p>
  * For example, the transform date(ts) is used to derive a date value from a timestamp column. The
  * transform name is "date" and its argument is a reference to the "ts" column.
+ *
+ * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface Transform extends Expression {
   /**
    * Returns the transform function name.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java
index 09592598a82f9..73aefa55ae5e1 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java
@@ -23,6 +23,8 @@
  * A physical representation of a data source scan for batch queries. This interface is used to
  * provide physical information, like how many partitions the scanned data has, and how to read
  * records from the partitions.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Batch {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java
index bedf30803f73f..03eec00168012 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java
@@ -31,6 +31,8 @@
  * {@link PartitionReaderFactory#createColumnarReader(InputPartition)} on executors to do
  * the actual reading. So {@link InputPartition} must be serializable while {@link PartitionReader}
  * doesn't need to be.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface InputPartition extends Serializable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java
index 254ee56501b9a..23fbd95800e26 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java
@@ -31,6 +31,8 @@
  * for normal data sources, or {@link org.apache.spark.sql.vectorized.ColumnarBatch} for columnar
  * data sources(whose {@link PartitionReaderFactory#supportColumnarReads(InputPartition)}
  * returns true).
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface PartitionReader<T> extends Closeable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java
index 7fef69ce2a5b8..9dded247e89f3 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java
@@ -29,6 +29,8 @@
  * If Spark fails to execute any methods in the implementations of this interface or in the returned
  * {@link PartitionReader} (by throwing an exception), corresponding Spark task would fail and
  * get retried until hitting the maximum retry times.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface PartitionReaderFactory extends Serializable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java
index c1584a58c117f..4146f217985b4 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java
@@ -34,6 +34,8 @@
  * implemented, if the {@link Table} that creates this {@link Scan} returns
  * {@link TableCapability#BATCH_READ} support in its {@link Table#capabilities()}.
  * </p>
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Scan {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/ScanBuilder.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/ScanBuilder.java
index ca2cd59b2e442..cb3eea7680058 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/ScanBuilder.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/ScanBuilder.java
@@ -23,6 +23,8 @@
  * An interface for building the {@link Scan}. Implementations can mixin SupportsPushDownXYZ
  * interfaces to do operator pushdown, and keep the operator pushdown result in the returned
  * {@link Scan}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface ScanBuilder {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Statistics.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Statistics.java
index 42778b4e28a1a..a4ef785da2c65 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Statistics.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Statistics.java
@@ -24,6 +24,8 @@
 /**
  * An interface to represent statistics for a data source, which is returned by
  * {@link SupportsReportStatistics#estimateStatistics()}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Statistics {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java
index bee9e5508ca66..6594af27734c7 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java
@@ -23,6 +23,8 @@
 /**
  * A mix-in interface for {@link ScanBuilder}. Data sources can implement this interface to
  * push down filters to the data source and reduce the size of the data to be read.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SupportsPushDownFilters extends ScanBuilder {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java
index 97143686d3efc..4f7da3c2a38f4 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java
@@ -24,6 +24,8 @@
  * A mix-in interface for {@link ScanBuilder}. Data sources can implement this
  * interface to push down required columns to the data source and only read these columns during
  * scan to reduce the size of the data to be read.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SupportsPushDownRequiredColumns extends ScanBuilder {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java
index 5bbc191730391..e7a27e07498fb 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java
@@ -26,6 +26,8 @@
  *
  * Note that, when a {@link Scan} implementation creates exactly one {@link InputPartition},
  * Spark may avoid adding a shuffle even if the reader does not implement this interface.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SupportsReportPartitioning extends Scan {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
index 3cd5ef60dc533..1e0c9ca7c7e4b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
@@ -23,9 +23,11 @@
  * A mix in interface for {@link Scan}. Data sources can implement this interface to
  * report statistics to Spark.
  *
- * As of Spark 2.4, statistics are reported to the optimizer before any operator is pushed to the
- * data source. Implementations that return more accurate statistics based on pushed operators will
- * not improve query performance until the planner can push operators before getting stats.
+ * As of Spark 3.0, statistics are reported to the optimizer after operators are pushed to the
+ * data source. Implementations may return more accurate statistics based on pushed operators
+ * which may improve query performance by providing better information to the optimizer.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SupportsReportStatistics extends Scan {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/ClusteredDistribution.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/ClusteredDistribution.java
index 23d3c6b563d1c..ed0354484d7be 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/ClusteredDistribution.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/ClusteredDistribution.java
@@ -24,6 +24,8 @@
  * A concrete implementation of {@link Distribution}. Represents a distribution where records that
  * share the same values for the {@link #clusteredColumns} will be produced by the same
  * {@link PartitionReader}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public class ClusteredDistribution implements Distribution {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java
index fd397c760f05a..264b268e247e1 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java
@@ -36,6 +36,8 @@
  * <ul>
  *   <li>{@link ClusteredDistribution}</li>
  * </ul>
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Distribution {}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java
index f0b9410ee6e61..c2c474696f667 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java
@@ -26,6 +26,8 @@
  * {@link SupportsReportPartitioning#outputPartitioning()}. Note that this should work
  * like a snapshot. Once created, it should be deterministic and always report the same number of
  * partitions and the same "satisfy" result for a certain distribution.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Partitioning {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java
index 8bd5273bb7d8e..c2ad9ec244a0d 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java
@@ -22,6 +22,8 @@
 
 /**
  * A variation on {@link PartitionReader} for use with continuous streaming processing.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface ContinuousPartitionReader<T> extends PartitionReader<T> {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReaderFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReaderFactory.java
index 962864da4aad8..385c6f655440f 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReaderFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReaderFactory.java
@@ -27,6 +27,8 @@
 /**
  * A variation on {@link PartitionReaderFactory} that returns {@link ContinuousPartitionReader}
  * instead of {@link PartitionReader}. It's used for continuous streaming processing.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface ContinuousPartitionReaderFactory extends PartitionReaderFactory {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java
index ee01a2553ae7a..a84578fe461a3 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java
@@ -23,6 +23,8 @@
 
 /**
  * A {@link SparkDataStream} for streaming queries with continuous mode.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface ContinuousStream extends SparkDataStream {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/MicroBatchStream.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/MicroBatchStream.java
index ceab0f75734d3..40ecbf0578ee5 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/MicroBatchStream.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/MicroBatchStream.java
@@ -25,6 +25,8 @@
 
 /**
  * A {@link SparkDataStream} for streaming queries with micro-batch mode.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface MicroBatchStream extends SparkDataStream {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java
index 400de2a659746..efb8ebb684f06 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java
@@ -25,6 +25,8 @@
  * During execution, offsets provided by the data source implementation will be logged and used as
  * restart checkpoints. Each source should provide an offset implementation which the source can use
  * to reconstruct a position in the stream up to which data has been seen/processed.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public abstract class Offset {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java
index 35ad3bbde5cbf..faee230467bea 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java
@@ -26,6 +26,8 @@
  * provide a method to merge these into a global Offset.
  *
  * These offsets must be serializable.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface PartitionOffset extends Serializable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadLimit.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadLimit.java
index 121ed1ad116f9..36f6e05e365d9 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadLimit.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadLimit.java
@@ -27,6 +27,7 @@
  * @see SupportsAdmissionControl#latestOffset(Offset, ReadLimit)
  * @see ReadAllAvailable
  * @see ReadMaxRows
+ * @since 3.0.0
  */
 @Evolving
 public interface ReadLimit {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java
index 1ba0c25ef4466..95703e255ea4e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java
@@ -25,6 +25,8 @@
  *
  * Data sources should implement concrete data stream interfaces:
  * {@link MicroBatchStream} and {@link ContinuousStream}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SparkDataStream {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java
index 3e8b14172d6b2..c4b073458eac8 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java
@@ -36,6 +36,8 @@
  * do it manually in their Spark applications if they want to retry.
  *
  * Please refer to the documentation of commit/abort methods for detailed specifications.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface BatchWrite {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java
index 59c69a18292d8..0022a71491738 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java
@@ -56,6 +56,8 @@
  * Spark only accepts the commit message that arrives first and ignore others.
  *
  * Note that, Currently the type `T` can only be {@link org.apache.spark.sql.catalyst.InternalRow}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface DataWriter<T> extends Closeable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java
index 310575df05d97..33b7ff3dd27b1 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java
@@ -31,6 +31,8 @@
  * Note that, the writer factory will be serialized and sent to executors, then the data writer
  * will be created on executors and do the actual writing. So this interface must be
  * serializable and {@link DataWriter} doesn't need to be.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface DataWriterFactory extends Serializable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java
index 831f4e5aac481..0638970df925e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java
@@ -24,6 +24,8 @@
 /**
  * This interface contains logical write information that data sources can use when generating a
  * {@link WriteBuilder}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface LogicalWriteInfo {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/PhysicalWriteInfo.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/PhysicalWriteInfo.java
index 55a092e39970e..248a6e69ac11f 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/PhysicalWriteInfo.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/PhysicalWriteInfo.java
@@ -23,6 +23,8 @@
 /**
  * This interface contains physical write information that data sources can use when
  * generating a {@link DataWriterFactory} or a {@link StreamingDataWriterFactory}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface PhysicalWriteInfo {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java
index 90e668d1935b1..422cd71d3453e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.connector.write;
 
+import org.apache.spark.annotation.Evolving;
+
 /**
  * Write builder trait for tables that support dynamic partition overwrite.
  * <p>
@@ -26,7 +28,10 @@
  * <p>
  * This is provided to implement SQL compatible with Hive table operations but is not recommended.
  * Instead, use the {@link SupportsOverwrite overwrite by filter API} to explicitly replace data.
+ *
+ * @since 3.0.0
  */
+@Evolving
 public interface SupportsDynamicOverwrite extends WriteBuilder {
   /**
    * Configures a write to dynamically replace partitions with data committed in the write.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java
index 6063a155ee209..b4e60257942da 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.connector.write;
 
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.sources.AlwaysTrue$;
 import org.apache.spark.sql.sources.Filter;
 
@@ -25,7 +26,10 @@
  * <p>
  * Overwriting data by filter will delete any data that matches the filter and replace it with data
  * that is committed in the write.
+ *
+ * @since 3.0.0
  */
+@Evolving
 public interface SupportsOverwrite extends WriteBuilder, SupportsTruncate {
   /**
    * Configures a write to replace data matching the filters with data committed in the write.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java
index 15a68375032d2..0763f657d3b91 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java
@@ -17,11 +17,16 @@
 
 package org.apache.spark.sql.connector.write;
 
+import org.apache.spark.annotation.Evolving;
+
 /**
  * Write builder trait for tables that support truncation.
  * <p>
  * Truncation removes all data in a table and replaces it with data that is committed in the write.
+ *
+ * @since 3.0.0
  */
+@Evolving
 public interface SupportsTruncate extends WriteBuilder {
   /**
    * Configures a write to replace all existing data with data committed in the write.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java
index a8d99a8f04b1b..5398ca46e9777 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java
@@ -28,6 +28,8 @@
  *
  * Unless modified by a mixin interface, the {@link BatchWrite} configured by this builder is to
  * append data without affecting existing data.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface WriteBuilder {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java
index 823f955bffb53..f340d20c3549f 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java
@@ -29,6 +29,8 @@
  *
  * This is an empty interface, data sources should define their own message class and use it when
  * generating messages at executor side and handling the messages at driver side.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface WriterCommitMessage extends Serializable {}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java
index 9946867e8ea65..0923d07e7e5a3 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java
@@ -33,6 +33,8 @@
  * Note that, the writer factory will be serialized and sent to executors, then the data writer
  * will be created on executors and do the actual writing. So this interface must be
  * serializable and {@link DataWriter} doesn't need to be.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface StreamingDataWriterFactory extends Serializable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java
index 4f930e1c158e5..e3dec3b2ff55e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java
@@ -40,6 +40,8 @@
  * do it manually in their Spark applications if they want to retry.
  *
  * Please refer to the documentation of commit/abort methods for detailed specifications.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface StreamingWrite {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/util/CaseInsensitiveStringMap.java b/sql/catalyst/src/main/java/org/apache/spark/sql/util/CaseInsensitiveStringMap.java
index c344a62be40c1..de7d98ba48c31 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/util/CaseInsensitiveStringMap.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/util/CaseInsensitiveStringMap.java
@@ -36,6 +36,8 @@
  * <p>
  * Methods that return keys in this map, like {@link #entrySet()} and {@link #keySet()}, return
  * keys converted to lower case. This map doesn't allow null key.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public class CaseInsensitiveStringMap implements Map<String, String> {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
index d2220dc2668f9..72fccd451ca12 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.vectorized;
 
-import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.complex.*;
 import org.apache.arrow.vector.holders.NullableVarCharHolder;
@@ -458,10 +457,8 @@ final boolean isNullAt(int rowId) {
 
     @Override
     final ColumnarArray getArray(int rowId) {
-      ArrowBuf offsets = accessor.getOffsetBuffer();
-      int index = rowId * ListVector.OFFSET_WIDTH;
-      int start = offsets.getInt(index);
-      int end = offsets.getInt(index + ListVector.OFFSET_WIDTH);
+      int start = accessor.getElementStartIndex(rowId);
+      int end = accessor.getElementEndIndex(rowId);
       return new ColumnarArray(arrayData, start, end - start);
     }
   }
diff --git a/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
new file mode 100644
index 0000000000000..e5f9b0cb7a6f6
--- /dev/null
+++ b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
@@ -0,0 +1,2401 @@
+[ {
+  "tz" : "Africa/Abidjan",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Africa/Accra",
+  "switches" : [ -62135596748, -59006361548, -55850687948, -52695014348, -46383580748, -43227907148, -40072233548, -33760799948, -30605126348, -27449452748, -21138019148, -17982345548, -14826671948, -12220070348, -12219983948, -12219897548, -12219811148, -12219724748, -12219638348, -12219551948, -12219465548, -12219379148, -12219292748, -2208988800 ],
+  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, 777548, 691148, 604748, 518348, 431948, 345548, 259148, 172748, 86348, -52, 0 ]
+}, {
+  "tz" : "Africa/Addis_Ababa",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Africa/Algiers",
+  "switches" : [ -62135597532, -59006362332, -55850688732, -52695015132, -46383581532, -43227907932, -40072234332, -33760800732, -30605127132, -27449453532, -21138019932, -17982346332, -14826672732, -12220071132, -12219984732, -12219898332, -12219811932, -12219725532, -12219639132, -12219552732, -12219466332, -12219379932, -12219293532, -2486679072, -2208988800 ],
+  "diffs" : [ -175668, -89268, -2868, 83532, 169932, 256332, 342732, 429132, 515532, 601932, 688332, 774732, 861132, 774732, 688332, 601932, 515532, 429132, 342732, 256332, 169932, 83532, -2868, -3039, 0 ]
+}, {
+  "tz" : "Africa/Asmara",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Africa/Asmera",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Africa/Bamako",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Africa/Bangui",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/Banjul",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Africa/Bissau",
+  "switches" : [ -62135593060, -59006357860, -55850684260, -52695010660, -46383577060, -43227903460, -40072229860, -33760796260, -30605122660, -27449449060, -21138015460, -17982341860, -14826668260, -12220066660, -12219980260, -12219893860, -12219807460, -12219721060, -12219634660, -12219548260, -12219461860, -12219375460, -12219289060, -2208988800 ],
+  "diffs" : [ -176540, -90140, -3740, 82660, 169060, 255460, 341860, 428260, 514660, 601060, 687460, 773860, 860260, 773860, 687460, 601060, 514660, 428260, 341860, 255460, 169060, 82660, -3740, 0 ]
+}, {
+  "tz" : "Africa/Blantyre",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
+}, {
+  "tz" : "Africa/Brazzaville",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/Bujumbura",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
+}, {
+  "tz" : "Africa/Cairo",
+  "switches" : [ -62135604309, -59006369109, -55850695509, -52695021909, -46383588309, -43227914709, -40072241109, -33760807509, -30605133909, -27449460309, -21138026709, -17982353109, -14826679509, -12220077909, -12219991509, -12219905109, -12219818709, -12219732309, -12219645909, -12219559509, -12219473109, -12219386709, -12219300309, -2208988800 ],
+  "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 777909, 691509, 605109, 518709, 432309, 345909, 259509, 173109, 86709, 309, 0 ]
+}, {
+  "tz" : "Africa/Casablanca",
+  "switches" : [ -62135594980, -59006359780, -55850686180, -52695012580, -46383578980, -43227905380, -40072231780, -33760798180, -30605124580, -27449450980, -21138017380, -17982343780, -14826670180, -12220068580, -12219982180, -12219895780, -12219809380, -12219722980, -12219636580, -12219550180, -12219463780, -12219377380, -12219290980, -2208988800 ],
+  "diffs" : [ -174620, -88220, -1820, 84580, 170980, 257380, 343780, 430180, 516580, 602980, 689380, 775780, 862180, 775780, 689380, 602980, 516580, 430180, 343780, 257380, 170980, 84580, -1820, 0 ]
+}, {
+  "tz" : "Africa/Ceuta",
+  "switches" : [ -62135595524, -59006360324, -55850686724, -52695013124, -46383579524, -43227905924, -40072232324, -33760798724, -30605125124, -27449451524, -21138017924, -17982344324, -14826670724, -12220069124, -12219982724, -12219896324, -12219809924, -12219723524, -12219637124, -12219550724, -12219464324, -12219377924, -12219291524, -2208988800 ],
+  "diffs" : [ -177676, -91276, -4876, 81524, 167924, 254324, 340724, 427124, 513524, 599924, 686324, 772724, 859124, 772724, 686324, 599924, 513524, 427124, 340724, 254324, 167924, 81524, -4876, 0 ]
+}, {
+  "tz" : "Africa/Conakry",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Africa/Dakar",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Africa/Dar_es_Salaam",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Africa/Djibouti",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Africa/Douala",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/El_Aaiun",
+  "switches" : [ -62135593632, -59006358432, -55850684832, -52695011232, -46383577632, -43227904032, -40072230432, -33760796832, -30605123232, -27449449632, -21138016032, -17982342432, -14826668832, -12220067232, -12219980832, -12219894432, -12219808032, -12219721632, -12219635232, -12219548832, -12219462432, -12219376032, -12219289632, -2208988800 ],
+  "diffs" : [ -175968, -89568, -3168, 83232, 169632, 256032, 342432, 428832, 515232, 601632, 688032, 774432, 860832, 774432, 688032, 601632, 515232, 428832, 342432, 256032, 169632, 83232, -3168, 0 ]
+}, {
+  "tz" : "Africa/Freetown",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Africa/Gaborone",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
+}, {
+  "tz" : "Africa/Harare",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
+}, {
+  "tz" : "Africa/Johannesburg",
+  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12220077120, -12219990720, -12219904320, -12219817920, -12219731520, -12219645120, -12219558720, -12219472320, -12219385920, -12219299520, -2458173120, -2208988800 ],
+  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, 777120, 690720, 604320, 517920, 431520, 345120, 258720, 172320, 85920, -480, -1800, 0 ]
+}, {
+  "tz" : "Africa/Juba",
+  "switches" : [ -62135604388, -59006369188, -55850695588, -52695021988, -46383588388, -43227914788, -40072241188, -33760807588, -30605133988, -27449460388, -21138026788, -17982353188, -14826679588, -12220077988, -12219991588, -12219905188, -12219818788, -12219732388, -12219645988, -12219559588, -12219473188, -12219386788, -12219300388, -2208988800 ],
+  "diffs" : [ -176012, -89612, -3212, 83188, 169588, 255988, 342388, 428788, 515188, 601588, 687988, 774388, 860788, 774388, 687988, 601588, 515188, 428788, 342388, 255988, 169588, 83188, -3212, 0 ]
+}, {
+  "tz" : "Africa/Kampala",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Africa/Khartoum",
+  "switches" : [ -62135604608, -59006369408, -55850695808, -52695022208, -46383588608, -43227915008, -40072241408, -33760807808, -30605134208, -27449460608, -21138027008, -17982353408, -14826679808, -12220078208, -12219991808, -12219905408, -12219819008, -12219732608, -12219646208, -12219559808, -12219473408, -12219387008, -12219300608, -2208988800 ],
+  "diffs" : [ -172192, -85792, 608, 87008, 173408, 259808, 346208, 432608, 519008, 605408, 691808, 778208, 864608, 778208, 691808, 605408, 519008, 432608, 346208, 259808, 173408, 87008, 608, 0 ]
+}, {
+  "tz" : "Africa/Kigali",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
+}, {
+  "tz" : "Africa/Kinshasa",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/Lagos",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/Libreville",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/Lome",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Africa/Luanda",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/Lubumbashi",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
+}, {
+  "tz" : "Africa/Lusaka",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
+}, {
+  "tz" : "Africa/Malabo",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/Maputo",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
+}, {
+  "tz" : "Africa/Maseru",
+  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12220077120, -12219990720, -12219904320, -12219817920, -12219731520, -12219645120, -12219558720, -12219472320, -12219385920, -12219299520, -2458173120, -2208988800 ],
+  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, 777120, 690720, 604320, 517920, 431520, 345120, 258720, 172320, 85920, -480, -1800, 0 ]
+}, {
+  "tz" : "Africa/Mbabane",
+  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12220077120, -12219990720, -12219904320, -12219817920, -12219731520, -12219645120, -12219558720, -12219472320, -12219385920, -12219299520, -2458173120, -2208988800 ],
+  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, 777120, 690720, 604320, 517920, 431520, 345120, 258720, 172320, 85920, -480, -1800, 0 ]
+}, {
+  "tz" : "Africa/Mogadishu",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Africa/Monrovia",
+  "switches" : [ -62135594212, -59006359012, -55850685412, -52695011812, -46383578212, -43227904612, -40072231012, -33760797412, -30605123812, -27449450212, -21138016612, -17982343012, -14826669412, -12220067812, -12219981412, -12219895012, -12219808612, -12219722212, -12219635812, -12219549412, -12219463012, -12219376612, -12219290212, -2208988800 ],
+  "diffs" : [ -175388, -88988, -2588, 83812, 170212, 256612, 343012, 429412, 515812, 602212, 688612, 775012, 861412, 775012, 688612, 602212, 515812, 429412, 343012, 256612, 170212, 83812, -2588, 0 ]
+}, {
+  "tz" : "Africa/Nairobi",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Africa/Ndjamena",
+  "switches" : [ -62135600412, -59006365212, -55850691612, -52695018012, -46383584412, -43227910812, -40072237212, -33760803612, -30605130012, -27449456412, -21138022812, -17982349212, -14826675612, -12220074012, -12219987612, -12219901212, -12219814812, -12219728412, -12219642012, -12219555612, -12219469212, -12219382812, -12219296412, -2208988800 ],
+  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 777612, 691212, 604812, 518412, 432012, 345612, 259212, 172812, 86412, 12, 0 ]
+}, {
+  "tz" : "Africa/Niamey",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/Nouakchott",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Africa/Ouagadougou",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Africa/Porto-Novo",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/Sao_Tome",
+  "switches" : [ -62135598416, -59006363216, -55850689616, -52695016016, -46383582416, -43227908816, -40072235216, -33760801616, -30605128016, -27449454416, -21138020816, -17982347216, -14826673616, -12220072016, -12219985616, -12219899216, -12219812816, -12219726416, -12219640016, -12219553616, -12219467216, -12219380816, -12219294416, -2713912016, -2208988800 ],
+  "diffs" : [ -171184, -84784, 1616, 88016, 174416, 260816, 347216, 433616, 520016, 606416, 692816, 779216, 865616, 779216, 692816, 606416, 520016, 433616, 347216, 260816, 174416, 88016, 1616, -2205, 0 ]
+}, {
+  "tz" : "Africa/Timbuktu",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Africa/Tripoli",
+  "switches" : [ -62135599964, -59006364764, -55850691164, -52695017564, -46383583964, -43227910364, -40072236764, -33760803164, -30605129564, -27449455964, -21138022364, -17982348764, -14826675164, -12220073564, -12219987164, -12219900764, -12219814364, -12219727964, -12219641564, -12219555164, -12219468764, -12219382364, -12219295964, -2208988800 ],
+  "diffs" : [ -176836, -90436, -4036, 82364, 168764, 255164, 341564, 427964, 514364, 600764, 687164, 773564, 859964, 773564, 687164, 600764, 514364, 427964, 341564, 255164, 168764, 82364, -4036, 0 ]
+}, {
+  "tz" : "Africa/Tunis",
+  "switches" : [ -62135599244, -59006364044, -55850690444, -52695016844, -46383583244, -43227909644, -40072236044, -33760802444, -30605128844, -27449455244, -21138021644, -17982348044, -14826674444, -12220072844, -12219986444, -12219900044, -12219813644, -12219727244, -12219640844, -12219554444, -12219468044, -12219381644, -12219295244, -2797202444, -2208988800 ],
+  "diffs" : [ -173956, -87556, -1156, 85244, 171644, 258044, 344444, 430844, 517244, 603644, 690044, 776444, 862844, 776444, 690044, 603644, 517244, 430844, 344444, 258044, 171644, 85244, -1156, -3039, 0 ]
+}, {
+  "tz" : "Africa/Windhoek",
+  "switches" : [ -62135600904, -59006365704, -55850692104, -52695018504, -46383584904, -43227911304, -40072237704, -33760804104, -30605130504, -27449456904, -21138023304, -17982349704, -14826676104, -12220074504, -12219988104, -12219901704, -12219815304, -12219728904, -12219642504, -12219556104, -12219469704, -12219383304, -12219296904, -2458170504, -2208988800 ],
+  "diffs" : [ -175896, -89496, -3096, 83304, 169704, 256104, 342504, 428904, 515304, 601704, 688104, 774504, 860904, 774504, 688104, 601704, 515304, 428904, 342504, 256104, 169704, 83304, -3096, -1800, 0 ]
+}, {
+  "tz" : "America/Adak",
+  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12220114402, -12220028002, -12219941602, -12219855202, -12219768802, -12219682402, -12219596002, -12219509602, -12219423202, -12219336802, -3225223727, -2208988800 ],
+  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 857602, 771202, 684802, 598402, 512002, 425602, 339202, 252802, 166402, 80002, -6398, 0 ]
+}, {
+  "tz" : "America/Anchorage",
+  "switches" : [ -62135647224, -59006412024, -55850738424, -52695064824, -46383631224, -43227957624, -40072284024, -33760850424, -30605176824, -27449503224, -21138069624, -17982396024, -14826722424, -12220120824, -12220034424, -12219948024, -12219861624, -12219775224, -12219688824, -12219602424, -12219516024, -12219429624, -12219343224, -3225223727, -2208988800 ],
+  "diffs" : [ -89976, -3576, 82824, 169224, 255624, 342024, 428424, 514824, 601224, 687624, 774024, 860424, 946824, 860424, 774024, 687624, 601224, 514824, 428424, 342024, 255624, 169224, 82824, -3576, 0 ]
+}, {
+  "tz" : "America/Anguilla",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Antigua",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Araguaina",
+  "switches" : [ -62135585232, -59006350032, -55850676432, -52695002832, -46383569232, -43227895632, -40072222032, -33760788432, -30605114832, -27449441232, -21138007632, -17982334032, -14826660432, -12220058832, -12219972432, -12219886032, -12219799632, -12219713232, -12219626832, -12219540432, -12219454032, -12219367632, -12219281232, -2208988800 ],
+  "diffs" : [ -173568, -87168, -768, 85632, 172032, 258432, 344832, 431232, 517632, 604032, 690432, 776832, 863232, 776832, 690432, 604032, 517632, 431232, 344832, 258432, 172032, 85632, -768, 0 ]
+}, {
+  "tz" : "America/Argentina/Buenos_Aires",
+  "switches" : [ -62135582772, -59006347572, -55850673972, -52695000372, -46383566772, -43227893172, -40072219572, -33760785972, -30605112372, -27449438772, -21138005172, -17982331572, -14826657972, -12220056372, -12219969972, -12219883572, -12219797172, -12219710772, -12219624372, -12219537972, -12219451572, -12219365172, -12219278772, -2372097972, -2208988800 ],
+  "diffs" : [ -176028, -89628, -3228, 83172, 169572, 255972, 342372, 428772, 515172, 601572, 687972, 774372, 860772, 774372, 687972, 601572, 515172, 428772, 342372, 255972, 169572, 83172, -3228, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Catamarca",
+  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12220054612, -12219968212, -12219881812, -12219795412, -12219709012, -12219622612, -12219536212, -12219449812, -12219363412, -12219277012, -2372096212, -2208988800 ],
+  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, 772612, 686212, 599812, 513412, 427012, 340612, 254212, 167812, 81412, -4988, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/ComodRivadavia",
+  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12220054612, -12219968212, -12219881812, -12219795412, -12219709012, -12219622612, -12219536212, -12219449812, -12219363412, -12219277012, -2372096212, -2208988800 ],
+  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, 772612, 686212, 599812, 513412, 427012, 340612, 254212, 167812, 81412, -4988, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Cordoba",
+  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12220054992, -12219968592, -12219882192, -12219795792, -12219709392, -12219622992, -12219536592, -12219450192, -12219363792, -12219277392, -2208988800 ],
+  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, 772992, 686592, 600192, 513792, 427392, 340992, 254592, 168192, 81792, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Jujuy",
+  "switches" : [ -62135581128, -59006345928, -55850672328, -52694998728, -46383565128, -43227891528, -40072217928, -33760784328, -30605110728, -27449437128, -21138003528, -17982329928, -14826656328, -12220054728, -12219968328, -12219881928, -12219795528, -12219709128, -12219622728, -12219536328, -12219449928, -12219363528, -12219277128, -2372096328, -2208988800 ],
+  "diffs" : [ -177672, -91272, -4872, 81528, 167928, 254328, 340728, 427128, 513528, 599928, 686328, 772728, 859128, 772728, 686328, 599928, 513528, 427128, 340728, 254328, 167928, 81528, -4872, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/La_Rioja",
+  "switches" : [ -62135580756, -59006345556, -55850671956, -52694998356, -46383564756, -43227891156, -40072217556, -33760783956, -30605110356, -27449436756, -21138003156, -17982329556, -14826655956, -12220054356, -12219967956, -12219881556, -12219795156, -12219708756, -12219622356, -12219535956, -12219449556, -12219363156, -12219276756, -2372095956, -2208988800 ],
+  "diffs" : [ -178044, -91644, -5244, 81156, 167556, 253956, 340356, 426756, 513156, 599556, 685956, 772356, 858756, 772356, 685956, 599556, 513156, 426756, 340356, 253956, 167556, 81156, -5244, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Mendoza",
+  "switches" : [ -62135580284, -59006345084, -55850671484, -52694997884, -46383564284, -43227890684, -40072217084, -33760783484, -30605109884, -27449436284, -21138002684, -17982329084, -14826655484, -12220053884, -12219967484, -12219881084, -12219794684, -12219708284, -12219621884, -12219535484, -12219449084, -12219362684, -12219276284, -2372095484, -2208988800 ],
+  "diffs" : [ -178516, -92116, -5716, 80684, 167084, 253484, 339884, 426284, 512684, 599084, 685484, 771884, 858284, 771884, 685484, 599084, 512684, 426284, 339884, 253484, 167084, 80684, -5716, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Rio_Gallegos",
+  "switches" : [ -62135580188, -59006344988, -55850671388, -52694997788, -46383564188, -43227890588, -40072216988, -33760783388, -30605109788, -27449436188, -21138002588, -17982328988, -14826655388, -12220053788, -12219967388, -12219880988, -12219794588, -12219708188, -12219621788, -12219535388, -12219448988, -12219362588, -12219276188, -2372095388, -2208988800 ],
+  "diffs" : [ -178612, -92212, -5812, 80588, 166988, 253388, 339788, 426188, 512588, 598988, 685388, 771788, 858188, 771788, 685388, 598988, 512588, 426188, 339788, 253388, 166988, 80588, -5812, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Salta",
+  "switches" : [ -62135581100, -59006345900, -55850672300, -52694998700, -46383565100, -43227891500, -40072217900, -33760784300, -30605110700, -27449437100, -21138003500, -17982329900, -14826656300, -12220054700, -12219968300, -12219881900, -12219795500, -12219709100, -12219622700, -12219536300, -12219449900, -12219363500, -12219277100, -2372096300, -2208988800 ],
+  "diffs" : [ -177700, -91300, -4900, 81500, 167900, 254300, 340700, 427100, 513500, 599900, 686300, 772700, 859100, 772700, 686300, 599900, 513500, 427100, 340700, 254300, 167900, 81500, -4900, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/San_Juan",
+  "switches" : [ -62135580356, -59006345156, -55850671556, -52694997956, -46383564356, -43227890756, -40072217156, -33760783556, -30605109956, -27449436356, -21138002756, -17982329156, -14826655556, -12220053956, -12219967556, -12219881156, -12219794756, -12219708356, -12219621956, -12219535556, -12219449156, -12219362756, -12219276356, -2372095556, -2208988800 ],
+  "diffs" : [ -178444, -92044, -5644, 80756, 167156, 253556, 339956, 426356, 512756, 599156, 685556, 771956, 858356, 771956, 685556, 599156, 512756, 426356, 339956, 253556, 167156, 80756, -5644, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/San_Luis",
+  "switches" : [ -62135580876, -59006345676, -55850672076, -52694998476, -46383564876, -43227891276, -40072217676, -33760784076, -30605110476, -27449436876, -21138003276, -17982329676, -14826656076, -12220054476, -12219968076, -12219881676, -12219795276, -12219708876, -12219622476, -12219536076, -12219449676, -12219363276, -12219276876, -2372096076, -2208988800 ],
+  "diffs" : [ -177924, -91524, -5124, 81276, 167676, 254076, 340476, 426876, 513276, 599676, 686076, 772476, 858876, 772476, 686076, 599676, 513276, 426876, 340476, 254076, 167676, 81276, -5124, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Tucuman",
+  "switches" : [ -62135581148, -59006345948, -55850672348, -52694998748, -46383565148, -43227891548, -40072217948, -33760784348, -30605110748, -27449437148, -21138003548, -17982329948, -14826656348, -12220054748, -12219968348, -12219881948, -12219795548, -12219709148, -12219622748, -12219536348, -12219449948, -12219363548, -12219277148, -2372096348, -2208988800 ],
+  "diffs" : [ -177652, -91252, -4852, 81548, 167948, 254348, 340748, 427148, 513548, 599948, 686348, 772748, 859148, 772748, 686348, 599948, 513548, 427148, 340748, 254348, 167948, 81548, -4852, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Ushuaia",
+  "switches" : [ -62135580408, -59006345208, -55850671608, -52694998008, -46383564408, -43227890808, -40072217208, -33760783608, -30605110008, -27449436408, -21138002808, -17982329208, -14826655608, -12220054008, -12219967608, -12219881208, -12219794808, -12219708408, -12219622008, -12219535608, -12219449208, -12219362808, -12219276408, -2372095608, -2208988800 ],
+  "diffs" : [ -178392, -91992, -5592, 80808, 167208, 253608, 340008, 426408, 512808, 599208, 685608, 772008, 858408, 772008, 685608, 599208, 512808, 426408, 340008, 253608, 167208, 80808, -5592, -4608, 0 ]
+}, {
+  "tz" : "America/Aruba",
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ]
+}, {
+  "tz" : "America/Asuncion",
+  "switches" : [ -62135582960, -59006347760, -55850674160, -52695000560, -46383566960, -43227893360, -40072219760, -33760786160, -30605112560, -27449438960, -21138005360, -17982331760, -14826658160, -12220056560, -12219970160, -12219883760, -12219797360, -12219710960, -12219624560, -12219538160, -12219451760, -12219365360, -12219278960, -2208988800 ],
+  "diffs" : [ -172240, -85840, 560, 86960, 173360, 259760, 346160, 432560, 518960, 605360, 691760, 778160, 864560, 778160, 691760, 605360, 518960, 432560, 346160, 259760, 173360, 86960, 560, 0 ]
+}, {
+  "tz" : "America/Atikokan",
+  "switches" : [ -62135574812, -59006339612, -55850666012, -52694992412, -46383558812, -43227885212, -40072211612, -33760778012, -30605104412, -27449430812, -21137997212, -17982323612, -14826650012, -12220048412, -12219962012, -12219875612, -12219789212, -12219702812, -12219616412, -12219530012, -12219443612, -12219357212, -12219270812, -2366733212, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, 773612, 687212, 600812, 514412, 428012, 341612, 255212, 168812, 82412, -3988, -3600, 0 ]
+}, {
+  "tz" : "America/Atka",
+  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12220114402, -12220028002, -12219941602, -12219855202, -12219768802, -12219682402, -12219596002, -12219509602, -12219423202, -12219336802, -3225223727, -2208988800 ],
+  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 857602, 771202, 684802, 598402, 512002, 425602, 339202, 252802, 166402, 80002, -6398, 0 ]
+}, {
+  "tz" : "America/Bahia",
+  "switches" : [ -62135587556, -59006352356, -55850678756, -52695005156, -46383571556, -43227897956, -40072224356, -33760790756, -30605117156, -27449443556, -21138009956, -17982336356, -14826662756, -12220061156, -12219974756, -12219888356, -12219801956, -12219715556, -12219629156, -12219542756, -12219456356, -12219369956, -12219283556, -2208988800 ],
+  "diffs" : [ -171244, -84844, 1556, 87956, 174356, 260756, 347156, 433556, 519956, 606356, 692756, 779156, 865556, 779156, 692756, 606356, 519956, 433556, 347156, 260756, 174356, 87956, 1556, 0 ]
+}, {
+  "tz" : "America/Bahia_Banderas",
+  "switches" : [ -62135571540, -59006336340, -55850662740, -52694989140, -46383555540, -43227881940, -40072208340, -33760774740, -30605101140, -27449427540, -21137993940, -17982320340, -14826646740, -12220045140, -12219958740, -12219872340, -12219785940, -12219699540, -12219613140, -12219526740, -12219440340, -12219353940, -12219267540, -2208988800 ],
+  "diffs" : [ -176460, -90060, -3660, 82740, 169140, 255540, 341940, 428340, 514740, 601140, 687540, 773940, 860340, 773940, 687540, 601140, 514740, 428340, 341940, 255540, 169140, 82740, -3660, 0 ]
+}, {
+  "tz" : "America/Barbados",
+  "switches" : [ -62135582491, -59006347291, -55850673691, -52695000091, -46383566491, -43227892891, -40072219291, -33760785691, -30605112091, -27449438491, -21138004891, -17982331291, -14826657691, -12220056091, -12219969691, -12219883291, -12219796891, -12219710491, -12219624091, -12219537691, -12219451291, -12219364891, -12219278491, -2208988800 ],
+  "diffs" : [ -172709, -86309, 91, 86491, 172891, 259291, 345691, 432091, 518491, 604891, 691291, 777691, 864091, 777691, 691291, 604891, 518491, 432091, 345691, 259291, 172891, 86491, 91, 0 ]
+}, {
+  "tz" : "America/Belem",
+  "switches" : [ -62135585164, -59006349964, -55850676364, -52695002764, -46383569164, -43227895564, -40072221964, -33760788364, -30605114764, -27449441164, -21138007564, -17982333964, -14826660364, -12220058764, -12219972364, -12219885964, -12219799564, -12219713164, -12219626764, -12219540364, -12219453964, -12219367564, -12219281164, -2208988800 ],
+  "diffs" : [ -173636, -87236, -836, 85564, 171964, 258364, 344764, 431164, 517564, 603964, 690364, 776764, 863164, 776764, 690364, 603964, 517564, 431164, 344764, 258364, 171964, 85564, -836, 0 ]
+}, {
+  "tz" : "America/Belize",
+  "switches" : [ -62135575632, -59006340432, -55850666832, -52694993232, -46383559632, -43227886032, -40072212432, -33760778832, -30605105232, -27449431632, -21137998032, -17982324432, -14826650832, -12220049232, -12219962832, -12219876432, -12219790032, -12219703632, -12219617232, -12219530832, -12219444432, -12219358032, -12219271632, -2208988800 ],
+  "diffs" : [ -172368, -85968, 432, 86832, 173232, 259632, 346032, 432432, 518832, 605232, 691632, 778032, 864432, 778032, 691632, 605232, 518832, 432432, 346032, 259632, 173232, 86832, 432, 0 ]
+}, {
+  "tz" : "America/Blanc-Sablon",
+  "switches" : [ -62135583092, -59006347892, -55850674292, -52695000692, -46383567092, -43227893492, -40072219892, -33760786292, -30605112692, -27449439092, -21138005492, -17982331892, -14826658292, -12220056692, -12219970292, -12219883892, -12219797492, -12219711092, -12219624692, -12219538292, -12219451892, -12219365492, -12219279092, -2713896692 ],
+  "diffs" : [ -172108, -85708, 692, 87092, 173492, 259892, 346292, 432692, 519092, 605492, 691892, 778292, 864692, 778292, 691892, 605492, 519092, 432692, 346292, 259892, 173492, 87092, 692, 0 ]
+}, {
+  "tz" : "America/Boa_Vista",
+  "switches" : [ -62135582240, -59006347040, -55850673440, -52694999840, -46383566240, -43227892640, -40072219040, -33760785440, -30605111840, -27449438240, -21138004640, -17982331040, -14826657440, -12220055840, -12219969440, -12219883040, -12219796640, -12219710240, -12219623840, -12219537440, -12219451040, -12219364640, -12219278240, -2208988800 ],
+  "diffs" : [ -172960, -86560, -160, 86240, 172640, 259040, 345440, 431840, 518240, 604640, 691040, 777440, 863840, 777440, 691040, 604640, 518240, 431840, 345440, 259040, 172640, 86240, -160, 0 ]
+}, {
+  "tz" : "America/Bogota",
+  "switches" : [ -62135579024, -59006343824, -55850670224, -52694996624, -46383563024, -43227889424, -40072215824, -33760782224, -30605108624, -27449435024, -21138001424, -17982327824, -14826654224, -12220052624, -12219966224, -12219879824, -12219793424, -12219707024, -12219620624, -12219534224, -12219447824, -12219361424, -12219275024, -2208988800 ],
+  "diffs" : [ -172576, -86176, 224, 86624, 173024, 259424, 345824, 432224, 518624, 605024, 691424, 777824, 864224, 777824, 691424, 605024, 518624, 432224, 345824, 259424, 173024, 86624, 224, 0 ]
+}, {
+  "tz" : "America/Boise",
+  "switches" : [ -62135568911, -59006333711, -55850660111, -52694986511, -46383552911, -43227879311, -40072205711, -33760772111, -30605098511, -27449424911, -21137991311, -17982317711, -14826644111, -12220042511, -12219956111, -12219869711, -12219783311, -12219696911, -12219610511, -12219524111, -12219437711, -12219351311, -12219264911, -2717640000, -2208988800 ],
+  "diffs" : [ -175489, -89089, -2689, 83711, 170111, 256511, 342911, 429311, 515711, 602111, 688511, 774911, 861311, 774911, 688511, 602111, 515711, 429311, 342911, 256511, 170111, 83711, -2689, -3600, 0 ]
+}, {
+  "tz" : "America/Buenos_Aires",
+  "switches" : [ -62135582772, -59006347572, -55850673972, -52695000372, -46383566772, -43227893172, -40072219572, -33760785972, -30605112372, -27449438772, -21138005172, -17982331572, -14826657972, -12220056372, -12219969972, -12219883572, -12219797172, -12219710772, -12219624372, -12219537972, -12219451572, -12219365172, -12219278772, -2372097972, -2208988800 ],
+  "diffs" : [ -176028, -89628, -3228, 83172, 169572, 255972, 342372, 428772, 515172, 601572, 687972, 774372, 860772, 774372, 687972, 601572, 515172, 428772, 342372, 255972, 169572, 83172, -3228, -4608, 0 ]
+}, {
+  "tz" : "America/Cambridge_Bay",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 802800, 716400, 630000, 543600, 457200, 370800, 284400, 198000, 111600, 25200, 0 ]
+}, {
+  "tz" : "America/Campo_Grande",
+  "switches" : [ -62135583692, -59006348492, -55850674892, -52695001292, -46383567692, -43227894092, -40072220492, -33760786892, -30605113292, -27449439692, -21138006092, -17982332492, -14826658892, -12220057292, -12219970892, -12219884492, -12219798092, -12219711692, -12219625292, -12219538892, -12219452492, -12219366092, -12219279692, -2208988800 ],
+  "diffs" : [ -171508, -85108, 1292, 87692, 174092, 260492, 346892, 433292, 519692, 606092, 692492, 778892, 865292, 778892, 692492, 606092, 519692, 433292, 346892, 260492, 174092, 87692, 1292, 0 ]
+}, {
+  "tz" : "America/Cancun",
+  "switches" : [ -62135575976, -59006340776, -55850667176, -52694993576, -46383559976, -43227886376, -40072212776, -33760779176, -30605105576, -27449431976, -21137998376, -17982324776, -14826651176, -12220049576, -12219963176, -12219876776, -12219790376, -12219703976, -12219617576, -12219531176, -12219444776, -12219358376, -12219271976, -2208988800 ],
+  "diffs" : [ -175624, -89224, -2824, 83576, 169976, 256376, 342776, 429176, 515576, 601976, 688376, 774776, 861176, 774776, 688376, 601976, 515576, 429176, 342776, 256376, 169976, 83576, -2824, 0 ]
+}, {
+  "tz" : "America/Caracas",
+  "switches" : [ -62135580736, -59006345536, -55850671936, -52694998336, -46383564736, -43227891136, -40072217536, -33760783936, -30605110336, -27449436736, -21138003136, -17982329536, -14826655936, -12220054336, -12219967936, -12219881536, -12219795136, -12219708736, -12219622336, -12219535936, -12219449536, -12219363136, -12219276736, -2524505536, -2208988800 ],
+  "diffs" : [ -174464, -88064, -1664, 84736, 171136, 257536, 343936, 430336, 516736, 603136, 689536, 775936, 862336, 775936, 689536, 603136, 516736, 430336, 343936, 257536, 171136, 84736, -1664, -1660, 0 ]
+}, {
+  "tz" : "America/Catamarca",
+  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12220054612, -12219968212, -12219881812, -12219795412, -12219709012, -12219622612, -12219536212, -12219449812, -12219363412, -12219277012, -2372096212, -2208988800 ],
+  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, 772612, 686212, 599812, 513412, 427012, 340612, 254212, 167812, 81412, -4988, -4608, 0 ]
+}, {
+  "tz" : "America/Cayenne",
+  "switches" : [ -62135584240, -59006349040, -55850675440, -52695001840, -46383568240, -43227894640, -40072221040, -33760787440, -30605113840, -27449440240, -21138006640, -17982333040, -14826659440, -12220057840, -12219971440, -12219885040, -12219798640, -12219712240, -12219625840, -12219539440, -12219453040, -12219366640, -12219280240, -2208988800 ],
+  "diffs" : [ -174560, -88160, -1760, 84640, 171040, 257440, 343840, 430240, 516640, 603040, 689440, 775840, 862240, 775840, 689440, 603040, 516640, 430240, 343840, 257440, 171040, 84640, -1760, 0 ]
+}, {
+  "tz" : "America/Cayman",
+  "switches" : [ -62135577712, -59006342512, -55850668912, -52694995312, -46383561712, -43227888112, -40072214512, -33760780912, -30605107312, -27449433712, -21138000112, -17982326512, -14826652912, -12220051312, -12219964912, -12219878512, -12219792112, -12219705712, -12219619312, -12219532912, -12219446512, -12219360112, -12219273712, -2524502512, -2208988800 ],
+  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, -1176, 0 ]
+}, {
+  "tz" : "America/Chicago",
+  "switches" : [ -62135575764, -59006340564, -55850666964, -52694993364, -46383559764, -43227886164, -40072212564, -33760778964, -30605105364, -27449431764, -21137998164, -17982324564, -14826650964, -12220049364, -12219962964, -12219876564, -12219790164, -12219703764, -12219617364, -12219530964, -12219444564, -12219358164, -12219271764, -2717647200 ],
+  "diffs" : [ -172236, -85836, 564, 86964, 173364, 259764, 346164, 432564, 518964, 605364, 691764, 778164, 864564, 778164, 691764, 605364, 518964, 432564, 346164, 259764, 173364, 86964, 564, 0 ]
+}, {
+  "tz" : "America/Chihuahua",
+  "switches" : [ -62135571340, -59006336140, -55850662540, -52694988940, -46383555340, -43227881740, -40072208140, -33760774540, -30605100940, -27449427340, -21137993740, -17982320140, -14826646540, -12220044940, -12219958540, -12219872140, -12219785740, -12219699340, -12219612940, -12219526540, -12219440140, -12219353740, -12219267340, -2208988800 ],
+  "diffs" : [ -173060, -86660, -260, 86140, 172540, 258940, 345340, 431740, 518140, 604540, 690940, 777340, 863740, 777340, 690940, 604540, 518140, 431740, 345340, 258940, 172540, 86140, -260, 0 ]
+}, {
+  "tz" : "America/Coral_Harbour",
+  "switches" : [ -62135574812, -59006339612, -55850666012, -52694992412, -46383558812, -43227885212, -40072211612, -33760778012, -30605104412, -27449430812, -21137997212, -17982323612, -14826650012, -12220048412, -12219962012, -12219875612, -12219789212, -12219702812, -12219616412, -12219530012, -12219443612, -12219357212, -12219270812, -2366733212, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, 773612, 687212, 600812, 514412, 428012, 341612, 255212, 168812, 82412, -3988, -3600, 0 ]
+}, {
+  "tz" : "America/Cordoba",
+  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12220054992, -12219968592, -12219882192, -12219795792, -12219709392, -12219622992, -12219536592, -12219450192, -12219363792, -12219277392, -2208988800 ],
+  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, 772992, 686592, 600192, 513792, 427392, 340992, 254592, 168192, 81792, -4608, 0 ]
+}, {
+  "tz" : "America/Costa_Rica",
+  "switches" : [ -62135576627, -59006341427, -55850667827, -52694994227, -46383560627, -43227887027, -40072213427, -33760779827, -30605106227, -27449432627, -21137999027, -17982325427, -14826651827, -12220050227, -12219963827, -12219877427, -12219791027, -12219704627, -12219618227, -12219531827, -12219445427, -12219359027, -12219272627, -2208988800 ],
+  "diffs" : [ -171373, -84973, 1427, 87827, 174227, 260627, 347027, 433427, 519827, 606227, 692627, 779027, 865427, 779027, 692627, 606227, 519827, 433427, 347027, 260627, 174227, 87827, 1427, 0 ]
+}, {
+  "tz" : "America/Creston",
+  "switches" : [ -62135568836, -59006333636, -55850660036, -52694986436, -46383552836, -43227879236, -40072205636, -33760772036, -30605098436, -27449424836, -21137991236, -17982317636, -14826644036, -12220042436, -12219956036, -12219869636, -12219783236, -12219696836, -12219610436, -12219524036, -12219437636, -12219351236, -12219264836, -2713882436 ],
+  "diffs" : [ -175564, -89164, -2764, 83636, 170036, 256436, 342836, 429236, 515636, 602036, 688436, 774836, 861236, 774836, 688436, 602036, 515636, 429236, 342836, 256436, 170036, 83636, -2764, 0 ]
+}, {
+  "tz" : "America/Cuiaba",
+  "switches" : [ -62135583340, -59006348140, -55850674540, -52695000940, -46383567340, -43227893740, -40072220140, -33760786540, -30605112940, -27449439340, -21138005740, -17982332140, -14826658540, -12220056940, -12219970540, -12219884140, -12219797740, -12219711340, -12219624940, -12219538540, -12219452140, -12219365740, -12219279340, -2208988800 ],
+  "diffs" : [ -171860, -85460, 940, 87340, 173740, 260140, 346540, 432940, 519340, 605740, 692140, 778540, 864940, 778540, 692140, 605740, 519340, 432940, 346540, 260140, 173740, 87340, 940, 0 ]
+}, {
+  "tz" : "America/Curacao",
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ]
+}, {
+  "tz" : "America/Danmarkshavn",
+  "switches" : [ -62135592320, -59006357120, -55850683520, -52695009920, -46383576320, -43227902720, -40072229120, -33760795520, -30605121920, -27449448320, -21138014720, -17982341120, -14826667520, -12220065920, -12219979520, -12219893120, -12219806720, -12219720320, -12219633920, -12219547520, -12219461120, -12219374720, -12219288320, -2208988800 ],
+  "diffs" : [ -177280, -90880, -4480, 81920, 168320, 254720, 341120, 427520, 513920, 600320, 686720, 773120, 859520, 773120, 686720, 600320, 513920, 427520, 341120, 254720, 168320, 81920, -4480, 0 ]
+}, {
+  "tz" : "America/Dawson",
+  "switches" : [ -62135563340, -59006328140, -55850654540, -52694980940, -46383547340, -43227873740, -40072200140, -33760766540, -30605092940, -27449419340, -21137985740, -17982312140, -14826638540, -12220036940, -12219950540, -12219864140, -12219777740, -12219691340, -12219604940, -12219518540, -12219432140, -12219345740, -12219259340, -2208988800 ],
+  "diffs" : [ -177460, -91060, -4660, 81740, 168140, 254540, 340940, 427340, 513740, 600140, 686540, 772940, 859340, 772940, 686540, 600140, 513740, 427340, 340940, 254540, 168140, 81740, -4660, 0 ]
+}, {
+  "tz" : "America/Dawson_Creek",
+  "switches" : [ -62135567944, -59006332744, -55850659144, -52694985544, -46383551944, -43227878344, -40072204744, -33760771144, -30605097544, -27449423944, -21137990344, -17982316744, -14826643144, -12220041544, -12219955144, -12219868744, -12219782344, -12219695944, -12219609544, -12219523144, -12219436744, -12219350344, -12219263944, -2713881544, -2208988800 ],
+  "diffs" : [ -176456, -90056, -3656, 82744, 169144, 255544, 341944, 428344, 514744, 601144, 687544, 773944, 860344, 773944, 687544, 601144, 514744, 428344, 341944, 255544, 169144, 82744, -3656, -3600, 0 ]
+}, {
+  "tz" : "America/Denver",
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12220045204, -12219958804, -12219872404, -12219786004, -12219699604, -12219613204, -12219526804, -12219440404, -12219354004, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 777604, 691204, 604804, 518404, 432004, 345604, 259204, 172804, 86404, 4, 0 ]
+}, {
+  "tz" : "America/Detroit",
+  "switches" : [ -62135576869, -59006341669, -55850668069, -52694994469, -46383560869, -43227887269, -40072213669, -33760780069, -30605106469, -27449432869, -21137999269, -17982325669, -14826652069, -12220050469, -12219964069, -12219877669, -12219791269, -12219704869, -12219618469, -12219532069, -12219445669, -12219359269, -12219272869, -2208988800 ],
+  "diffs" : [ -174731, -88331, -1931, 84469, 170869, 257269, 343669, 430069, 516469, 602869, 689269, 775669, 862069, 775669, 689269, 602869, 516469, 430069, 343669, 257269, 170869, 84469, -1931, 0 ]
+}, {
+  "tz" : "America/Dominica",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Edmonton",
+  "switches" : [ -62135569568, -59006334368, -55850660768, -52694987168, -46383553568, -43227879968, -40072206368, -33760772768, -30605099168, -27449425568, -21137991968, -17982318368, -14826644768, -12220043168, -12219956768, -12219870368, -12219783968, -12219697568, -12219611168, -12219524768, -12219438368, -12219351968, -12219265568, -2208988800 ],
+  "diffs" : [ -174832, -88432, -2032, 84368, 170768, 257168, 343568, 429968, 516368, 602768, 689168, 775568, 861968, 775568, 689168, 602768, 516368, 429968, 343568, 257168, 170768, 84368, -2032, 0 ]
+}, {
+  "tz" : "America/Eirunepe",
+  "switches" : [ -62135580032, -59006344832, -55850671232, -52694997632, -46383564032, -43227890432, -40072216832, -33760783232, -30605109632, -27449436032, -21138002432, -17982328832, -14826655232, -12220053632, -12219967232, -12219880832, -12219794432, -12219708032, -12219621632, -12219535232, -12219448832, -12219362432, -12219276032, -2208988800 ],
+  "diffs" : [ -171568, -85168, 1232, 87632, 174032, 260432, 346832, 433232, 519632, 606032, 692432, 778832, 865232, 778832, 692432, 606032, 519632, 433232, 346832, 260432, 174032, 87632, 1232, 0 ]
+}, {
+  "tz" : "America/El_Salvador",
+  "switches" : [ -62135575392, -59006340192, -55850666592, -52694992992, -46383559392, -43227885792, -40072212192, -33760778592, -30605104992, -27449431392, -21137997792, -17982324192, -14826650592, -12220048992, -12219962592, -12219876192, -12219789792, -12219703392, -12219616992, -12219530592, -12219444192, -12219357792, -12219271392, -2208988800 ],
+  "diffs" : [ -172608, -86208, 192, 86592, 172992, 259392, 345792, 432192, 518592, 604992, 691392, 777792, 864192, 777792, 691392, 604992, 518592, 432192, 345792, 259392, 172992, 86592, 192, 0 ]
+}, {
+  "tz" : "America/Ensenada",
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12220042316, -12219955916, -12219869516, -12219783116, -12219696716, -12219610316, -12219523916, -12219437516, -12219351116, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 778316, 691916, 605516, 519116, 432716, 346316, 259916, 173516, 87116, 716, 0 ]
+}, {
+  "tz" : "America/Fort_Nelson",
+  "switches" : [ -62135567353, -59006332153, -55850658553, -52694984953, -46383551353, -43227877753, -40072204153, -33760770553, -30605096953, -27449423353, -21137989753, -17982316153, -14826642553, -12220040953, -12219954553, -12219868153, -12219781753, -12219695353, -12219608953, -12219522553, -12219436153, -12219349753, -12219263353, -2713880953, -2208988800 ],
+  "diffs" : [ -177047, -90647, -4247, 82153, 168553, 254953, 341353, 427753, 514153, 600553, 686953, 773353, 859753, 773353, 686953, 600553, 514153, 427753, 341353, 254953, 168553, 82153, -4247, -3600, 0 ]
+}, {
+  "tz" : "America/Fort_Wayne",
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12220049722, -12219963322, -12219876922, -12219790522, -12219704122, -12219617722, -12219531322, -12219444922, -12219358522, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, 774922, 688522, 602122, 515722, 429322, 342922, 256522, 170122, 83722, -2678, -3600, 0 ]
+}, {
+  "tz" : "America/Fortaleza",
+  "switches" : [ -62135587560, -59006352360, -55850678760, -52695005160, -46383571560, -43227897960, -40072224360, -33760790760, -30605117160, -27449443560, -21138009960, -17982336360, -14826662760, -12220061160, -12219974760, -12219888360, -12219801960, -12219715560, -12219629160, -12219542760, -12219456360, -12219369960, -12219283560, -2208988800 ],
+  "diffs" : [ -171240, -84840, 1560, 87960, 174360, 260760, 347160, 433560, 519960, 606360, 692760, 779160, 865560, 779160, 692760, 606360, 519960, 433560, 347160, 260760, 174360, 87960, 1560, 0 ]
+}, {
+  "tz" : "America/Glace_Bay",
+  "switches" : [ -62135582412, -59006347212, -55850673612, -52695000012, -46383566412, -43227892812, -40072219212, -33760785612, -30605112012, -27449438412, -21138004812, -17982331212, -14826657612, -12220056012, -12219969612, -12219883212, -12219796812, -12219710412, -12219624012, -12219537612, -12219451212, -12219364812, -12219278412, -2208988800 ],
+  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 777612, 691212, 604812, 518412, 432012, 345612, 259212, 172812, 86412, 12, 0 ]
+}, {
+  "tz" : "America/Godthab",
+  "switches" : [ -62135584384, -59006349184, -55850675584, -52695001984, -46383568384, -43227894784, -40072221184, -33760787584, -30605113984, -27449440384, -21138006784, -17982333184, -14826659584, -12220057984, -12219971584, -12219885184, -12219798784, -12219712384, -12219625984, -12219539584, -12219453184, -12219366784, -12219280384, -2208988800 ],
+  "diffs" : [ -174416, -88016, -1616, 84784, 171184, 257584, 343984, 430384, 516784, 603184, 689584, 775984, 862384, 775984, 689584, 603184, 516784, 430384, 343984, 257584, 171184, 84784, -1616, 0 ]
+}, {
+  "tz" : "America/Goose_Bay",
+  "switches" : [ -62135582300, -59006347100, -55850673500, -52694999900, -46383566300, -43227892700, -40072219100, -33760785500, -30605111900, -27449438300, -21138004700, -17982331100, -14826657500, -12220055900, -12219969500, -12219883100, -12219796700, -12219710300, -12219623900, -12219537500, -12219451100, -12219364700, -12219278300, -2713895900, -2208988800 ],
+  "diffs" : [ -172900, -86500, -100, 86300, 172700, 259100, 345500, 431900, 518300, 604700, 691100, 777500, 863900, 777500, 691100, 604700, 518300, 431900, 345500, 259100, 172700, 86300, -100, 1748, 0 ]
+}, {
+  "tz" : "America/Grand_Turk",
+  "switches" : [ -62135579728, -59006344528, -55850670928, -52694997328, -46383563728, -43227890128, -40072216528, -33760782928, -30605109328, -27449435728, -21138002128, -17982328528, -14826654928, -12220053328, -12219966928, -12219880528, -12219794128, -12219707728, -12219621328, -12219534928, -12219448528, -12219362128, -12219275728, -2524504528, -2208988800 ],
+  "diffs" : [ -171872, -85472, 928, 87328, 173728, 260128, 346528, 432928, 519328, 605728, 692128, 778528, 864928, 778528, 692128, 605728, 519328, 432928, 346528, 260128, 173728, 87328, 928, -430, 0 ]
+}, {
+  "tz" : "America/Grenada",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Guadeloupe",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Guatemala",
+  "switches" : [ -62135575076, -59006339876, -55850666276, -52694992676, -46383559076, -43227885476, -40072211876, -33760778276, -30605104676, -27449431076, -21137997476, -17982323876, -14826650276, -12220048676, -12219962276, -12219875876, -12219789476, -12219703076, -12219616676, -12219530276, -12219443876, -12219357476, -12219271076, -2208988800 ],
+  "diffs" : [ -172924, -86524, -124, 86276, 172676, 259076, 345476, 431876, 518276, 604676, 691076, 777476, 863876, 777476, 691076, 604676, 518276, 431876, 345476, 259076, 172676, 86276, -124, 0 ]
+}, {
+  "tz" : "America/Guayaquil",
+  "switches" : [ -62135577640, -59006342440, -55850668840, -52694995240, -46383561640, -43227888040, -40072214440, -33760780840, -30605107240, -27449433640, -21138000040, -17982326440, -14826652840, -12220051240, -12219964840, -12219878440, -12219792040, -12219705640, -12219619240, -12219532840, -12219446440, -12219360040, -12219273640, -2524502440, -2208988800 ],
+  "diffs" : [ -173960, -87560, -1160, 85240, 171640, 258040, 344440, 430840, 517240, 603640, 690040, 776440, 862840, 776440, 690040, 603640, 517240, 430840, 344440, 258040, 171640, 85240, -1160, -840, 0 ]
+}, {
+  "tz" : "America/Guyana",
+  "switches" : [ -62135582840, -59006347640, -55850674040, -52695000440, -46383566840, -43227893240, -40072219640, -33760786040, -30605112440, -27449438840, -21138005240, -17982331640, -14826658040, -12220056440, -12219970040, -12219883640, -12219797240, -12219710840, -12219624440, -12219538040, -12219451640, -12219365240, -12219278840, -2208988800 ],
+  "diffs" : [ -172360, -85960, 440, 86840, 173240, 259640, 346040, 432440, 518840, 605240, 691640, 778040, 864440, 778040, 691640, 605240, 518840, 432440, 346040, 259640, 173240, 86840, 440, 0 ]
+}, {
+  "tz" : "America/Halifax",
+  "switches" : [ -62135581536, -59006346336, -55850672736, -52694999136, -46383565536, -43227891936, -40072218336, -33760784736, -30605111136, -27449437536, -21138003936, -17982330336, -14826656736, -12220055136, -12219968736, -12219882336, -12219795936, -12219709536, -12219623136, -12219536736, -12219450336, -12219363936, -12219277536, -2208988800 ],
+  "diffs" : [ -173664, -87264, -864, 85536, 171936, 258336, 344736, 431136, 517536, 603936, 690336, 776736, 863136, 776736, 690336, 603936, 517536, 431136, 344736, 258336, 171936, 85536, -864, 0 ]
+}, {
+  "tz" : "America/Havana",
+  "switches" : [ -62135577032, -59006341832, -55850668232, -52694994632, -46383561032, -43227887432, -40072213832, -33760780232, -30605106632, -27449433032, -21137999432, -17982325832, -14826652232, -12220050632, -12219964232, -12219877832, -12219791432, -12219705032, -12219618632, -12219532232, -12219445832, -12219359432, -12219273032, -2524501832, -2208988800 ],
+  "diffs" : [ -174568, -88168, -1768, 84632, 171032, 257432, 343832, 430232, 516632, 603032, 689432, 775832, 862232, 775832, 689432, 603032, 516632, 430232, 343832, 257432, 171032, 84632, -1768, -1776, 0 ]
+}, {
+  "tz" : "America/Hermosillo",
+  "switches" : [ -62135570168, -59006334968, -55850661368, -52694987768, -46383554168, -43227880568, -40072206968, -33760773368, -30605099768, -27449426168, -21137992568, -17982318968, -14826645368, -12220043768, -12219957368, -12219870968, -12219784568, -12219698168, -12219611768, -12219525368, -12219438968, -12219352568, -12219266168, -2208988800 ],
+  "diffs" : [ -174232, -87832, -1432, 84968, 171368, 257768, 344168, 430568, 516968, 603368, 689768, 776168, 862568, 776168, 689768, 603368, 516968, 430568, 344168, 257768, 171368, 84968, -1432, 0 ]
+}, {
+  "tz" : "America/Indiana/Indianapolis",
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12220049722, -12219963322, -12219876922, -12219790522, -12219704122, -12219617722, -12219531322, -12219444922, -12219358522, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, 774922, 688522, 602122, 515722, 429322, 342922, 256522, 170122, 83722, -2678, -3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Knox",
+  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12220049610, -12219963210, -12219876810, -12219790410, -12219704010, -12219617610, -12219531210, -12219444810, -12219358410, -12219272010, -2717647200 ],
+  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 778410, 692010, 605610, 519210, 432810, 346410, 260010, 173610, 87210, 810, 0 ]
+}, {
+  "tz" : "America/Indiana/Marengo",
+  "switches" : [ -62135576077, -59006340877, -55850667277, -52694993677, -46383560077, -43227886477, -40072212877, -33760779277, -30605105677, -27449432077, -21137998477, -17982324877, -14826651277, -12220049677, -12219963277, -12219876877, -12219790477, -12219704077, -12219617677, -12219531277, -12219444877, -12219358477, -12219272077, -2717647200, -2208988800 ],
+  "diffs" : [ -175523, -89123, -2723, 83677, 170077, 256477, 342877, 429277, 515677, 602077, 688477, 774877, 861277, 774877, 688477, 602077, 515677, 429277, 342877, 256477, 170077, 83677, -2723, -3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Petersburg",
+  "switches" : [ -62135575853, -59006340653, -55850667053, -52694993453, -46383559853, -43227886253, -40072212653, -33760779053, -30605105453, -27449431853, -21137998253, -17982324653, -14826651053, -12220049453, -12219963053, -12219876653, -12219790253, -12219703853, -12219617453, -12219531053, -12219444653, -12219358253, -12219271853, -2717647200, -2208988800 ],
+  "diffs" : [ -175747, -89347, -2947, 83453, 169853, 256253, 342653, 429053, 515453, 601853, 688253, 774653, 861053, 774653, 688253, 601853, 515453, 429053, 342653, 256253, 169853, 83453, -2947, -3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Tell_City",
+  "switches" : [ -62135575977, -59006340777, -55850667177, -52694993577, -46383559977, -43227886377, -40072212777, -33760779177, -30605105577, -27449431977, -21137998377, -17982324777, -14826651177, -12220049577, -12219963177, -12219876777, -12219790377, -12219703977, -12219617577, -12219531177, -12219444777, -12219358377, -12219271977, -2717647200 ],
+  "diffs" : [ -172023, -85623, 777, 87177, 173577, 259977, 346377, 432777, 519177, 605577, 691977, 778377, 864777, 778377, 691977, 605577, 519177, 432777, 346377, 259977, 173577, 87177, 777, 0 ]
+}, {
+  "tz" : "America/Indiana/Vevay",
+  "switches" : [ -62135576384, -59006341184, -55850667584, -52694993984, -46383560384, -43227886784, -40072213184, -33760779584, -30605105984, -27449432384, -21137998784, -17982325184, -14826651584, -12220049984, -12219963584, -12219877184, -12219790784, -12219704384, -12219617984, -12219531584, -12219445184, -12219358784, -12219272384, -2717647200, -2208988800 ],
+  "diffs" : [ -175216, -88816, -2416, 83984, 170384, 256784, 343184, 429584, 515984, 602384, 688784, 775184, 861584, 775184, 688784, 602384, 515984, 429584, 343184, 256784, 170384, 83984, -2416, -3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Vincennes",
+  "switches" : [ -62135575793, -59006340593, -55850666993, -52694993393, -46383559793, -43227886193, -40072212593, -33760778993, -30605105393, -27449431793, -21137998193, -17982324593, -14826650993, -12220049393, -12219962993, -12219876593, -12219790193, -12219703793, -12219617393, -12219530993, -12219444593, -12219358193, -12219271793, -2717647200, -2208988800 ],
+  "diffs" : [ -175807, -89407, -3007, 83393, 169793, 256193, 342593, 428993, 515393, 601793, 688193, 774593, 860993, 774593, 688193, 601793, 515393, 428993, 342593, 256193, 169793, 83393, -3007, -3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Winamac",
+  "switches" : [ -62135576015, -59006340815, -55850667215, -52694993615, -46383560015, -43227886415, -40072212815, -33760779215, -30605105615, -27449432015, -21137998415, -17982324815, -14826651215, -12220049615, -12219963215, -12219876815, -12219790415, -12219704015, -12219617615, -12219531215, -12219444815, -12219358415, -12219272015, -2717647200, -2208988800 ],
+  "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, -3600, 0 ]
+}, {
+  "tz" : "America/Indianapolis",
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12220049722, -12219963322, -12219876922, -12219790522, -12219704122, -12219617722, -12219531322, -12219444922, -12219358522, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, 774922, 688522, 602122, 515722, 429322, 342922, 256522, 170122, 83722, -2678, -3600, 0 ]
+}, {
+  "tz" : "America/Inuvik",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 802800, 716400, 630000, 543600, 457200, 370800, 284400, 198000, 111600, 25200, 0 ]
+}, {
+  "tz" : "America/Iqaluit",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -154800, -68400, 18000, 104400, 190800, 277200, 363600, 450000, 536400, 622800, 709200, 795600, 882000, 795600, 709200, 622800, 536400, 450000, 363600, 277200, 190800, 104400, 18000, 0 ]
+}, {
+  "tz" : "America/Jamaica",
+  "switches" : [ -62135578370, -59006343170, -55850669570, -52694995970, -46383562370, -43227888770, -40072215170, -33760781570, -30605107970, -27449434370, -21138000770, -17982327170, -14826653570, -12220051970, -12219965570, -12219879170, -12219792770, -12219706370, -12219619970, -12219533570, -12219447170, -12219360770, -12219274370, -2208988800 ],
+  "diffs" : [ -173230, -86830, -430, 85970, 172370, 258770, 345170, 431570, 517970, 604370, 690770, 777170, 863570, 777170, 690770, 604370, 517970, 431570, 345170, 258770, 172370, 85970, -430, 0 ]
+}, {
+  "tz" : "America/Jujuy",
+  "switches" : [ -62135581128, -59006345928, -55850672328, -52694998728, -46383565128, -43227891528, -40072217928, -33760784328, -30605110728, -27449437128, -21138003528, -17982329928, -14826656328, -12220054728, -12219968328, -12219881928, -12219795528, -12219709128, -12219622728, -12219536328, -12219449928, -12219363528, -12219277128, -2372096328, -2208988800 ],
+  "diffs" : [ -177672, -91272, -4872, 81528, 167928, 254328, 340728, 427128, 513528, 599928, 686328, 772728, 859128, 772728, 686328, 599928, 513528, 427128, 340728, 254328, 167928, 81528, -4872, -4608, 0 ]
+}, {
+  "tz" : "America/Juneau",
+  "switches" : [ -62135650939, -59006415739, -55850742139, -52695068539, -46383634939, -43227961339, -40072287739, -33760854139, -30605180539, -27449506939, -21138073339, -17982399739, -14826726139, -12220124539, -12220038139, -12219951739, -12219865339, -12219778939, -12219692539, -12219606139, -12219519739, -12219433339, -12219346939, -3225223727, -2208988800 ],
+  "diffs" : [ -86261, 139, 86539, 172939, 259339, 345739, 432139, 518539, 604939, 691339, 777739, 864139, 950539, 864139, 777739, 691339, 604939, 518539, 432139, 345739, 259339, 172939, 86539, 139, 0 ]
+}, {
+  "tz" : "America/Kentucky/Louisville",
+  "switches" : [ -62135576218, -59006341018, -55850667418, -52694993818, -46383560218, -43227886618, -40072213018, -33760779418, -30605105818, -27449432218, -21137998618, -17982325018, -14826651418, -12220049818, -12219963418, -12219877018, -12219790618, -12219704218, -12219617818, -12219531418, -12219445018, -12219358618, -12219272218, -2717647200, -2208988800 ],
+  "diffs" : [ -175382, -88982, -2582, 83818, 170218, 256618, 343018, 429418, 515818, 602218, 688618, 775018, 861418, 775018, 688618, 602218, 515818, 429418, 343018, 256618, 170218, 83818, -2582, -3600, 0 ]
+}, {
+  "tz" : "America/Kentucky/Monticello",
+  "switches" : [ -62135576436, -59006341236, -55850667636, -52694994036, -46383560436, -43227886836, -40072213236, -33760779636, -30605106036, -27449432436, -21137998836, -17982325236, -14826651636, -12220050036, -12219963636, -12219877236, -12219790836, -12219704436, -12219618036, -12219531636, -12219445236, -12219358836, -12219272436, -2717647200, -2208988800 ],
+  "diffs" : [ -175164, -88764, -2364, 84036, 170436, 256836, 343236, 429636, 516036, 602436, 688836, 775236, 861636, 775236, 688836, 602436, 516036, 429636, 343236, 256836, 170436, 84036, -2364, -3600, 0 ]
+}, {
+  "tz" : "America/Knox_IN",
+  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12220049610, -12219963210, -12219876810, -12219790410, -12219704010, -12219617610, -12219531210, -12219444810, -12219358410, -12219272010, -2717647200 ],
+  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 778410, 692010, 605610, 519210, 432810, 346410, 260010, 173610, 87210, 810, 0 ]
+}, {
+  "tz" : "America/Kralendijk",
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ]
+}, {
+  "tz" : "America/La_Paz",
+  "switches" : [ -62135580444, -59006345244, -55850671644, -52694998044, -46383564444, -43227890844, -40072217244, -33760783644, -30605110044, -27449436444, -21138002844, -17982329244, -14826655644, -12220054044, -12219967644, -12219881244, -12219794844, -12219708444, -12219622044, -12219535644, -12219449244, -12219362844, -12219276444, -2208988800 ],
+  "diffs" : [ -174756, -88356, -1956, 84444, 170844, 257244, 343644, 430044, 516444, 602844, 689244, 775644, 862044, 775644, 689244, 602844, 516444, 430044, 343644, 257244, 170844, 84444, -1956, 0 ]
+}, {
+  "tz" : "America/Lima",
+  "switches" : [ -62135578308, -59006343108, -55850669508, -52694995908, -46383562308, -43227888708, -40072215108, -33760781508, -30605107908, -27449434308, -21138000708, -17982327108, -14826653508, -12220051908, -12219965508, -12219879108, -12219792708, -12219706308, -12219619908, -12219533508, -12219447108, -12219360708, -12219274308, -2524503108, -2208988800 ],
+  "diffs" : [ -173292, -86892, -492, 85908, 172308, 258708, 345108, 431508, 517908, 604308, 690708, 777108, 863508, 777108, 690708, 604308, 517908, 431508, 345108, 258708, 172308, 85908, -492, -516, 0 ]
+}, {
+  "tz" : "America/Los_Angeles",
+  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12220042022, -12219955622, -12219869222, -12219782822, -12219696422, -12219610022, -12219523622, -12219437222, -12219350822, -12219264422, -2717640000 ],
+  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 778022, 691622, 605222, 518822, 432422, 346022, 259622, 173222, 86822, 422, 0 ]
+}, {
+  "tz" : "America/Louisville",
+  "switches" : [ -62135576218, -59006341018, -55850667418, -52694993818, -46383560218, -43227886618, -40072213018, -33760779418, -30605105818, -27449432218, -21137998618, -17982325018, -14826651418, -12220049818, -12219963418, -12219877018, -12219790618, -12219704218, -12219617818, -12219531418, -12219445018, -12219358618, -12219272218, -2717647200, -2208988800 ],
+  "diffs" : [ -175382, -88982, -2582, 83818, 170218, 256618, 343018, 429418, 515818, 602218, 688618, 775018, 861418, 775018, 688618, 602218, 515818, 429418, 343018, 256618, 170218, 83818, -2582, -3600, 0 ]
+}, {
+  "tz" : "America/Lower_Princes",
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ]
+}, {
+  "tz" : "America/Maceio",
+  "switches" : [ -62135588228, -59006353028, -55850679428, -52695005828, -46383572228, -43227898628, -40072225028, -33760791428, -30605117828, -27449444228, -21138010628, -17982337028, -14826663428, -12220061828, -12219975428, -12219889028, -12219802628, -12219716228, -12219629828, -12219543428, -12219457028, -12219370628, -12219284228, -2208988800 ],
+  "diffs" : [ -170572, -84172, 2228, 88628, 175028, 261428, 347828, 434228, 520628, 607028, 693428, 779828, 866228, 779828, 693428, 607028, 520628, 434228, 347828, 261428, 175028, 88628, 2228, 0 ]
+}, {
+  "tz" : "America/Managua",
+  "switches" : [ -62135576092, -59006340892, -55850667292, -52694993692, -46383560092, -43227886492, -40072212892, -33760779292, -30605105692, -27449432092, -21137998492, -17982324892, -14826651292, -12220049692, -12219963292, -12219876892, -12219790492, -12219704092, -12219617692, -12219531292, -12219444892, -12219358492, -12219272092, -2524500892, -2208988800 ],
+  "diffs" : [ -171908, -85508, 892, 87292, 173692, 260092, 346492, 432892, 519292, 605692, 692092, 778492, 864892, 778492, 692092, 605692, 519292, 432892, 346492, 260092, 173692, 87292, 892, 888, 0 ]
+}, {
+  "tz" : "America/Manaus",
+  "switches" : [ -62135582396, -59006347196, -55850673596, -52694999996, -46383566396, -43227892796, -40072219196, -33760785596, -30605111996, -27449438396, -21138004796, -17982331196, -14826657596, -12220055996, -12219969596, -12219883196, -12219796796, -12219710396, -12219623996, -12219537596, -12219451196, -12219364796, -12219278396, -2208988800 ],
+  "diffs" : [ -172804, -86404, -4, 86396, 172796, 259196, 345596, 431996, 518396, 604796, 691196, 777596, 863996, 777596, 691196, 604796, 518396, 431996, 345596, 259196, 172796, 86396, -4, 0 ]
+}, {
+  "tz" : "America/Marigot",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Martinique",
+  "switches" : [ -62135582140, -59006346940, -55850673340, -52694999740, -46383566140, -43227892540, -40072218940, -33760785340, -30605111740, -27449438140, -21138004540, -17982330940, -14826657340, -12220055740, -12219969340, -12219882940, -12219796540, -12219710140, -12219623740, -12219537340, -12219450940, -12219364540, -12219278140, -2208988800 ],
+  "diffs" : [ -173060, -86660, -260, 86140, 172540, 258940, 345340, 431740, 518140, 604540, 690940, 777340, 863740, 777340, 690940, 604540, 518140, 431740, 345340, 258940, 172540, 86140, -260, 0 ]
+}, {
+  "tz" : "America/Matamoros",
+  "switches" : [ -62135572800, -59006337600, -55850664000, -52694990400, -46383556800, -43227883200, -40072209600, -33760776000, -30605102400, -27449428800, -21137995200, -17982321600, -14826648000, -12220046400, -12219960000, -12219873600, -12219787200, -12219700800, -12219614400, -12219528000, -12219441600, -12219355200, -12219268800, -2208988800 ],
+  "diffs" : [ -175200, -88800, -2400, 84000, 170400, 256800, 343200, 429600, 516000, 602400, 688800, 775200, 861600, 775200, 688800, 602400, 516000, 429600, 343200, 256800, 170400, 84000, -2400, 0 ]
+}, {
+  "tz" : "America/Mazatlan",
+  "switches" : [ -62135571260, -59006336060, -55850662460, -52694988860, -46383555260, -43227881660, -40072208060, -33760774460, -30605100860, -27449427260, -21137993660, -17982320060, -14826646460, -12220044860, -12219958460, -12219872060, -12219785660, -12219699260, -12219612860, -12219526460, -12219440060, -12219353660, -12219267260, -2208988800 ],
+  "diffs" : [ -173140, -86740, -340, 86060, 172460, 258860, 345260, 431660, 518060, 604460, 690860, 777260, 863660, 777260, 690860, 604460, 518060, 431660, 345260, 258860, 172460, 86060, -340, 0 ]
+}, {
+  "tz" : "America/Mendoza",
+  "switches" : [ -62135580284, -59006345084, -55850671484, -52694997884, -46383564284, -43227890684, -40072217084, -33760783484, -30605109884, -27449436284, -21138002684, -17982329084, -14826655484, -12220053884, -12219967484, -12219881084, -12219794684, -12219708284, -12219621884, -12219535484, -12219449084, -12219362684, -12219276284, -2372095484, -2208988800 ],
+  "diffs" : [ -178516, -92116, -5716, 80684, 167084, 253484, 339884, 426284, 512684, 599084, 685484, 771884, 858284, 771884, 685484, 599084, 512684, 426284, 339884, 253484, 167084, 80684, -5716, -4608, 0 ]
+}, {
+  "tz" : "America/Menominee",
+  "switches" : [ -62135575773, -59006340573, -55850666973, -52694993373, -46383559773, -43227886173, -40072212573, -33760778973, -30605105373, -27449431773, -21137998173, -17982324573, -14826650973, -12220049373, -12219962973, -12219876573, -12219790173, -12219703773, -12219617373, -12219530973, -12219444573, -12219358173, -12219271773, -2659759773 ],
+  "diffs" : [ -172227, -85827, 573, 86973, 173373, 259773, 346173, 432573, 518973, 605373, 691773, 778173, 864573, 778173, 691773, 605373, 518973, 432573, 346173, 259773, 173373, 86973, 573, 0 ]
+}, {
+  "tz" : "America/Merida",
+  "switches" : [ -62135575292, -59006340092, -55850666492, -52694992892, -46383559292, -43227885692, -40072212092, -33760778492, -30605104892, -27449431292, -21137997692, -17982324092, -14826650492, -12220048892, -12219962492, -12219876092, -12219789692, -12219703292, -12219616892, -12219530492, -12219444092, -12219357692, -12219271292, -2208988800 ],
+  "diffs" : [ -172708, -86308, 92, 86492, 172892, 259292, 345692, 432092, 518492, 604892, 691292, 777692, 864092, 777692, 691292, 604892, 518492, 432092, 345692, 259292, 172892, 86492, 92, 0 ]
+}, {
+  "tz" : "America/Metlakatla",
+  "switches" : [ -62135651622, -59006416422, -55850742822, -52695069222, -46383635622, -43227962022, -40072288422, -33760854822, -30605181222, -27449507622, -21138074022, -17982400422, -14826726822, -12220125222, -12220038822, -12219952422, -12219866022, -12219779622, -12219693222, -12219606822, -12219520422, -12219434022, -12219347622, -3225223727, -2208988800 ],
+  "diffs" : [ -85578, 822, 87222, 173622, 260022, 346422, 432822, 519222, 605622, 692022, 778422, 864822, 951222, 864822, 778422, 692022, 605622, 519222, 432822, 346422, 260022, 173622, 87222, 822, 0 ]
+}, {
+  "tz" : "America/Mexico_City",
+  "switches" : [ -62135573004, -59006337804, -55850664204, -52694990604, -46383557004, -43227883404, -40072209804, -33760776204, -30605102604, -27449429004, -21137995404, -17982321804, -14826648204, -12220046604, -12219960204, -12219873804, -12219787404, -12219701004, -12219614604, -12219528204, -12219441804, -12219355404, -12219269004, -2208988800 ],
+  "diffs" : [ -174996, -88596, -2196, 84204, 170604, 257004, 343404, 429804, 516204, 602604, 689004, 775404, 861804, 775404, 689004, 602604, 516204, 429804, 343404, 257004, 170604, 84204, -2196, 0 ]
+}, {
+  "tz" : "America/Miquelon",
+  "switches" : [ -62135583320, -59006348120, -55850674520, -52695000920, -46383567320, -43227893720, -40072220120, -33760786520, -30605112920, -27449439320, -21138005720, -17982332120, -14826658520, -12220056920, -12219970520, -12219884120, -12219797720, -12219711320, -12219624920, -12219538520, -12219452120, -12219365720, -12219279320, -2208988800 ],
+  "diffs" : [ -175480, -89080, -2680, 83720, 170120, 256520, 342920, 429320, 515720, 602120, 688520, 774920, 861320, 774920, 688520, 602120, 515720, 429320, 342920, 256520, 170120, 83720, -2680, 0 ]
+}, {
+  "tz" : "America/Moncton",
+  "switches" : [ -62135581252, -59006346052, -55850672452, -52694998852, -46383565252, -43227891652, -40072218052, -33760784452, -30605110852, -27449437252, -21138003652, -17982330052, -14826656452, -12220054852, -12219968452, -12219882052, -12219795652, -12219709252, -12219622852, -12219536452, -12219450052, -12219363652, -12219277252, -2715882052, -2208988800 ],
+  "diffs" : [ -173948, -87548, -1148, 85252, 171652, 258052, 344452, 430852, 517252, 603652, 690052, 776452, 862852, 776452, 690052, 603652, 517252, 430852, 344452, 258052, 171652, 85252, -1148, -3600, 0 ]
+}, {
+  "tz" : "America/Monterrey",
+  "switches" : [ -62135572724, -59006337524, -55850663924, -52694990324, -46383556724, -43227883124, -40072209524, -33760775924, -30605102324, -27449428724, -21137995124, -17982321524, -14826647924, -12220046324, -12219959924, -12219873524, -12219787124, -12219700724, -12219614324, -12219527924, -12219441524, -12219355124, -12219268724, -2208988800 ],
+  "diffs" : [ -175276, -88876, -2476, 83924, 170324, 256724, 343124, 429524, 515924, 602324, 688724, 775124, 861524, 775124, 688724, 602324, 515924, 429524, 343124, 256724, 170324, 83924, -2476, 0 ]
+}, {
+  "tz" : "America/Montevideo",
+  "switches" : [ -62135583309, -59006348109, -55850674509, -52695000909, -46383567309, -43227893709, -40072220109, -33760786509, -30605112909, -27449439309, -21138005709, -17982332109, -14826658509, -12220056909, -12219970509, -12219884109, -12219797709, -12219711309, -12219624909, -12219538509, -12219452109, -12219365709, -12219279309, -2208988800 ],
+  "diffs" : [ -175491, -89091, -2691, 83709, 170109, 256509, 342909, 429309, 515709, 602109, 688509, 774909, 861309, 774909, 688509, 602109, 515709, 429309, 342909, 256509, 170109, 83709, -2691, 0 ]
+}, {
+  "tz" : "America/Montreal",
+  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12220051348, -12219964948, -12219878548, -12219792148, -12219705748, -12219619348, -12219532948, -12219446548, -12219360148, -12219273748, -2366736148 ],
+  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, 776548, 690148, 603748, 517348, 430948, 344548, 258148, 171748, 85348, -1052, 0 ]
+}, {
+  "tz" : "America/Montserrat",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Nassau",
+  "switches" : [ -62135578230, -59006343030, -55850669430, -52694995830, -46383562230, -43227888630, -40072215030, -33760781430, -30605107830, -27449434230, -21138000630, -17982327030, -14826653430, -12220051830, -12219965430, -12219879030, -12219792630, -12219706230, -12219619830, -12219533430, -12219447030, -12219360630, -12219274230, -2208988800 ],
+  "diffs" : [ -173370, -86970, -570, 85830, 172230, 258630, 345030, 431430, 517830, 604230, 690630, 777030, 863430, 777030, 690630, 604230, 517830, 431430, 345030, 258630, 172230, 85830, -570, 0 ]
+}, {
+  "tz" : "America/New_York",
+  "switches" : [ -62135579038, -59006343838, -55850670238, -52694996638, -46383563038, -43227889438, -40072215838, -33760782238, -30605108638, -27449435038, -21138001438, -17982327838, -14826654238, -12220052638, -12219966238, -12219879838, -12219793438, -12219707038, -12219620638, -12219534238, -12219447838, -12219361438, -12219275038, -2717650800 ],
+  "diffs" : [ -172562, -86162, 238, 86638, 173038, 259438, 345838, 432238, 518638, 605038, 691438, 777838, 864238, 777838, 691438, 605038, 518638, 432238, 345838, 259438, 173038, 86638, 238, 0 ]
+}, {
+  "tz" : "America/Nipigon",
+  "switches" : [ -62135575616, -59006340416, -55850666816, -52694993216, -46383559616, -43227886016, -40072212416, -33760778816, -30605105216, -27449431616, -21137998016, -17982324416, -14826650816, -12220049216, -12219962816, -12219876416, -12219790016, -12219703616, -12219617216, -12219530816, -12219444416, -12219358016, -12219271616, -2366734016 ],
+  "diffs" : [ -175984, -89584, -3184, 83216, 169616, 256016, 342416, 428816, 515216, 601616, 688016, 774416, 860816, 774416, 688016, 601616, 515216, 428816, 342416, 256016, 169616, 83216, -3184, 0 ]
+}, {
+  "tz" : "America/Nome",
+  "switches" : [ -62135643502, -59006408302, -55850734702, -52695061102, -46383627502, -43227953902, -40072280302, -33760846702, -30605173102, -27449499502, -21138065902, -17982392302, -14826718702, -12220117102, -12220030702, -12219944302, -12219857902, -12219771502, -12219685102, -12219598702, -12219512302, -12219425902, -12219339502, -3225223727, -2208988800 ],
+  "diffs" : [ -93698, -7298, 79102, 165502, 251902, 338302, 424702, 511102, 597502, 683902, 770302, 856702, 943102, 856702, 770302, 683902, 597502, 511102, 424702, 338302, 251902, 165502, 79102, -7298, 0 ]
+}, {
+  "tz" : "America/Noronha",
+  "switches" : [ -62135589020, -59006353820, -55850680220, -52695006620, -46383573020, -43227899420, -40072225820, -33760792220, -30605118620, -27449445020, -21138011420, -17982337820, -14826664220, -12220062620, -12219976220, -12219889820, -12219803420, -12219717020, -12219630620, -12219544220, -12219457820, -12219371420, -12219285020, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ]
+}, {
+  "tz" : "America/North_Dakota/Beulah",
+  "switches" : [ -62135572373, -59006337173, -55850663573, -52694989973, -46383556373, -43227882773, -40072209173, -33760775573, -30605101973, -27449428373, -21137994773, -17982321173, -14826647573, -12220045973, -12219959573, -12219873173, -12219786773, -12219700373, -12219613973, -12219527573, -12219441173, -12219354773, -12219268373, -2717643600, -2208988800 ],
+  "diffs" : [ -175627, -89227, -2827, 83573, 169973, 256373, 342773, 429173, 515573, 601973, 688373, 774773, 861173, 774773, 688373, 601973, 515573, 429173, 342773, 256373, 169973, 83573, -2827, -3600, 0 ]
+}, {
+  "tz" : "America/North_Dakota/Center",
+  "switches" : [ -62135572488, -59006337288, -55850663688, -52694990088, -46383556488, -43227882888, -40072209288, -33760775688, -30605102088, -27449428488, -21137994888, -17982321288, -14826647688, -12220046088, -12219959688, -12219873288, -12219786888, -12219700488, -12219614088, -12219527688, -12219441288, -12219354888, -12219268488, -2717643600, -2208988800 ],
+  "diffs" : [ -175512, -89112, -2712, 83688, 170088, 256488, 342888, 429288, 515688, 602088, 688488, 774888, 861288, 774888, 688488, 602088, 515688, 429288, 342888, 256488, 170088, 83688, -2712, -3600, 0 ]
+}, {
+  "tz" : "America/North_Dakota/New_Salem",
+  "switches" : [ -62135572461, -59006337261, -55850663661, -52694990061, -46383556461, -43227882861, -40072209261, -33760775661, -30605102061, -27449428461, -21137994861, -17982321261, -14826647661, -12220046061, -12219959661, -12219873261, -12219786861, -12219700461, -12219614061, -12219527661, -12219441261, -12219354861, -12219268461, -2717643600, -2208988800 ],
+  "diffs" : [ -175539, -89139, -2739, 83661, 170061, 256461, 342861, 429261, 515661, 602061, 688461, 774861, 861261, 774861, 688461, 602061, 515661, 429261, 342861, 256461, 170061, 83661, -2739, -3600, 0 ]
+}, {
+  "tz" : "America/Ojinaga",
+  "switches" : [ -62135571740, -59006336540, -55850662940, -52694989340, -46383555740, -43227882140, -40072208540, -33760774940, -30605101340, -27449427740, -21137994140, -17982320540, -14826646940, -12220045340, -12219958940, -12219872540, -12219786140, -12219699740, -12219613340, -12219526940, -12219440540, -12219354140, -12219267740, -2208988800 ],
+  "diffs" : [ -172660, -86260, 140, 86540, 172940, 259340, 345740, 432140, 518540, 604940, 691340, 777740, 864140, 777740, 691340, 604940, 518540, 432140, 345740, 259340, 172940, 86540, 140, 0 ]
+}, {
+  "tz" : "America/Panama",
+  "switches" : [ -62135577712, -59006342512, -55850668912, -52694995312, -46383561712, -43227888112, -40072214512, -33760780912, -30605107312, -27449433712, -21138000112, -17982326512, -14826652912, -12220051312, -12219964912, -12219878512, -12219792112, -12219705712, -12219619312, -12219532912, -12219446512, -12219360112, -12219273712, -2524502512, -2208988800 ],
+  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, -1176, 0 ]
+}, {
+  "tz" : "America/Pangnirtung",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -154800, -68400, 18000, 104400, 190800, 277200, 363600, 450000, 536400, 622800, 709200, 795600, 882000, 795600, 709200, 622800, 536400, 450000, 363600, 277200, 190800, 104400, 18000, 0 ]
+}, {
+  "tz" : "America/Paramaribo",
+  "switches" : [ -62135583560, -59006348360, -55850674760, -52695001160, -46383567560, -43227893960, -40072220360, -33760786760, -30605113160, -27449439560, -21138005960, -17982332360, -14826658760, -12220057160, -12219970760, -12219884360, -12219797960, -12219711560, -12219625160, -12219538760, -12219452360, -12219365960, -12219279560, -2208988800 ],
+  "diffs" : [ -175240, -88840, -2440, 83960, 170360, 256760, 343160, 429560, 515960, 602360, 688760, 775160, 861560, 775160, 688760, 602360, 515960, 429560, 343160, 256760, 170360, 83960, -2440, 0 ]
+}, {
+  "tz" : "America/Phoenix",
+  "switches" : [ -62135569902, -59006334702, -55850661102, -52694987502, -46383553902, -43227880302, -40072206702, -33760773102, -30605099502, -27449425902, -21137992302, -17982318702, -14826645102, -12220043502, -12219957102, -12219870702, -12219784302, -12219697902, -12219611502, -12219525102, -12219438702, -12219352302, -12219265902, -2717643600 ],
+  "diffs" : [ -174498, -88098, -1698, 84702, 171102, 257502, 343902, 430302, 516702, 603102, 689502, 775902, 862302, 775902, 689502, 603102, 516702, 430302, 343902, 257502, 171102, 84702, -1698, 0 ]
+}, {
+  "tz" : "America/Port-au-Prince",
+  "switches" : [ -62135579440, -59006344240, -55850670640, -52694997040, -46383563440, -43227889840, -40072216240, -33760782640, -30605109040, -27449435440, -21138001840, -17982328240, -14826654640, -12220053040, -12219966640, -12219880240, -12219793840, -12219707440, -12219621040, -12219534640, -12219448240, -12219361840, -12219275440, -2524504240, -2208988800 ],
+  "diffs" : [ -172160, -85760, 640, 87040, 173440, 259840, 346240, 432640, 519040, 605440, 691840, 778240, 864640, 778240, 691840, 605440, 519040, 432640, 346240, 259840, 173440, 87040, 640, 660, 0 ]
+}, {
+  "tz" : "America/Port_of_Spain",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Porto_Acre",
+  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12220054128, -12219967728, -12219881328, -12219794928, -12219708528, -12219622128, -12219535728, -12219449328, -12219362928, -12219276528, -2208988800 ],
+  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 779328, 692928, 606528, 520128, 433728, 347328, 260928, 174528, 88128, 1728, 0 ]
+}, {
+  "tz" : "America/Porto_Velho",
+  "switches" : [ -62135581464, -59006346264, -55850672664, -52694999064, -46383565464, -43227891864, -40072218264, -33760784664, -30605111064, -27449437464, -21138003864, -17982330264, -14826656664, -12220055064, -12219968664, -12219882264, -12219795864, -12219709464, -12219623064, -12219536664, -12219450264, -12219363864, -12219277464, -2208988800 ],
+  "diffs" : [ -173736, -87336, -936, 85464, 171864, 258264, 344664, 431064, 517464, 603864, 690264, 776664, 863064, 776664, 690264, 603864, 517464, 431064, 344664, 258264, 171864, 85464, -936, 0 ]
+}, {
+  "tz" : "America/Puerto_Rico",
+  "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ],
+  "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ]
+}, {
+  "tz" : "America/Punta_Arenas",
+  "switches" : [ -62135579780, -59006344580, -55850670980, -52694997380, -46383563780, -43227890180, -40072216580, -33760782980, -30605109380, -27449435780, -21138002180, -17982328580, -14826654980, -12220053380, -12219966980, -12219880580, -12219794180, -12219707780, -12219621380, -12219534980, -12219448580, -12219362180, -12219275780, -2524504580, -2208988800 ],
+  "diffs" : [ -179020, -92620, -6220, 80180, 166580, 252980, 339380, 425780, 512180, 598580, 684980, 771380, 857780, 771380, 684980, 598580, 512180, 425780, 339380, 252980, 166580, 80180, -6220, -6166, 0 ]
+}, {
+  "tz" : "America/Rainy_River",
+  "switches" : [ -62135574104, -59006338904, -55850665304, -52694991704, -46383558104, -43227884504, -40072210904, -33760777304, -30605103704, -27449430104, -21137996504, -17982322904, -14826649304, -12220047704, -12219961304, -12219874904, -12219788504, -12219702104, -12219615704, -12219529304, -12219442904, -12219356504, -12219270104, -2366732504 ],
+  "diffs" : [ -173896, -87496, -1096, 85304, 171704, 258104, 344504, 430904, 517304, 603704, 690104, 776504, 862904, 776504, 690104, 603704, 517304, 430904, 344504, 258104, 171704, 85304, -1096, 0 ]
+}, {
+  "tz" : "America/Rankin_Inlet",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -151200, -64800, 21600, 108000, 194400, 280800, 367200, 453600, 540000, 626400, 712800, 799200, 885600, 799200, 712800, 626400, 540000, 453600, 367200, 280800, 194400, 108000, 21600, 0 ]
+}, {
+  "tz" : "America/Recife",
+  "switches" : [ -62135588424, -59006353224, -55850679624, -52695006024, -46383572424, -43227898824, -40072225224, -33760791624, -30605118024, -27449444424, -21138010824, -17982337224, -14826663624, -12220062024, -12219975624, -12219889224, -12219802824, -12219716424, -12219630024, -12219543624, -12219457224, -12219370824, -12219284424, -2208988800 ],
+  "diffs" : [ -170376, -83976, 2424, 88824, 175224, 261624, 348024, 434424, 520824, 607224, 693624, 780024, 866424, 780024, 693624, 607224, 520824, 434424, 348024, 261624, 175224, 88824, 2424, 0 ]
+}, {
+  "tz" : "America/Regina",
+  "switches" : [ -62135571684, -59006336484, -55850662884, -52694989284, -46383555684, -43227882084, -40072208484, -33760774884, -30605101284, -27449427684, -21137994084, -17982320484, -14826646884, -12220045284, -12219958884, -12219872484, -12219786084, -12219699684, -12219613284, -12219526884, -12219440484, -12219354084, -12219267684, -2208988800 ],
+  "diffs" : [ -176316, -89916, -3516, 82884, 169284, 255684, 342084, 428484, 514884, 601284, 687684, 774084, 860484, 774084, 687684, 601284, 514884, 428484, 342084, 255684, 169284, 82884, -3516, 0 ]
+}, {
+  "tz" : "America/Resolute",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -151200, -64800, 21600, 108000, 194400, 280800, 367200, 453600, 540000, 626400, 712800, 799200, 885600, 799200, 712800, 626400, 540000, 453600, 367200, 280800, 194400, 108000, 21600, 0 ]
+}, {
+  "tz" : "America/Rio_Branco",
+  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12220054128, -12219967728, -12219881328, -12219794928, -12219708528, -12219622128, -12219535728, -12219449328, -12219362928, -12219276528, -2208988800 ],
+  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 779328, 692928, 606528, 520128, 433728, 347328, 260928, 174528, 88128, 1728, 0 ]
+}, {
+  "tz" : "America/Rosario",
+  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12220054992, -12219968592, -12219882192, -12219795792, -12219709392, -12219622992, -12219536592, -12219450192, -12219363792, -12219277392, -2208988800 ],
+  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, 772992, 686592, 600192, 513792, 427392, 340992, 254592, 168192, 81792, -4608, 0 ]
+}, {
+  "tz" : "America/Santa_Isabel",
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12220042316, -12219955916, -12219869516, -12219783116, -12219696716, -12219610316, -12219523916, -12219437516, -12219351116, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 778316, 691916, 605516, 519116, 432716, 346316, 259916, 173516, 87116, 716, 0 ]
+}, {
+  "tz" : "America/Santarem",
+  "switches" : [ -62135583672, -59006348472, -55850674872, -52695001272, -46383567672, -43227894072, -40072220472, -33760786872, -30605113272, -27449439672, -21138006072, -17982332472, -14826658872, -12220057272, -12219970872, -12219884472, -12219798072, -12219711672, -12219625272, -12219538872, -12219452472, -12219366072, -12219279672, -2208988800 ],
+  "diffs" : [ -175128, -88728, -2328, 84072, 170472, 256872, 343272, 429672, 516072, 602472, 688872, 775272, 861672, 775272, 688872, 602472, 516072, 429672, 343272, 256872, 170472, 84072, -2328, 0 ]
+}, {
+  "tz" : "America/Santiago",
+  "switches" : [ -62135579834, -59006344634, -55850671034, -52694997434, -46383563834, -43227890234, -40072216634, -33760783034, -30605109434, -27449435834, -21138002234, -17982328634, -14826655034, -12220053434, -12219967034, -12219880634, -12219794234, -12219707834, -12219621434, -12219535034, -12219448634, -12219362234, -12219275834, -2208988800 ],
+  "diffs" : [ -175366, -88966, -2566, 83834, 170234, 256634, 343034, 429434, 515834, 602234, 688634, 775034, 861434, 775034, 688634, 602234, 515834, 429434, 343034, 256634, 170234, 83834, -2566, 0 ]
+}, {
+  "tz" : "America/Santo_Domingo",
+  "switches" : [ -62135580024, -59006344824, -55850671224, -52694997624, -46383564024, -43227890424, -40072216824, -33760783224, -30605109624, -27449436024, -21138002424, -17982328824, -14826655224, -12220053624, -12219967224, -12219880824, -12219794424, -12219708024, -12219621624, -12219535224, -12219448824, -12219362424, -12219276024, -2524504824, -2208988800 ],
+  "diffs" : [ -175176, -88776, -2376, 84024, 170424, 256824, 343224, 429624, 516024, 602424, 688824, 775224, 861624, 775224, 688824, 602424, 516024, 429624, 343224, 256824, 170424, 84024, -2376, -2400, 0 ]
+}, {
+  "tz" : "America/Sao_Paulo",
+  "switches" : [ -62135585612, -59006350412, -55850676812, -52695003212, -46383569612, -43227896012, -40072222412, -33760788812, -30605115212, -27449441612, -21138008012, -17982334412, -14826660812, -12220059212, -12219972812, -12219886412, -12219800012, -12219713612, -12219627212, -12219540812, -12219454412, -12219368012, -12219281612, -2208988800 ],
+  "diffs" : [ -173188, -86788, -388, 86012, 172412, 258812, 345212, 431612, 518012, 604412, 690812, 777212, 863612, 777212, 690812, 604412, 518012, 431612, 345212, 258812, 172412, 86012, -388, 0 ]
+}, {
+  "tz" : "America/Scoresbysund",
+  "switches" : [ -62135591528, -59006356328, -55850682728, -52695009128, -46383575528, -43227901928, -40072228328, -33760794728, -30605121128, -27449447528, -21138013928, -17982340328, -14826666728, -12220065128, -12219978728, -12219892328, -12219805928, -12219719528, -12219633128, -12219546728, -12219460328, -12219373928, -12219287528, -2208988800 ],
+  "diffs" : [ -174472, -88072, -1672, 84728, 171128, 257528, 343928, 430328, 516728, 603128, 689528, 775928, 862328, 775928, 689528, 603128, 516728, 430328, 343928, 257528, 171128, 84728, -1672, 0 ]
+}, {
+  "tz" : "America/Shiprock",
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12220045204, -12219958804, -12219872404, -12219786004, -12219699604, -12219613204, -12219526804, -12219440404, -12219354004, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 777604, 691204, 604804, 518404, 432004, 345604, 259204, 172804, 86404, 4, 0 ]
+}, {
+  "tz" : "America/Sitka",
+  "switches" : [ -62135650727, -59006415527, -55850741927, -52695068327, -46383634727, -43227961127, -40072287527, -33760853927, -30605180327, -27449506727, -21138073127, -17982399527, -14826725927, -12220124327, -12220037927, -12219951527, -12219865127, -12219778727, -12219692327, -12219605927, -12219519527, -12219433127, -12219346727, -3225223727, -2208988800 ],
+  "diffs" : [ -86473, -73, 86327, 172727, 259127, 345527, 431927, 518327, 604727, 691127, 777527, 863927, 950327, 863927, 777527, 691127, 604727, 518327, 431927, 345527, 259127, 172727, 86327, -73, 0 ]
+}, {
+  "tz" : "America/St_Barthelemy",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/St_Johns",
+  "switches" : [ -62135584148, -59006348948, -55850675348, -52695001748, -46383568148, -43227894548, -40072220948, -33760787348, -30605113748, -27449440148, -21138006548, -17982332948, -14826659348, -12220057748, -12219971348, -12219884948, -12219798548, -12219712148, -12219625748, -12219539348, -12219452948, -12219366548, -12219280148, -2208988800 ],
+  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, 777548, 691148, 604748, 518348, 431948, 345548, 259148, 172748, 86348, -52, 0 ]
+}, {
+  "tz" : "America/St_Kitts",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/St_Lucia",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/St_Thomas",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/St_Vincent",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Swift_Current",
+  "switches" : [ -62135570920, -59006335720, -55850662120, -52694988520, -46383554920, -43227881320, -40072207720, -33760774120, -30605100520, -27449426920, -21137993320, -17982319720, -14826646120, -12220044520, -12219958120, -12219871720, -12219785320, -12219698920, -12219612520, -12219526120, -12219439720, -12219353320, -12219266920, -2208988800 ],
+  "diffs" : [ -177080, -90680, -4280, 82120, 168520, 254920, 341320, 427720, 514120, 600520, 686920, 773320, 859720, 773320, 686920, 600520, 514120, 427720, 341320, 254920, 168520, 82120, -4280, 0 ]
+}, {
+  "tz" : "America/Tegucigalpa",
+  "switches" : [ -62135575868, -59006340668, -55850667068, -52694993468, -46383559868, -43227886268, -40072212668, -33760779068, -30605105468, -27449431868, -21137998268, -17982324668, -14826651068, -12220049468, -12219963068, -12219876668, -12219790268, -12219703868, -12219617468, -12219531068, -12219444668, -12219358268, -12219271868, -2208988800 ],
+  "diffs" : [ -172132, -85732, 668, 87068, 173468, 259868, 346268, 432668, 519068, 605468, 691868, 778268, 864668, 778268, 691868, 605468, 519068, 432668, 346268, 259868, 173468, 87068, 668, 0 ]
+}, {
+  "tz" : "America/Thule",
+  "switches" : [ -62135580292, -59006345092, -55850671492, -52694997892, -46383564292, -43227890692, -40072217092, -33760783492, -30605109892, -27449436292, -21138002692, -17982329092, -14826655492, -12220053892, -12219967492, -12219881092, -12219794692, -12219708292, -12219621892, -12219535492, -12219449092, -12219362692, -12219276292, -2208988800 ],
+  "diffs" : [ -174908, -88508, -2108, 84292, 170692, 257092, 343492, 429892, 516292, 602692, 689092, 775492, 861892, 775492, 689092, 602692, 516292, 429892, 343492, 257092, 170692, 84292, -2108, 0 ]
+}, {
+  "tz" : "America/Thunder_Bay",
+  "switches" : [ -62135575380, -59006340180, -55850666580, -52694992980, -46383559380, -43227885780, -40072212180, -33760778580, -30605104980, -27449431380, -21137997780, -17982324180, -14826650580, -12220048980, -12219962580, -12219876180, -12219789780, -12219703380, -12219616980, -12219530580, -12219444180, -12219357780, -12219271380, -2366733780, -2208988800 ],
+  "diffs" : [ -176220, -89820, -3420, 82980, 169380, 255780, 342180, 428580, 514980, 601380, 687780, 774180, 860580, 774180, 687780, 601380, 514980, 428580, 342180, 255780, 169380, 82980, -3420, -3600, 0 ]
+}, {
+  "tz" : "America/Tijuana",
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12220042316, -12219955916, -12219869516, -12219783116, -12219696716, -12219610316, -12219523916, -12219437516, -12219351116, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 778316, 691916, 605516, 519116, 432716, 346316, 259916, 173516, 87116, 716, 0 ]
+}, {
+  "tz" : "America/Toronto",
+  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12220051348, -12219964948, -12219878548, -12219792148, -12219705748, -12219619348, -12219532948, -12219446548, -12219360148, -12219273748, -2366736148 ],
+  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, 776548, 690148, 603748, 517348, 430948, 344548, 258148, 171748, 85348, -1052, 0 ]
+}, {
+  "tz" : "America/Tortola",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Vancouver",
+  "switches" : [ -62135567252, -59006332052, -55850658452, -52694984852, -46383551252, -43227877652, -40072204052, -33760770452, -30605096852, -27449423252, -21137989652, -17982316052, -14826642452, -12220040852, -12219954452, -12219868052, -12219781652, -12219695252, -12219608852, -12219522452, -12219436052, -12219349652, -12219263252, -2713880852 ],
+  "diffs" : [ -173548, -87148, -748, 85652, 172052, 258452, 344852, 431252, 517652, 604052, 690452, 776852, 863252, 776852, 690452, 604052, 517652, 431252, 344852, 258452, 172052, 85652, -748, 0 ]
+}, {
+  "tz" : "America/Virgin",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
+}, {
+  "tz" : "America/Whitehorse",
+  "switches" : [ -62135564388, -59006329188, -55850655588, -52694981988, -46383548388, -43227874788, -40072201188, -33760767588, -30605093988, -27449420388, -21137986788, -17982313188, -14826639588, -12220037988, -12219951588, -12219865188, -12219778788, -12219692388, -12219605988, -12219519588, -12219433188, -12219346788, -12219260388, -2208988800 ],
+  "diffs" : [ -176412, -90012, -3612, 82788, 169188, 255588, 341988, 428388, 514788, 601188, 687588, 773988, 860388, 773988, 687588, 601188, 514788, 428388, 341988, 255588, 169188, 82788, -3612, 0 ]
+}, {
+  "tz" : "America/Winnipeg",
+  "switches" : [ -62135573484, -59006338284, -55850664684, -52694991084, -46383557484, -43227883884, -40072210284, -33760776684, -30605103084, -27449429484, -21137995884, -17982322284, -14826648684, -12220047084, -12219960684, -12219874284, -12219787884, -12219701484, -12219615084, -12219528684, -12219442284, -12219355884, -12219269484, -2602258284 ],
+  "diffs" : [ -174516, -88116, -1716, 84684, 171084, 257484, 343884, 430284, 516684, 603084, 689484, 775884, 862284, 775884, 689484, 603084, 516684, 430284, 343884, 257484, 171084, 84684, -1716, 0 ]
+}, {
+  "tz" : "America/Yakutat",
+  "switches" : [ -62135649665, -59006414465, -55850740865, -52695067265, -46383633665, -43227960065, -40072286465, -33760852865, -30605179265, -27449505665, -21138072065, -17982398465, -14826724865, -12220123265, -12220036865, -12219950465, -12219864065, -12219777665, -12219691265, -12219604865, -12219518465, -12219432065, -12219345665, -3225223727, -2208988800 ],
+  "diffs" : [ -87535, -1135, 85265, 171665, 258065, 344465, 430865, 517265, 603665, 690065, 776465, 862865, 949265, 862865, 776465, 690065, 603665, 517265, 430865, 344465, 258065, 171665, 85265, -1135, 0 ]
+}, {
+  "tz" : "America/Yellowknife",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 802800, 716400, 630000, 543600, 457200, 370800, 284400, 198000, 111600, 25200, 0 ]
+}, {
+  "tz" : "Antarctica/Casey",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -201600, -115200, -28800, 57600, 144000, 230400, 316800, 403200, 489600, 576000, 662400, 748800, 835200, 748800, 662400, 576000, 489600, 403200, 316800, 230400, 144000, 57600, -28800, 0 ]
+}, {
+  "tz" : "Antarctica/Davis",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -198000, -111600, -25200, 61200, 147600, 234000, 320400, 406800, 493200, 579600, 666000, 752400, 838800, 752400, 666000, 579600, 493200, 406800, 320400, 234000, 147600, 61200, -25200, 0 ]
+}, {
+  "tz" : "Antarctica/DumontDUrville",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -208800, -122400, -36000, 50400, 136800, 223200, 309600, 396000, 482400, 568800, 655200, 741600, 828000, 741600, 655200, 568800, 482400, 396000, 309600, 223200, 136800, 50400, -36000, 0 ]
+}, {
+  "tz" : "Antarctica/Macquarie",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2214259200, -2208988800 ],
+  "diffs" : [ -212400, -126000, -39600, 46800, 133200, 219600, 306000, 392400, 478800, 565200, 651600, 738000, 824400, 738000, 651600, 565200, 478800, 392400, 306000, 219600, 133200, 46800, -39600, -3600, 0 ]
+}, {
+  "tz" : "Antarctica/Mawson",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -190800, -104400, -18000, 68400, 154800, 241200, 327600, 414000, 500400, 586800, 673200, 759600, 846000, 759600, 673200, 586800, 500400, 414000, 327600, 241200, 154800, 68400, -18000, 0 ]
+}, {
+  "tz" : "Antarctica/McMurdo",
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12220112344, -12220025944, -12219939544, -12219853144, -12219766744, -12219680344, -12219593944, -12219507544, -12219421144, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, 776344, 689944, 603544, 517144, 430744, 344344, 257944, 171544, 85144, -1256, -1800, 0 ]
+}, {
+  "tz" : "Antarctica/Palmer",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -162000, -75600, 10800, 97200, 183600, 270000, 356400, 442800, 529200, 615600, 702000, 788400, 874800, 788400, 702000, 615600, 529200, 442800, 356400, 270000, 183600, 97200, 10800, 0 ]
+}, {
+  "tz" : "Antarctica/Rothera",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -162000, -75600, 10800, 97200, 183600, 270000, 356400, 442800, 529200, 615600, 702000, 788400, 874800, 788400, 702000, 615600, 529200, 442800, 356400, 270000, 183600, 97200, 10800, 0 ]
+}, {
+  "tz" : "Antarctica/South_Pole",
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12220112344, -12220025944, -12219939544, -12219853144, -12219766744, -12219680344, -12219593944, -12219507544, -12219421144, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, 776344, 689944, 603544, 517144, 430744, 344344, 257944, 171544, 85144, -1256, -1800, 0 ]
+}, {
+  "tz" : "Antarctica/Syowa",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -183600, -97200, -10800, 75600, 162000, 248400, 334800, 421200, 507600, 594000, 680400, 766800, 853200, 766800, 680400, 594000, 507600, 421200, 334800, 248400, 162000, 75600, -10800, 0 ]
+}, {
+  "tz" : "Antarctica/Troll",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Antarctica/Vostok",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -194400, -108000, -21600, 64800, 151200, 237600, 324000, 410400, 496800, 583200, 669600, 756000, 842400, 756000, 669600, 583200, 496800, 410400, 324000, 237600, 151200, 64800, -21600, 0 ]
+}, {
+  "tz" : "Arctic/Longyearbyen",
+  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12220072980, -12219986580, -12219900180, -12219813780, -12219727380, -12219640980, -12219554580, -12219468180, -12219381780, -12219295380, -2366757780 ],
+  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, 776580, 690180, 603780, 517380, 430980, 344580, 258180, 171780, 85380, -1020, 0 ]
+}, {
+  "tz" : "Asia/Aden",
+  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12220081612, -12219995212, -12219908812, -12219822412, -12219736012, -12219649612, -12219563212, -12219476812, -12219390412, -12219304012, -2208988800 ],
+  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 778012, 691612, 605212, 518812, 432412, 346012, 259612, 173212, 86812, 412, 0 ]
+}, {
+  "tz" : "Asia/Almaty",
+  "switches" : [ -62135615268, -59006380068, -55850706468, -52695032868, -46383599268, -43227925668, -40072252068, -33760818468, -30605144868, -27449471268, -21138037668, -17982364068, -14826690468, -12220088868, -12220002468, -12219916068, -12219829668, -12219743268, -12219656868, -12219570468, -12219484068, -12219397668, -12219311268, -2208988800 ],
+  "diffs" : [ -175932, -89532, -3132, 83268, 169668, 256068, 342468, 428868, 515268, 601668, 688068, 774468, 860868, 774468, 688068, 601668, 515268, 428868, 342468, 256068, 169668, 83268, -3132, 0 ]
+}, {
+  "tz" : "Asia/Amman",
+  "switches" : [ -62135605424, -59006370224, -55850696624, -52695023024, -46383589424, -43227915824, -40072242224, -33760808624, -30605135024, -27449461424, -21138027824, -17982354224, -14826680624, -12220079024, -12219992624, -12219906224, -12219819824, -12219733424, -12219647024, -12219560624, -12219474224, -12219387824, -12219301424, -2208988800 ],
+  "diffs" : [ -171376, -84976, 1424, 87824, 174224, 260624, 347024, 433424, 519824, 606224, 692624, 779024, 865424, 779024, 692624, 606224, 519824, 433424, 347024, 260624, 174224, 87824, 1424, 0 ]
+}, {
+  "tz" : "Asia/Anadyr",
+  "switches" : [ -62135639396, -59006404196, -55850730596, -52695056996, -46383623396, -43227949796, -40072276196, -33760842596, -30605168996, -27449495396, -21138061796, -17982388196, -14826714596, -12220112996, -12220026596, -12219940196, -12219853796, -12219767396, -12219680996, -12219594596, -12219508196, -12219421796, -12219335396, -2208988800 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, 776996, 690596, 604196, 517796, 431396, 344996, 258596, 172196, 85796, -604, 0 ]
+}, {
+  "tz" : "Asia/Aqtau",
+  "switches" : [ -62135608864, -59006373664, -55850700064, -52695026464, -46383592864, -43227919264, -40072245664, -33760812064, -30605138464, -27449464864, -21138031264, -17982357664, -14826684064, -12220082464, -12219996064, -12219909664, -12219823264, -12219736864, -12219650464, -12219564064, -12219477664, -12219391264, -12219304864, -2208988800 ],
+  "diffs" : [ -178736, -92336, -5936, 80464, 166864, 253264, 339664, 426064, 512464, 598864, 685264, 771664, 858064, 771664, 685264, 598864, 512464, 426064, 339664, 253264, 166864, 80464, -5936, 0 ]
+}, {
+  "tz" : "Asia/Aqtobe",
+  "switches" : [ -62135610520, -59006375320, -55850701720, -52695028120, -46383594520, -43227920920, -40072247320, -33760813720, -30605140120, -27449466520, -21138032920, -17982359320, -14826685720, -12220084120, -12219997720, -12219911320, -12219824920, -12219738520, -12219652120, -12219565720, -12219479320, -12219392920, -12219306520, -2208988800 ],
+  "diffs" : [ -177080, -90680, -4280, 82120, 168520, 254920, 341320, 427720, 514120, 600520, 686920, 773320, 859720, 773320, 686920, 600520, 514120, 427720, 341320, 254920, 168520, 82120, -4280, 0 ]
+}, {
+  "tz" : "Asia/Ashgabat",
+  "switches" : [ -62135610812, -59006375612, -55850702012, -52695028412, -46383594812, -43227921212, -40072247612, -33760814012, -30605140412, -27449466812, -21138033212, -17982359612, -14826686012, -12220084412, -12219998012, -12219911612, -12219825212, -12219738812, -12219652412, -12219566012, -12219479612, -12219393212, -12219306812, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, 773612, 687212, 600812, 514412, 428012, 341612, 255212, 168812, 82412, -3988, 0 ]
+}, {
+  "tz" : "Asia/Ashkhabad",
+  "switches" : [ -62135610812, -59006375612, -55850702012, -52695028412, -46383594812, -43227921212, -40072247612, -33760814012, -30605140412, -27449466812, -21138033212, -17982359612, -14826686012, -12220084412, -12219998012, -12219911612, -12219825212, -12219738812, -12219652412, -12219566012, -12219479612, -12219393212, -12219306812, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, 773612, 687212, 600812, 514412, 428012, 341612, 255212, 168812, 82412, -3988, 0 ]
+}, {
+  "tz" : "Asia/Atyrau",
+  "switches" : [ -62135609264, -59006374064, -55850700464, -52695026864, -46383593264, -43227919664, -40072246064, -33760812464, -30605138864, -27449465264, -21138031664, -17982358064, -14826684464, -12220082864, -12219996464, -12219910064, -12219823664, -12219737264, -12219650864, -12219564464, -12219478064, -12219391664, -12219305264, -2208988800 ],
+  "diffs" : [ -178336, -91936, -5536, 80864, 167264, 253664, 340064, 426464, 512864, 599264, 685664, 772064, 858464, 772064, 685664, 599264, 512864, 426464, 340064, 253664, 167264, 80864, -5536, 0 ]
+}, {
+  "tz" : "Asia/Baghdad",
+  "switches" : [ -62135607460, -59006372260, -55850698660, -52695025060, -46383591460, -43227917860, -40072244260, -33760810660, -30605137060, -27449463460, -21138029860, -17982356260, -14826682660, -12220081060, -12219994660, -12219908260, -12219821860, -12219735460, -12219649060, -12219562660, -12219476260, -12219389860, -12219303460, -2524532260, -2208988800 ],
+  "diffs" : [ -172940, -86540, -140, 86260, 172660, 259060, 345460, 431860, 518260, 604660, 691060, 777460, 863860, 777460, 691060, 604660, 518260, 431860, 345460, 259060, 172660, 86260, -140, -144, 0 ]
+}, {
+  "tz" : "Asia/Bahrain",
+  "switches" : [ -62135609168, -59006373968, -55850700368, -52695026768, -46383593168, -43227919568, -40072245968, -33760812368, -30605138768, -27449465168, -21138031568, -17982357968, -14826684368, -12220082768, -12219996368, -12219909968, -12219823568, -12219737168, -12219650768, -12219564368, -12219477968, -12219391568, -12219305168, -2208988800 ],
+  "diffs" : [ -171232, -84832, 1568, 87968, 174368, 260768, 347168, 433568, 519968, 606368, 692768, 779168, 865568, 779168, 692768, 606368, 519968, 433568, 347168, 260768, 174368, 87968, 1568, 0 ]
+}, {
+  "tz" : "Asia/Baku",
+  "switches" : [ -62135608764, -59006373564, -55850699964, -52695026364, -46383592764, -43227919164, -40072245564, -33760811964, -30605138364, -27449464764, -21138031164, -17982357564, -14826683964, -12220082364, -12219995964, -12219909564, -12219823164, -12219736764, -12219650364, -12219563964, -12219477564, -12219391164, -12219304764, -2208988800 ],
+  "diffs" : [ -175236, -88836, -2436, 83964, 170364, 256764, 343164, 429564, 515964, 602364, 688764, 775164, 861564, 775164, 688764, 602364, 515964, 429564, 343164, 256764, 170364, 83964, -2436, 0 ]
+}, {
+  "tz" : "Asia/Bangkok",
+  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12220094524, -12220008124, -12219921724, -12219835324, -12219748924, -12219662524, -12219576124, -12219489724, -12219403324, -12219316924, -2208988800 ],
+  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, 776524, 690124, 603724, 517324, 430924, 344524, 258124, 171724, 85324, -1076, 0 ]
+}, {
+  "tz" : "Asia/Barnaul",
+  "switches" : [ -62135616900, -59006381700, -55850708100, -52695034500, -46383600900, -43227927300, -40072253700, -33760820100, -30605146500, -27449472900, -21138039300, -17982365700, -14826692100, -12220090500, -12220004100, -12219917700, -12219831300, -12219744900, -12219658500, -12219572100, -12219485700, -12219399300, -12219312900, -2208988800 ],
+  "diffs" : [ -177900, -91500, -5100, 81300, 167700, 254100, 340500, 426900, 513300, 599700, 686100, 772500, 858900, 772500, 686100, 599700, 513300, 426900, 340500, 254100, 167700, 81300, -5100, 0 ]
+}, {
+  "tz" : "Asia/Beirut",
+  "switches" : [ -62135605320, -59006370120, -55850696520, -52695022920, -46383589320, -43227915720, -40072242120, -33760808520, -30605134920, -27449461320, -21138027720, -17982354120, -14826680520, -12220078920, -12219992520, -12219906120, -12219819720, -12219733320, -12219646920, -12219560520, -12219474120, -12219387720, -12219301320, -2840149320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
+}, {
+  "tz" : "Asia/Bishkek",
+  "switches" : [ -62135614704, -59006379504, -55850705904, -52695032304, -46383598704, -43227925104, -40072251504, -33760817904, -30605144304, -27449470704, -21138037104, -17982363504, -14826689904, -12220088304, -12220001904, -12219915504, -12219829104, -12219742704, -12219656304, -12219569904, -12219483504, -12219397104, -12219310704, -2208988800 ],
+  "diffs" : [ -176496, -90096, -3696, 82704, 169104, 255504, 341904, 428304, 514704, 601104, 687504, 773904, 860304, 773904, 687504, 601104, 514704, 428304, 341904, 255504, 169104, 82704, -3696, 0 ]
+}, {
+  "tz" : "Asia/Brunei",
+  "switches" : [ -62135624380, -59006389180, -55850715580, -52695041980, -46383608380, -43227934780, -40072261180, -33760827580, -30605153980, -27449480380, -21138046780, -17982373180, -14826699580, -12220097980, -12220011580, -12219925180, -12219838780, -12219752380, -12219665980, -12219579580, -12219493180, -12219406780, -12219320380, -2208988800 ],
+  "diffs" : [ -174020, -87620, -1220, 85180, 171580, 257980, 344380, 430780, 517180, 603580, 689980, 776380, 862780, 776380, 689980, 603580, 517180, 430780, 344380, 257980, 171580, 85180, -1220, 0 ]
+}, {
+  "tz" : "Asia/Calcutta",
+  "switches" : [ -62135618008, -59006382808, -55850709208, -52695035608, -46383602008, -43227928408, -40072254808, -33760821208, -30605147608, -27449474008, -21138040408, -17982366808, -14826693208, -12220091608, -12220005208, -12219918808, -12219832408, -12219746008, -12219659608, -12219573208, -12219486808, -12219400408, -12219314008, -3645237208, -3155694800, -2208988800 ],
+  "diffs" : [ -171392, -84992, 1408, 87808, 174208, 260608, 347008, 433408, 519808, 606208, 692608, 779008, 865408, 779008, 692608, 606208, 519808, 433408, 347008, 260608, 174208, 87808, 1408, 1400, -530, 0 ]
+}, {
+  "tz" : "Asia/Chita",
+  "switches" : [ -62135624032, -59006388832, -55850715232, -52695041632, -46383608032, -43227934432, -40072260832, -33760827232, -30605153632, -27449480032, -21138046432, -17982372832, -14826699232, -12220097632, -12220011232, -12219924832, -12219838432, -12219752032, -12219665632, -12219579232, -12219492832, -12219406432, -12219320032, -2208988800 ],
+  "diffs" : [ -177968, -91568, -5168, 81232, 167632, 254032, 340432, 426832, 513232, 599632, 686032, 772432, 858832, 772432, 686032, 599632, 513232, 426832, 340432, 254032, 167632, 81232, -5168, 0 ]
+}, {
+  "tz" : "Asia/Choibalsan",
+  "switches" : [ -62135624280, -59006389080, -55850715480, -52695041880, -46383608280, -43227934680, -40072261080, -33760827480, -30605153880, -27449480280, -21138046680, -17982373080, -14826699480, -12220097880, -12220011480, -12219925080, -12219838680, -12219752280, -12219665880, -12219579480, -12219493080, -12219406680, -12219320280, -2208988800 ],
+  "diffs" : [ -174120, -87720, -1320, 85080, 171480, 257880, 344280, 430680, 517080, 603480, 689880, 776280, 862680, 776280, 689880, 603480, 517080, 430680, 344280, 257880, 171480, 85080, -1320, 0 ]
+}, {
+  "tz" : "Asia/Chongqing",
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12220099543, -12220013143, -12219926743, -12219840343, -12219753943, -12219667543, -12219581143, -12219494743, -12219408343, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 777943, 691543, 605143, 518743, 432343, 345943, 259543, 173143, 86743, 343, 0 ]
+}, {
+  "tz" : "Asia/Chungking",
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12220099543, -12220013143, -12219926743, -12219840343, -12219753943, -12219667543, -12219581143, -12219494743, -12219408343, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 777943, 691543, 605143, 518743, 432343, 345943, 259543, 173143, 86743, 343, 0 ]
+}, {
+  "tz" : "Asia/Colombo",
+  "switches" : [ -62135615964, -59006380764, -55850707164, -52695033564, -46383599964, -43227926364, -40072252764, -33760819164, -30605145564, -27449471964, -21138038364, -17982364764, -14826691164, -12220089564, -12220003164, -12219916764, -12219830364, -12219743964, -12219657564, -12219571164, -12219484764, -12219398364, -12219311964, -2840159964, -2208988800 ],
+  "diffs" : [ -173436, -87036, -636, 85764, 172164, 258564, 344964, 431364, 517764, 604164, 690564, 776964, 863364, 776964, 690564, 604164, 517764, 431364, 344964, 258564, 172164, 85764, -636, -628, 0 ]
+}, {
+  "tz" : "Asia/Dacca",
+  "switches" : [ -62135618500, -59006383300, -55850709700, -52695036100, -46383602500, -43227928900, -40072255300, -33760821700, -30605148100, -27449474500, -21138040900, -17982367300, -14826693700, -12220092100, -12220005700, -12219919300, -12219832900, -12219746500, -12219660100, -12219573700, -12219487300, -12219400900, -12219314500, -2524543300, -2208988800 ],
+  "diffs" : [ -172700, -86300, 100, 86500, 172900, 259300, 345700, 432100, 518500, 604900, 691300, 777700, 864100, 777700, 691300, 604900, 518500, 432100, 345700, 259300, 172900, 86500, 100, -400, 0 ]
+}, {
+  "tz" : "Asia/Damascus",
+  "switches" : [ -62135605512, -59006370312, -55850696712, -52695023112, -46383589512, -43227915912, -40072242312, -33760808712, -30605135112, -27449461512, -21138027912, -17982354312, -14826680712, -12220079112, -12219992712, -12219906312, -12219819912, -12219733512, -12219647112, -12219560712, -12219474312, -12219387912, -12219301512, -2208988800 ],
+  "diffs" : [ -171288, -84888, 1512, 87912, 174312, 260712, 347112, 433512, 519912, 606312, 692712, 779112, 865512, 779112, 692712, 606312, 519912, 433512, 347112, 260712, 174312, 87912, 1512, 0 ]
+}, {
+  "tz" : "Asia/Dhaka",
+  "switches" : [ -62135618500, -59006383300, -55850709700, -52695036100, -46383602500, -43227928900, -40072255300, -33760821700, -30605148100, -27449474500, -21138040900, -17982367300, -14826693700, -12220092100, -12220005700, -12219919300, -12219832900, -12219746500, -12219660100, -12219573700, -12219487300, -12219400900, -12219314500, -2524543300, -2208988800 ],
+  "diffs" : [ -172700, -86300, 100, 86500, 172900, 259300, 345700, 432100, 518500, 604900, 691300, 777700, 864100, 777700, 691300, 604900, 518500, 432100, 345700, 259300, 172900, 86500, 100, -400, 0 ]
+}, {
+  "tz" : "Asia/Dili",
+  "switches" : [ -62135626940, -59006391740, -55850718140, -52695044540, -46383610940, -43227937340, -40072263740, -33760830140, -30605156540, -27449482940, -21138049340, -17982375740, -14826702140, -12220100540, -12220014140, -12219927740, -12219841340, -12219754940, -12219668540, -12219582140, -12219495740, -12219409340, -12219322940, -2208988800 ],
+  "diffs" : [ -175060, -88660, -2260, 84140, 170540, 256940, 343340, 429740, 516140, 602540, 688940, 775340, 861740, 775340, 688940, 602540, 516140, 429740, 343340, 256940, 170540, 84140, -2260, 0 ]
+}, {
+  "tz" : "Asia/Dubai",
+  "switches" : [ -62135610072, -59006374872, -55850701272, -52695027672, -46383594072, -43227920472, -40072246872, -33760813272, -30605139672, -27449466072, -21138032472, -17982358872, -14826685272, -12220083672, -12219997272, -12219910872, -12219824472, -12219738072, -12219651672, -12219565272, -12219478872, -12219392472, -12219306072, -2208988800 ],
+  "diffs" : [ -173928, -87528, -1128, 85272, 171672, 258072, 344472, 430872, 517272, 603672, 690072, 776472, 862872, 776472, 690072, 603672, 517272, 430872, 344472, 258072, 171672, 85272, -1128, 0 ]
+}, {
+  "tz" : "Asia/Dushanbe",
+  "switches" : [ -62135613312, -59006378112, -55850704512, -52695030912, -46383597312, -43227923712, -40072250112, -33760816512, -30605142912, -27449469312, -21138035712, -17982362112, -14826688512, -12220086912, -12220000512, -12219914112, -12219827712, -12219741312, -12219654912, -12219568512, -12219482112, -12219395712, -12219309312, -2208988800 ],
+  "diffs" : [ -174288, -87888, -1488, 84912, 171312, 257712, 344112, 430512, 516912, 603312, 689712, 776112, 862512, 776112, 689712, 603312, 516912, 430512, 344112, 257712, 171312, 84912, -1488, 0 ]
+}, {
+  "tz" : "Asia/Famagusta",
+  "switches" : [ -62135604948, -59006369748, -55850696148, -52695022548, -46383588948, -43227915348, -40072241748, -33760808148, -30605134548, -27449460948, -21138027348, -17982353748, -14826680148, -12220078548, -12219992148, -12219905748, -12219819348, -12219732948, -12219646548, -12219560148, -12219473748, -12219387348, -12219300948, -2208988800 ],
+  "diffs" : [ -171852, -85452, 948, 87348, 173748, 260148, 346548, 432948, 519348, 605748, 692148, 778548, 864948, 778548, 692148, 605748, 519348, 432948, 346548, 260148, 173748, 87348, 948, 0 ]
+}, {
+  "tz" : "Asia/Gaza",
+  "switches" : [ -62135605072, -59006369872, -55850696272, -52695022672, -46383589072, -43227915472, -40072241872, -33760808272, -30605134672, -27449461072, -21138027472, -17982353872, -14826680272, -12220078672, -12219992272, -12219905872, -12219819472, -12219733072, -12219646672, -12219560272, -12219473872, -12219387472, -12219301072, -2208988800 ],
+  "diffs" : [ -171728, -85328, 1072, 87472, 173872, 260272, 346672, 433072, 519472, 605872, 692272, 778672, 865072, 778672, 692272, 605872, 519472, 433072, 346672, 260272, 173872, 87472, 1072, 0 ]
+}, {
+  "tz" : "Asia/Harbin",
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12220099543, -12220013143, -12219926743, -12219840343, -12219753943, -12219667543, -12219581143, -12219494743, -12219408343, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 777943, 691543, 605143, 518743, 432343, 345943, 259543, 173143, 86743, 343, 0 ]
+}, {
+  "tz" : "Asia/Hebron",
+  "switches" : [ -62135605223, -59006370023, -55850696423, -52695022823, -46383589223, -43227915623, -40072242023, -33760808423, -30605134823, -27449461223, -21138027623, -17982354023, -14826680423, -12220078823, -12219992423, -12219906023, -12219819623, -12219733223, -12219646823, -12219560423, -12219474023, -12219387623, -12219301223, -2208988800 ],
+  "diffs" : [ -171577, -85177, 1223, 87623, 174023, 260423, 346823, 433223, 519623, 606023, 692423, 778823, 865223, 778823, 692423, 606023, 519623, 433223, 346823, 260423, 174023, 87623, 1223, 0 ]
+}, {
+  "tz" : "Asia/Ho_Chi_Minh",
+  "switches" : [ -62135622400, -59006387200, -55850713600, -52695040000, -46383606400, -43227932800, -40072259200, -33760825600, -30605152000, -27449478400, -21138044800, -17982371200, -14826697600, -12220096000, -12220009600, -12219923200, -12219836800, -12219750400, -12219664000, -12219577600, -12219491200, -12219404800, -12219318400, -2208988800 ],
+  "diffs" : [ -172400, -86000, 400, 86800, 173200, 259600, 346000, 432400, 518800, 605200, 691600, 778000, 864400, 778000, 691600, 605200, 518800, 432400, 346000, 259600, 173200, 86800, 400, 0 ]
+}, {
+  "tz" : "Asia/Hong_Kong",
+  "switches" : [ -62135624202, -59006389002, -55850715402, -52695041802, -46383608202, -43227934602, -40072261002, -33760827402, -30605153802, -27449480202, -21138046602, -17982373002, -14826699402, -12220097802, -12220011402, -12219925002, -12219838602, -12219752202, -12219665802, -12219579402, -12219493002, -12219406602, -12219320202, -2208988800 ],
+  "diffs" : [ -174198, -87798, -1398, 85002, 171402, 257802, 344202, 430602, 517002, 603402, 689802, 776202, 862602, 776202, 689802, 603402, 517002, 430602, 344202, 257802, 171402, 85002, -1398, 0 ]
+}, {
+  "tz" : "Asia/Hovd",
+  "switches" : [ -62135618796, -59006383596, -55850709996, -52695036396, -46383602796, -43227929196, -40072255596, -33760821996, -30605148396, -27449474796, -21138041196, -17982367596, -14826693996, -12220092396, -12220005996, -12219919596, -12219833196, -12219746796, -12219660396, -12219573996, -12219487596, -12219401196, -12219314796, -2208988800 ],
+  "diffs" : [ -176004, -89604, -3204, 83196, 169596, 255996, 342396, 428796, 515196, 601596, 687996, 774396, 860796, 774396, 687996, 601596, 515196, 428796, 342396, 255996, 169596, 83196, -3204, 0 ]
+}, {
+  "tz" : "Asia/Irkutsk",
+  "switches" : [ -62135621825, -59006386625, -55850713025, -52695039425, -46383605825, -43227932225, -40072258625, -33760825025, -30605151425, -27449477825, -21138044225, -17982370625, -14826697025, -12220095425, -12220009025, -12219922625, -12219836225, -12219749825, -12219663425, -12219577025, -12219490625, -12219404225, -12219317825, -2208988800 ],
+  "diffs" : [ -176575, -90175, -3775, 82625, 169025, 255425, 341825, 428225, 514625, 601025, 687425, 773825, 860225, 773825, 687425, 601025, 514625, 428225, 341825, 255425, 169025, 82625, -3775, 0 ]
+}, {
+  "tz" : "Asia/Istanbul",
+  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12220077352, -12219990952, -12219904552, -12219818152, -12219731752, -12219645352, -12219558952, -12219472552, -12219386152, -12219299752, -2840147752, -2208988800 ],
+  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, 773752, 687352, 600952, 514552, 428152, 341752, 255352, 168952, 82552, -3848, -3784, 0 ]
+}, {
+  "tz" : "Asia/Jakarta",
+  "switches" : [ -62135622432, -59006387232, -55850713632, -52695040032, -46383606432, -43227932832, -40072259232, -33760825632, -30605152032, -27449478432, -21138044832, -17982371232, -14826697632, -12220096032, -12220009632, -12219923232, -12219836832, -12219750432, -12219664032, -12219577632, -12219491232, -12219404832, -12219318432, -2208988800 ],
+  "diffs" : [ -172368, -85968, 432, 86832, 173232, 259632, 346032, 432432, 518832, 605232, 691632, 778032, 864432, 778032, 691632, 605232, 518832, 432432, 346032, 259632, 173232, 86832, 432, 0 ]
+}, {
+  "tz" : "Asia/Jayapura",
+  "switches" : [ -62135630568, -59006395368, -55850721768, -52695048168, -46383614568, -43227940968, -40072267368, -33760833768, -30605160168, -27449486568, -21138052968, -17982379368, -14826705768, -12220104168, -12220017768, -12219931368, -12219844968, -12219758568, -12219672168, -12219585768, -12219499368, -12219412968, -12219326568, -2208988800 ],
+  "diffs" : [ -171432, -85032, 1368, 87768, 174168, 260568, 346968, 433368, 519768, 606168, 692568, 778968, 865368, 778968, 692568, 606168, 519768, 433368, 346968, 260568, 174168, 87768, 1368, 0 ]
+}, {
+  "tz" : "Asia/Jerusalem",
+  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12220078854, -12219992454, -12219906054, -12219819654, -12219733254, -12219646854, -12219560454, -12219474054, -12219387654, -12219301254, -2840149254, -2208988800 ],
+  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 778854, 692454, 606054, 519654, 433254, 346854, 260454, 174054, 87654, 1254, 1240, 0 ]
+}, {
+  "tz" : "Asia/Kabul",
+  "switches" : [ -62135613408, -59006378208, -55850704608, -52695031008, -46383597408, -43227923808, -40072250208, -33760816608, -30605143008, -27449469408, -21138035808, -17982362208, -14826688608, -12220087008, -12220000608, -12219914208, -12219827808, -12219741408, -12219655008, -12219568608, -12219482208, -12219395808, -12219309408, -2524538208, -2208988800 ],
+  "diffs" : [ -172392, -85992, 408, 86808, 173208, 259608, 346008, 432408, 518808, 605208, 691608, 778008, 864408, 778008, 691608, 605208, 518808, 432408, 346008, 259608, 173208, 86808, 408, -1800, 0 ]
+}, {
+  "tz" : "Asia/Kamchatka",
+  "switches" : [ -62135634876, -59006399676, -55850726076, -52695052476, -46383618876, -43227945276, -40072271676, -33760838076, -30605164476, -27449490876, -21138057276, -17982383676, -14826710076, -12220108476, -12220022076, -12219935676, -12219849276, -12219762876, -12219676476, -12219590076, -12219503676, -12219417276, -12219330876, -2208988800 ],
+  "diffs" : [ -177924, -91524, -5124, 81276, 167676, 254076, 340476, 426876, 513276, 599676, 686076, 772476, 858876, 772476, 686076, 599676, 513276, 426876, 340476, 254076, 167676, 81276, -5124, 0 ]
+}, {
+  "tz" : "Asia/Karachi",
+  "switches" : [ -62135612892, -59006377692, -55850704092, -52695030492, -46383596892, -43227923292, -40072249692, -33760816092, -30605142492, -27449468892, -21138035292, -17982361692, -14826688092, -12220086492, -12220000092, -12219913692, -12219827292, -12219740892, -12219654492, -12219568092, -12219481692, -12219395292, -12219308892, -2208988800 ],
+  "diffs" : [ -174708, -88308, -1908, 84492, 170892, 257292, 343692, 430092, 516492, 602892, 689292, 775692, 862092, 775692, 689292, 602892, 516492, 430092, 343692, 257292, 170892, 84492, -1908, 0 ]
+}, {
+  "tz" : "Asia/Kashgar",
+  "switches" : [ -62135617820, -59006382620, -55850709020, -52695035420, -46383601820, -43227928220, -40072254620, -33760821020, -30605147420, -27449473820, -21138040220, -17982366620, -14826693020, -12220091420, -12220005020, -12219918620, -12219832220, -12219745820, -12219659420, -12219573020, -12219486620, -12219400220, -12219313820, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ]
+}, {
+  "tz" : "Asia/Kathmandu",
+  "switches" : [ -62135617276, -59006382076, -55850708476, -52695034876, -46383601276, -43227927676, -40072254076, -33760820476, -30605146876, -27449473276, -21138039676, -17982366076, -14826692476, -12220090876, -12220004476, -12219918076, -12219831676, -12219745276, -12219658876, -12219572476, -12219486076, -12219399676, -12219313276, -2208988800 ],
+  "diffs" : [ -173024, -86624, -224, 86176, 172576, 258976, 345376, 431776, 518176, 604576, 690976, 777376, 863776, 777376, 690976, 604576, 518176, 431776, 345376, 258976, 172576, 86176, -224, 0 ]
+}, {
+  "tz" : "Asia/Katmandu",
+  "switches" : [ -62135617276, -59006382076, -55850708476, -52695034876, -46383601276, -43227927676, -40072254076, -33760820476, -30605146876, -27449473276, -21138039676, -17982366076, -14826692476, -12220090876, -12220004476, -12219918076, -12219831676, -12219745276, -12219658876, -12219572476, -12219486076, -12219399676, -12219313276, -2208988800 ],
+  "diffs" : [ -173024, -86624, -224, 86176, 172576, 258976, 345376, 431776, 518176, 604576, 690976, 777376, 863776, 777376, 690976, 604576, 518176, 431776, 345376, 258976, 172576, 86176, -224, 0 ]
+}, {
+  "tz" : "Asia/Khandyga",
+  "switches" : [ -62135629333, -59006394133, -55850720533, -52695046933, -46383613333, -43227939733, -40072266133, -33760832533, -30605158933, -27449485333, -21138051733, -17982378133, -14826704533, -12220102933, -12220016533, -12219930133, -12219843733, -12219757333, -12219670933, -12219584533, -12219498133, -12219411733, -12219325333, -2208988800 ],
+  "diffs" : [ -172667, -86267, 133, 86533, 172933, 259333, 345733, 432133, 518533, 604933, 691333, 777733, 864133, 777733, 691333, 604933, 518533, 432133, 345733, 259333, 172933, 86533, 133, 0 ]
+}, {
+  "tz" : "Asia/Kolkata",
+  "switches" : [ -62135618008, -59006382808, -55850709208, -52695035608, -46383602008, -43227928408, -40072254808, -33760821208, -30605147608, -27449474008, -21138040408, -17982366808, -14826693208, -12220091608, -12220005208, -12219918808, -12219832408, -12219746008, -12219659608, -12219573208, -12219486808, -12219400408, -12219314008, -3645237208, -3155694800, -2208988800 ],
+  "diffs" : [ -171392, -84992, 1408, 87808, 174208, 260608, 347008, 433408, 519808, 606208, 692608, 779008, 865408, 779008, 692608, 606208, 519808, 433408, 347008, 260608, 174208, 87808, 1408, 1400, -530, 0 ]
+}, {
+  "tz" : "Asia/Krasnoyarsk",
+  "switches" : [ -62135619086, -59006383886, -55850710286, -52695036686, -46383603086, -43227929486, -40072255886, -33760822286, -30605148686, -27449475086, -21138041486, -17982367886, -14826694286, -12220092686, -12220006286, -12219919886, -12219833486, -12219747086, -12219660686, -12219574286, -12219487886, -12219401486, -12219315086, -2208988800 ],
+  "diffs" : [ -175714, -89314, -2914, 83486, 169886, 256286, 342686, 429086, 515486, 601886, 688286, 774686, 861086, 774686, 688286, 601886, 515486, 429086, 342686, 256286, 169886, 83486, -2914, 0 ]
+}, {
+  "tz" : "Asia/Kuala_Lumpur",
+  "switches" : [ -62135621206, -59006386006, -55850712406, -52695038806, -46383605206, -43227931606, -40072258006, -33760824406, -30605150806, -27449477206, -21138043606, -17982370006, -14826696406, -12220094806, -12220008406, -12219922006, -12219835606, -12219749206, -12219662806, -12219576406, -12219490006, -12219403606, -12219317206, -2208988800 ],
+  "diffs" : [ -177194, -90794, -4394, 82006, 168406, 254806, 341206, 427606, 514006, 600406, 686806, 773206, 859606, 773206, 686806, 600406, 514006, 427606, 341206, 254806, 168406, 82006, -4394, 0 ]
+}, {
+  "tz" : "Asia/Kuching",
+  "switches" : [ -62135623280, -59006388080, -55850714480, -52695040880, -46383607280, -43227933680, -40072260080, -33760826480, -30605152880, -27449479280, -21138045680, -17982372080, -14826698480, -12220096880, -12220010480, -12219924080, -12219837680, -12219751280, -12219664880, -12219578480, -12219492080, -12219405680, -12219319280, -2208988800 ],
+  "diffs" : [ -175120, -88720, -2320, 84080, 170480, 256880, 343280, 429680, 516080, 602480, 688880, 775280, 861680, 775280, 688880, 602480, 516080, 429680, 343280, 256880, 170480, 84080, -2320, 0 ]
+}, {
+  "tz" : "Asia/Kuwait",
+  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12220081612, -12219995212, -12219908812, -12219822412, -12219736012, -12219649612, -12219563212, -12219476812, -12219390412, -12219304012, -2208988800 ],
+  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 778012, 691612, 605212, 518812, 432412, 346012, 259612, 173212, 86812, 412, 0 ]
+}, {
+  "tz" : "Asia/Macao",
+  "switches" : [ -62135624050, -59006388850, -55850715250, -52695041650, -46383608050, -43227934450, -40072260850, -33760827250, -30605153650, -27449480050, -21138046450, -17982372850, -14826699250, -12220097650, -12220011250, -12219924850, -12219838450, -12219752050, -12219665650, -12219579250, -12219492850, -12219406450, -12219320050, -2208988800 ],
+  "diffs" : [ -174350, -87950, -1550, 84850, 171250, 257650, 344050, 430450, 516850, 603250, 689650, 776050, 862450, 776050, 689650, 603250, 516850, 430450, 344050, 257650, 171250, 84850, -1550, 0 ]
+}, {
+  "tz" : "Asia/Macau",
+  "switches" : [ -62135624050, -59006388850, -55850715250, -52695041650, -46383608050, -43227934450, -40072260850, -33760827250, -30605153650, -27449480050, -21138046450, -17982372850, -14826699250, -12220097650, -12220011250, -12219924850, -12219838450, -12219752050, -12219665650, -12219579250, -12219492850, -12219406450, -12219320050, -2208988800 ],
+  "diffs" : [ -174350, -87950, -1550, 84850, 171250, 257650, 344050, 430450, 516850, 603250, 689650, 776050, 862450, 776050, 689650, 603250, 516850, 430450, 344050, 257650, 171250, 84850, -1550, 0 ]
+}, {
+  "tz" : "Asia/Magadan",
+  "switches" : [ -62135632992, -59006397792, -55850724192, -52695050592, -46383616992, -43227943392, -40072269792, -33760836192, -30605162592, -27449488992, -21138055392, -17982381792, -14826708192, -12220106592, -12220020192, -12219933792, -12219847392, -12219760992, -12219674592, -12219588192, -12219501792, -12219415392, -12219328992, -2208988800 ],
+  "diffs" : [ -176208, -89808, -3408, 82992, 169392, 255792, 342192, 428592, 514992, 601392, 687792, 774192, 860592, 774192, 687792, 601392, 514992, 428592, 342192, 255792, 169392, 82992, -3408, 0 ]
+}, {
+  "tz" : "Asia/Makassar",
+  "switches" : [ -62135625456, -59006390256, -55850716656, -52695043056, -46383609456, -43227935856, -40072262256, -33760828656, -30605155056, -27449481456, -21138047856, -17982374256, -14826700656, -12220099056, -12220012656, -12219926256, -12219839856, -12219753456, -12219667056, -12219580656, -12219494256, -12219407856, -12219321456, -2208988800 ],
+  "diffs" : [ -172944, -86544, -144, 86256, 172656, 259056, 345456, 431856, 518256, 604656, 691056, 777456, 863856, 777456, 691056, 604656, 518256, 431856, 345456, 259056, 172656, 86256, -144, 0 ]
+}, {
+  "tz" : "Asia/Manila",
+  "switches" : [ -62135539440, -59006304240, -55850630640, -52694957040, -46383523440, -43227849840, -40072176240, -33760742640, -30605069040, -27449395440, -21137961840, -17982288240, -14826614640, -12220013040, -12219926640, -12219840240, -12219753840, -12219667440, -12219581040, -12219494640, -12219408240, -12219321840, -12219235440, -3944621040, -2229321840 ],
+  "diffs" : [ -258960, -172560, -86160, 240, 86640, 173040, 259440, 345840, 432240, 518640, 605040, 691440, 777840, 691440, 605040, 518640, 432240, 345840, 259440, 173040, 86640, 240, -86160, 240, 0 ]
+}, {
+  "tz" : "Asia/Muscat",
+  "switches" : [ -62135610072, -59006374872, -55850701272, -52695027672, -46383594072, -43227920472, -40072246872, -33760813272, -30605139672, -27449466072, -21138032472, -17982358872, -14826685272, -12220083672, -12219997272, -12219910872, -12219824472, -12219738072, -12219651672, -12219565272, -12219478872, -12219392472, -12219306072, -2208988800 ],
+  "diffs" : [ -173928, -87528, -1128, 85272, 171672, 258072, 344472, 430872, 517272, 603672, 690072, 776472, 862872, 776472, 690072, 603672, 517272, 430872, 344472, 258072, 171672, 85272, -1128, 0 ]
+}, {
+  "tz" : "Asia/Nicosia",
+  "switches" : [ -62135604808, -59006369608, -55850696008, -52695022408, -46383588808, -43227915208, -40072241608, -33760808008, -30605134408, -27449460808, -21138027208, -17982353608, -14826680008, -12220078408, -12219992008, -12219905608, -12219819208, -12219732808, -12219646408, -12219560008, -12219473608, -12219387208, -12219300808, -2208988800 ],
+  "diffs" : [ -171992, -85592, 808, 87208, 173608, 260008, 346408, 432808, 519208, 605608, 692008, 778408, 864808, 778408, 692008, 605608, 519208, 432808, 346408, 260008, 173608, 87208, 808, 0 ]
+}, {
+  "tz" : "Asia/Novokuznetsk",
+  "switches" : [ -62135617728, -59006382528, -55850708928, -52695035328, -46383601728, -43227928128, -40072254528, -33760820928, -30605147328, -27449473728, -21138040128, -17982366528, -14826692928, -12220091328, -12220004928, -12219918528, -12219832128, -12219745728, -12219659328, -12219572928, -12219486528, -12219400128, -12219313728, -2208988800 ],
+  "diffs" : [ -177072, -90672, -4272, 82128, 168528, 254928, 341328, 427728, 514128, 600528, 686928, 773328, 859728, 773328, 686928, 600528, 514128, 427728, 341328, 254928, 168528, 82128, -4272, 0 ]
+}, {
+  "tz" : "Asia/Novosibirsk",
+  "switches" : [ -62135616700, -59006381500, -55850707900, -52695034300, -46383600700, -43227927100, -40072253500, -33760819900, -30605146300, -27449472700, -21138039100, -17982365500, -14826691900, -12220090300, -12220003900, -12219917500, -12219831100, -12219744700, -12219658300, -12219571900, -12219485500, -12219399100, -12219312700, -2208988800 ],
+  "diffs" : [ -178100, -91700, -5300, 81100, 167500, 253900, 340300, 426700, 513100, 599500, 685900, 772300, 858700, 772300, 685900, 599500, 513100, 426700, 340300, 253900, 167500, 81100, -5300, 0 ]
+}, {
+  "tz" : "Asia/Omsk",
+  "switches" : [ -62135614410, -59006379210, -55850705610, -52695032010, -46383598410, -43227924810, -40072251210, -33760817610, -30605144010, -27449470410, -21138036810, -17982363210, -14826689610, -12220088010, -12220001610, -12219915210, -12219828810, -12219742410, -12219656010, -12219569610, -12219483210, -12219396810, -12219310410, -2208988800 ],
+  "diffs" : [ -176790, -90390, -3990, 82410, 168810, 255210, 341610, 428010, 514410, 600810, 687210, 773610, 860010, 773610, 687210, 600810, 514410, 428010, 341610, 255210, 168810, 82410, -3990, 0 ]
+}, {
+  "tz" : "Asia/Oral",
+  "switches" : [ -62135609124, -59006373924, -55850700324, -52695026724, -46383593124, -43227919524, -40072245924, -33760812324, -30605138724, -27449465124, -21138031524, -17982357924, -14826684324, -12220082724, -12219996324, -12219909924, -12219823524, -12219737124, -12219650724, -12219564324, -12219477924, -12219391524, -12219305124, -2208988800 ],
+  "diffs" : [ -178476, -92076, -5676, 80724, 167124, 253524, 339924, 426324, 512724, 599124, 685524, 771924, 858324, 771924, 685524, 599124, 512724, 426324, 339924, 253524, 167124, 80724, -5676, 0 ]
+}, {
+  "tz" : "Asia/Phnom_Penh",
+  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12220094524, -12220008124, -12219921724, -12219835324, -12219748924, -12219662524, -12219576124, -12219489724, -12219403324, -12219316924, -2208988800 ],
+  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, 776524, 690124, 603724, 517324, 430924, 344524, 258124, 171724, 85324, -1076, 0 ]
+}, {
+  "tz" : "Asia/Pontianak",
+  "switches" : [ -62135623040, -59006387840, -55850714240, -52695040640, -46383607040, -43227933440, -40072259840, -33760826240, -30605152640, -27449479040, -21138045440, -17982371840, -14826698240, -12220096640, -12220010240, -12219923840, -12219837440, -12219751040, -12219664640, -12219578240, -12219491840, -12219405440, -12219319040, -2208988800 ],
+  "diffs" : [ -171760, -85360, 1040, 87440, 173840, 260240, 346640, 433040, 519440, 605840, 692240, 778640, 865040, 778640, 692240, 605840, 519440, 433040, 346640, 260240, 173840, 87440, 1040, 0 ]
+}, {
+  "tz" : "Asia/Pyongyang",
+  "switches" : [ -62135626980, -59006391780, -55850718180, -52695044580, -46383610980, -43227937380, -40072263780, -33760830180, -30605156580, -27449482980, -21138049380, -17982375780, -14826702180, -12220100580, -12220014180, -12219927780, -12219841380, -12219754980, -12219668580, -12219582180, -12219495780, -12219409380, -12219322980, -2208988800 ],
+  "diffs" : [ -175020, -88620, -2220, 84180, 170580, 256980, 343380, 429780, 516180, 602580, 688980, 775380, 861780, 775380, 688980, 602580, 516180, 429780, 343380, 256980, 170580, 84180, -2220, 0 ]
+}, {
+  "tz" : "Asia/Qatar",
+  "switches" : [ -62135609168, -59006373968, -55850700368, -52695026768, -46383593168, -43227919568, -40072245968, -33760812368, -30605138768, -27449465168, -21138031568, -17982357968, -14826684368, -12220082768, -12219996368, -12219909968, -12219823568, -12219737168, -12219650768, -12219564368, -12219477968, -12219391568, -12219305168, -2208988800 ],
+  "diffs" : [ -171232, -84832, 1568, 87968, 174368, 260768, 347168, 433568, 519968, 606368, 692768, 779168, 865568, 779168, 692768, 606368, 519968, 433568, 347168, 260768, 174368, 87968, 1568, 0 ]
+}, {
+  "tz" : "Asia/Qostanay",
+  "switches" : [ -62135612068, -59006376868, -55850703268, -52695029668, -46383596068, -43227922468, -40072248868, -33760815268, -30605141668, -27449468068, -21138034468, -17982360868, -14826687268, -12220085668, -12219999268, -12219912868, -12219826468, -12219740068, -12219653668, -12219567268, -12219480868, -12219394468, -12219308068, -2208988800 ],
+  "diffs" : [ -179132, -92732, -6332, 80068, 166468, 252868, 339268, 425668, 512068, 598468, 684868, 771268, 857668, 771268, 684868, 598468, 512068, 425668, 339268, 252868, 166468, 80068, -6332, 0 ]
+}, {
+  "tz" : "Asia/Qyzylorda",
+  "switches" : [ -62135612512, -59006377312, -55850703712, -52695030112, -46383596512, -43227922912, -40072249312, -33760815712, -30605142112, -27449468512, -21138034912, -17982361312, -14826687712, -12220086112, -12219999712, -12219913312, -12219826912, -12219740512, -12219654112, -12219567712, -12219481312, -12219394912, -12219308512, -2208988800 ],
+  "diffs" : [ -175088, -88688, -2288, 84112, 170512, 256912, 343312, 429712, 516112, 602512, 688912, 775312, 861712, 775312, 688912, 602512, 516112, 429712, 343312, 256912, 170512, 84112, -2288, 0 ]
+}, {
+  "tz" : "Asia/Rangoon",
+  "switches" : [ -62135619887, -59006384687, -55850711087, -52695037487, -46383603887, -43227930287, -40072256687, -33760823087, -30605149487, -27449475887, -21138042287, -17982368687, -14826695087, -12220093487, -12220007087, -12219920687, -12219834287, -12219747887, -12219661487, -12219575087, -12219488687, -12219402287, -12219315887, -2208988800 ],
+  "diffs" : [ -173113, -86713, -313, 86087, 172487, 258887, 345287, 431687, 518087, 604487, 690887, 777287, 863687, 777287, 690887, 604487, 518087, 431687, 345287, 258887, 172487, 86087, -313, 0 ]
+}, {
+  "tz" : "Asia/Riyadh",
+  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12220081612, -12219995212, -12219908812, -12219822412, -12219736012, -12219649612, -12219563212, -12219476812, -12219390412, -12219304012, -2208988800 ],
+  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 778012, 691612, 605212, 518812, 432412, 346012, 259612, 173212, 86812, 412, 0 ]
+}, {
+  "tz" : "Asia/Saigon",
+  "switches" : [ -62135622400, -59006387200, -55850713600, -52695040000, -46383606400, -43227932800, -40072259200, -33760825600, -30605152000, -27449478400, -21138044800, -17982371200, -14826697600, -12220096000, -12220009600, -12219923200, -12219836800, -12219750400, -12219664000, -12219577600, -12219491200, -12219404800, -12219318400, -2208988800 ],
+  "diffs" : [ -172400, -86000, 400, 86800, 173200, 259600, 346000, 432400, 518800, 605200, 691600, 778000, 864400, 778000, 691600, 605200, 518800, 432400, 346000, 259600, 173200, 86800, 400, 0 ]
+}, {
+  "tz" : "Asia/Sakhalin",
+  "switches" : [ -62135631048, -59006395848, -55850722248, -52695048648, -46383615048, -43227941448, -40072267848, -33760834248, -30605160648, -27449487048, -21138053448, -17982379848, -14826706248, -12220104648, -12220018248, -12219931848, -12219845448, -12219759048, -12219672648, -12219586248, -12219499848, -12219413448, -12219327048, -2208988800 ],
+  "diffs" : [ -178152, -91752, -5352, 81048, 167448, 253848, 340248, 426648, 513048, 599448, 685848, 772248, 858648, 772248, 685848, 599448, 513048, 426648, 340248, 253848, 167448, 81048, -5352, 0 ]
+}, {
+  "tz" : "Asia/Samarkand",
+  "switches" : [ -62135612873, -59006377673, -55850704073, -52695030473, -46383596873, -43227923273, -40072249673, -33760816073, -30605142473, -27449468873, -21138035273, -17982361673, -14826688073, -12220086473, -12220000073, -12219913673, -12219827273, -12219740873, -12219654473, -12219568073, -12219481673, -12219395273, -12219308873, -2208988800 ],
+  "diffs" : [ -174727, -88327, -1927, 84473, 170873, 257273, 343673, 430073, 516473, 602873, 689273, 775673, 862073, 775673, 689273, 602873, 516473, 430073, 343673, 257273, 170873, 84473, -1927, 0 ]
+}, {
+  "tz" : "Asia/Seoul",
+  "switches" : [ -62135627272, -59006392072, -55850718472, -52695044872, -46383611272, -43227937672, -40072264072, -33760830472, -30605156872, -27449483272, -21138049672, -17982376072, -14826702472, -12220100872, -12220014472, -12219928072, -12219841672, -12219755272, -12219668872, -12219582472, -12219496072, -12219409672, -12219323272, -2208988800 ],
+  "diffs" : [ -174728, -88328, -1928, 84472, 170872, 257272, 343672, 430072, 516472, 602872, 689272, 775672, 862072, 775672, 689272, 602872, 516472, 430072, 343672, 257272, 170872, 84472, -1928, 0 ]
+}, {
+  "tz" : "Asia/Shanghai",
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12220099543, -12220013143, -12219926743, -12219840343, -12219753943, -12219667543, -12219581143, -12219494743, -12219408343, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 777943, 691543, 605143, 518743, 432343, 345943, 259543, 173143, 86743, 343, 0 ]
+}, {
+  "tz" : "Asia/Singapore",
+  "switches" : [ -62135621725, -59006386525, -55850712925, -52695039325, -46383605725, -43227932125, -40072258525, -33760824925, -30605151325, -27449477725, -21138044125, -17982370525, -14826696925, -12220095325, -12220008925, -12219922525, -12219836125, -12219749725, -12219663325, -12219576925, -12219490525, -12219404125, -12219317725, -2208988800 ],
+  "diffs" : [ -176675, -90275, -3875, 82525, 168925, 255325, 341725, 428125, 514525, 600925, 687325, 773725, 860125, 773725, 687325, 600925, 514525, 428125, 341725, 255325, 168925, 82525, -3875, 0 ]
+}, {
+  "tz" : "Asia/Srednekolymsk",
+  "switches" : [ -62135633692, -59006398492, -55850724892, -52695051292, -46383617692, -43227944092, -40072270492, -33760836892, -30605163292, -27449489692, -21138056092, -17982382492, -14826708892, -12220107292, -12220020892, -12219934492, -12219848092, -12219761692, -12219675292, -12219588892, -12219502492, -12219416092, -12219329692, -2208988800 ],
+  "diffs" : [ -175508, -89108, -2708, 83692, 170092, 256492, 342892, 429292, 515692, 602092, 688492, 774892, 861292, 774892, 688492, 602092, 515692, 429292, 342892, 256492, 170092, 83692, -2708, 0 ]
+}, {
+  "tz" : "Asia/Taipei",
+  "switches" : [ -62135625960, -59006390760, -55850717160, -52695043560, -46383609960, -43227936360, -40072262760, -33760829160, -30605155560, -27449481960, -21138048360, -17982374760, -14826701160, -12220099560, -12220013160, -12219926760, -12219840360, -12219753960, -12219667560, -12219581160, -12219494760, -12219408360, -12219321960, -2335248360 ],
+  "diffs" : [ -172440, -86040, 360, 86760, 173160, 259560, 345960, 432360, 518760, 605160, 691560, 777960, 864360, 777960, 691560, 605160, 518760, 432360, 345960, 259560, 173160, 86760, 360, 0 ]
+}, {
+  "tz" : "Asia/Tashkent",
+  "switches" : [ -62135613431, -59006378231, -55850704631, -52695031031, -46383597431, -43227923831, -40072250231, -33760816631, -30605143031, -27449469431, -21138035831, -17982362231, -14826688631, -12220087031, -12220000631, -12219914231, -12219827831, -12219741431, -12219655031, -12219568631, -12219482231, -12219395831, -12219309431, -2208988800 ],
+  "diffs" : [ -174169, -87769, -1369, 85031, 171431, 257831, 344231, 430631, 517031, 603431, 689831, 776231, 862631, 776231, 689831, 603431, 517031, 430631, 344231, 257831, 171431, 85031, -1369, 0 ]
+}, {
+  "tz" : "Asia/Tbilisi",
+  "switches" : [ -62135607551, -59006372351, -55850698751, -52695025151, -46383591551, -43227917951, -40072244351, -33760810751, -30605137151, -27449463551, -21138029951, -17982356351, -14826682751, -12220081151, -12219994751, -12219908351, -12219821951, -12219735551, -12219649151, -12219562751, -12219476351, -12219389951, -12219303551, -2208988800 ],
+  "diffs" : [ -176449, -90049, -3649, 82751, 169151, 255551, 341951, 428351, 514751, 601151, 687551, 773951, 860351, 773951, 687551, 601151, 514751, 428351, 341951, 255551, 169151, 82751, -3649, 0 ]
+}, {
+  "tz" : "Asia/Tehran",
+  "switches" : [ -62135609144, -59006373944, -55850700344, -52695026744, -46383593144, -43227919544, -40072245944, -33760812344, -30605138744, -27449465144, -21138031544, -17982357944, -14826684344, -12220082744, -12219996344, -12219909944, -12219823544, -12219737144, -12219650744, -12219564344, -12219477944, -12219391544, -12219305144, -2208988800 ],
+  "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, 777344, 690944, 604544, 518144, 431744, 345344, 258944, 172544, 86144, -256, 0 ]
+}, {
+  "tz" : "Asia/Tel_Aviv",
+  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12220078854, -12219992454, -12219906054, -12219819654, -12219733254, -12219646854, -12219560454, -12219474054, -12219387654, -12219301254, -2840149254, -2208988800 ],
+  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 778854, 692454, 606054, 519654, 433254, 346854, 260454, 174054, 87654, 1254, 1240, 0 ]
+}, {
+  "tz" : "Asia/Thimbu",
+  "switches" : [ -62135618316, -59006383116, -55850709516, -52695035916, -46383602316, -43227928716, -40072255116, -33760821516, -30605147916, -27449474316, -21138040716, -17982367116, -14826693516, -12220091916, -12220005516, -12219919116, -12219832716, -12219746316, -12219659916, -12219573516, -12219487116, -12219400716, -12219314316, -2208988800 ],
+  "diffs" : [ -172884, -86484, -84, 86316, 172716, 259116, 345516, 431916, 518316, 604716, 691116, 777516, 863916, 777516, 691116, 604716, 518316, 431916, 345516, 259116, 172716, 86316, -84, 0 ]
+}, {
+  "tz" : "Asia/Thimphu",
+  "switches" : [ -62135618316, -59006383116, -55850709516, -52695035916, -46383602316, -43227928716, -40072255116, -33760821516, -30605147916, -27449474316, -21138040716, -17982367116, -14826693516, -12220091916, -12220005516, -12219919116, -12219832716, -12219746316, -12219659916, -12219573516, -12219487116, -12219400716, -12219314316, -2208988800 ],
+  "diffs" : [ -172884, -86484, -84, 86316, 172716, 259116, 345516, 431916, 518316, 604716, 691116, 777516, 863916, 777516, 691116, 604716, 518316, 431916, 345516, 259116, 172716, 86316, -84, 0 ]
+}, {
+  "tz" : "Asia/Tokyo",
+  "switches" : [ -62135630339, -59006395139, -55850721539, -52695047939, -46383614339, -43227940739, -40072267139, -33760833539, -30605159939, -27449486339, -21138052739, -17982379139, -14826705539, -12220103939, -12220017539, -12219931139, -12219844739, -12219758339, -12219671939, -12219585539, -12219499139, -12219412739, -12219326339, -2587712400 ],
+  "diffs" : [ -171661, -85261, 1139, 87539, 173939, 260339, 346739, 433139, 519539, 605939, 692339, 778739, 865139, 778739, 692339, 605939, 519539, 433139, 346739, 260339, 173939, 87539, 1139, 0 ]
+}, {
+  "tz" : "Asia/Tomsk",
+  "switches" : [ -62135617191, -59006381991, -55850708391, -52695034791, -46383601191, -43227927591, -40072253991, -33760820391, -30605146791, -27449473191, -21138039591, -17982365991, -14826692391, -12220090791, -12220004391, -12219917991, -12219831591, -12219745191, -12219658791, -12219572391, -12219485991, -12219399591, -12219313191, -2208988800 ],
+  "diffs" : [ -177609, -91209, -4809, 81591, 167991, 254391, 340791, 427191, 513591, 599991, 686391, 772791, 859191, 772791, 686391, 599991, 513591, 427191, 340791, 254391, 167991, 81591, -4809, 0 ]
+}, {
+  "tz" : "Asia/Ujung_Pandang",
+  "switches" : [ -62135625456, -59006390256, -55850716656, -52695043056, -46383609456, -43227935856, -40072262256, -33760828656, -30605155056, -27449481456, -21138047856, -17982374256, -14826700656, -12220099056, -12220012656, -12219926256, -12219839856, -12219753456, -12219667056, -12219580656, -12219494256, -12219407856, -12219321456, -2208988800 ],
+  "diffs" : [ -172944, -86544, -144, 86256, 172656, 259056, 345456, 431856, 518256, 604656, 691056, 777456, 863856, 777456, 691056, 604656, 518256, 431856, 345456, 259056, 172656, 86256, -144, 0 ]
+}, {
+  "tz" : "Asia/Ulaanbaatar",
+  "switches" : [ -62135622452, -59006387252, -55850713652, -52695040052, -46383606452, -43227932852, -40072259252, -33760825652, -30605152052, -27449478452, -21138044852, -17982371252, -14826697652, -12220096052, -12220009652, -12219923252, -12219836852, -12219750452, -12219664052, -12219577652, -12219491252, -12219404852, -12219318452, -2208988800 ],
+  "diffs" : [ -175948, -89548, -3148, 83252, 169652, 256052, 342452, 428852, 515252, 601652, 688052, 774452, 860852, 774452, 688052, 601652, 515252, 428852, 342452, 256052, 169652, 83252, -3148, 0 ]
+}, {
+  "tz" : "Asia/Ulan_Bator",
+  "switches" : [ -62135622452, -59006387252, -55850713652, -52695040052, -46383606452, -43227932852, -40072259252, -33760825652, -30605152052, -27449478452, -21138044852, -17982371252, -14826697652, -12220096052, -12220009652, -12219923252, -12219836852, -12219750452, -12219664052, -12219577652, -12219491252, -12219404852, -12219318452, -2208988800 ],
+  "diffs" : [ -175948, -89548, -3148, 83252, 169652, 256052, 342452, 428852, 515252, 601652, 688052, 774452, 860852, 774452, 688052, 601652, 515252, 428852, 342452, 256052, 169652, 83252, -3148, 0 ]
+}, {
+  "tz" : "Asia/Urumqi",
+  "switches" : [ -62135617820, -59006382620, -55850709020, -52695035420, -46383601820, -43227928220, -40072254620, -33760821020, -30605147420, -27449473820, -21138040220, -17982366620, -14826693020, -12220091420, -12220005020, -12219918620, -12219832220, -12219745820, -12219659420, -12219573020, -12219486620, -12219400220, -12219313820, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ]
+}, {
+  "tz" : "Asia/Ust-Nera",
+  "switches" : [ -62135631174, -59006395974, -55850722374, -52695048774, -46383615174, -43227941574, -40072267974, -33760834374, -30605160774, -27449487174, -21138053574, -17982379974, -14826706374, -12220104774, -12220018374, -12219931974, -12219845574, -12219759174, -12219672774, -12219586374, -12219499974, -12219413574, -12219327174, -2208988800 ],
+  "diffs" : [ -174426, -88026, -1626, 84774, 171174, 257574, 343974, 430374, 516774, 603174, 689574, 775974, 862374, 775974, 689574, 603174, 516774, 430374, 343974, 257574, 171174, 84774, -1626, 0 ]
+}, {
+  "tz" : "Asia/Vientiane",
+  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12220094524, -12220008124, -12219921724, -12219835324, -12219748924, -12219662524, -12219576124, -12219489724, -12219403324, -12219316924, -2208988800 ],
+  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, 776524, 690124, 603724, 517324, 430924, 344524, 258124, 171724, 85324, -1076, 0 ]
+}, {
+  "tz" : "Asia/Vladivostok",
+  "switches" : [ -62135628451, -59006393251, -55850719651, -52695046051, -46383612451, -43227938851, -40072265251, -33760831651, -30605158051, -27449484451, -21138050851, -17982377251, -14826703651, -12220102051, -12220015651, -12219929251, -12219842851, -12219756451, -12219670051, -12219583651, -12219497251, -12219410851, -12219324451, -2208988800 ],
+  "diffs" : [ -177149, -90749, -4349, 82051, 168451, 254851, 341251, 427651, 514051, 600451, 686851, 773251, 859651, 773251, 686851, 600451, 514051, 427651, 341251, 254851, 168451, 82051, -4349, 0 ]
+}, {
+  "tz" : "Asia/Yakutsk",
+  "switches" : [ -62135627938, -59006392738, -55850719138, -52695045538, -46383611938, -43227938338, -40072264738, -33760831138, -30605157538, -27449483938, -21138050338, -17982376738, -14826703138, -12220101538, -12220015138, -12219928738, -12219842338, -12219755938, -12219669538, -12219583138, -12219496738, -12219410338, -12219323938, -2208988800 ],
+  "diffs" : [ -174062, -87662, -1262, 85138, 171538, 257938, 344338, 430738, 517138, 603538, 689938, 776338, 862738, 776338, 689938, 603538, 517138, 430738, 344338, 257938, 171538, 85138, -1262, 0 ]
+}, {
+  "tz" : "Asia/Yangon",
+  "switches" : [ -62135619887, -59006384687, -55850711087, -52695037487, -46383603887, -43227930287, -40072256687, -33760823087, -30605149487, -27449475887, -21138042287, -17982368687, -14826695087, -12220093487, -12220007087, -12219920687, -12219834287, -12219747887, -12219661487, -12219575087, -12219488687, -12219402287, -12219315887, -2208988800 ],
+  "diffs" : [ -173113, -86713, -313, 86087, 172487, 258887, 345287, 431687, 518087, 604487, 690887, 777287, 863687, 777287, 690887, 604487, 518087, 431687, 345287, 258887, 172487, 86087, -313, 0 ]
+}, {
+  "tz" : "Asia/Yekaterinburg",
+  "switches" : [ -62135611353, -59006376153, -55850702553, -52695028953, -46383595353, -43227921753, -40072248153, -33760814553, -30605140953, -27449467353, -21138033753, -17982360153, -14826686553, -12220084953, -12219998553, -12219912153, -12219825753, -12219739353, -12219652953, -12219566553, -12219480153, -12219393753, -12219307353, -2208988800 ],
+  "diffs" : [ -176247, -89847, -3447, 82953, 169353, 255753, 342153, 428553, 514953, 601353, 687753, 774153, 860553, 774153, 687753, 601353, 514953, 428553, 342153, 255753, 169353, 82953, -3447, 0 ]
+}, {
+  "tz" : "Asia/Yerevan",
+  "switches" : [ -62135607480, -59006372280, -55850698680, -52695025080, -46383591480, -43227917880, -40072244280, -33760810680, -30605137080, -27449463480, -21138029880, -17982356280, -14826682680, -12220081080, -12219994680, -12219908280, -12219821880, -12219735480, -12219649080, -12219562680, -12219476280, -12219389880, -12219303480, -2208988800 ],
+  "diffs" : [ -176520, -90120, -3720, 82680, 169080, 255480, 341880, 428280, 514680, 601080, 687480, 773880, 860280, 773880, 687480, 601080, 514680, 428280, 341880, 255480, 169080, 82680, -3720, 0 ]
+}, {
+  "tz" : "Atlantic/Azores",
+  "switches" : [ -62135590640, -59006355440, -55850681840, -52695008240, -46383574640, -43227901040, -40072227440, -33760793840, -30605120240, -27449446640, -21138013040, -17982339440, -14826665840, -12220064240, -12219977840, -12219891440, -12219805040, -12219718640, -12219632240, -12219545840, -12219459440, -12219373040, -12219286640, -2713904240, -2208988800 ],
+  "diffs" : [ -175360, -88960, -2560, 83840, 170240, 256640, 343040, 429440, 515840, 602240, 688640, 775040, 861440, 775040, 688640, 602240, 515840, 429440, 343040, 256640, 170240, 83840, -2560, -3272, 0 ]
+}, {
+  "tz" : "Atlantic/Bermuda",
+  "switches" : [ -62135581242, -59006346042, -55850672442, -52694998842, -46383565242, -43227891642, -40072218042, -33760784442, -30605110842, -27449437242, -21138003642, -17982330042, -14826656442, -12220054842, -12219968442, -12219882042, -12219795642, -12219709242, -12219622842, -12219536442, -12219450042, -12219363642, -12219277242, -2208988800 ],
+  "diffs" : [ -173958, -87558, -1158, 85242, 171642, 258042, 344442, 430842, 517242, 603642, 690042, 776442, 862842, 776442, 690042, 603642, 517242, 430842, 344442, 258042, 171642, 85242, -1158, 0 ]
+}, {
+  "tz" : "Atlantic/Canary",
+  "switches" : [ -62135593104, -59006357904, -55850684304, -52695010704, -46383577104, -43227903504, -40072229904, -33760796304, -30605122704, -27449449104, -21138015504, -17982341904, -14826668304, -12220066704, -12219980304, -12219893904, -12219807504, -12219721104, -12219634704, -12219548304, -12219461904, -12219375504, -12219289104, -2208988800 ],
+  "diffs" : [ -176496, -90096, -3696, 82704, 169104, 255504, 341904, 428304, 514704, 601104, 687504, 773904, 860304, 773904, 687504, 601104, 514704, 428304, 341904, 255504, 169104, 82704, -3696, 0 ]
+}, {
+  "tz" : "Atlantic/Cape_Verde",
+  "switches" : [ -62135591156, -59006355956, -55850682356, -52695008756, -46383575156, -43227901556, -40072227956, -33760794356, -30605120756, -27449447156, -21138013556, -17982339956, -14826666356, -12220064756, -12219978356, -12219891956, -12219805556, -12219719156, -12219632756, -12219546356, -12219459956, -12219373556, -12219287156, -2208988800 ],
+  "diffs" : [ -174844, -88444, -2044, 84356, 170756, 257156, 343556, 429956, 516356, 602756, 689156, 775556, 861956, 775556, 689156, 602756, 516356, 429956, 343556, 257156, 170756, 84356, -2044, 0 ]
+}, {
+  "tz" : "Atlantic/Faeroe",
+  "switches" : [ -62135595176, -59006359976, -55850686376, -52695012776, -46383579176, -43227905576, -40072231976, -33760798376, -30605124776, -27449451176, -21138017576, -17982343976, -14826670376, -12220068776, -12219982376, -12219895976, -12219809576, -12219723176, -12219636776, -12219550376, -12219463976, -12219377576, -12219291176, -2208988800 ],
+  "diffs" : [ -174424, -88024, -1624, 84776, 171176, 257576, 343976, 430376, 516776, 603176, 689576, 775976, 862376, 775976, 689576, 603176, 516776, 430376, 343976, 257576, 171176, 84776, -1624, 0 ]
+}, {
+  "tz" : "Atlantic/Faroe",
+  "switches" : [ -62135595176, -59006359976, -55850686376, -52695012776, -46383579176, -43227905576, -40072231976, -33760798376, -30605124776, -27449451176, -21138017576, -17982343976, -14826670376, -12220068776, -12219982376, -12219895976, -12219809576, -12219723176, -12219636776, -12219550376, -12219463976, -12219377576, -12219291176, -2208988800 ],
+  "diffs" : [ -174424, -88024, -1624, 84776, 171176, 257576, 343976, 430376, 516776, 603176, 689576, 775976, 862376, 775976, 689576, 603176, 516776, 430376, 343976, 257576, 171176, 84776, -1624, 0 ]
+}, {
+  "tz" : "Atlantic/Jan_Mayen",
+  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12220072980, -12219986580, -12219900180, -12219813780, -12219727380, -12219640980, -12219554580, -12219468180, -12219381780, -12219295380, -2366757780 ],
+  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, 776580, 690180, 603780, 517380, 430980, 344580, 258180, 171780, 85380, -1020, 0 ]
+}, {
+  "tz" : "Atlantic/Madeira",
+  "switches" : [ -62135592744, -59006357544, -55850683944, -52695010344, -46383576744, -43227903144, -40072229544, -33760795944, -30605122344, -27449448744, -21138015144, -17982341544, -14826667944, -12220066344, -12219979944, -12219893544, -12219807144, -12219720744, -12219634344, -12219547944, -12219461544, -12219375144, -12219288744, -2208988800 ],
+  "diffs" : [ -176856, -90456, -4056, 82344, 168744, 255144, 341544, 427944, 514344, 600744, 687144, 773544, 859944, 773544, 687144, 600744, 514344, 427944, 341544, 255144, 168744, 82344, -4056, 0 ]
+}, {
+  "tz" : "Atlantic/Reykjavik",
+  "switches" : [ -62135591520, -59006356320, -55850682720, -52695009120, -46383575520, -43227901920, -40072228320, -33760794720, -30605121120, -27449447520, -21138013920, -17982340320, -14826666720, -12220065120, -12219978720, -12219892320, -12219805920, -12219719520, -12219633120, -12219546720, -12219460320, -12219373920, -12219287520, -2208988800 ],
+  "diffs" : [ -178080, -91680, -5280, 81120, 167520, 253920, 340320, 426720, 513120, 599520, 685920, 772320, 858720, 772320, 685920, 599520, 513120, 426720, 340320, 253920, 167520, 81120, -5280, 0 ]
+}, {
+  "tz" : "Atlantic/South_Georgia",
+  "switches" : [ -62135588032, -59006352832, -55850679232, -52695005632, -46383572032, -43227898432, -40072224832, -33760791232, -30605117632, -27449444032, -21138010432, -17982336832, -14826663232, -12220061632, -12219975232, -12219888832, -12219802432, -12219716032, -12219629632, -12219543232, -12219456832, -12219370432, -12219284032, -2524512832 ],
+  "diffs" : [ -174368, -87968, -1568, 84832, 171232, 257632, 344032, 430432, 516832, 603232, 689632, 776032, 862432, 776032, 689632, 603232, 516832, 430432, 344032, 257632, 171232, 84832, -1568, 0 ]
+}, {
+  "tz" : "Atlantic/St_Helena",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
+}, {
+  "tz" : "Atlantic/Stanley",
+  "switches" : [ -62135582916, -59006347716, -55850674116, -52695000516, -46383566916, -43227893316, -40072219716, -33760786116, -30605112516, -27449438916, -21138005316, -17982331716, -14826658116, -12220056516, -12219970116, -12219883716, -12219797316, -12219710916, -12219624516, -12219538116, -12219451716, -12219365316, -12219278916, -2208988800 ],
+  "diffs" : [ -175884, -89484, -3084, 83316, 169716, 256116, 342516, 428916, 515316, 601716, 688116, 774516, 860916, 774516, 688116, 601716, 515316, 428916, 342516, 256116, 169716, 83316, -3084, 0 ]
+}, {
+  "tz" : "Australia/ACT",
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12220106692, -12220020292, -12219933892, -12219847492, -12219761092, -12219674692, -12219588292, -12219501892, -12219415492, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 777892, 691492, 605092, 518692, 432292, 345892, 259492, 173092, 86692, 292, 0 ]
+}, {
+  "tz" : "Australia/Adelaide",
+  "switches" : [ -62135630060, -59006394860, -55850721260, -52695047660, -46383614060, -43227940460, -40072266860, -33760833260, -30605159660, -27449486060, -21138052460, -17982378860, -14826705260, -12220103660, -12220017260, -12219930860, -12219844460, -12219758060, -12219671660, -12219585260, -12219498860, -12219412460, -12219326060, -2364110060, -2230189200 ],
+  "diffs" : [ -173740, -87340, -940, 85460, 171860, 258260, 344660, 431060, 517460, 603860, 690260, 776660, 863060, 776660, 690260, 603860, 517460, 431060, 344660, 258260, 171860, 85460, -940, -1800, 0 ]
+}, {
+  "tz" : "Australia/Brisbane",
+  "switches" : [ -62135633528, -59006398328, -55850724728, -52695051128, -46383617528, -43227943928, -40072270328, -33760836728, -30605163128, -27449489528, -21138055928, -17982382328, -14826708728, -12220107128, -12220020728, -12219934328, -12219847928, -12219761528, -12219675128, -12219588728, -12219502328, -12219415928, -12219329528, -2366791928 ],
+  "diffs" : [ -172072, -85672, 728, 87128, 173528, 259928, 346328, 432728, 519128, 605528, 691928, 778328, 864728, 778328, 691928, 605528, 519128, 432728, 346328, 259928, 173528, 87128, 728, 0 ]
+}, {
+  "tz" : "Australia/Broken_Hill",
+  "switches" : [ -62135630748, -59006395548, -55850721948, -52695048348, -46383614748, -43227941148, -40072267548, -33760833948, -30605160348, -27449486748, -21138053148, -17982379548, -14826705948, -12220104348, -12220017948, -12219931548, -12219845148, -12219758748, -12219672348, -12219585948, -12219499548, -12219413148, -12219326748, -2364110748, -2314951200, -2230189200 ],
+  "diffs" : [ -173052, -86652, -252, 86148, 172548, 258948, 345348, 431748, 518148, 604548, 690948, 777348, 863748, 777348, 690948, 604548, 518148, 431748, 345348, 258948, 172548, 86148, -252, 1800, -1800, 0 ]
+}, {
+  "tz" : "Australia/Canberra",
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12220106692, -12220020292, -12219933892, -12219847492, -12219761092, -12219674692, -12219588292, -12219501892, -12219415492, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 777892, 691492, 605092, 518692, 432292, 345892, 259492, 173092, 86692, 292, 0 ]
+}, {
+  "tz" : "Australia/Currie",
+  "switches" : [ -62135631328, -59006396128, -55850722528, -52695048928, -46383615328, -43227941728, -40072268128, -33760834528, -30605160928, -27449487328, -21138053728, -17982380128, -14826706528, -12220104928, -12220018528, -12219932128, -12219845728, -12219759328, -12219672928, -12219586528, -12219500128, -12219413728, -12219327328, -2345794528 ],
+  "diffs" : [ -174272, -87872, -1472, 84928, 171328, 257728, 344128, 430528, 516928, 603328, 689728, 776128, 862528, 776128, 689728, 603328, 516928, 430528, 344128, 257728, 171328, 84928, -1472, 0 ]
+}, {
+  "tz" : "Australia/Darwin",
+  "switches" : [ -62135628200, -59006393000, -55850719400, -52695045800, -46383612200, -43227938600, -40072265000, -33760831400, -30605157800, -27449484200, -21138050600, -17982377000, -14826703400, -12220101800, -12220015400, -12219929000, -12219842600, -12219756200, -12219669800, -12219583400, -12219497000, -12219410600, -12219324200, -2364108200, -2230189200 ],
+  "diffs" : [ -175600, -89200, -2800, 83600, 170000, 256400, 342800, 429200, 515600, 602000, 688400, 774800, 861200, 774800, 688400, 602000, 515600, 429200, 342800, 256400, 170000, 83600, -2800, -1800, 0 ]
+}, {
+  "tz" : "Australia/Eucla",
+  "switches" : [ -62135627728, -59006392528, -55850718928, -52695045328, -46383611728, -43227938128, -40072264528, -33760830928, -30605157328, -27449483728, -21138050128, -17982376528, -14826702928, -12220101328, -12220014928, -12219928528, -12219842128, -12219755728, -12219669328, -12219582928, -12219496528, -12219410128, -12219323728, -2337928528 ],
+  "diffs" : [ -173372, -86972, -572, 85828, 172228, 258628, 345028, 431428, 517828, 604228, 690628, 777028, 863428, 777028, 690628, 604228, 517828, 431428, 345028, 258628, 172228, 85828, -572, 0 ]
+}, {
+  "tz" : "Australia/Hobart",
+  "switches" : [ -62135632156, -59006396956, -55850723356, -52695049756, -46383616156, -43227942556, -40072268956, -33760835356, -30605161756, -27449488156, -21138054556, -17982380956, -14826707356, -12220105756, -12220019356, -12219932956, -12219846556, -12219760156, -12219673756, -12219587356, -12219500956, -12219414556, -12219328156, -2345795356 ],
+  "diffs" : [ -173444, -87044, -644, 85756, 172156, 258556, 344956, 431356, 517756, 604156, 690556, 776956, 863356, 776956, 690556, 604156, 517756, 431356, 344956, 258556, 172156, 85756, -644, 0 ]
+}, {
+  "tz" : "Australia/LHI",
+  "switches" : [ -62135634980, -59006399780, -55850726180, -52695052580, -46383618980, -43227945380, -40072271780, -33760838180, -30605164580, -27449490980, -21138057380, -17982383780, -14826710180, -12220108580, -12220022180, -12219935780, -12219849380, -12219762980, -12219676580, -12219590180, -12219503780, -12219417380, -12219330980, -2364114980, -2208988800 ],
+  "diffs" : [ -172420, -86020, 380, 86780, 173180, 259580, 345980, 432380, 518780, 605180, 691580, 777980, 864380, 777980, 691580, 605180, 518780, 432380, 345980, 259580, 173180, 86780, 380, -1800, 0 ]
+}, {
+  "tz" : "Australia/Lindeman",
+  "switches" : [ -62135632556, -59006397356, -55850723756, -52695050156, -46383616556, -43227942956, -40072269356, -33760835756, -30605162156, -27449488556, -21138054956, -17982381356, -14826707756, -12220106156, -12220019756, -12219933356, -12219846956, -12219760556, -12219674156, -12219587756, -12219501356, -12219414956, -12219328556, -2366790956 ],
+  "diffs" : [ -173044, -86644, -244, 86156, 172556, 258956, 345356, 431756, 518156, 604556, 690956, 777356, 863756, 777356, 690956, 604556, 518156, 431756, 345356, 258956, 172556, 86156, -244, 0 ]
+}, {
+  "tz" : "Australia/Lord_Howe",
+  "switches" : [ -62135634980, -59006399780, -55850726180, -52695052580, -46383618980, -43227945380, -40072271780, -33760838180, -30605164580, -27449490980, -21138057380, -17982383780, -14826710180, -12220108580, -12220022180, -12219935780, -12219849380, -12219762980, -12219676580, -12219590180, -12219503780, -12219417380, -12219330980, -2364114980, -2208988800 ],
+  "diffs" : [ -172420, -86020, 380, 86780, 173180, 259580, 345980, 432380, 518780, 605180, 691580, 777980, 864380, 777980, 691580, 605180, 518780, 432380, 345980, 259580, 173180, 86780, 380, -1800, 0 ]
+}, {
+  "tz" : "Australia/Melbourne",
+  "switches" : [ -62135631592, -59006396392, -55850722792, -52695049192, -46383615592, -43227941992, -40072268392, -33760834792, -30605161192, -27449487592, -21138053992, -17982380392, -14826706792, -12220105192, -12220018792, -12219932392, -12219845992, -12219759592, -12219673192, -12219586792, -12219500392, -12219413992, -12219327592, -2364111592 ],
+  "diffs" : [ -174008, -87608, -1208, 85192, 171592, 257992, 344392, 430792, 517192, 603592, 689992, 776392, 862792, 776392, 689992, 603592, 517192, 430792, 344392, 257992, 171592, 85192, -1208, 0 ]
+}, {
+  "tz" : "Australia/NSW",
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12220106692, -12220020292, -12219933892, -12219847492, -12219761092, -12219674692, -12219588292, -12219501892, -12219415492, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 777892, 691492, 605092, 518692, 432292, 345892, 259492, 173092, 86692, 292, 0 ]
+}, {
+  "tz" : "Australia/North",
+  "switches" : [ -62135628200, -59006393000, -55850719400, -52695045800, -46383612200, -43227938600, -40072265000, -33760831400, -30605157800, -27449484200, -21138050600, -17982377000, -14826703400, -12220101800, -12220015400, -12219929000, -12219842600, -12219756200, -12219669800, -12219583400, -12219497000, -12219410600, -12219324200, -2364108200, -2230189200 ],
+  "diffs" : [ -175600, -89200, -2800, 83600, 170000, 256400, 342800, 429200, 515600, 602000, 688400, 774800, 861200, 774800, 688400, 602000, 515600, 429200, 342800, 256400, 170000, 83600, -2800, -1800, 0 ]
+}, {
+  "tz" : "Australia/Perth",
+  "switches" : [ -62135624604, -59006389404, -55850715804, -52695042204, -46383608604, -43227935004, -40072261404, -33760827804, -30605154204, -27449480604, -21138047004, -17982373404, -14826699804, -12220098204, -12220011804, -12219925404, -12219839004, -12219752604, -12219666204, -12219579804, -12219493404, -12219407004, -12219320604, -2337925404 ],
+  "diffs" : [ -173796, -87396, -996, 85404, 171804, 258204, 344604, 431004, 517404, 603804, 690204, 776604, 863004, 776604, 690204, 603804, 517404, 431004, 344604, 258204, 171804, 85404, -996, 0 ]
+}, {
+  "tz" : "Australia/Queensland",
+  "switches" : [ -62135633528, -59006398328, -55850724728, -52695051128, -46383617528, -43227943928, -40072270328, -33760836728, -30605163128, -27449489528, -21138055928, -17982382328, -14826708728, -12220107128, -12220020728, -12219934328, -12219847928, -12219761528, -12219675128, -12219588728, -12219502328, -12219415928, -12219329528, -2366791928 ],
+  "diffs" : [ -172072, -85672, 728, 87128, 173528, 259928, 346328, 432728, 519128, 605528, 691928, 778328, 864728, 778328, 691928, 605528, 519128, 432728, 346328, 259928, 173528, 87128, 728, 0 ]
+}, {
+  "tz" : "Australia/South",
+  "switches" : [ -62135630060, -59006394860, -55850721260, -52695047660, -46383614060, -43227940460, -40072266860, -33760833260, -30605159660, -27449486060, -21138052460, -17982378860, -14826705260, -12220103660, -12220017260, -12219930860, -12219844460, -12219758060, -12219671660, -12219585260, -12219498860, -12219412460, -12219326060, -2364110060, -2230189200 ],
+  "diffs" : [ -173740, -87340, -940, 85460, 171860, 258260, 344660, 431060, 517460, 603860, 690260, 776660, 863060, 776660, 690260, 603860, 517460, 431060, 344660, 258260, 171860, 85460, -940, -1800, 0 ]
+}, {
+  "tz" : "Australia/Sydney",
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12220106692, -12220020292, -12219933892, -12219847492, -12219761092, -12219674692, -12219588292, -12219501892, -12219415492, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 777892, 691492, 605092, 518692, 432292, 345892, 259492, 173092, 86692, 292, 0 ]
+}, {
+  "tz" : "Australia/Tasmania",
+  "switches" : [ -62135632156, -59006396956, -55850723356, -52695049756, -46383616156, -43227942556, -40072268956, -33760835356, -30605161756, -27449488156, -21138054556, -17982380956, -14826707356, -12220105756, -12220019356, -12219932956, -12219846556, -12219760156, -12219673756, -12219587356, -12219500956, -12219414556, -12219328156, -2345795356 ],
+  "diffs" : [ -173444, -87044, -644, 85756, 172156, 258556, 344956, 431356, 517756, 604156, 690556, 776956, 863356, 776956, 690556, 604156, 517756, 431356, 344956, 258556, 172156, 85756, -644, 0 ]
+}, {
+  "tz" : "Australia/Victoria",
+  "switches" : [ -62135631592, -59006396392, -55850722792, -52695049192, -46383615592, -43227941992, -40072268392, -33760834792, -30605161192, -27449487592, -21138053992, -17982380392, -14826706792, -12220105192, -12220018792, -12219932392, -12219845992, -12219759592, -12219673192, -12219586792, -12219500392, -12219413992, -12219327592, -2364111592 ],
+  "diffs" : [ -174008, -87608, -1208, 85192, 171592, 257992, 344392, 430792, 517192, 603592, 689992, 776392, 862792, 776392, 689992, 603592, 517192, 430792, 344392, 257992, 171592, 85192, -1208, 0 ]
+}, {
+  "tz" : "Australia/West",
+  "switches" : [ -62135624604, -59006389404, -55850715804, -52695042204, -46383608604, -43227935004, -40072261404, -33760827804, -30605154204, -27449480604, -21138047004, -17982373404, -14826699804, -12220098204, -12220011804, -12219925404, -12219839004, -12219752604, -12219666204, -12219579804, -12219493404, -12219407004, -12219320604, -2337925404 ],
+  "diffs" : [ -173796, -87396, -996, 85404, 171804, 258204, 344604, 431004, 517404, 603804, 690204, 776604, 863004, 776604, 690204, 603804, 517404, 431004, 344604, 258204, 171804, 85404, -996, 0 ]
+}, {
+  "tz" : "Australia/Yancowinna",
+  "switches" : [ -62135630748, -59006395548, -55850721948, -52695048348, -46383614748, -43227941148, -40072267548, -33760833948, -30605160348, -27449486748, -21138053148, -17982379548, -14826705948, -12220104348, -12220017948, -12219931548, -12219845148, -12219758748, -12219672348, -12219585948, -12219499548, -12219413148, -12219326748, -2364110748, -2314951200, -2230189200 ],
+  "diffs" : [ -173052, -86652, -252, 86148, 172548, 258948, 345348, 431748, 518148, 604548, 690948, 777348, 863748, 777348, 690948, 604548, 518148, 431748, 345348, 258948, 172548, 86148, -252, 1800, -1800, 0 ]
+}, {
+  "tz" : "Brazil/Acre",
+  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12220054128, -12219967728, -12219881328, -12219794928, -12219708528, -12219622128, -12219535728, -12219449328, -12219362928, -12219276528, -2208988800 ],
+  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 779328, 692928, 606528, 520128, 433728, 347328, 260928, 174528, 88128, 1728, 0 ]
+}, {
+  "tz" : "Brazil/DeNoronha",
+  "switches" : [ -62135589020, -59006353820, -55850680220, -52695006620, -46383573020, -43227899420, -40072225820, -33760792220, -30605118620, -27449445020, -21138011420, -17982337820, -14826664220, -12220062620, -12219976220, -12219889820, -12219803420, -12219717020, -12219630620, -12219544220, -12219457820, -12219371420, -12219285020, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ]
+}, {
+  "tz" : "Brazil/East",
+  "switches" : [ -62135585612, -59006350412, -55850676812, -52695003212, -46383569612, -43227896012, -40072222412, -33760788812, -30605115212, -27449441612, -21138008012, -17982334412, -14826660812, -12220059212, -12219972812, -12219886412, -12219800012, -12219713612, -12219627212, -12219540812, -12219454412, -12219368012, -12219281612, -2208988800 ],
+  "diffs" : [ -173188, -86788, -388, 86012, 172412, 258812, 345212, 431612, 518012, 604412, 690812, 777212, 863612, 777212, 690812, 604412, 518012, 431612, 345212, 258812, 172412, 86012, -388, 0 ]
+}, {
+  "tz" : "Brazil/West",
+  "switches" : [ -62135582396, -59006347196, -55850673596, -52694999996, -46383566396, -43227892796, -40072219196, -33760785596, -30605111996, -27449438396, -21138004796, -17982331196, -14826657596, -12220055996, -12219969596, -12219883196, -12219796796, -12219710396, -12219623996, -12219537596, -12219451196, -12219364796, -12219278396, -2208988800 ],
+  "diffs" : [ -172804, -86404, -4, 86396, 172796, 259196, 345596, 431996, 518396, 604796, 691196, 777596, 863996, 777596, 691196, 604796, 518396, 431996, 345596, 259196, 172796, 86396, -4, 0 ]
+}, {
+  "tz" : "CET",
+  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12220074000, -12219987600, -12219901200, -12219814800, -12219728400, -12219642000, -12219555600, -12219469200, -12219382800, -12219296400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "CST6CDT",
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12220048800, -12219962400, -12219876000, -12219789600, -12219703200, -12219616800, -12219530400, -12219444000, -12219357600, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Canada/Atlantic",
+  "switches" : [ -62135581536, -59006346336, -55850672736, -52694999136, -46383565536, -43227891936, -40072218336, -33760784736, -30605111136, -27449437536, -21138003936, -17982330336, -14826656736, -12220055136, -12219968736, -12219882336, -12219795936, -12219709536, -12219623136, -12219536736, -12219450336, -12219363936, -12219277536, -2208988800 ],
+  "diffs" : [ -173664, -87264, -864, 85536, 171936, 258336, 344736, 431136, 517536, 603936, 690336, 776736, 863136, 776736, 690336, 603936, 517536, 431136, 344736, 258336, 171936, 85536, -864, 0 ]
+}, {
+  "tz" : "Canada/Central",
+  "switches" : [ -62135573484, -59006338284, -55850664684, -52694991084, -46383557484, -43227883884, -40072210284, -33760776684, -30605103084, -27449429484, -21137995884, -17982322284, -14826648684, -12220047084, -12219960684, -12219874284, -12219787884, -12219701484, -12219615084, -12219528684, -12219442284, -12219355884, -12219269484, -2602258284 ],
+  "diffs" : [ -174516, -88116, -1716, 84684, 171084, 257484, 343884, 430284, 516684, 603084, 689484, 775884, 862284, 775884, 689484, 603084, 516684, 430284, 343884, 257484, 171084, 84684, -1716, 0 ]
+}, {
+  "tz" : "Canada/Eastern",
+  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12220051348, -12219964948, -12219878548, -12219792148, -12219705748, -12219619348, -12219532948, -12219446548, -12219360148, -12219273748, -2366736148 ],
+  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, 776548, 690148, 603748, 517348, 430948, 344548, 258148, 171748, 85348, -1052, 0 ]
+}, {
+  "tz" : "Canada/Mountain",
+  "switches" : [ -62135569568, -59006334368, -55850660768, -52694987168, -46383553568, -43227879968, -40072206368, -33760772768, -30605099168, -27449425568, -21137991968, -17982318368, -14826644768, -12220043168, -12219956768, -12219870368, -12219783968, -12219697568, -12219611168, -12219524768, -12219438368, -12219351968, -12219265568, -2208988800 ],
+  "diffs" : [ -174832, -88432, -2032, 84368, 170768, 257168, 343568, 429968, 516368, 602768, 689168, 775568, 861968, 775568, 689168, 602768, 516368, 429968, 343568, 257168, 170768, 84368, -2032, 0 ]
+}, {
+  "tz" : "Canada/Newfoundland",
+  "switches" : [ -62135584148, -59006348948, -55850675348, -52695001748, -46383568148, -43227894548, -40072220948, -33760787348, -30605113748, -27449440148, -21138006548, -17982332948, -14826659348, -12220057748, -12219971348, -12219884948, -12219798548, -12219712148, -12219625748, -12219539348, -12219452948, -12219366548, -12219280148, -2208988800 ],
+  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, 777548, 691148, 604748, 518348, 431948, 345548, 259148, 172748, 86348, -52, 0 ]
+}, {
+  "tz" : "Canada/Pacific",
+  "switches" : [ -62135567252, -59006332052, -55850658452, -52694984852, -46383551252, -43227877652, -40072204052, -33760770452, -30605096852, -27449423252, -21137989652, -17982316052, -14826642452, -12220040852, -12219954452, -12219868052, -12219781652, -12219695252, -12219608852, -12219522452, -12219436052, -12219349652, -12219263252, -2713880852 ],
+  "diffs" : [ -173548, -87148, -748, 85652, 172052, 258452, 344852, 431252, 517652, 604052, 690452, 776852, 863252, 776852, 690452, 604052, 517652, 431252, 344852, 258452, 172052, 85652, -748, 0 ]
+}, {
+  "tz" : "Canada/Saskatchewan",
+  "switches" : [ -62135571684, -59006336484, -55850662884, -52694989284, -46383555684, -43227882084, -40072208484, -33760774884, -30605101284, -27449427684, -21137994084, -17982320484, -14826646884, -12220045284, -12219958884, -12219872484, -12219786084, -12219699684, -12219613284, -12219526884, -12219440484, -12219354084, -12219267684, -2208988800 ],
+  "diffs" : [ -176316, -89916, -3516, 82884, 169284, 255684, 342084, 428484, 514884, 601284, 687684, 774084, 860484, 774084, 687684, 601284, 514884, 428484, 342084, 255684, 169284, 82884, -3516, 0 ]
+}, {
+  "tz" : "Canada/Yukon",
+  "switches" : [ -62135564388, -59006329188, -55850655588, -52694981988, -46383548388, -43227874788, -40072201188, -33760767588, -30605093988, -27449420388, -21137986788, -17982313188, -14826639588, -12220037988, -12219951588, -12219865188, -12219778788, -12219692388, -12219605988, -12219519588, -12219433188, -12219346788, -12219260388, -2208988800 ],
+  "diffs" : [ -176412, -90012, -3612, 82788, 169188, 255588, 341988, 428388, 514788, 601188, 687588, 773988, 860388, 773988, 687588, 601188, 514788, 428388, 341988, 255588, 169188, 82788, -3612, 0 ]
+}, {
+  "tz" : "Chile/Continental",
+  "switches" : [ -62135579834, -59006344634, -55850671034, -52694997434, -46383563834, -43227890234, -40072216634, -33760783034, -30605109434, -27449435834, -21138002234, -17982328634, -14826655034, -12220053434, -12219967034, -12219880634, -12219794234, -12219707834, -12219621434, -12219535034, -12219448634, -12219362234, -12219275834, -2208988800 ],
+  "diffs" : [ -175366, -88966, -2566, 83834, 170234, 256634, 343034, 429434, 515834, 602234, 688634, 775034, 861434, 775034, 688634, 602234, 515834, 429434, 343034, 256634, 170234, 83834, -2566, 0 ]
+}, {
+  "tz" : "Chile/EasterIsland",
+  "switches" : [ -62135570552, -59006335352, -55850661752, -52694988152, -46383554552, -43227880952, -40072207352, -33760773752, -30605100152, -27449426552, -21137992952, -17982319352, -14826645752, -12220044152, -12219957752, -12219871352, -12219784952, -12219698552, -12219612152, -12219525752, -12219439352, -12219352952, -12219266552, -2208988800 ],
+  "diffs" : [ -177448, -91048, -4648, 81752, 168152, 254552, 340952, 427352, 513752, 600152, 686552, 772952, 859352, 772952, 686552, 600152, 513752, 427352, 340952, 254552, 168152, 81752, -4648, 0 ]
+}, {
+  "tz" : "Cuba",
+  "switches" : [ -62135577032, -59006341832, -55850668232, -52694994632, -46383561032, -43227887432, -40072213832, -33760780232, -30605106632, -27449433032, -21137999432, -17982325832, -14826652232, -12220050632, -12219964232, -12219877832, -12219791432, -12219705032, -12219618632, -12219532232, -12219445832, -12219359432, -12219273032, -2524501832, -2208988800 ],
+  "diffs" : [ -174568, -88168, -1768, 84632, 171032, 257432, 343832, 430232, 516632, 603032, 689432, 775832, 862232, 775832, 689432, 603032, 516632, 430232, 343832, 257432, 171032, 84632, -1768, -1776, 0 ]
+}, {
+  "tz" : "EET",
+  "switches" : [ -62135604000, -59006368800, -55850695200, -52695021600, -46383588000, -43227914400, -40072240800, -33760807200, -30605133600, -27449460000, -21138026400, -17982352800, -14826679200, -12220077600, -12219991200, -12219904800, -12219818400, -12219732000, -12219645600, -12219559200, -12219472800, -12219386400, -12219300000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "EST5EDT",
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12220052400, -12219966000, -12219879600, -12219793200, -12219706800, -12219620400, -12219534000, -12219447600, -12219361200, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Egypt",
+  "switches" : [ -62135604309, -59006369109, -55850695509, -52695021909, -46383588309, -43227914709, -40072241109, -33760807509, -30605133909, -27449460309, -21138026709, -17982353109, -14826679509, -12220077909, -12219991509, -12219905109, -12219818709, -12219732309, -12219645909, -12219559509, -12219473109, -12219386709, -12219300309, -2208988800 ],
+  "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 777909, 691509, 605109, 518709, 432309, 345909, 259509, 173109, 86709, 309, 0 ]
+}, {
+  "tz" : "Eire",
+  "switches" : [ -62135595300, -59006360100, -55850686500, -52695012900, -46383579300, -43227905700, -40072232100, -33760798500, -30605124900, -27449451300, -21138017700, -17982344100, -14826670500, -12220068900, -12219982500, -12219896100, -12219809700, -12219723300, -12219636900, -12219550500, -12219464100, -12219377700, -12219291300, -2821649700, -2208988800 ],
+  "diffs" : [ -174300, -87900, -1500, 84900, 171300, 257700, 344100, 430500, 516900, 603300, 689700, 776100, 862500, 776100, 689700, 603300, 516900, 430500, 344100, 257700, 171300, 84900, -1500, -1521, 0 ]
+}, {
+  "tz" : "Etc/GMT",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+0",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+1",
+  "switches" : [ -62135593200, -59006358000, -55850684400, -52695010800, -46383577200, -43227903600, -40072230000, -33760796400, -30605122800, -27449449200, -21138015600, -17982342000, -14826668400, -12220066800, -12219980400, -12219894000, -12219807600, -12219721200, -12219634800, -12219548400, -12219462000, -12219375600, -12219289200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+10",
+  "switches" : [ -62135560800, -59006325600, -55850652000, -52694978400, -46383544800, -43227871200, -40072197600, -33760764000, -30605090400, -27449416800, -21137983200, -17982309600, -14826636000, -12220034400, -12219948000, -12219861600, -12219775200, -12219688800, -12219602400, -12219516000, -12219429600, -12219343200, -12219256800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+11",
+  "switches" : [ -62135557200, -59006322000, -55850648400, -52694974800, -46383541200, -43227867600, -40072194000, -33760760400, -30605086800, -27449413200, -21137979600, -17982306000, -14826632400, -12220030800, -12219944400, -12219858000, -12219771600, -12219685200, -12219598800, -12219512400, -12219426000, -12219339600, -12219253200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+12",
+  "switches" : [ -62135553600, -59006318400, -55850644800, -52694971200, -46383537600, -43227864000, -40072190400, -33760756800, -30605083200, -27449409600, -21137976000, -17982302400, -14826628800, -12220027200, -12219940800, -12219854400, -12219768000, -12219681600, -12219595200, -12219508800, -12219422400, -12219336000, -12219249600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+2",
+  "switches" : [ -62135589600, -59006354400, -55850680800, -52695007200, -46383573600, -43227900000, -40072226400, -33760792800, -30605119200, -27449445600, -21138012000, -17982338400, -14826664800, -12220063200, -12219976800, -12219890400, -12219804000, -12219717600, -12219631200, -12219544800, -12219458400, -12219372000, -12219285600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+3",
+  "switches" : [ -62135586000, -59006350800, -55850677200, -52695003600, -46383570000, -43227896400, -40072222800, -33760789200, -30605115600, -27449442000, -21138008400, -17982334800, -14826661200, -12220059600, -12219973200, -12219886800, -12219800400, -12219714000, -12219627600, -12219541200, -12219454800, -12219368400, -12219282000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+4",
+  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12220056000, -12219969600, -12219883200, -12219796800, -12219710400, -12219624000, -12219537600, -12219451200, -12219364800, -12219278400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+5",
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12220052400, -12219966000, -12219879600, -12219793200, -12219706800, -12219620400, -12219534000, -12219447600, -12219361200, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+6",
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12220048800, -12219962400, -12219876000, -12219789600, -12219703200, -12219616800, -12219530400, -12219444000, -12219357600, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+7",
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12220045200, -12219958800, -12219872400, -12219786000, -12219699600, -12219613200, -12219526800, -12219440400, -12219354000, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+8",
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12220041600, -12219955200, -12219868800, -12219782400, -12219696000, -12219609600, -12219523200, -12219436800, -12219350400, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT+9",
+  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12220038000, -12219951600, -12219865200, -12219778800, -12219692400, -12219606000, -12219519600, -12219433200, -12219346800, -12219260400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-0",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-1",
+  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12220074000, -12219987600, -12219901200, -12219814800, -12219728400, -12219642000, -12219555600, -12219469200, -12219382800, -12219296400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-10",
+  "switches" : [ -62135632800, -59006397600, -55850724000, -52695050400, -46383616800, -43227943200, -40072269600, -33760836000, -30605162400, -27449488800, -21138055200, -17982381600, -14826708000, -12220106400, -12220020000, -12219933600, -12219847200, -12219760800, -12219674400, -12219588000, -12219501600, -12219415200, -12219328800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-11",
+  "switches" : [ -62135636400, -59006401200, -55850727600, -52695054000, -46383620400, -43227946800, -40072273200, -33760839600, -30605166000, -27449492400, -21138058800, -17982385200, -14826711600, -12220110000, -12220023600, -12219937200, -12219850800, -12219764400, -12219678000, -12219591600, -12219505200, -12219418800, -12219332400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-12",
+  "switches" : [ -62135640000, -59006404800, -55850731200, -52695057600, -46383624000, -43227950400, -40072276800, -33760843200, -30605169600, -27449496000, -21138062400, -17982388800, -14826715200, -12220113600, -12220027200, -12219940800, -12219854400, -12219768000, -12219681600, -12219595200, -12219508800, -12219422400, -12219336000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-13",
+  "switches" : [ -62135643600, -59006408400, -55850734800, -52695061200, -46383627600, -43227954000, -40072280400, -33760846800, -30605173200, -27449499600, -21138066000, -17982392400, -14826718800, -12220117200, -12220030800, -12219944400, -12219858000, -12219771600, -12219685200, -12219598800, -12219512400, -12219426000, -12219339600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-14",
+  "switches" : [ -62135647200, -59006412000, -55850738400, -52695064800, -46383631200, -43227957600, -40072284000, -33760850400, -30605176800, -27449503200, -21138069600, -17982396000, -14826722400, -12220120800, -12220034400, -12219948000, -12219861600, -12219775200, -12219688800, -12219602400, -12219516000, -12219429600, -12219343200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-2",
+  "switches" : [ -62135604000, -59006368800, -55850695200, -52695021600, -46383588000, -43227914400, -40072240800, -33760807200, -30605133600, -27449460000, -21138026400, -17982352800, -14826679200, -12220077600, -12219991200, -12219904800, -12219818400, -12219732000, -12219645600, -12219559200, -12219472800, -12219386400, -12219300000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-3",
+  "switches" : [ -62135607600, -59006372400, -55850698800, -52695025200, -46383591600, -43227918000, -40072244400, -33760810800, -30605137200, -27449463600, -21138030000, -17982356400, -14826682800, -12220081200, -12219994800, -12219908400, -12219822000, -12219735600, -12219649200, -12219562800, -12219476400, -12219390000, -12219303600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-4",
+  "switches" : [ -62135611200, -59006376000, -55850702400, -52695028800, -46383595200, -43227921600, -40072248000, -33760814400, -30605140800, -27449467200, -21138033600, -17982360000, -14826686400, -12220084800, -12219998400, -12219912000, -12219825600, -12219739200, -12219652800, -12219566400, -12219480000, -12219393600, -12219307200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-5",
+  "switches" : [ -62135614800, -59006379600, -55850706000, -52695032400, -46383598800, -43227925200, -40072251600, -33760818000, -30605144400, -27449470800, -21138037200, -17982363600, -14826690000, -12220088400, -12220002000, -12219915600, -12219829200, -12219742800, -12219656400, -12219570000, -12219483600, -12219397200, -12219310800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-6",
+  "switches" : [ -62135618400, -59006383200, -55850709600, -52695036000, -46383602400, -43227928800, -40072255200, -33760821600, -30605148000, -27449474400, -21138040800, -17982367200, -14826693600, -12220092000, -12220005600, -12219919200, -12219832800, -12219746400, -12219660000, -12219573600, -12219487200, -12219400800, -12219314400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-7",
+  "switches" : [ -62135622000, -59006386800, -55850713200, -52695039600, -46383606000, -43227932400, -40072258800, -33760825200, -30605151600, -27449478000, -21138044400, -17982370800, -14826697200, -12220095600, -12220009200, -12219922800, -12219836400, -12219750000, -12219663600, -12219577200, -12219490800, -12219404400, -12219318000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-8",
+  "switches" : [ -62135625600, -59006390400, -55850716800, -52695043200, -46383609600, -43227936000, -40072262400, -33760828800, -30605155200, -27449481600, -21138048000, -17982374400, -14826700800, -12220099200, -12220012800, -12219926400, -12219840000, -12219753600, -12219667200, -12219580800, -12219494400, -12219408000, -12219321600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT-9",
+  "switches" : [ -62135629200, -59006394000, -55850720400, -52695046800, -46383613200, -43227939600, -40072266000, -33760832400, -30605158800, -27449485200, -21138051600, -17982378000, -14826704400, -12220102800, -12220016400, -12219930000, -12219843600, -12219757200, -12219670800, -12219584400, -12219498000, -12219411600, -12219325200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/GMT0",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/Greenwich",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/UCT",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/UTC",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/Universal",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Etc/Zulu",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Europe/Amsterdam",
+  "switches" : [ -62135597972, -59006362772, -55850689172, -52695015572, -46383581972, -43227908372, -40072234772, -33760801172, -30605127572, -27449453972, -21138020372, -17982346772, -14826673172, -12220071572, -12219985172, -12219898772, -12219812372, -12219725972, -12219639572, -12219553172, -12219466772, -12219380372, -12219293972, -2208988800 ],
+  "diffs" : [ -175228, -88828, -2428, 83972, 170372, 256772, 343172, 429572, 515972, 602372, 688772, 775172, 861572, 775172, 688772, 602372, 515972, 429572, 343172, 256772, 170372, 83972, -2428, 0 ]
+}, {
+  "tz" : "Europe/Andorra",
+  "switches" : [ -62135597164, -59006361964, -55850688364, -52695014764, -46383581164, -43227907564, -40072233964, -33760800364, -30605126764, -27449453164, -21138019564, -17982345964, -14826672364, -12220070764, -12219984364, -12219897964, -12219811564, -12219725164, -12219638764, -12219552364, -12219465964, -12219379564, -12219293164, -2208988800 ],
+  "diffs" : [ -176036, -89636, -3236, 83164, 169564, 255964, 342364, 428764, 515164, 601564, 687964, 774364, 860764, 774364, 687964, 601564, 515164, 428764, 342364, 255964, 169564, 83164, -3236, 0 ]
+}, {
+  "tz" : "Europe/Astrakhan",
+  "switches" : [ -62135608332, -59006373132, -55850699532, -52695025932, -46383592332, -43227918732, -40072245132, -33760811532, -30605137932, -27449464332, -21138030732, -17982357132, -14826683532, -12220081932, -12219995532, -12219909132, -12219822732, -12219736332, -12219649932, -12219563532, -12219477132, -12219390732, -12219304332, -2208988800 ],
+  "diffs" : [ -175668, -89268, -2868, 83532, 169932, 256332, 342732, 429132, 515532, 601932, 688332, 774732, 861132, 774732, 688332, 601932, 515532, 429132, 342732, 256332, 169932, 83532, -2868, 0 ]
+}, {
+  "tz" : "Europe/Athens",
+  "switches" : [ -62135602492, -59006367292, -55850693692, -52695020092, -46383586492, -43227912892, -40072239292, -33760805692, -30605132092, -27449458492, -21138024892, -17982351292, -14826677692, -12220076092, -12219989692, -12219903292, -12219816892, -12219730492, -12219644092, -12219557692, -12219471292, -12219384892, -12219298492, -2208988800 ],
+  "diffs" : [ -174308, -87908, -1508, 84892, 171292, 257692, 344092, 430492, 516892, 603292, 689692, 776092, 862492, 776092, 689692, 603292, 516892, 430492, 344092, 257692, 171292, 84892, -1508, 0 ]
+}, {
+  "tz" : "Europe/Belfast",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
+}, {
+  "tz" : "Europe/Belgrade",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
+}, {
+  "tz" : "Europe/Berlin",
+  "switches" : [ -62135600008, -59006364808, -55850691208, -52695017608, -46383584008, -43227910408, -40072236808, -33760803208, -30605129608, -27449456008, -21138022408, -17982348808, -14826675208, -12220073608, -12219987208, -12219900808, -12219814408, -12219728008, -12219641608, -12219555208, -12219468808, -12219382408, -12219296008, -2422054408 ],
+  "diffs" : [ -173192, -86792, -392, 86008, 172408, 258808, 345208, 431608, 518008, 604408, 690808, 777208, 863608, 777208, 690808, 604408, 518008, 431608, 345208, 258808, 172408, 86008, -392, 0 ]
+}, {
+  "tz" : "Europe/Bratislava",
+  "switches" : [ -62135600264, -59006365064, -55850691464, -52695017864, -46383584264, -43227910664, -40072237064, -33760803464, -30605129864, -27449456264, -21138022664, -17982349064, -14826675464, -12220073864, -12219987464, -12219901064, -12219814664, -12219728264, -12219641864, -12219555464, -12219469064, -12219382664, -12219296264, -2469401864 ],
+  "diffs" : [ -172936, -86536, -136, 86264, 172664, 259064, 345464, 431864, 518264, 604664, 691064, 777464, 863864, 777464, 691064, 604664, 518264, 431864, 345464, 259064, 172664, 86264, -136, 0 ]
+}, {
+  "tz" : "Europe/Brussels",
+  "switches" : [ -62135597850, -59006362650, -55850689050, -52695015450, -46383581850, -43227908250, -40072234650, -33760801050, -30605127450, -27449453850, -21138020250, -17982346650, -14826673050, -12220071450, -12219985050, -12219898650, -12219812250, -12219725850, -12219639450, -12219553050, -12219466650, -12219380250, -12219293850, -2450995200, -2208988800 ],
+  "diffs" : [ -175350, -88950, -2550, 83850, 170250, 256650, 343050, 429450, 515850, 602250, 688650, 775050, 861450, 775050, 688650, 602250, 515850, 429450, 343050, 256650, 170250, 83850, -2550, -3600, 0 ]
+}, {
+  "tz" : "Europe/Bucharest",
+  "switches" : [ -62135603064, -59006367864, -55850694264, -52695020664, -46383587064, -43227913464, -40072239864, -33760806264, -30605132664, -27449459064, -21138025464, -17982351864, -14826678264, -12220076664, -12219990264, -12219903864, -12219817464, -12219731064, -12219644664, -12219558264, -12219471864, -12219385464, -12219299064, -2208988800 ],
+  "diffs" : [ -173736, -87336, -936, 85464, 171864, 258264, 344664, 431064, 517464, 603864, 690264, 776664, 863064, 776664, 690264, 603864, 517464, 431064, 344664, 258264, 171864, 85464, -936, 0 ]
+}, {
+  "tz" : "Europe/Budapest",
+  "switches" : [ -62135601380, -59006366180, -55850692580, -52695018980, -46383585380, -43227911780, -40072238180, -33760804580, -30605130980, -27449457380, -21138023780, -17982350180, -14826676580, -12220074980, -12219988580, -12219902180, -12219815780, -12219729380, -12219642980, -12219556580, -12219470180, -12219383780, -12219297380, -2500938980 ],
+  "diffs" : [ -171820, -85420, 980, 87380, 173780, 260180, 346580, 432980, 519380, 605780, 692180, 778580, 864980, 778580, 692180, 605780, 519380, 432980, 346580, 260180, 173780, 87380, 980, 0 ]
+}, {
+  "tz" : "Europe/Busingen",
+  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12220072448, -12219986048, -12219899648, -12219813248, -12219726848, -12219640448, -12219554048, -12219467648, -12219381248, -12219294848, -3675198848, -2385246586 ],
+  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, 776048, 689648, 603248, 516848, 430448, 344048, 257648, 171248, 84848, -1552, -1814, 0 ]
+}, {
+  "tz" : "Europe/Chisinau",
+  "switches" : [ -62135603720, -59006368520, -55850694920, -52695021320, -46383587720, -43227914120, -40072240520, -33760806920, -30605133320, -27449459720, -21138026120, -17982352520, -14826678920, -12220077320, -12219990920, -12219904520, -12219818120, -12219731720, -12219645320, -12219558920, -12219472520, -12219386120, -12219299720, -2840147720, -2208988800 ],
+  "diffs" : [ -173080, -86680, -280, 86120, 172520, 258920, 345320, 431720, 518120, 604520, 690920, 777320, 863720, 777320, 690920, 604520, 518120, 431720, 345320, 258920, 172520, 86120, -280, -300, 0 ]
+}, {
+  "tz" : "Europe/Copenhagen",
+  "switches" : [ -62135599820, -59006364620, -55850691020, -52695017420, -46383583820, -43227910220, -40072236620, -33760803020, -30605129420, -27449455820, -21138022220, -17982348620, -14826675020, -12220073420, -12219987020, -12219900620, -12219814220, -12219727820, -12219641420, -12219555020, -12219468620, -12219382220, -12219295820, -2398294220 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ]
+}, {
+  "tz" : "Europe/Dublin",
+  "switches" : [ -62135595300, -59006360100, -55850686500, -52695012900, -46383579300, -43227905700, -40072232100, -33760798500, -30605124900, -27449451300, -21138017700, -17982344100, -14826670500, -12220068900, -12219982500, -12219896100, -12219809700, -12219723300, -12219636900, -12219550500, -12219464100, -12219377700, -12219291300, -2821649700, -2208988800 ],
+  "diffs" : [ -174300, -87900, -1500, 84900, 171300, 257700, 344100, 430500, 516900, 603300, 689700, 776100, 862500, 776100, 689700, 603300, 516900, 430500, 344100, 257700, 171300, 84900, -1500, -1521, 0 ]
+}, {
+  "tz" : "Europe/Gibraltar",
+  "switches" : [ -62135595516, -59006360316, -55850686716, -52695013116, -46383579516, -43227905916, -40072232316, -33760798716, -30605125116, -27449451516, -21138017916, -17982344316, -14826670716, -12220069116, -12219982716, -12219896316, -12219809916, -12219723516, -12219637116, -12219550716, -12219464316, -12219377916, -12219291516, -2821649916, -2208988800 ],
+  "diffs" : [ -177684, -91284, -4884, 81516, 167916, 254316, 340716, 427116, 513516, 599916, 686316, 772716, 859116, 772716, 686316, 599916, 513516, 427116, 340716, 254316, 167916, 81516, -4884, -3600, 0 ]
+}, {
+  "tz" : "Europe/Guernsey",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
+}, {
+  "tz" : "Europe/Helsinki",
+  "switches" : [ -62135602789, -59006367589, -55850693989, -52695020389, -46383586789, -43227913189, -40072239589, -33760805989, -30605132389, -27449458789, -21138025189, -17982351589, -14826677989, -12220076389, -12219989989, -12219903589, -12219817189, -12219730789, -12219644389, -12219557989, -12219471589, -12219385189, -12219298789, -2208988800 ],
+  "diffs" : [ -174011, -87611, -1211, 85189, 171589, 257989, 344389, 430789, 517189, 603589, 689989, 776389, 862789, 776389, 689989, 603589, 517189, 430789, 344389, 257989, 171589, 85189, -1211, 0 ]
+}, {
+  "tz" : "Europe/Isle_of_Man",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
+}, {
+  "tz" : "Europe/Istanbul",
+  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12220077352, -12219990952, -12219904552, -12219818152, -12219731752, -12219645352, -12219558952, -12219472552, -12219386152, -12219299752, -2840147752, -2208988800 ],
+  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, 773752, 687352, 600952, 514552, 428152, 341752, 255352, 168952, 82552, -3848, -3784, 0 ]
+}, {
+  "tz" : "Europe/Jersey",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
+}, {
+  "tz" : "Europe/Kaliningrad",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2422056120, -2208988800 ],
+  "diffs" : [ -175080, -88680, -2280, 84120, 170520, 256920, 343320, 429720, 516120, 602520, 688920, 775320, 861720, 775320, 688920, 602520, 516120, 429720, 343320, 256920, 170520, 84120, -2280, -3600, 0 ]
+}, {
+  "tz" : "Europe/Kiev",
+  "switches" : [ -62135604124, -59006368924, -55850695324, -52695021724, -46383588124, -43227914524, -40072240924, -33760807324, -30605133724, -27449460124, -21138026524, -17982352924, -14826679324, -12220077724, -12219991324, -12219904924, -12219818524, -12219732124, -12219645724, -12219559324, -12219472924, -12219386524, -12219300124, -2208988800 ],
+  "diffs" : [ -172676, -86276, 124, 86524, 172924, 259324, 345724, 432124, 518524, 604924, 691324, 777724, 864124, 777724, 691324, 604924, 518524, 432124, 345724, 259324, 172924, 86524, 124, 0 ]
+}, {
+  "tz" : "Europe/Kirov",
+  "switches" : [ -62135608728, -59006373528, -55850699928, -52695026328, -46383592728, -43227919128, -40072245528, -33760811928, -30605138328, -27449464728, -21138031128, -17982357528, -14826683928, -12220082328, -12219995928, -12219909528, -12219823128, -12219736728, -12219650328, -12219563928, -12219477528, -12219391128, -12219304728, -2208988800 ],
+  "diffs" : [ -171672, -85272, 1128, 87528, 173928, 260328, 346728, 433128, 519528, 605928, 692328, 778728, 865128, 778728, 692328, 605928, 519528, 433128, 346728, 260328, 173928, 87528, 1128, 0 ]
+}, {
+  "tz" : "Europe/Lisbon",
+  "switches" : [ -62135594595, -59006359395, -55850685795, -52695012195, -46383578595, -43227904995, -40072231395, -33760797795, -30605124195, -27449450595, -21138016995, -17982343395, -14826669795, -12220068195, -12219981795, -12219895395, -12219808995, -12219722595, -12219636195, -12219549795, -12219463395, -12219376995, -12219290595, -2208988800 ],
+  "diffs" : [ -175005, -88605, -2205, 84195, 170595, 256995, 343395, 429795, 516195, 602595, 688995, 775395, 861795, 775395, 688995, 602595, 516195, 429795, 343395, 256995, 170595, 84195, -2205, 0 ]
+}, {
+  "tz" : "Europe/Ljubljana",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
+}, {
+  "tz" : "Europe/London",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
+}, {
+  "tz" : "Europe/Luxembourg",
+  "switches" : [ -62135598276, -59006363076, -55850689476, -52695015876, -46383582276, -43227908676, -40072235076, -33760801476, -30605127876, -27449454276, -21138020676, -17982347076, -14826673476, -12220071876, -12219985476, -12219899076, -12219812676, -12219726276, -12219639876, -12219553476, -12219467076, -12219380676, -12219294276, -2208988800 ],
+  "diffs" : [ -174924, -88524, -2124, 84276, 170676, 257076, 343476, 429876, 516276, 602676, 689076, 775476, 861876, 775476, 689076, 602676, 516276, 429876, 343476, 257076, 170676, 84276, -2124, 0 ]
+}, {
+  "tz" : "Europe/Madrid",
+  "switches" : [ -62135595916, -59006360716, -55850687116, -52695013516, -46383579916, -43227906316, -40072232716, -33760799116, -30605125516, -27449451916, -21138018316, -17982344716, -14826671116, -12220069516, -12219983116, -12219896716, -12219810316, -12219723916, -12219637516, -12219551116, -12219464716, -12219378316, -12219291916, -2208988800 ],
+  "diffs" : [ -177284, -90884, -4484, 81916, 168316, 254716, 341116, 427516, 513916, 600316, 686716, 773116, 859516, 773116, 686716, 600316, 513916, 427516, 341116, 254716, 168316, 81916, -4484, 0 ]
+}, {
+  "tz" : "Europe/Malta",
+  "switches" : [ -62135600284, -59006365084, -55850691484, -52695017884, -46383584284, -43227910684, -40072237084, -33760803484, -30605129884, -27449456284, -21138022684, -17982349084, -14826675484, -12220073884, -12219987484, -12219901084, -12219814684, -12219728284, -12219641884, -12219555484, -12219469084, -12219382684, -12219296284, -2403478684 ],
+  "diffs" : [ -172916, -86516, -116, 86284, 172684, 259084, 345484, 431884, 518284, 604684, 691084, 777484, 863884, 777484, 691084, 604684, 518284, 431884, 345484, 259084, 172684, 86284, -116, 0 ]
+}, {
+  "tz" : "Europe/Mariehamn",
+  "switches" : [ -62135602789, -59006367589, -55850693989, -52695020389, -46383586789, -43227913189, -40072239589, -33760805989, -30605132389, -27449458789, -21138025189, -17982351589, -14826677989, -12220076389, -12219989989, -12219903589, -12219817189, -12219730789, -12219644389, -12219557989, -12219471589, -12219385189, -12219298789, -2208988800 ],
+  "diffs" : [ -174011, -87611, -1211, 85189, 171589, 257989, 344389, 430789, 517189, 603589, 689989, 776389, 862789, 776389, 689989, 603589, 517189, 430789, 344389, 257989, 171589, 85189, -1211, 0 ]
+}, {
+  "tz" : "Europe/Minsk",
+  "switches" : [ -62135603416, -59006368216, -55850694616, -52695021016, -46383587416, -43227913816, -40072240216, -33760806616, -30605133016, -27449459416, -21138025816, -17982352216, -14826678616, -12220077016, -12219990616, -12219904216, -12219817816, -12219731416, -12219645016, -12219558616, -12219472216, -12219385816, -12219299416, -2840147416, -2208988800 ],
+  "diffs" : [ -176984, -90584, -4184, 82216, 168616, 255016, 341416, 427816, 514216, 600616, 687016, 773416, 859816, 773416, 687016, 600616, 514216, 427816, 341416, 255016, 168616, 82216, -4184, -4200, 0 ]
+}, {
+  "tz" : "Europe/Monaco",
+  "switches" : [ -62135598572, -59006363372, -55850689772, -52695016172, -46383582572, -43227908972, -40072235372, -33760801772, -30605128172, -27449454572, -21138020972, -17982347372, -14826673772, -12220072172, -12219985772, -12219899372, -12219812972, -12219726572, -12219640172, -12219553772, -12219467372, -12219380972, -12219294572, -2486680172, -2208988800 ],
+  "diffs" : [ -174628, -88228, -1828, 84572, 170972, 257372, 343772, 430172, 516572, 602972, 689372, 775772, 862172, 775772, 689372, 602972, 516572, 430172, 343772, 257372, 170972, 84572, -1828, -3039, 0 ]
+}, {
+  "tz" : "Europe/Moscow",
+  "switches" : [ -62135605817, -59006370617, -55850697017, -52695023417, -46383589817, -43227916217, -40072242617, -33760809017, -30605135417, -27449461817, -21138028217, -17982354617, -14826681017, -12220079417, -12219993017, -12219906617, -12219820217, -12219733817, -12219647417, -12219561017, -12219474617, -12219388217, -12219301817, -2208988800 ],
+  "diffs" : [ -174583, -88183, -1783, 84617, 171017, 257417, 343817, 430217, 516617, 603017, 689417, 775817, 862217, 775817, 689417, 603017, 516617, 430217, 343817, 257417, 171017, 84617, -1783, 0 ]
+}, {
+  "tz" : "Europe/Nicosia",
+  "switches" : [ -62135604808, -59006369608, -55850696008, -52695022408, -46383588808, -43227915208, -40072241608, -33760808008, -30605134408, -27449460808, -21138027208, -17982353608, -14826680008, -12220078408, -12219992008, -12219905608, -12219819208, -12219732808, -12219646408, -12219560008, -12219473608, -12219387208, -12219300808, -2208988800 ],
+  "diffs" : [ -171992, -85592, 808, 87208, 173608, 260008, 346408, 432808, 519208, 605608, 692008, 778408, 864808, 778408, 692008, 605608, 519208, 432808, 346408, 260008, 173608, 87208, 808, 0 ]
+}, {
+  "tz" : "Europe/Oslo",
+  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12220072980, -12219986580, -12219900180, -12219813780, -12219727380, -12219640980, -12219554580, -12219468180, -12219381780, -12219295380, -2366757780 ],
+  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, 776580, 690180, 603780, 517380, 430980, 344580, 258180, 171780, 85380, -1020, 0 ]
+}, {
+  "tz" : "Europe/Paris",
+  "switches" : [ -62135597361, -59006362161, -55850688561, -52695014961, -46383581361, -43227907761, -40072234161, -33760800561, -30605126961, -27449453361, -21138019761, -17982346161, -14826672561, -12220070961, -12219984561, -12219898161, -12219811761, -12219725361, -12219638961, -12219552561, -12219466161, -12219379761, -12219293361, -2208988800 ],
+  "diffs" : [ -175839, -89439, -3039, 83361, 169761, 256161, 342561, 428961, 515361, 601761, 688161, 774561, 860961, 774561, 688161, 601761, 515361, 428961, 342561, 256161, 169761, 83361, -3039, 0 ]
+}, {
+  "tz" : "Europe/Podgorica",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
+}, {
+  "tz" : "Europe/Prague",
+  "switches" : [ -62135600264, -59006365064, -55850691464, -52695017864, -46383584264, -43227910664, -40072237064, -33760803464, -30605129864, -27449456264, -21138022664, -17982349064, -14826675464, -12220073864, -12219987464, -12219901064, -12219814664, -12219728264, -12219641864, -12219555464, -12219469064, -12219382664, -12219296264, -2469401864 ],
+  "diffs" : [ -172936, -86536, -136, 86264, 172664, 259064, 345464, 431864, 518264, 604664, 691064, 777464, 863864, 777464, 691064, 604664, 518264, 431864, 345464, 259064, 172664, 86264, -136, 0 ]
+}, {
+  "tz" : "Europe/Riga",
+  "switches" : [ -62135602594, -59006367394, -55850693794, -52695020194, -46383586594, -43227912994, -40072239394, -33760805794, -30605132194, -27449458594, -21138024994, -17982351394, -14826677794, -12220076194, -12219989794, -12219903394, -12219816994, -12219730594, -12219644194, -12219557794, -12219471394, -12219384994, -12219298594, -2208988800 ],
+  "diffs" : [ -174206, -87806, -1406, 84994, 171394, 257794, 344194, 430594, 516994, 603394, 689794, 776194, 862594, 776194, 689794, 603394, 516994, 430594, 344194, 257794, 171394, 84994, -1406, 0 ]
+}, {
+  "tz" : "Europe/Rome",
+  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12220073396, -12219986996, -12219900596, -12219814196, -12219727796, -12219641396, -12219554996, -12219468596, -12219382196, -12219295796, -2403565200 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, 776996, 690596, 604196, 517796, 431396, 344996, 258596, 172196, 85796, -604, 0 ]
+}, {
+  "tz" : "Europe/Samara",
+  "switches" : [ -62135608820, -59006373620, -55850700020, -52695026420, -46383592820, -43227919220, -40072245620, -33760812020, -30605138420, -27449464820, -21138031220, -17982357620, -14826684020, -12220082420, -12219996020, -12219909620, -12219823220, -12219736820, -12219650420, -12219564020, -12219477620, -12219391220, -12219304820, -2208988800 ],
+  "diffs" : [ -175180, -88780, -2380, 84020, 170420, 256820, 343220, 429620, 516020, 602420, 688820, 775220, 861620, 775220, 688820, 602420, 516020, 429620, 343220, 256820, 170420, 84020, -2380, 0 ]
+}, {
+  "tz" : "Europe/San_Marino",
+  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12220073396, -12219986996, -12219900596, -12219814196, -12219727796, -12219641396, -12219554996, -12219468596, -12219382196, -12219295796, -2403565200 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, 776996, 690596, 604196, 517796, 431396, 344996, 258596, 172196, 85796, -604, 0 ]
+}, {
+  "tz" : "Europe/Sarajevo",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
+}, {
+  "tz" : "Europe/Saratov",
+  "switches" : [ -62135607858, -59006372658, -55850699058, -52695025458, -46383591858, -43227918258, -40072244658, -33760811058, -30605137458, -27449463858, -21138030258, -17982356658, -14826683058, -12220081458, -12219995058, -12219908658, -12219822258, -12219735858, -12219649458, -12219563058, -12219476658, -12219390258, -12219303858, -2208988800 ],
+  "diffs" : [ -176142, -89742, -3342, 83058, 169458, 255858, 342258, 428658, 515058, 601458, 687858, 774258, 860658, 774258, 687858, 601458, 515058, 428658, 342258, 255858, 169458, 83058, -3342, 0 ]
+}, {
+  "tz" : "Europe/Simferopol",
+  "switches" : [ -62135604984, -59006369784, -55850696184, -52695022584, -46383588984, -43227915384, -40072241784, -33760808184, -30605134584, -27449460984, -21138027384, -17982353784, -14826680184, -12220078584, -12219992184, -12219905784, -12219819384, -12219732984, -12219646584, -12219560184, -12219473784, -12219387384, -12219300984, -2840148984, -2208988800 ],
+  "diffs" : [ -175416, -89016, -2616, 83784, 170184, 256584, 342984, 429384, 515784, 602184, 688584, 774984, 861384, 774984, 688584, 602184, 515784, 429384, 342984, 256584, 170184, 83784, -2616, -2640, 0 ]
+}, {
+  "tz" : "Europe/Skopje",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
+}, {
+  "tz" : "Europe/Sofia",
+  "switches" : [ -62135602396, -59006367196, -55850693596, -52695019996, -46383586396, -43227912796, -40072239196, -33760805596, -30605131996, -27449458396, -21138024796, -17982351196, -14826677596, -12220075996, -12219989596, -12219903196, -12219816796, -12219730396, -12219643996, -12219557596, -12219471196, -12219384796, -12219298396, -2840146396, -2369527016 ],
+  "diffs" : [ -174404, -88004, -1604, 84796, 171196, 257596, 343996, 430396, 516796, 603196, 689596, 775996, 862396, 775996, 689596, 603196, 516796, 430396, 343996, 257596, 171196, 84796, -1604, -184, 0 ]
+}, {
+  "tz" : "Europe/Stockholm",
+  "switches" : [ -62135601132, -59006365932, -55850692332, -52695018732, -46383585132, -43227911532, -40072237932, -33760804332, -30605130732, -27449457132, -21138023532, -17982349932, -14826676332, -12220074732, -12219988332, -12219901932, -12219815532, -12219729132, -12219642732, -12219556332, -12219469932, -12219383532, -12219297132, -2871681132, -2208992414 ],
+  "diffs" : [ -172068, -85668, 732, 87132, 173532, 259932, 346332, 432732, 519132, 605532, 691932, 778332, 864732, 778332, 691932, 605532, 519132, 432732, 346332, 259932, 173532, 87132, 732, 14, 0 ]
+}, {
+  "tz" : "Europe/Tallinn",
+  "switches" : [ -62135602740, -59006367540, -55850693940, -52695020340, -46383586740, -43227913140, -40072239540, -33760805940, -30605132340, -27449458740, -21138025140, -17982351540, -14826677940, -12220076340, -12219989940, -12219903540, -12219817140, -12219730740, -12219644340, -12219557940, -12219471540, -12219385140, -12219298740, -2208988800 ],
+  "diffs" : [ -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, 862740, 776340, 689940, 603540, 517140, 430740, 344340, 257940, 171540, 85140, -1260, 0 ]
+}, {
+  "tz" : "Europe/Tirane",
+  "switches" : [ -62135601560, -59006366360, -55850692760, -52695019160, -46383585560, -43227911960, -40072238360, -33760804760, -30605131160, -27449457560, -21138023960, -17982350360, -14826676760, -12220075160, -12219988760, -12219902360, -12219815960, -12219729560, -12219643160, -12219556760, -12219470360, -12219383960, -12219297560, -2208988800 ],
+  "diffs" : [ -171640, -85240, 1160, 87560, 173960, 260360, 346760, 433160, 519560, 605960, 692360, 778760, 865160, 778760, 692360, 605960, 519560, 433160, 346760, 260360, 173960, 87560, 1160, 0 ]
+}, {
+  "tz" : "Europe/Tiraspol",
+  "switches" : [ -62135603720, -59006368520, -55850694920, -52695021320, -46383587720, -43227914120, -40072240520, -33760806920, -30605133320, -27449459720, -21138026120, -17982352520, -14826678920, -12220077320, -12219990920, -12219904520, -12219818120, -12219731720, -12219645320, -12219558920, -12219472520, -12219386120, -12219299720, -2840147720, -2208988800 ],
+  "diffs" : [ -173080, -86680, -280, 86120, 172520, 258920, 345320, 431720, 518120, 604520, 690920, 777320, 863720, 777320, 690920, 604520, 518120, 431720, 345320, 258920, 172520, 86120, -280, -300, 0 ]
+}, {
+  "tz" : "Europe/Ulyanovsk",
+  "switches" : [ -62135608416, -59006373216, -55850699616, -52695026016, -46383592416, -43227918816, -40072245216, -33760811616, -30605138016, -27449464416, -21138030816, -17982357216, -14826683616, -12220082016, -12219995616, -12219909216, -12219822816, -12219736416, -12219650016, -12219563616, -12219477216, -12219390816, -12219304416, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Europe/Uzhgorod",
+  "switches" : [ -62135602152, -59006366952, -55850693352, -52695019752, -46383586152, -43227912552, -40072238952, -33760805352, -30605131752, -27449458152, -21138024552, -17982350952, -14826677352, -12220075752, -12219989352, -12219902952, -12219816552, -12219730152, -12219643752, -12219557352, -12219470952, -12219384552, -12219298152, -2500939752, -2208988800 ],
+  "diffs" : [ -174648, -88248, -1848, 84552, 170952, 257352, 343752, 430152, 516552, 602952, 689352, 775752, 862152, 775752, 689352, 602952, 516552, 430152, 343752, 257352, 170952, 84552, -1848, -3600, 0 ]
+}, {
+  "tz" : "Europe/Vaduz",
+  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12220072448, -12219986048, -12219899648, -12219813248, -12219726848, -12219640448, -12219554048, -12219467648, -12219381248, -12219294848, -3675198848, -2385246586 ],
+  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, 776048, 689648, 603248, 516848, 430448, 344048, 257648, 171248, 84848, -1552, -1814, 0 ]
+}, {
+  "tz" : "Europe/Vatican",
+  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12220073396, -12219986996, -12219900596, -12219814196, -12219727796, -12219641396, -12219554996, -12219468596, -12219382196, -12219295796, -2403565200 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, 776996, 690596, 604196, 517796, 431396, 344996, 258596, 172196, 85796, -604, 0 ]
+}, {
+  "tz" : "Europe/Vienna",
+  "switches" : [ -62135600721, -59006365521, -55850691921, -52695018321, -46383584721, -43227911121, -40072237521, -33760803921, -30605130321, -27449456721, -21138023121, -17982349521, -14826675921, -12220074321, -12219987921, -12219901521, -12219815121, -12219728721, -12219642321, -12219555921, -12219469521, -12219383121, -12219296721, -2422055121 ],
+  "diffs" : [ -172479, -86079, 321, 86721, 173121, 259521, 345921, 432321, 518721, 605121, 691521, 777921, 864321, 777921, 691521, 605121, 518721, 432321, 345921, 259521, 173121, 86721, 321, 0 ]
+}, {
+  "tz" : "Europe/Vilnius",
+  "switches" : [ -62135602876, -59006367676, -55850694076, -52695020476, -46383586876, -43227913276, -40072239676, -33760806076, -30605132476, -27449458876, -21138025276, -17982351676, -14826678076, -12220076476, -12219990076, -12219903676, -12219817276, -12219730876, -12219644476, -12219558076, -12219471676, -12219385276, -12219298876, -2840146876, -2208988800 ],
+  "diffs" : [ -173924, -87524, -1124, 85276, 171676, 258076, 344476, 430876, 517276, 603676, 690076, 776476, 862876, 776476, 690076, 603676, 517276, 430876, 344476, 258076, 171676, 85276, -1124, -2160, 0 ]
+}, {
+  "tz" : "Europe/Volgograd",
+  "switches" : [ -62135607460, -59006372260, -55850698660, -52695025060, -46383591460, -43227917860, -40072244260, -33760810660, -30605137060, -27449463460, -21138029860, -17982356260, -14826682660, -12220081060, -12219994660, -12219908260, -12219821860, -12219735460, -12219649060, -12219562660, -12219476260, -12219389860, -12219303460, -2208988800 ],
+  "diffs" : [ -176540, -90140, -3740, 82660, 169060, 255460, 341860, 428260, 514660, 601060, 687460, 773860, 860260, 773860, 687460, 601060, 514660, 428260, 341860, 255460, 169060, 82660, -3740, 0 ]
+}, {
+  "tz" : "Europe/Warsaw",
+  "switches" : [ -62135601840, -59006366640, -55850693040, -52695019440, -46383585840, -43227912240, -40072238640, -33760805040, -30605131440, -27449457840, -21138024240, -17982350640, -14826677040, -12220075440, -12219989040, -12219902640, -12219816240, -12219729840, -12219643440, -12219557040, -12219470640, -12219384240, -12219297840, -2208988800 ],
+  "diffs" : [ -171360, -84960, 1440, 87840, 174240, 260640, 347040, 433440, 519840, 606240, 692640, 779040, 865440, 779040, 692640, 606240, 519840, 433440, 347040, 260640, 174240, 87840, 1440, 0 ]
+}, {
+  "tz" : "Europe/Zagreb",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
+}, {
+  "tz" : "Europe/Zaporozhye",
+  "switches" : [ -62135605240, -59006370040, -55850696440, -52695022840, -46383589240, -43227915640, -40072242040, -33760808440, -30605134840, -27449461240, -21138027640, -17982354040, -14826680440, -12220078840, -12219992440, -12219906040, -12219819640, -12219733240, -12219646840, -12219560440, -12219474040, -12219387640, -12219301240, -2840149240, -2208988800 ],
+  "diffs" : [ -171560, -85160, 1240, 87640, 174040, 260440, 346840, 433240, 519640, 606040, 692440, 778840, 865240, 778840, 692440, 606040, 519640, 433240, 346840, 260440, 174040, 87640, 1240, 1200, 0 ]
+}, {
+  "tz" : "Europe/Zurich",
+  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12220072448, -12219986048, -12219899648, -12219813248, -12219726848, -12219640448, -12219554048, -12219467648, -12219381248, -12219294848, -3675198848, -2385246586 ],
+  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, 776048, 689648, 603248, 516848, 430448, 344048, 257648, 171248, 84848, -1552, -1814, 0 ]
+}, {
+  "tz" : "GB",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
+}, {
+  "tz" : "GB-Eire",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
+}, {
+  "tz" : "GMT",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "GMT0",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Greenwich",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Hongkong",
+  "switches" : [ -62135624202, -59006389002, -55850715402, -52695041802, -46383608202, -43227934602, -40072261002, -33760827402, -30605153802, -27449480202, -21138046602, -17982373002, -14826699402, -12220097802, -12220011402, -12219925002, -12219838602, -12219752202, -12219665802, -12219579402, -12219493002, -12219406602, -12219320202, -2208988800 ],
+  "diffs" : [ -174198, -87798, -1398, 85002, 171402, 257802, 344202, 430602, 517002, 603402, 689802, 776202, 862602, 776202, 689802, 603402, 517002, 430602, 344202, 257802, 171402, 85002, -1398, 0 ]
+}, {
+  "tz" : "Iceland",
+  "switches" : [ -62135591520, -59006356320, -55850682720, -52695009120, -46383575520, -43227901920, -40072228320, -33760794720, -30605121120, -27449447520, -21138013920, -17982340320, -14826666720, -12220065120, -12219978720, -12219892320, -12219805920, -12219719520, -12219633120, -12219546720, -12219460320, -12219373920, -12219287520, -2208988800 ],
+  "diffs" : [ -178080, -91680, -5280, 81120, 167520, 253920, 340320, 426720, 513120, 599520, 685920, 772320, 858720, 772320, 685920, 599520, 513120, 426720, 340320, 253920, 167520, 81120, -5280, 0 ]
+}, {
+  "tz" : "Indian/Antananarivo",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Indian/Chagos",
+  "switches" : [ -62135614180, -59006378980, -55850705380, -52695031780, -46383598180, -43227924580, -40072250980, -33760817380, -30605143780, -27449470180, -21138036580, -17982362980, -14826689380, -12220087780, -12220001380, -12219914980, -12219828580, -12219742180, -12219655780, -12219569380, -12219482980, -12219396580, -12219310180, -2208988800 ],
+  "diffs" : [ -177020, -90620, -4220, 82180, 168580, 254980, 341380, 427780, 514180, 600580, 686980, 773380, 859780, 773380, 686980, 600580, 514180, 427780, 341380, 254980, 168580, 82180, -4220, 0 ]
+}, {
+  "tz" : "Indian/Christmas",
+  "switches" : [ -62135622172, -59006386972, -55850713372, -52695039772, -46383606172, -43227932572, -40072258972, -33760825372, -30605151772, -27449478172, -21138044572, -17982370972, -14826697372, -12220095772, -12220009372, -12219922972, -12219836572, -12219750172, -12219663772, -12219577372, -12219490972, -12219404572, -12219318172, -2364102172 ],
+  "diffs" : [ -172628, -86228, 172, 86572, 172972, 259372, 345772, 432172, 518572, 604972, 691372, 777772, 864172, 777772, 691372, 604972, 518572, 432172, 345772, 259372, 172972, 86572, 172, 0 ]
+}, {
+  "tz" : "Indian/Cocos",
+  "switches" : [ -62135620060, -59006384860, -55850711260, -52695037660, -46383604060, -43227930460, -40072256860, -33760823260, -30605149660, -27449476060, -21138042460, -17982368860, -14826695260, -12220093660, -12220007260, -12219920860, -12219834460, -12219748060, -12219661660, -12219575260, -12219488860, -12219402460, -12219316060, -2209012060 ],
+  "diffs" : [ -172940, -86540, -140, 86260, 172660, 259060, 345460, 431860, 518260, 604660, 691060, 777460, 863860, 777460, 691060, 604660, 518260, 431860, 345460, 259060, 172660, 86260, -140, 0 ]
+}, {
+  "tz" : "Indian/Comoro",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Indian/Kerguelen",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -190800, -104400, -18000, 68400, 154800, 241200, 327600, 414000, 500400, 586800, 673200, 759600, 846000, 759600, 673200, 586800, 500400, 414000, 327600, 241200, 154800, 68400, -18000, 0 ]
+}, {
+  "tz" : "Indian/Mahe",
+  "switches" : [ -62135610108, -59006374908, -55850701308, -52695027708, -46383594108, -43227920508, -40072246908, -33760813308, -30605139708, -27449466108, -21138032508, -17982358908, -14826685308, -12220083708, -12219997308, -12219910908, -12219824508, -12219738108, -12219651708, -12219565308, -12219478908, -12219392508, -12219306108, -2208988800 ],
+  "diffs" : [ -173892, -87492, -1092, 85308, 171708, 258108, 344508, 430908, 517308, 603708, 690108, 776508, 862908, 776508, 690108, 603708, 517308, 430908, 344508, 258108, 171708, 85308, -1092, 0 ]
+}, {
+  "tz" : "Indian/Maldives",
+  "switches" : [ -62135614440, -59006379240, -55850705640, -52695032040, -46383598440, -43227924840, -40072251240, -33760817640, -30605144040, -27449470440, -21138036840, -17982363240, -14826689640, -12220088040, -12220001640, -12219915240, -12219828840, -12219742440, -12219656040, -12219569640, -12219483240, -12219396840, -12219310440, -2208988800 ],
+  "diffs" : [ -173160, -86760, -360, 86040, 172440, 258840, 345240, 431640, 518040, 604440, 690840, 777240, 863640, 777240, 690840, 604440, 518040, 431640, 345240, 258840, 172440, 86040, -360, 0 ]
+}, {
+  "tz" : "Indian/Mauritius",
+  "switches" : [ -62135610600, -59006375400, -55850701800, -52695028200, -46383594600, -43227921000, -40072247400, -33760813800, -30605140200, -27449466600, -21138033000, -17982359400, -14826685800, -12220084200, -12219997800, -12219911400, -12219825000, -12219738600, -12219652200, -12219565800, -12219479400, -12219393000, -12219306600, -2208988800 ],
+  "diffs" : [ -173400, -87000, -600, 85800, 172200, 258600, 345000, 431400, 517800, 604200, 690600, 777000, 863400, 777000, 690600, 604200, 517800, 431400, 345000, 258600, 172200, 85800, -600, 0 ]
+}, {
+  "tz" : "Indian/Mayotte",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
+}, {
+  "tz" : "Indian/Reunion",
+  "switches" : [ -62135610112, -59006374912, -55850701312, -52695027712, -46383594112, -43227920512, -40072246912, -33760813312, -30605139712, -27449466112, -21138032512, -17982358912, -14826685312, -12220083712, -12219997312, -12219910912, -12219824512, -12219738112, -12219651712, -12219565312, -12219478912, -12219392512, -12219306112, -2208988800 ],
+  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, 0 ]
+}, {
+  "tz" : "Iran",
+  "switches" : [ -62135609144, -59006373944, -55850700344, -52695026744, -46383593144, -43227919544, -40072245944, -33760812344, -30605138744, -27449465144, -21138031544, -17982357944, -14826684344, -12220082744, -12219996344, -12219909944, -12219823544, -12219737144, -12219650744, -12219564344, -12219477944, -12219391544, -12219305144, -2208988800 ],
+  "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, 777344, 690944, 604544, 518144, 431744, 345344, 258944, 172544, 86144, -256, 0 ]
+}, {
+  "tz" : "Israel",
+  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12220078854, -12219992454, -12219906054, -12219819654, -12219733254, -12219646854, -12219560454, -12219474054, -12219387654, -12219301254, -2840149254, -2208988800 ],
+  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 778854, 692454, 606054, 519654, 433254, 346854, 260454, 174054, 87654, 1254, 1240, 0 ]
+}, {
+  "tz" : "Jamaica",
+  "switches" : [ -62135578370, -59006343170, -55850669570, -52694995970, -46383562370, -43227888770, -40072215170, -33760781570, -30605107970, -27449434370, -21138000770, -17982327170, -14826653570, -12220051970, -12219965570, -12219879170, -12219792770, -12219706370, -12219619970, -12219533570, -12219447170, -12219360770, -12219274370, -2208988800 ],
+  "diffs" : [ -173230, -86830, -430, 85970, 172370, 258770, 345170, 431570, 517970, 604370, 690770, 777170, 863570, 777170, 690770, 604370, 517970, 431570, 345170, 258770, 172370, 85970, -430, 0 ]
+}, {
+  "tz" : "Japan",
+  "switches" : [ -62135630339, -59006395139, -55850721539, -52695047939, -46383614339, -43227940739, -40072267139, -33760833539, -30605159939, -27449486339, -21138052739, -17982379139, -14826705539, -12220103939, -12220017539, -12219931139, -12219844739, -12219758339, -12219671939, -12219585539, -12219499139, -12219412739, -12219326339, -2587712400 ],
+  "diffs" : [ -171661, -85261, 1139, 87539, 173939, 260339, 346739, 433139, 519539, 605939, 692339, 778739, 865139, 778739, 692339, 605939, 519539, 433139, 346739, 260339, 173939, 87539, 1139, 0 ]
+}, {
+  "tz" : "Kwajalein",
+  "switches" : [ -62135636960, -59006401760, -55850728160, -52695054560, -46383620960, -43227947360, -40072273760, -33760840160, -30605166560, -27449492960, -21138059360, -17982385760, -14826712160, -12220110560, -12220024160, -12219937760, -12219851360, -12219764960, -12219678560, -12219592160, -12219505760, -12219419360, -12219332960, -2208988800 ],
+  "diffs" : [ -175840, -89440, -3040, 83360, 169760, 256160, 342560, 428960, 515360, 601760, 688160, 774560, 860960, 774560, 688160, 601760, 515360, 428960, 342560, 256160, 169760, 83360, -3040, 0 ]
+}, {
+  "tz" : "Libya",
+  "switches" : [ -62135599964, -59006364764, -55850691164, -52695017564, -46383583964, -43227910364, -40072236764, -33760803164, -30605129564, -27449455964, -21138022364, -17982348764, -14826675164, -12220073564, -12219987164, -12219900764, -12219814364, -12219727964, -12219641564, -12219555164, -12219468764, -12219382364, -12219295964, -2208988800 ],
+  "diffs" : [ -176836, -90436, -4036, 82364, 168764, 255164, 341564, 427964, 514364, 600764, 687164, 773564, 859964, 773564, 687164, 600764, 514364, 427964, 341564, 255164, 168764, 82364, -4036, 0 ]
+}, {
+  "tz" : "MET",
+  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12220074000, -12219987600, -12219901200, -12219814800, -12219728400, -12219642000, -12219555600, -12219469200, -12219382800, -12219296400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "MST7MDT",
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12220045200, -12219958800, -12219872400, -12219786000, -12219699600, -12219613200, -12219526800, -12219440400, -12219354000, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Mexico/BajaNorte",
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12220042316, -12219955916, -12219869516, -12219783116, -12219696716, -12219610316, -12219523916, -12219437516, -12219351116, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 778316, 691916, 605516, 519116, 432716, 346316, 259916, 173516, 87116, 716, 0 ]
+}, {
+  "tz" : "Mexico/BajaSur",
+  "switches" : [ -62135571260, -59006336060, -55850662460, -52694988860, -46383555260, -43227881660, -40072208060, -33760774460, -30605100860, -27449427260, -21137993660, -17982320060, -14826646460, -12220044860, -12219958460, -12219872060, -12219785660, -12219699260, -12219612860, -12219526460, -12219440060, -12219353660, -12219267260, -2208988800 ],
+  "diffs" : [ -173140, -86740, -340, 86060, 172460, 258860, 345260, 431660, 518060, 604460, 690860, 777260, 863660, 777260, 690860, 604460, 518060, 431660, 345260, 258860, 172460, 86060, -340, 0 ]
+}, {
+  "tz" : "Mexico/General",
+  "switches" : [ -62135573004, -59006337804, -55850664204, -52694990604, -46383557004, -43227883404, -40072209804, -33760776204, -30605102604, -27449429004, -21137995404, -17982321804, -14826648204, -12220046604, -12219960204, -12219873804, -12219787404, -12219701004, -12219614604, -12219528204, -12219441804, -12219355404, -12219269004, -2208988800 ],
+  "diffs" : [ -174996, -88596, -2196, 84204, 170604, 257004, 343404, 429804, 516204, 602604, 689004, 775404, 861804, 775404, 689004, 602604, 516204, 429804, 343404, 257004, 170604, 84204, -2196, 0 ]
+}, {
+  "tz" : "NZ",
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12220112344, -12220025944, -12219939544, -12219853144, -12219766744, -12219680344, -12219593944, -12219507544, -12219421144, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, 776344, 689944, 603544, 517144, 430744, 344344, 257944, 171544, 85144, -1256, -1800, 0 ]
+}, {
+  "tz" : "NZ-CHAT",
+  "switches" : [ -62135640828, -59006405628, -55850732028, -52695058428, -46383624828, -43227951228, -40072277628, -33760844028, -30605170428, -27449496828, -21138063228, -17982389628, -14826716028, -12220114428, -12220028028, -12219941628, -12219855228, -12219768828, -12219682428, -12219596028, -12219509628, -12219423228, -12219336828, -3192437628, -2208988800 ],
+  "diffs" : [ -174672, -88272, -1872, 84528, 170928, 257328, 343728, 430128, 516528, 602928, 689328, 775728, 862128, 775728, 689328, 602928, 516528, 430128, 343728, 257328, 170928, 84528, -1872, -1800, 0 ]
+}, {
+  "tz" : "Navajo",
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12220045204, -12219958804, -12219872404, -12219786004, -12219699604, -12219613204, -12219526804, -12219440404, -12219354004, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 777604, 691204, 604804, 518404, 432004, 345604, 259204, 172804, 86404, 4, 0 ]
+}, {
+  "tz" : "PRC",
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12220099543, -12220013143, -12219926743, -12219840343, -12219753943, -12219667543, -12219581143, -12219494743, -12219408343, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 777943, 691543, 605143, 518743, 432343, 345943, 259543, 173143, 86743, 343, 0 ]
+}, {
+  "tz" : "PST8PDT",
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12220041600, -12219955200, -12219868800, -12219782400, -12219696000, -12219609600, -12219523200, -12219436800, -12219350400, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Pacific/Apia",
+  "switches" : [ -62135641984, -59006406784, -55850733184, -52695059584, -46383625984, -43227952384, -40072278784, -33760845184, -30605171584, -27449497984, -21138064384, -17982390784, -14826717184, -12220115584, -12220029184, -12219942784, -12219856384, -12219769984, -12219683584, -12219597184, -12219510784, -12219424384, -12219337984, -2445424384, -2208988800 ],
+  "diffs" : [ -174416, -88016, -1616, 84784, 171184, 257584, 343984, 430384, 516784, 603184, 689584, 775984, 862384, 775984, 689584, 603184, 516784, 430384, 343984, 257584, 171184, 84784, -1616, -88016, 0 ]
+}, {
+  "tz" : "Pacific/Auckland",
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12220112344, -12220025944, -12219939544, -12219853144, -12219766744, -12219680344, -12219593944, -12219507544, -12219421144, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, 776344, 689944, 603544, 517144, 430744, 344344, 257944, 171544, 85144, -1256, -1800, 0 ]
+}, {
+  "tz" : "Pacific/Bougainville",
+  "switches" : [ -62135634136, -59006398936, -55850725336, -52695051736, -46383618136, -43227944536, -40072270936, -33760837336, -30605163736, -27449490136, -21138056536, -17982382936, -14826709336, -12220107736, -12220021336, -12219934936, -12219848536, -12219762136, -12219675736, -12219589336, -12219502936, -12219416536, -12219330136, -2840178136, -2366790512, -2208988800 ],
+  "diffs" : [ -175064, -88664, -2264, 84136, 170536, 256936, 343336, 429736, 516136, 602536, 688936, 775336, 861736, 775336, 688936, 602536, 516136, 429736, 343336, 256936, 170536, 84136, -2264, -4288, -3600, 0 ]
+}, {
+  "tz" : "Pacific/Chatham",
+  "switches" : [ -62135640828, -59006405628, -55850732028, -52695058428, -46383624828, -43227951228, -40072277628, -33760844028, -30605170428, -27449496828, -21138063228, -17982389628, -14826716028, -12220114428, -12220028028, -12219941628, -12219855228, -12219768828, -12219682428, -12219596028, -12219509628, -12219423228, -12219336828, -3192437628, -2208988800 ],
+  "diffs" : [ -174672, -88272, -1872, 84528, 170928, 257328, 343728, 430128, 516528, 602928, 689328, 775728, 862128, 775728, 689328, 602928, 516528, 430128, 343728, 257328, 170928, 84528, -1872, -1800, 0 ]
+}, {
+  "tz" : "Pacific/Chuuk",
+  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12220020428, -12219934028, -12219847628, -12219761228, -12219674828, -12219588428, -12219502028, -12219415628, -12219329228, -12219242828, -3944628428, -2208988800 ],
+  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, 691628, 605228, 518828, 432428, 346028, 259628, 173228, 86828, 428, -85972, 428, 0 ]
+}, {
+  "tz" : "Pacific/Easter",
+  "switches" : [ -62135570552, -59006335352, -55850661752, -52694988152, -46383554552, -43227880952, -40072207352, -33760773752, -30605100152, -27449426552, -21137992952, -17982319352, -14826645752, -12220044152, -12219957752, -12219871352, -12219784952, -12219698552, -12219612152, -12219525752, -12219439352, -12219352952, -12219266552, -2208988800 ],
+  "diffs" : [ -177448, -91048, -4648, 81752, 168152, 254552, 340952, 427352, 513752, 600152, 686552, 772952, 859352, 772952, 686552, 600152, 513752, 427352, 340952, 254552, 168152, 81752, -4648, 0 ]
+}, {
+  "tz" : "Pacific/Efate",
+  "switches" : [ -62135637196, -59006401996, -55850728396, -52695054796, -46383621196, -43227947596, -40072273996, -33760840396, -30605166796, -27449493196, -21138059596, -17982385996, -14826712396, -12220110796, -12220024396, -12219937996, -12219851596, -12219765196, -12219678796, -12219592396, -12219505996, -12219419596, -12219333196, -2208988800 ],
+  "diffs" : [ -172004, -85604, 796, 87196, 173596, 259996, 346396, 432796, 519196, 605596, 691996, 778396, 864796, 778396, 691996, 605596, 519196, 432796, 346396, 259996, 173596, 87196, 796, 0 ]
+}, {
+  "tz" : "Pacific/Enderbury",
+  "switches" : [ -62135555740, -59006320540, -55850646940, -52694973340, -46383539740, -43227866140, -40072192540, -33760758940, -30605085340, -27449411740, -21137978140, -17982304540, -14826630940, -12220029340, -12219942940, -12219856540, -12219770140, -12219683740, -12219597340, -12219510940, -12219424540, -12219338140, -12219251740, -2208988800 ],
+  "diffs" : [ -260660, -174260, -87860, -1460, 84940, 171340, 257740, 344140, 430540, 516940, 603340, 689740, 776140, 689740, 603340, 516940, 430540, 344140, 257740, 171340, 84940, -1460, -87860, 0 ]
+}, {
+  "tz" : "Pacific/Fakaofo",
+  "switches" : [ -62135555704, -59006320504, -55850646904, -52694973304, -46383539704, -43227866104, -40072192504, -33760758904, -30605085304, -27449411704, -21137978104, -17982304504, -14826630904, -12220029304, -12219942904, -12219856504, -12219770104, -12219683704, -12219597304, -12219510904, -12219424504, -12219338104, -12219251704, -2208988800 ],
+  "diffs" : [ -260696, -174296, -87896, -1496, 84904, 171304, 257704, 344104, 430504, 516904, 603304, 689704, 776104, 689704, 603304, 516904, 430504, 344104, 257704, 171304, 84904, -1496, -87896, 0 ]
+}, {
+  "tz" : "Pacific/Fiji",
+  "switches" : [ -62135639744, -59006404544, -55850730944, -52695057344, -46383623744, -43227950144, -40072276544, -33760842944, -30605169344, -27449495744, -21138062144, -17982388544, -14826714944, -12220113344, -12220026944, -12219940544, -12219854144, -12219767744, -12219681344, -12219594944, -12219508544, -12219422144, -12219335744, -2208988800 ],
+  "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, 777344, 690944, 604544, 518144, 431744, 345344, 258944, 172544, 86144, -256, 0 ]
+}, {
+  "tz" : "Pacific/Funafuti",
+  "switches" : [ -62135639812, -59006404612, -55850731012, -52695057412, -46383623812, -43227950212, -40072276612, -33760843012, -30605169412, -27449495812, -21138062212, -17982388612, -14826715012, -12220113412, -12220027012, -12219940612, -12219854212, -12219767812, -12219681412, -12219595012, -12219508612, -12219422212, -12219335812, -2208988800 ],
+  "diffs" : [ -172988, -86588, -188, 86212, 172612, 259012, 345412, 431812, 518212, 604612, 691012, 777412, 863812, 777412, 691012, 604612, 518212, 431812, 345412, 259012, 172612, 86212, -188, 0 ]
+}, {
+  "tz" : "Pacific/Galapagos",
+  "switches" : [ -62135575296, -59006340096, -55850666496, -52694992896, -46383559296, -43227885696, -40072212096, -33760778496, -30605104896, -27449431296, -21137997696, -17982324096, -14826650496, -12220048896, -12219962496, -12219876096, -12219789696, -12219703296, -12219616896, -12219530496, -12219444096, -12219357696, -12219271296, -2208988800 ],
+  "diffs" : [ -172704, -86304, 96, 86496, 172896, 259296, 345696, 432096, 518496, 604896, 691296, 777696, 864096, 777696, 691296, 604896, 518496, 432096, 345696, 259296, 172896, 86496, 96, 0 ]
+}, {
+  "tz" : "Pacific/Gambier",
+  "switches" : [ -62135564412, -59006329212, -55850655612, -52694982012, -46383548412, -43227874812, -40072201212, -33760767612, -30605094012, -27449420412, -21137986812, -17982313212, -14826639612, -12220038012, -12219951612, -12219865212, -12219778812, -12219692412, -12219606012, -12219519612, -12219433212, -12219346812, -12219260412, -2208988800 ],
+  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 777612, 691212, 604812, 518412, 432012, 345612, 259212, 172812, 86412, 12, 0 ]
+}, {
+  "tz" : "Pacific/Guadalcanal",
+  "switches" : [ -62135635188, -59006399988, -55850726388, -52695052788, -46383619188, -43227945588, -40072271988, -33760838388, -30605164788, -27449491188, -21138057588, -17982383988, -14826710388, -12220108788, -12220022388, -12219935988, -12219849588, -12219763188, -12219676788, -12219590388, -12219503988, -12219417588, -12219331188, -2208988800 ],
+  "diffs" : [ -174012, -87612, -1212, 85188, 171588, 257988, 344388, 430788, 517188, 603588, 689988, 776388, 862788, 776388, 689988, 603588, 517188, 430788, 344388, 257988, 171588, 85188, -1212, 0 ]
+}, {
+  "tz" : "Pacific/Guam",
+  "switches" : [ -62135545140, -59006309940, -55850636340, -52694962740, -46383529140, -43227855540, -40072181940, -33760748340, -30605074740, -27449401140, -21137967540, -17982293940, -14826620340, -12220018740, -12219932340, -12219845940, -12219759540, -12219673140, -12219586740, -12219500340, -12219413940, -12219327540, -12219241140, -3944626740, -2208988800 ],
+  "diffs" : [ -260460, -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, 689940, 603540, 517140, 430740, 344340, 257940, 171540, 85140, -1260, -87660, -1260, 0 ]
+}, {
+  "tz" : "Pacific/Honolulu",
+  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12220032514, -12219946114, -12219859714, -12219773314, -12219686914, -12219600514, -12219514114, -12219427714, -12219341314, -12219254914, -2334101314, -2208988800 ],
+  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, 775714, 689314, 602914, 516514, 430114, 343714, 257314, 170914, 84514, -1886, -1800, 0 ]
+}, {
+  "tz" : "Pacific/Johnston",
+  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12220032514, -12219946114, -12219859714, -12219773314, -12219686914, -12219600514, -12219514114, -12219427714, -12219341314, -12219254914, -2334101314, -2208988800 ],
+  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, 775714, 689314, 602914, 516514, 430114, 343714, 257314, 170914, 84514, -1886, -1800, 0 ]
+}, {
+  "tz" : "Pacific/Kiritimati",
+  "switches" : [ -62135559040, -59006323840, -55850650240, -52694976640, -46383543040, -43227869440, -40072195840, -33760762240, -30605088640, -27449415040, -21137981440, -17982307840, -14826634240, -12220032640, -12219946240, -12219859840, -12219773440, -12219687040, -12219600640, -12219514240, -12219427840, -12219341440, -12219255040, -2208988800 ],
+  "diffs" : [ -260960, -174560, -88160, -1760, 84640, 171040, 257440, 343840, 430240, 516640, 603040, 689440, 775840, 689440, 603040, 516640, 430240, 343840, 257440, 171040, 84640, -1760, -88160, 0 ]
+}, {
+  "tz" : "Pacific/Kosrae",
+  "switches" : [ -62135549516, -59006314316, -55850640716, -52694967116, -46383533516, -43227859916, -40072186316, -33760752716, -30605079116, -27449405516, -21137971916, -17982298316, -14826624716, -12220023116, -12219936716, -12219850316, -12219763916, -12219677516, -12219591116, -12219504716, -12219418316, -12219331916, -12219245516, -3944631116, -2208988800 ],
+  "diffs" : [ -259684, -173284, -86884, -484, 85916, 172316, 258716, 345116, 431516, 517916, 604316, 690716, 777116, 690716, 604316, 517916, 431516, 345116, 258716, 172316, 85916, -484, -86884, -484, 0 ]
+}, {
+  "tz" : "Pacific/Kwajalein",
+  "switches" : [ -62135636960, -59006401760, -55850728160, -52695054560, -46383620960, -43227947360, -40072273760, -33760840160, -30605166560, -27449492960, -21138059360, -17982385760, -14826712160, -12220110560, -12220024160, -12219937760, -12219851360, -12219764960, -12219678560, -12219592160, -12219505760, -12219419360, -12219332960, -2208988800 ],
+  "diffs" : [ -175840, -89440, -3040, 83360, 169760, 256160, 342560, 428960, 515360, 601760, 688160, 774560, 860960, 774560, 688160, 601760, 515360, 428960, 342560, 256160, 169760, 83360, -3040, 0 ]
+}, {
+  "tz" : "Pacific/Majuro",
+  "switches" : [ -62135637888, -59006402688, -55850729088, -52695055488, -46383621888, -43227948288, -40072274688, -33760841088, -30605167488, -27449493888, -21138060288, -17982386688, -14826713088, -12220111488, -12220025088, -12219938688, -12219852288, -12219765888, -12219679488, -12219593088, -12219506688, -12219420288, -12219333888, -2208988800 ],
+  "diffs" : [ -174912, -88512, -2112, 84288, 170688, 257088, 343488, 429888, 516288, 602688, 689088, 775488, 861888, 775488, 689088, 602688, 516288, 429888, 343488, 257088, 170688, 84288, -2112, 0 ]
+}, {
+  "tz" : "Pacific/Marquesas",
+  "switches" : [ -62135563320, -59006328120, -55850654520, -52694980920, -46383547320, -43227873720, -40072200120, -33760766520, -30605092920, -27449419320, -21137985720, -17982312120, -14826638520, -12220036920, -12219950520, -12219864120, -12219777720, -12219691320, -12219604920, -12219518520, -12219432120, -12219345720, -12219259320, -2208988800 ],
+  "diffs" : [ -172080, -85680, 720, 87120, 173520, 259920, 346320, 432720, 519120, 605520, 691920, 778320, 864720, 778320, 691920, 605520, 519120, 432720, 346320, 259920, 173520, 87120, 720, 0 ]
+}, {
+  "tz" : "Pacific/Midway",
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12220115832, -12220029432, -12219943032, -12219856632, -12219770232, -12219683832, -12219597432, -12219511032, -12219424632, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 862632, 776232, 689832, 603432, 517032, 430632, 344232, 257832, 171432, 85032, -1368, 0 ]
+}, {
+  "tz" : "Pacific/Nauru",
+  "switches" : [ -62135636860, -59006401660, -55850728060, -52695054460, -46383620860, -43227947260, -40072273660, -33760840060, -30605166460, -27449492860, -21138059260, -17982385660, -14826712060, -12220110460, -12220024060, -12219937660, -12219851260, -12219764860, -12219678460, -12219592060, -12219505660, -12219419260, -12219332860, -2208988800 ],
+  "diffs" : [ -175940, -89540, -3140, 83260, 169660, 256060, 342460, 428860, 515260, 601660, 688060, 774460, 860860, 774460, 688060, 601660, 515260, 428860, 342460, 256060, 169660, 83260, -3140, 0 ]
+}, {
+  "tz" : "Pacific/Niue",
+  "switches" : [ -62135556020, -59006320820, -55850647220, -52694973620, -46383540020, -43227866420, -40072192820, -33760759220, -30605085620, -27449412020, -21137978420, -17982304820, -14826631220, -12220029620, -12219943220, -12219856820, -12219770420, -12219684020, -12219597620, -12219511220, -12219424820, -12219338420, -12219252020, -2208988800 ],
+  "diffs" : [ -173980, -87580, -1180, 85220, 171620, 258020, 344420, 430820, 517220, 603620, 690020, 776420, 862820, 776420, 690020, 603620, 517220, 430820, 344420, 258020, 171620, 85220, -1180, 0 ]
+}, {
+  "tz" : "Pacific/Norfolk",
+  "switches" : [ -62135637112, -59006401912, -55850728312, -52695054712, -46383621112, -43227947512, -40072273912, -33760840312, -30605166712, -27449493112, -21138059512, -17982385912, -14826712312, -12220110712, -12220024312, -12219937912, -12219851512, -12219765112, -12219678712, -12219592312, -12219505912, -12219419512, -12219333112, -2208988800 ],
+  "diffs" : [ -172088, -85688, 712, 87112, 173512, 259912, 346312, 432712, 519112, 605512, 691912, 778312, 864712, 778312, 691912, 605512, 519112, 432712, 346312, 259912, 173512, 87112, 712, 0 ]
+}, {
+  "tz" : "Pacific/Noumea",
+  "switches" : [ -62135636748, -59006401548, -55850727948, -52695054348, -46383620748, -43227947148, -40072273548, -33760839948, -30605166348, -27449492748, -21138059148, -17982385548, -14826711948, -12220110348, -12220023948, -12219937548, -12219851148, -12219764748, -12219678348, -12219591948, -12219505548, -12219419148, -12219332748, -2208988800 ],
+  "diffs" : [ -172452, -86052, 348, 86748, 173148, 259548, 345948, 432348, 518748, 605148, 691548, 777948, 864348, 777948, 691548, 605148, 518748, 432348, 345948, 259548, 173148, 86748, 348, 0 ]
+}, {
+  "tz" : "Pacific/Pago_Pago",
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12220115832, -12220029432, -12219943032, -12219856632, -12219770232, -12219683832, -12219597432, -12219511032, -12219424632, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 862632, 776232, 689832, 603432, 517032, 430632, 344232, 257832, 171432, 85032, -1368, 0 ]
+}, {
+  "tz" : "Pacific/Palau",
+  "switches" : [ -62135542676, -59006307476, -55850633876, -52694960276, -46383526676, -43227853076, -40072179476, -33760745876, -30605072276, -27449398676, -21137965076, -17982291476, -14826617876, -12220016276, -12219929876, -12219843476, -12219757076, -12219670676, -12219584276, -12219497876, -12219411476, -12219325076, -12219238676, -3944624276, -2208988800 ],
+  "diffs" : [ -259324, -172924, -86524, -124, 86276, 172676, 259076, 345476, 431876, 518276, 604676, 691076, 777476, 691076, 604676, 518276, 431876, 345476, 259076, 172676, 86276, -124, -86524, -124, 0 ]
+}, {
+  "tz" : "Pacific/Pitcairn",
+  "switches" : [ -62135565580, -59006330380, -55850656780, -52694983180, -46383549580, -43227875980, -40072202380, -33760768780, -30605095180, -27449421580, -21137987980, -17982314380, -14826640780, -12220039180, -12219952780, -12219866380, -12219779980, -12219693580, -12219607180, -12219520780, -12219434380, -12219347980, -12219261580, -2208988800 ],
+  "diffs" : [ -175220, -88820, -2420, 83980, 170380, 256780, 343180, 429580, 515980, 602380, 688780, 775180, 861580, 775180, 688780, 602380, 515980, 429580, 343180, 256780, 170380, 83980, -2420, 0 ]
+}, {
+  "tz" : "Pacific/Pohnpei",
+  "switches" : [ -62135548372, -59006313172, -55850639572, -52694965972, -46383532372, -43227858772, -40072185172, -33760751572, -30605077972, -27449404372, -21137970772, -17982297172, -14826623572, -12220021972, -12219935572, -12219849172, -12219762772, -12219676372, -12219589972, -12219503572, -12219417172, -12219330772, -12219244372, -3944629972, -2208988800 ],
+  "diffs" : [ -260828, -174428, -88028, -1628, 84772, 171172, 257572, 343972, 430372, 516772, 603172, 689572, 775972, 689572, 603172, 516772, 430372, 343972, 257572, 171172, 84772, -1628, -88028, -1628, 0 ]
+}, {
+  "tz" : "Pacific/Ponape",
+  "switches" : [ -62135548372, -59006313172, -55850639572, -52694965972, -46383532372, -43227858772, -40072185172, -33760751572, -30605077972, -27449404372, -21137970772, -17982297172, -14826623572, -12220021972, -12219935572, -12219849172, -12219762772, -12219676372, -12219589972, -12219503572, -12219417172, -12219330772, -12219244372, -3944629972, -2208988800 ],
+  "diffs" : [ -260828, -174428, -88028, -1628, 84772, 171172, 257572, 343972, 430372, 516772, 603172, 689572, 775972, 689572, 603172, 516772, 430372, 343972, 257572, 171172, 84772, -1628, -88028, -1628, 0 ]
+}, {
+  "tz" : "Pacific/Port_Moresby",
+  "switches" : [ -62135632120, -59006396920, -55850723320, -52695049720, -46383616120, -43227942520, -40072268920, -33760835320, -30605161720, -27449488120, -21138054520, -17982380920, -14826707320, -12220105720, -12220019320, -12219932920, -12219846520, -12219760120, -12219673720, -12219587320, -12219500920, -12219414520, -12219328120, -2840176120, -2366790512 ],
+  "diffs" : [ -173480, -87080, -680, 85720, 172120, 258520, 344920, 431320, 517720, 604120, 690520, 776920, 863320, 776920, 690520, 604120, 517720, 431320, 344920, 258520, 172120, 85720, -680, -688, 0 ]
+}, {
+  "tz" : "Pacific/Rarotonga",
+  "switches" : [ -62135558456, -59006323256, -55850649656, -52694976056, -46383542456, -43227868856, -40072195256, -33760761656, -30605088056, -27449414456, -21137980856, -17982307256, -14826633656, -12220032056, -12219945656, -12219859256, -12219772856, -12219686456, -12219600056, -12219513656, -12219427256, -12219340856, -12219254456, -2208988800 ],
+  "diffs" : [ -175144, -88744, -2344, 84056, 170456, 256856, 343256, 429656, 516056, 602456, 688856, 775256, 861656, 775256, 688856, 602456, 516056, 429656, 343256, 256856, 170456, 84056, -2344, 0 ]
+}, {
+  "tz" : "Pacific/Saipan",
+  "switches" : [ -62135545140, -59006309940, -55850636340, -52694962740, -46383529140, -43227855540, -40072181940, -33760748340, -30605074740, -27449401140, -21137967540, -17982293940, -14826620340, -12220018740, -12219932340, -12219845940, -12219759540, -12219673140, -12219586740, -12219500340, -12219413940, -12219327540, -12219241140, -3944626740, -2208988800 ],
+  "diffs" : [ -260460, -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, 689940, 603540, 517140, 430740, 344340, 257940, 171540, 85140, -1260, -87660, -1260, 0 ]
+}, {
+  "tz" : "Pacific/Samoa",
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12220115832, -12220029432, -12219943032, -12219856632, -12219770232, -12219683832, -12219597432, -12219511032, -12219424632, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 862632, 776232, 689832, 603432, 517032, 430632, 344232, 257832, 171432, 85032, -1368, 0 ]
+}, {
+  "tz" : "Pacific/Tahiti",
+  "switches" : [ -62135560904, -59006325704, -55850652104, -52694978504, -46383544904, -43227871304, -40072197704, -33760764104, -30605090504, -27449416904, -21137983304, -17982309704, -14826636104, -12220034504, -12219948104, -12219861704, -12219775304, -12219688904, -12219602504, -12219516104, -12219429704, -12219343304, -12219256904, -2208988800 ],
+  "diffs" : [ -172696, -86296, 104, 86504, 172904, 259304, 345704, 432104, 518504, 604904, 691304, 777704, 864104, 777704, 691304, 604904, 518504, 432104, 345704, 259304, 172904, 86504, 104, 0 ]
+}, {
+  "tz" : "Pacific/Tarawa",
+  "switches" : [ -62135638324, -59006403124, -55850729524, -52695055924, -46383622324, -43227948724, -40072275124, -33760841524, -30605167924, -27449494324, -21138060724, -17982387124, -14826713524, -12220111924, -12220025524, -12219939124, -12219852724, -12219766324, -12219679924, -12219593524, -12219507124, -12219420724, -12219334324, -2208988800 ],
+  "diffs" : [ -174476, -88076, -1676, 84724, 171124, 257524, 343924, 430324, 516724, 603124, 689524, 775924, 862324, 775924, 689524, 603124, 516724, 430324, 343924, 257524, 171124, 84724, -1676, 0 ]
+}, {
+  "tz" : "Pacific/Tongatapu",
+  "switches" : [ -62135641160, -59006405960, -55850732360, -52695058760, -46383625160, -43227951560, -40072277960, -33760844360, -30605170760, -27449497160, -21138063560, -17982389960, -14826716360, -12220114760, -12220028360, -12219941960, -12219855560, -12219769160, -12219682760, -12219596360, -12219509960, -12219423560, -12219337160, -2208988800 ],
+  "diffs" : [ -175240, -88840, -2440, 83960, 170360, 256760, 343160, 429560, 515960, 602360, 688760, 775160, 861560, 775160, 688760, 602360, 515960, 429560, 343160, 256760, 170360, 83960, -2440, 0 ]
+}, {
+  "tz" : "Pacific/Truk",
+  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12220020428, -12219934028, -12219847628, -12219761228, -12219674828, -12219588428, -12219502028, -12219415628, -12219329228, -12219242828, -3944628428, -2208988800 ],
+  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, 691628, 605228, 518828, 432428, 346028, 259628, 173228, 86828, 428, -85972, 428, 0 ]
+}, {
+  "tz" : "Pacific/Wake",
+  "switches" : [ -62135636788, -59006401588, -55850727988, -52695054388, -46383620788, -43227947188, -40072273588, -33760839988, -30605166388, -27449492788, -21138059188, -17982385588, -14826711988, -12220110388, -12220023988, -12219937588, -12219851188, -12219764788, -12219678388, -12219591988, -12219505588, -12219419188, -12219332788, -2208988800 ],
+  "diffs" : [ -176012, -89612, -3212, 83188, 169588, 255988, 342388, 428788, 515188, 601588, 687988, 774388, 860788, 774388, 687988, 601588, 515188, 428788, 342388, 255988, 169588, 83188, -3212, 0 ]
+}, {
+  "tz" : "Pacific/Wallis",
+  "switches" : [ -62135640920, -59006405720, -55850732120, -52695058520, -46383624920, -43227951320, -40072277720, -33760844120, -30605170520, -27449496920, -21138063320, -17982389720, -14826716120, -12220114520, -12220028120, -12219941720, -12219855320, -12219768920, -12219682520, -12219596120, -12219509720, -12219423320, -12219336920, -2208988800 ],
+  "diffs" : [ -171880, -85480, 920, 87320, 173720, 260120, 346520, 432920, 519320, 605720, 692120, 778520, 864920, 778520, 692120, 605720, 519320, 432920, 346520, 260120, 173720, 87320, 920, 0 ]
+}, {
+  "tz" : "Pacific/Yap",
+  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12220020428, -12219934028, -12219847628, -12219761228, -12219674828, -12219588428, -12219502028, -12219415628, -12219329228, -12219242828, -3944628428, -2208988800 ],
+  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, 691628, 605228, 518828, 432428, 346028, 259628, 173228, 86828, 428, -85972, 428, 0 ]
+}, {
+  "tz" : "Poland",
+  "switches" : [ -62135601840, -59006366640, -55850693040, -52695019440, -46383585840, -43227912240, -40072238640, -33760805040, -30605131440, -27449457840, -21138024240, -17982350640, -14826677040, -12220075440, -12219989040, -12219902640, -12219816240, -12219729840, -12219643440, -12219557040, -12219470640, -12219384240, -12219297840, -2208988800 ],
+  "diffs" : [ -171360, -84960, 1440, 87840, 174240, 260640, 347040, 433440, 519840, 606240, 692640, 779040, 865440, 779040, 692640, 606240, 519840, 433440, 347040, 260640, 174240, 87840, 1440, 0 ]
+}, {
+  "tz" : "Portugal",
+  "switches" : [ -62135594595, -59006359395, -55850685795, -52695012195, -46383578595, -43227904995, -40072231395, -33760797795, -30605124195, -27449450595, -21138016995, -17982343395, -14826669795, -12220068195, -12219981795, -12219895395, -12219808995, -12219722595, -12219636195, -12219549795, -12219463395, -12219376995, -12219290595, -2208988800 ],
+  "diffs" : [ -175005, -88605, -2205, 84195, 170595, 256995, 343395, 429795, 516195, 602595, 688995, 775395, 861795, 775395, 688995, 602595, 516195, 429795, 343395, 256995, 170595, 84195, -2205, 0 ]
+}, {
+  "tz" : "ROK",
+  "switches" : [ -62135627272, -59006392072, -55850718472, -52695044872, -46383611272, -43227937672, -40072264072, -33760830472, -30605156872, -27449483272, -21138049672, -17982376072, -14826702472, -12220100872, -12220014472, -12219928072, -12219841672, -12219755272, -12219668872, -12219582472, -12219496072, -12219409672, -12219323272, -2208988800 ],
+  "diffs" : [ -174728, -88328, -1928, 84472, 170872, 257272, 343672, 430072, 516472, 602872, 689272, 775672, 862072, 775672, 689272, 602872, 516472, 430072, 343672, 257272, 170872, 84472, -1928, 0 ]
+}, {
+  "tz" : "Singapore",
+  "switches" : [ -62135621725, -59006386525, -55850712925, -52695039325, -46383605725, -43227932125, -40072258525, -33760824925, -30605151325, -27449477725, -21138044125, -17982370525, -14826696925, -12220095325, -12220008925, -12219922525, -12219836125, -12219749725, -12219663325, -12219576925, -12219490525, -12219404125, -12219317725, -2208988800 ],
+  "diffs" : [ -176675, -90275, -3875, 82525, 168925, 255325, 341725, 428125, 514525, 600925, 687325, 773725, 860125, 773725, 687325, 600925, 514525, 428125, 341725, 255325, 168925, 82525, -3875, 0 ]
+}, {
+  "tz" : "SystemV/AST4",
+  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12220056000, -12219969600, -12219883200, -12219796800, -12219710400, -12219624000, -12219537600, -12219451200, -12219364800, -12219278400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/AST4ADT",
+  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12220056000, -12219969600, -12219883200, -12219796800, -12219710400, -12219624000, -12219537600, -12219451200, -12219364800, -12219278400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/CST6",
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12220048800, -12219962400, -12219876000, -12219789600, -12219703200, -12219616800, -12219530400, -12219444000, -12219357600, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/CST6CDT",
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12220048800, -12219962400, -12219876000, -12219789600, -12219703200, -12219616800, -12219530400, -12219444000, -12219357600, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/EST5",
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12220052400, -12219966000, -12219879600, -12219793200, -12219706800, -12219620400, -12219534000, -12219447600, -12219361200, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/EST5EDT",
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12220052400, -12219966000, -12219879600, -12219793200, -12219706800, -12219620400, -12219534000, -12219447600, -12219361200, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/HST10",
+  "switches" : [ -62135560800, -59006325600, -55850652000, -52694978400, -46383544800, -43227871200, -40072197600, -33760764000, -30605090400, -27449416800, -21137983200, -17982309600, -14826636000, -12220034400, -12219948000, -12219861600, -12219775200, -12219688800, -12219602400, -12219516000, -12219429600, -12219343200, -12219256800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/MST7",
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12220045200, -12219958800, -12219872400, -12219786000, -12219699600, -12219613200, -12219526800, -12219440400, -12219354000, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/MST7MDT",
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12220045200, -12219958800, -12219872400, -12219786000, -12219699600, -12219613200, -12219526800, -12219440400, -12219354000, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/PST8",
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12220041600, -12219955200, -12219868800, -12219782400, -12219696000, -12219609600, -12219523200, -12219436800, -12219350400, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/PST8PDT",
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12220041600, -12219955200, -12219868800, -12219782400, -12219696000, -12219609600, -12219523200, -12219436800, -12219350400, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/YST9",
+  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12220038000, -12219951600, -12219865200, -12219778800, -12219692400, -12219606000, -12219519600, -12219433200, -12219346800, -12219260400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "SystemV/YST9YDT",
+  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12220038000, -12219951600, -12219865200, -12219778800, -12219692400, -12219606000, -12219519600, -12219433200, -12219346800, -12219260400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Turkey",
+  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12220077352, -12219990952, -12219904552, -12219818152, -12219731752, -12219645352, -12219558952, -12219472552, -12219386152, -12219299752, -2840147752, -2208988800 ],
+  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, 773752, 687352, 600952, 514552, 428152, 341752, 255352, 168952, 82552, -3848, -3784, 0 ]
+}, {
+  "tz" : "UCT",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "US/Alaska",
+  "switches" : [ -62135647224, -59006412024, -55850738424, -52695064824, -46383631224, -43227957624, -40072284024, -33760850424, -30605176824, -27449503224, -21138069624, -17982396024, -14826722424, -12220120824, -12220034424, -12219948024, -12219861624, -12219775224, -12219688824, -12219602424, -12219516024, -12219429624, -12219343224, -3225223727, -2208988800 ],
+  "diffs" : [ -89976, -3576, 82824, 169224, 255624, 342024, 428424, 514824, 601224, 687624, 774024, 860424, 946824, 860424, 774024, 687624, 601224, 514824, 428424, 342024, 255624, 169224, 82824, -3576, 0 ]
+}, {
+  "tz" : "US/Aleutian",
+  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12220114402, -12220028002, -12219941602, -12219855202, -12219768802, -12219682402, -12219596002, -12219509602, -12219423202, -12219336802, -3225223727, -2208988800 ],
+  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 857602, 771202, 684802, 598402, 512002, 425602, 339202, 252802, 166402, 80002, -6398, 0 ]
+}, {
+  "tz" : "US/Arizona",
+  "switches" : [ -62135569902, -59006334702, -55850661102, -52694987502, -46383553902, -43227880302, -40072206702, -33760773102, -30605099502, -27449425902, -21137992302, -17982318702, -14826645102, -12220043502, -12219957102, -12219870702, -12219784302, -12219697902, -12219611502, -12219525102, -12219438702, -12219352302, -12219265902, -2717643600 ],
+  "diffs" : [ -174498, -88098, -1698, 84702, 171102, 257502, 343902, 430302, 516702, 603102, 689502, 775902, 862302, 775902, 689502, 603102, 516702, 430302, 343902, 257502, 171102, 84702, -1698, 0 ]
+}, {
+  "tz" : "US/Central",
+  "switches" : [ -62135575764, -59006340564, -55850666964, -52694993364, -46383559764, -43227886164, -40072212564, -33760778964, -30605105364, -27449431764, -21137998164, -17982324564, -14826650964, -12220049364, -12219962964, -12219876564, -12219790164, -12219703764, -12219617364, -12219530964, -12219444564, -12219358164, -12219271764, -2717647200 ],
+  "diffs" : [ -172236, -85836, 564, 86964, 173364, 259764, 346164, 432564, 518964, 605364, 691764, 778164, 864564, 778164, 691764, 605364, 518964, 432564, 346164, 259764, 173364, 86964, 564, 0 ]
+}, {
+  "tz" : "US/East-Indiana",
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12220049722, -12219963322, -12219876922, -12219790522, -12219704122, -12219617722, -12219531322, -12219444922, -12219358522, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, 774922, 688522, 602122, 515722, 429322, 342922, 256522, 170122, 83722, -2678, -3600, 0 ]
+}, {
+  "tz" : "US/Eastern",
+  "switches" : [ -62135579038, -59006343838, -55850670238, -52694996638, -46383563038, -43227889438, -40072215838, -33760782238, -30605108638, -27449435038, -21138001438, -17982327838, -14826654238, -12220052638, -12219966238, -12219879838, -12219793438, -12219707038, -12219620638, -12219534238, -12219447838, -12219361438, -12219275038, -2717650800 ],
+  "diffs" : [ -172562, -86162, 238, 86638, 173038, 259438, 345838, 432238, 518638, 605038, 691438, 777838, 864238, 777838, 691438, 605038, 518638, 432238, 345838, 259438, 173038, 86638, 238, 0 ]
+}, {
+  "tz" : "US/Hawaii",
+  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12220032514, -12219946114, -12219859714, -12219773314, -12219686914, -12219600514, -12219514114, -12219427714, -12219341314, -12219254914, -2334101314, -2208988800 ],
+  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, 775714, 689314, 602914, 516514, 430114, 343714, 257314, 170914, 84514, -1886, -1800, 0 ]
+}, {
+  "tz" : "US/Indiana-Starke",
+  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12220049610, -12219963210, -12219876810, -12219790410, -12219704010, -12219617610, -12219531210, -12219444810, -12219358410, -12219272010, -2717647200 ],
+  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 778410, 692010, 605610, 519210, 432810, 346410, 260010, 173610, 87210, 810, 0 ]
+}, {
+  "tz" : "US/Michigan",
+  "switches" : [ -62135576869, -59006341669, -55850668069, -52694994469, -46383560869, -43227887269, -40072213669, -33760780069, -30605106469, -27449432869, -21137999269, -17982325669, -14826652069, -12220050469, -12219964069, -12219877669, -12219791269, -12219704869, -12219618469, -12219532069, -12219445669, -12219359269, -12219272869, -2208988800 ],
+  "diffs" : [ -174731, -88331, -1931, 84469, 170869, 257269, 343669, 430069, 516469, 602869, 689269, 775669, 862069, 775669, 689269, 602869, 516469, 430069, 343669, 257269, 170869, 84469, -1931, 0 ]
+}, {
+  "tz" : "US/Mountain",
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12220045204, -12219958804, -12219872404, -12219786004, -12219699604, -12219613204, -12219526804, -12219440404, -12219354004, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 777604, 691204, 604804, 518404, 432004, 345604, 259204, 172804, 86404, 4, 0 ]
+}, {
+  "tz" : "US/Pacific",
+  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12220042022, -12219955622, -12219869222, -12219782822, -12219696422, -12219610022, -12219523622, -12219437222, -12219350822, -12219264422, -2717640000 ],
+  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 778022, 691622, 605222, 518822, 432422, 346022, 259622, 173222, 86822, 422, 0 ]
+}, {
+  "tz" : "US/Pacific-New",
+  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12220042022, -12219955622, -12219869222, -12219782822, -12219696422, -12219610022, -12219523622, -12219437222, -12219350822, -12219264422, -2717640000 ],
+  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 778022, 691622, 605222, 518822, 432422, 346022, 259622, 173222, 86822, 422, 0 ]
+}, {
+  "tz" : "US/Samoa",
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12220115832, -12220029432, -12219943032, -12219856632, -12219770232, -12219683832, -12219597432, -12219511032, -12219424632, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 862632, 776232, 689832, 603432, 517032, 430632, 344232, 257832, 171432, 85032, -1368, 0 ]
+}, {
+  "tz" : "UTC",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Universal",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "W-SU",
+  "switches" : [ -62135605817, -59006370617, -55850697017, -52695023417, -46383589817, -43227916217, -40072242617, -33760809017, -30605135417, -27449461817, -21138028217, -17982354617, -14826681017, -12220079417, -12219993017, -12219906617, -12219820217, -12219733817, -12219647417, -12219561017, -12219474617, -12219388217, -12219301817, -2208988800 ],
+  "diffs" : [ -174583, -88183, -1783, 84617, 171017, 257417, 343817, 430217, 516617, 603017, 689417, 775817, 862217, 775817, 689417, 603017, 516617, 430217, 343817, 257417, 171017, 84617, -1783, 0 ]
+}, {
+  "tz" : "WET",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+}, {
+  "tz" : "Zulu",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
+} ]
\ No newline at end of file
diff --git a/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json b/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json
new file mode 100644
index 0000000000000..eddc16f9ae767
--- /dev/null
+++ b/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json
@@ -0,0 +1,2401 @@
+[ {
+  "tz" : "Africa/Abidjan",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Accra",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 172852, 86452, 52, -86348, -172748, -259148, -345548, -431948, -518348, -604748, -691148, -777548, -863948, 52, 0 ]
+}, {
+  "tz" : "Africa/Addis_Ababa",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Algiers",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2486682111, -2208988800 ],
+  "diffs" : [ 175668, 89268, 2868, -83532, -169932, -256332, -342732, -429132, -515532, -601932, -688332, -774732, -861132, 2868, 3039, 0 ]
+}, {
+  "tz" : "Africa/Asmara",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Asmera",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Bamako",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Bangui",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Banjul",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Bissau",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 176540, 90140, 3740, -82660, -169060, -255460, -341860, -428260, -514660, -601060, -687460, -773860, -860260, 3740, 0 ]
+}, {
+  "tz" : "Africa/Blantyre",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Brazzaville",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Bujumbura",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Cairo",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172491, 86091, -309, -86709, -173109, -259509, -345909, -432309, -518709, -605109, -691509, -777909, -864309, -309, 0 ]
+}, {
+  "tz" : "Africa/Casablanca",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 174620, 88220, 1820, -84580, -170980, -257380, -343780, -430180, -516580, -602980, -689380, -775780, -862180, 1820, 0 ]
+}, {
+  "tz" : "Africa/Ceuta",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 177676, 91276, 4876, -81524, -167924, -254324, -340724, -427124, -513524, -599924, -686324, -772724, -859124, 4876, 0 ]
+}, {
+  "tz" : "Africa/Conakry",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Dakar",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Dar_es_Salaam",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Djibouti",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Douala",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/El_Aaiun",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 175968, 89568, 3168, -83232, -169632, -256032, -342432, -428832, -515232, -601632, -688032, -774432, -860832, 3168, 0 ]
+}, {
+  "tz" : "Africa/Freetown",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Gaborone",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Harare",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Johannesburg",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2458174920, -2208988800 ],
+  "diffs" : [ 173280, 86880, 480, -85920, -172320, -258720, -345120, -431520, -517920, -604320, -690720, -777120, -863520, 480, 1800, 0 ]
+}, {
+  "tz" : "Africa/Juba",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 176012, 89612, 3212, -83188, -169588, -255988, -342388, -428788, -515188, -601588, -687988, -774388, -860788, 3212, 0 ]
+}, {
+  "tz" : "Africa/Kampala",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Khartoum",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172192, 85792, -608, -87008, -173408, -259808, -346208, -432608, -519008, -605408, -691808, -778208, -864608, -608, 0 ]
+}, {
+  "tz" : "Africa/Kigali",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Kinshasa",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Lagos",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Libreville",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Lome",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Luanda",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Lubumbashi",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Lusaka",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Malabo",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Maputo",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Maseru",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2458174920, -2208988800 ],
+  "diffs" : [ 173280, 86880, 480, -85920, -172320, -258720, -345120, -431520, -517920, -604320, -690720, -777120, -863520, 480, 1800, 0 ]
+}, {
+  "tz" : "Africa/Mbabane",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2458174920, -2208988800 ],
+  "diffs" : [ 173280, 86880, 480, -85920, -172320, -258720, -345120, -431520, -517920, -604320, -690720, -777120, -863520, 480, 1800, 0 ]
+}, {
+  "tz" : "Africa/Mogadishu",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Monrovia",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 175388, 88988, 2588, -83812, -170212, -256612, -343012, -429412, -515812, -602212, -688612, -775012, -861412, 2588, 0 ]
+}, {
+  "tz" : "Africa/Nairobi",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Ndjamena",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 172788, 86388, -12, -86412, -172812, -259212, -345612, -432012, -518412, -604812, -691212, -777612, -864012, -12, 0 ]
+}, {
+  "tz" : "Africa/Niamey",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Nouakchott",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Ouagadougou",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Porto-Novo",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Sao_Tome",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2713914221, -2208988800 ],
+  "diffs" : [ 171184, 84784, -1616, -88016, -174416, -260816, -347216, -433616, -520016, -606416, -692816, -779216, -865616, -1616, 2205, 0 ]
+}, {
+  "tz" : "Africa/Timbuktu",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Tripoli",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 176836, 90436, 4036, -82364, -168764, -255164, -341564, -427964, -514364, -600764, -687164, -773564, -859964, 4036, 0 ]
+}, {
+  "tz" : "Africa/Tunis",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2797205483, -2208988800 ],
+  "diffs" : [ 173956, 87556, 1156, -85244, -171644, -258044, -344444, -430844, -517244, -603644, -690044, -776444, -862844, 1156, 3039, 0 ]
+}, {
+  "tz" : "Africa/Windhoek",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2458172304, -2208988800 ],
+  "diffs" : [ 175896, 89496, 3096, -83304, -169704, -256104, -342504, -428904, -515304, -601704, -688104, -774504, -860904, 3096, 1800, 0 ]
+}, {
+  "tz" : "America/Adak",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -3225230125, -2208988800 ],
+  "diffs" : [ 92798, 6398, -80002, -166402, -252802, -339202, -425602, -512002, -598402, -684802, -771202, -857602, -944002, -80002, 6398, 0 ]
+}, {
+  "tz" : "America/Anchorage",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225227303, -2208988800 ],
+  "diffs" : [ 89976, 3576, -82824, -169224, -255624, -342024, -428424, -514824, -601224, -687624, -774024, -860424, -946824, -82824, 3576, 0 ]
+}, {
+  "tz" : "America/Anguilla",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Antigua",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Araguaina",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 173568, 87168, 768, -85632, -172032, -258432, -344832, -431232, -517632, -604032, -690432, -776832, -863232, 768, 0 ]
+}, {
+  "tz" : "America/Argentina/Buenos_Aires",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372102580, -2208988800 ],
+  "diffs" : [ 176028, 89628, 3228, -83172, -169572, -255972, -342372, -428772, -515172, -601572, -687972, -774372, -860772, 3228, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Catamarca",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100820, -2208988800 ],
+  "diffs" : [ 177788, 91388, 4988, -81412, -167812, -254212, -340612, -427012, -513412, -599812, -686212, -772612, -859012, 4988, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/ComodRivadavia",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100820, -2208988800 ],
+  "diffs" : [ 177788, 91388, 4988, -81412, -167812, -254212, -340612, -427012, -513412, -599812, -686212, -772612, -859012, 4988, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Cordoba",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 177408, 91008, 4608, -81792, -168192, -254592, -340992, -427392, -513792, -600192, -686592, -772992, -859392, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Jujuy",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100936, -2208988800 ],
+  "diffs" : [ 177672, 91272, 4872, -81528, -167928, -254328, -340728, -427128, -513528, -599928, -686328, -772728, -859128, 4872, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/La_Rioja",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100564, -2208988800 ],
+  "diffs" : [ 178044, 91644, 5244, -81156, -167556, -253956, -340356, -426756, -513156, -599556, -685956, -772356, -858756, 5244, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Mendoza",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100092, -2208988800 ],
+  "diffs" : [ 178516, 92116, 5716, -80684, -167084, -253484, -339884, -426284, -512684, -599084, -685484, -771884, -858284, 5716, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Rio_Gallegos",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372099996, -2208988800 ],
+  "diffs" : [ 178612, 92212, 5812, -80588, -166988, -253388, -339788, -426188, -512588, -598988, -685388, -771788, -858188, 5812, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Salta",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100908, -2208988800 ],
+  "diffs" : [ 177700, 91300, 4900, -81500, -167900, -254300, -340700, -427100, -513500, -599900, -686300, -772700, -859100, 4900, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/San_Juan",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100164, -2208988800 ],
+  "diffs" : [ 178444, 92044, 5644, -80756, -167156, -253556, -339956, -426356, -512756, -599156, -685556, -771956, -858356, 5644, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/San_Luis",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100684, -2208988800 ],
+  "diffs" : [ 177924, 91524, 5124, -81276, -167676, -254076, -340476, -426876, -513276, -599676, -686076, -772476, -858876, 5124, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Tucuman",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100956, -2208988800 ],
+  "diffs" : [ 177652, 91252, 4852, -81548, -167948, -254348, -340748, -427148, -513548, -599948, -686348, -772748, -859148, 4852, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Ushuaia",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100216, -2208988800 ],
+  "diffs" : [ 178392, 91992, 5592, -80808, -167208, -253608, -340008, -426408, -512808, -599208, -685608, -772008, -858408, 5592, 4608, 0 ]
+}, {
+  "tz" : "America/Aruba",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ]
+}, {
+  "tz" : "America/Asuncion",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172240, 85840, -560, -86960, -173360, -259760, -346160, -432560, -518960, -605360, -691760, -778160, -864560, -560, 0 ]
+}, {
+  "tz" : "America/Atikokan",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736812, -2208988800 ],
+  "diffs" : [ 176788, 90388, 3988, -82412, -168812, -255212, -341612, -428012, -514412, -600812, -687212, -773612, -860012, 3988, 3600, 0 ]
+}, {
+  "tz" : "America/Atka",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -3225230125, -2208988800 ],
+  "diffs" : [ 92798, 6398, -80002, -166402, -252802, -339202, -425602, -512002, -598402, -684802, -771202, -857602, -944002, -80002, 6398, 0 ]
+}, {
+  "tz" : "America/Bahia",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 171244, 84844, -1556, -87956, -174356, -260756, -347156, -433556, -519956, -606356, -692756, -779156, -865556, -1556, 0 ]
+}, {
+  "tz" : "America/Bahia_Banderas",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 176460, 90060, 3660, -82740, -169140, -255540, -341940, -428340, -514740, -601140, -687540, -773940, -860340, 3660, 0 ]
+}, {
+  "tz" : "America/Barbados",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172709, 86309, -91, -86491, -172891, -259291, -345691, -432091, -518491, -604891, -691291, -777691, -864091, -91, 0 ]
+}, {
+  "tz" : "America/Belem",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 173636, 87236, 836, -85564, -171964, -258364, -344764, -431164, -517564, -603964, -690364, -776764, -863164, 836, 0 ]
+}, {
+  "tz" : "America/Belize",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172368, 85968, -432, -86832, -173232, -259632, -346032, -432432, -518832, -605232, -691632, -778032, -864432, -432, 0 ]
+}, {
+  "tz" : "America/Blanc-Sablon",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2713896692 ],
+  "diffs" : [ 172108, 85708, -692, -87092, -173492, -259892, -346292, -432692, -519092, -605492, -691892, -778292, -864692, -692, 0 ]
+}, {
+  "tz" : "America/Boa_Vista",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172960, 86560, 160, -86240, -172640, -259040, -345440, -431840, -518240, -604640, -691040, -777440, -863840, 160, 0 ]
+}, {
+  "tz" : "America/Bogota",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 172576, 86176, -224, -86624, -173024, -259424, -345824, -432224, -518624, -605024, -691424, -777824, -864224, -224, 0 ]
+}, {
+  "tz" : "America/Boise",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600, -2208988800 ],
+  "diffs" : [ 175489, 89089, 2689, -83711, -170111, -256511, -342911, -429311, -515711, -602111, -688511, -774911, -861311, 2689, 3600, 0 ]
+}, {
+  "tz" : "America/Buenos_Aires",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372102580, -2208988800 ],
+  "diffs" : [ 176028, 89628, 3228, -83172, -169572, -255972, -342372, -428772, -515172, -601572, -687972, -774372, -860772, 3228, 4608, 0 ]
+}, {
+  "tz" : "America/Cambridge_Bay",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 147600, 61200, -25200, -111600, -198000, -284400, -370800, -457200, -543600, -630000, -716400, -802800, -889200, -25200, 0 ]
+}, {
+  "tz" : "America/Campo_Grande",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 171508, 85108, -1292, -87692, -174092, -260492, -346892, -433292, -519692, -606092, -692492, -778892, -865292, -1292, 0 ]
+}, {
+  "tz" : "America/Cancun",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 175624, 89224, 2824, -83576, -169976, -256376, -342776, -429176, -515576, -601976, -688376, -774776, -861176, 2824, 0 ]
+}, {
+  "tz" : "America/Caracas",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2524507196, -2208988800 ],
+  "diffs" : [ 174464, 88064, 1664, -84736, -171136, -257536, -343936, -430336, -516736, -603136, -689536, -775936, -862336, 1664, 1660, 0 ]
+}, {
+  "tz" : "America/Catamarca",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100820, -2208988800 ],
+  "diffs" : [ 177788, 91388, 4988, -81412, -167812, -254212, -340612, -427012, -513412, -599812, -686212, -772612, -859012, 4988, 4608, 0 ]
+}, {
+  "tz" : "America/Cayenne",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 174560, 88160, 1760, -84640, -171040, -257440, -343840, -430240, -516640, -603040, -689440, -775840, -862240, 1760, 0 ]
+}, {
+  "tz" : "America/Cayman",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503688, -2208988800 ],
+  "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 1176, 0 ]
+}, {
+  "tz" : "America/Chicago",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 172236, 85836, -564, -86964, -173364, -259764, -346164, -432564, -518964, -605364, -691764, -778164, -864564, -564, 0 ]
+}, {
+  "tz" : "America/Chihuahua",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 173060, 86660, 260, -86140, -172540, -258940, -345340, -431740, -518140, -604540, -690940, -777340, -863740, 260, 0 ]
+}, {
+  "tz" : "America/Coral_Harbour",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736812, -2208988800 ],
+  "diffs" : [ 176788, 90388, 3988, -82412, -168812, -255212, -341612, -428012, -514412, -600812, -687212, -773612, -860012, 3988, 3600, 0 ]
+}, {
+  "tz" : "America/Cordoba",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 177408, 91008, 4608, -81792, -168192, -254592, -340992, -427392, -513792, -600192, -686592, -772992, -859392, 4608, 0 ]
+}, {
+  "tz" : "America/Costa_Rica",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 171373, 84973, -1427, -87827, -174227, -260627, -347027, -433427, -519827, -606227, -692627, -779027, -865427, -1427, 0 ]
+}, {
+  "tz" : "America/Creston",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2713882436 ],
+  "diffs" : [ 175564, 89164, 2764, -83636, -170036, -256436, -342836, -429236, -515636, -602036, -688436, -774836, -861236, 2764, 0 ]
+}, {
+  "tz" : "America/Cuiaba",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 171860, 85460, -940, -87340, -173740, -260140, -346540, -432940, -519340, -605740, -692140, -778540, -864940, -940, 0 ]
+}, {
+  "tz" : "America/Curacao",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ]
+}, {
+  "tz" : "America/Danmarkshavn",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 177280, 90880, 4480, -81920, -168320, -254720, -341120, -427520, -513920, -600320, -686720, -773120, -859520, 4480, 0 ]
+}, {
+  "tz" : "America/Dawson",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 177460, 91060, 4660, -81740, -168140, -254540, -340940, -427340, -513740, -600140, -686540, -772940, -859340, 4660, 0 ]
+}, {
+  "tz" : "America/Dawson_Creek",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2713885144, -2208988800 ],
+  "diffs" : [ 176456, 90056, 3656, -82744, -169144, -255544, -341944, -428344, -514744, -601144, -687544, -773944, -860344, 3656, 3600, 0 ]
+}, {
+  "tz" : "America/Denver",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 172796, 86396, -4, -86404, -172804, -259204, -345604, -432004, -518404, -604804, -691204, -777604, -864004, -4, 0 ]
+}, {
+  "tz" : "America/Detroit",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 174731, 88331, 1931, -84469, -170869, -257269, -343669, -430069, -516469, -602869, -689269, -775669, -862069, 1931, 0 ]
+}, {
+  "tz" : "America/Dominica",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Edmonton",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 174832, 88432, 2032, -84368, -170768, -257168, -343568, -429968, -516368, -602768, -689168, -775568, -861968, 2032, 0 ]
+}, {
+  "tz" : "America/Eirunepe",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 171568, 85168, -1232, -87632, -174032, -260432, -346832, -433232, -519632, -606032, -692432, -778832, -865232, -1232, 0 ]
+}, {
+  "tz" : "America/El_Salvador",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172608, 86208, -192, -86592, -172992, -259392, -345792, -432192, -518592, -604992, -691392, -777792, -864192, -192, 0 ]
+}, {
+  "tz" : "America/Ensenada",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 172084, 85684, -716, -87116, -173516, -259916, -346316, -432716, -519116, -605516, -691916, -778316, -864716, -716, 0 ]
+}, {
+  "tz" : "America/Fort_Nelson",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2713884553, -2208988800 ],
+  "diffs" : [ 177047, 90647, 4247, -82153, -168553, -254953, -341353, -427753, -514153, -600553, -686953, -773353, -859753, 4247, 3600, 0 ]
+}, {
+  "tz" : "America/Fort_Wayne",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175478, 89078, 2678, -83722, -170122, -256522, -342922, -429322, -515722, -602122, -688522, -774922, -861322, 2678, 3600, 0 ]
+}, {
+  "tz" : "America/Fortaleza",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 171240, 84840, -1560, -87960, -174360, -260760, -347160, -433560, -519960, -606360, -692760, -779160, -865560, -1560, 0 ]
+}, {
+  "tz" : "America/Glace_Bay",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172788, 86388, -12, -86412, -172812, -259212, -345612, -432012, -518412, -604812, -691212, -777612, -864012, -12, 0 ]
+}, {
+  "tz" : "America/Godthab",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 174416, 88016, 1616, -84784, -171184, -257584, -343984, -430384, -516784, -603184, -689584, -775984, -862384, 1616, 0 ]
+}, {
+  "tz" : "America/Goose_Bay",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2713894152, -2208988800 ],
+  "diffs" : [ 172900, 86500, 100, -86300, -172700, -259100, -345500, -431900, -518300, -604700, -691100, -777500, -863900, 100, -1748, 0 ]
+}, {
+  "tz" : "America/Grand_Turk",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524504958, -2208988800 ],
+  "diffs" : [ 171872, 85472, -928, -87328, -173728, -260128, -346528, -432928, -519328, -605728, -692128, -778528, -864928, -928, 430, 0 ]
+}, {
+  "tz" : "America/Grenada",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Guadeloupe",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Guatemala",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172924, 86524, 124, -86276, -172676, -259076, -345476, -431876, -518276, -604676, -691076, -777476, -863876, 124, 0 ]
+}, {
+  "tz" : "America/Guayaquil",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503280, -2208988800 ],
+  "diffs" : [ 173960, 87560, 1160, -85240, -171640, -258040, -344440, -430840, -517240, -603640, -690040, -776440, -862840, 1160, 840, 0 ]
+}, {
+  "tz" : "America/Guyana",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172360, 85960, -440, -86840, -173240, -259640, -346040, -432440, -518840, -605240, -691640, -778040, -864440, -440, 0 ]
+}, {
+  "tz" : "America/Halifax",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173664, 87264, 864, -85536, -171936, -258336, -344736, -431136, -517536, -603936, -690336, -776736, -863136, 864, 0 ]
+}, {
+  "tz" : "America/Havana",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503608, -2208988800 ],
+  "diffs" : [ 174568, 88168, 1768, -84632, -171032, -257432, -343832, -430232, -516632, -603032, -689432, -775832, -862232, 1768, 1776, 0 ]
+}, {
+  "tz" : "America/Hermosillo",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 174232, 87832, 1432, -84968, -171368, -257768, -344168, -430568, -516968, -603368, -689768, -776168, -862568, 1432, 0 ]
+}, {
+  "tz" : "America/Indiana/Indianapolis",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175478, 89078, 2678, -83722, -170122, -256522, -342922, -429322, -515722, -602122, -688522, -774922, -861322, 2678, 3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Knox",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 171990, 85590, -810, -87210, -173610, -260010, -346410, -432810, -519210, -605610, -692010, -778410, -864810, -810, 0 ]
+}, {
+  "tz" : "America/Indiana/Marengo",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175523, 89123, 2723, -83677, -170077, -256477, -342877, -429277, -515677, -602077, -688477, -774877, -861277, 2723, 3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Petersburg",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175747, 89347, 2947, -83453, -169853, -256253, -342653, -429053, -515453, -601853, -688253, -774653, -861053, 2947, 3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Tell_City",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 172023, 85623, -777, -87177, -173577, -259977, -346377, -432777, -519177, -605577, -691977, -778377, -864777, -777, 0 ]
+}, {
+  "tz" : "America/Indiana/Vevay",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175216, 88816, 2416, -83984, -170384, -256784, -343184, -429584, -515984, -602384, -688784, -775184, -861584, 2416, 3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Vincennes",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175807, 89407, 3007, -83393, -169793, -256193, -342593, -428993, -515393, -601793, -688193, -774593, -860993, 3007, 3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Winamac",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 3600, 0 ]
+}, {
+  "tz" : "America/Indianapolis",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175478, 89078, 2678, -83722, -170122, -256522, -342922, -429322, -515722, -602122, -688522, -774922, -861322, 2678, 3600, 0 ]
+}, {
+  "tz" : "America/Inuvik",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 147600, 61200, -25200, -111600, -198000, -284400, -370800, -457200, -543600, -630000, -716400, -802800, -889200, -25200, 0 ]
+}, {
+  "tz" : "America/Iqaluit",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 154800, 68400, -18000, -104400, -190800, -277200, -363600, -450000, -536400, -622800, -709200, -795600, -882000, -18000, 0 ]
+}, {
+  "tz" : "America/Jamaica",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 173230, 86830, 430, -85970, -172370, -258770, -345170, -431570, -517970, -604370, -690770, -777170, -863570, 430, 0 ]
+}, {
+  "tz" : "America/Jujuy",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100936, -2208988800 ],
+  "diffs" : [ 177672, 91272, 4872, -81528, -167928, -254328, -340728, -427128, -513528, -599928, -686328, -772728, -859128, 4872, 4608, 0 ]
+}, {
+  "tz" : "America/Juneau",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225223588, -2208988800 ],
+  "diffs" : [ 86261, -139, -86539, -172939, -259339, -345739, -432139, -518539, -604939, -691339, -777739, -864139, -950539, -86539, -139, 0 ]
+}, {
+  "tz" : "America/Kentucky/Louisville",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175382, 88982, 2582, -83818, -170218, -256618, -343018, -429418, -515818, -602218, -688618, -775018, -861418, 2582, 3600, 0 ]
+}, {
+  "tz" : "America/Kentucky/Monticello",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175164, 88764, 2364, -84036, -170436, -256836, -343236, -429636, -516036, -602436, -688836, -775236, -861636, 2364, 3600, 0 ]
+}, {
+  "tz" : "America/Knox_IN",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 171990, 85590, -810, -87210, -173610, -260010, -346410, -432810, -519210, -605610, -692010, -778410, -864810, -810, 0 ]
+}, {
+  "tz" : "America/Kralendijk",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ]
+}, {
+  "tz" : "America/La_Paz",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174756, 88356, 1956, -84444, -170844, -257244, -343644, -430044, -516444, -602844, -689244, -775644, -862044, 1956, 0 ]
+}, {
+  "tz" : "America/Lima",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503624, -2208988800 ],
+  "diffs" : [ 173292, 86892, 492, -85908, -172308, -258708, -345108, -431508, -517908, -604308, -690708, -777108, -863508, 492, 516, 0 ]
+}, {
+  "tz" : "America/Los_Angeles",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2717640000 ],
+  "diffs" : [ 172378, 85978, -422, -86822, -173222, -259622, -346022, -432422, -518822, -605222, -691622, -778022, -864422, -422, 0 ]
+}, {
+  "tz" : "America/Louisville",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175382, 88982, 2582, -83818, -170218, -256618, -343018, -429418, -515818, -602218, -688618, -775018, -861418, 2582, 3600, 0 ]
+}, {
+  "tz" : "America/Lower_Princes",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ]
+}, {
+  "tz" : "America/Maceio",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 170572, 84172, -2228, -88628, -175028, -261428, -347828, -434228, -520628, -607028, -693428, -779828, -866228, -2228, 0 ]
+}, {
+  "tz" : "America/Managua",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2524500004, -2208988800 ],
+  "diffs" : [ 171908, 85508, -892, -87292, -173692, -260092, -346492, -432892, -519292, -605692, -692092, -778492, -864892, -892, -888, 0 ]
+}, {
+  "tz" : "America/Manaus",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172804, 86404, 4, -86396, -172796, -259196, -345596, -431996, -518396, -604796, -691196, -777596, -863996, 4, 0 ]
+}, {
+  "tz" : "America/Marigot",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Martinique",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173060, 86660, 260, -86140, -172540, -258940, -345340, -431740, -518140, -604540, -690940, -777340, -863740, 260, 0 ]
+}, {
+  "tz" : "America/Matamoros",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 175200, 88800, 2400, -84000, -170400, -256800, -343200, -429600, -516000, -602400, -688800, -775200, -861600, 2400, 0 ]
+}, {
+  "tz" : "America/Mazatlan",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 173140, 86740, 340, -86060, -172460, -258860, -345260, -431660, -518060, -604460, -690860, -777260, -863660, 340, 0 ]
+}, {
+  "tz" : "America/Mendoza",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100092, -2208988800 ],
+  "diffs" : [ 178516, 92116, 5716, -80684, -167084, -253484, -339884, -426284, -512684, -599084, -685484, -771884, -858284, 5716, 4608, 0 ]
+}, {
+  "tz" : "America/Menominee",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2659759773 ],
+  "diffs" : [ 172227, 85827, -573, -86973, -173373, -259773, -346173, -432573, -518973, -605373, -691773, -778173, -864573, -573, 0 ]
+}, {
+  "tz" : "America/Merida",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172708, 86308, -92, -86492, -172892, -259292, -345692, -432092, -518492, -604892, -691292, -777692, -864092, -92, 0 ]
+}, {
+  "tz" : "America/Metlakatla",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225222905, -2208988800 ],
+  "diffs" : [ 85578, -822, -87222, -173622, -260022, -346422, -432822, -519222, -605622, -692022, -778422, -864822, -951222, -87222, -822, 0 ]
+}, {
+  "tz" : "America/Mexico_City",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 174996, 88596, 2196, -84204, -170604, -257004, -343404, -429804, -516204, -602604, -689004, -775404, -861804, 2196, 0 ]
+}, {
+  "tz" : "America/Miquelon",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 175480, 89080, 2680, -83720, -170120, -256520, -342920, -429320, -515720, -602120, -688520, -774920, -861320, 2680, 0 ]
+}, {
+  "tz" : "America/Moncton",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2715885652, -2208988800 ],
+  "diffs" : [ 173948, 87548, 1148, -85252, -171652, -258052, -344452, -430852, -517252, -603652, -690052, -776452, -862852, 1148, 3600, 0 ]
+}, {
+  "tz" : "America/Monterrey",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 175276, 88876, 2476, -83924, -170324, -256724, -343124, -429524, -515924, -602324, -688724, -775124, -861524, 2476, 0 ]
+}, {
+  "tz" : "America/Montevideo",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 175491, 89091, 2691, -83709, -170109, -256509, -342909, -429309, -515709, -602109, -688509, -774909, -861309, 2691, 0 ]
+}, {
+  "tz" : "America/Montreal",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736148 ],
+  "diffs" : [ 173852, 87452, 1052, -85348, -171748, -258148, -344548, -430948, -517348, -603748, -690148, -776548, -862948, 1052, 0 ]
+}, {
+  "tz" : "America/Montserrat",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Nassau",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 173370, 86970, 570, -85830, -172230, -258630, -345030, -431430, -517830, -604230, -690630, -777030, -863430, 570, 0 ]
+}, {
+  "tz" : "America/New_York",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800 ],
+  "diffs" : [ 172562, 86162, -238, -86638, -173038, -259438, -345838, -432238, -518638, -605038, -691438, -777838, -864238, -238, 0 ]
+}, {
+  "tz" : "America/Nipigon",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366734016 ],
+  "diffs" : [ 175984, 89584, 3184, -83216, -169616, -256016, -342416, -428816, -515216, -601616, -688016, -774416, -860816, 3184, 0 ]
+}, {
+  "tz" : "America/Nome",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225231025, -2208988800 ],
+  "diffs" : [ 93698, 7298, -79102, -165502, -251902, -338302, -424702, -511102, -597502, -683902, -770302, -856702, -943102, -79102, 7298, 0 ]
+}, {
+  "tz" : "America/Noronha",
+  "switches" : [ -62135762400, -59006440800, -55850680800, -52694920800, -46383400800, -43227640800, -40071880800, -33760360800, -30604600800, -27448840800, -21137320800, -17981560800, -14825800800, -12219285600, -2208988800 ],
+  "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ]
+}, {
+  "tz" : "America/North_Dakota/Beulah",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200, -2208988800 ],
+  "diffs" : [ 175627, 89227, 2827, -83573, -169973, -256373, -342773, -429173, -515573, -601973, -688373, -774773, -861173, 2827, 3600, 0 ]
+}, {
+  "tz" : "America/North_Dakota/Center",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200, -2208988800 ],
+  "diffs" : [ 175512, 89112, 2712, -83688, -170088, -256488, -342888, -429288, -515688, -602088, -688488, -774888, -861288, 2712, 3600, 0 ]
+}, {
+  "tz" : "America/North_Dakota/New_Salem",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200, -2208988800 ],
+  "diffs" : [ 175539, 89139, 2739, -83661, -170061, -256461, -342861, -429261, -515661, -602061, -688461, -774861, -861261, 2739, 3600, 0 ]
+}, {
+  "tz" : "America/Ojinaga",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 172660, 86260, -140, -86540, -172940, -259340, -345740, -432140, -518540, -604940, -691340, -777740, -864140, -140, 0 ]
+}, {
+  "tz" : "America/Panama",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503688, -2208988800 ],
+  "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 1176, 0 ]
+}, {
+  "tz" : "America/Pangnirtung",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 154800, 68400, -18000, -104400, -190800, -277200, -363600, -450000, -536400, -622800, -709200, -795600, -882000, -18000, 0 ]
+}, {
+  "tz" : "America/Paramaribo",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 175240, 88840, 2440, -83960, -170360, -256760, -343160, -429560, -515960, -602360, -688760, -775160, -861560, 2440, 0 ]
+}, {
+  "tz" : "America/Phoenix",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 174498, 88098, 1698, -84702, -171102, -257502, -343902, -430302, -516702, -603102, -689502, -775902, -862302, 1698, 0 ]
+}, {
+  "tz" : "America/Port-au-Prince",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503580, -2208988800 ],
+  "diffs" : [ 172160, 85760, -640, -87040, -173440, -259840, -346240, -432640, -519040, -605440, -691840, -778240, -864640, -640, -660, 0 ]
+}, {
+  "tz" : "America/Port_of_Spain",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Porto_Acre",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 171072, 84672, -1728, -88128, -174528, -260928, -347328, -433728, -520128, -606528, -692928, -779328, -865728, -1728, 0 ]
+}, {
+  "tz" : "America/Porto_Velho",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173736, 87336, 936, -85464, -171864, -258264, -344664, -431064, -517464, -603864, -690264, -776664, -863064, 936, 0 ]
+}, {
+  "tz" : "America/Puerto_Rico",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ],
+  "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ]
+}, {
+  "tz" : "America/Punta_Arenas",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2524510746, -2208988800 ],
+  "diffs" : [ 179020, 92620, 6220, -80180, -166580, -252980, -339380, -425780, -512180, -598580, -684980, -771380, -857780, 6220, 6166, 0 ]
+}, {
+  "tz" : "America/Rainy_River",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2366732504 ],
+  "diffs" : [ 173896, 87496, 1096, -85304, -171704, -258104, -344504, -430904, -517304, -603704, -690104, -776504, -862904, 1096, 0 ]
+}, {
+  "tz" : "America/Rankin_Inlet",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 151200, 64800, -21600, -108000, -194400, -280800, -367200, -453600, -540000, -626400, -712800, -799200, -885600, -21600, 0 ]
+}, {
+  "tz" : "America/Recife",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 170376, 83976, -2424, -88824, -175224, -261624, -348024, -434424, -520824, -607224, -693624, -780024, -866424, -2424, 0 ]
+}, {
+  "tz" : "America/Regina",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 176316, 89916, 3516, -82884, -169284, -255684, -342084, -428484, -514884, -601284, -687684, -774084, -860484, 3516, 0 ]
+}, {
+  "tz" : "America/Resolute",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 151200, 64800, -21600, -108000, -194400, -280800, -367200, -453600, -540000, -626400, -712800, -799200, -885600, -21600, 0 ]
+}, {
+  "tz" : "America/Rio_Branco",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 171072, 84672, -1728, -88128, -174528, -260928, -347328, -433728, -520128, -606528, -692928, -779328, -865728, -1728, 0 ]
+}, {
+  "tz" : "America/Rosario",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 177408, 91008, 4608, -81792, -168192, -254592, -340992, -427392, -513792, -600192, -686592, -772992, -859392, 4608, 0 ]
+}, {
+  "tz" : "America/Santa_Isabel",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 172084, 85684, -716, -87116, -173516, -259916, -346316, -432716, -519116, -605516, -691916, -778316, -864716, -716, 0 ]
+}, {
+  "tz" : "America/Santarem",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 175128, 88728, 2328, -84072, -170472, -256872, -343272, -429672, -516072, -602472, -688872, -775272, -861672, 2328, 0 ]
+}, {
+  "tz" : "America/Santiago",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 175366, 88966, 2566, -83834, -170234, -256634, -343034, -429434, -515834, -602234, -688634, -775034, -861434, 2566, 0 ]
+}, {
+  "tz" : "America/Santo_Domingo",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2524507224, -2208988800 ],
+  "diffs" : [ 175176, 88776, 2376, -84024, -170424, -256824, -343224, -429624, -516024, -602424, -688824, -775224, -861624, 2376, 2400, 0 ]
+}, {
+  "tz" : "America/Sao_Paulo",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 173188, 86788, 388, -86012, -172412, -258812, -345212, -431612, -518012, -604412, -690812, -777212, -863612, 388, 0 ]
+}, {
+  "tz" : "America/Scoresbysund",
+  "switches" : [ -62135766000, -59006444400, -55850684400, -52694924400, -46383404400, -43227644400, -40071884400, -33760364400, -30604604400, -27448844400, -21137324400, -17981564400, -14825804400, -12219289200, -2208988800 ],
+  "diffs" : [ 174472, 88072, 1672, -84728, -171128, -257528, -343928, -430328, -516728, -603128, -689528, -775928, -862328, 1672, 0 ]
+}, {
+  "tz" : "America/Shiprock",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 172796, 86396, -4, -86404, -172804, -259204, -345604, -432004, -518404, -604804, -691204, -777604, -864004, -4, 0 ]
+}, {
+  "tz" : "America/Sitka",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225223800, -2208988800 ],
+  "diffs" : [ 86473, 73, -86327, -172727, -259127, -345527, -431927, -518327, -604727, -691127, -777527, -863927, -950327, -86327, 73, 0 ]
+}, {
+  "tz" : "America/St_Barthelemy",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/St_Johns",
+  "switches" : [ -62135757000, -59006435400, -55850675400, -52694915400, -46383395400, -43227635400, -40071875400, -33760355400, -30604595400, -27448835400, -21137315400, -17981555400, -14825795400, -12219280200, -2208988800 ],
+  "diffs" : [ 172852, 86452, 52, -86348, -172748, -259148, -345548, -431948, -518348, -604748, -691148, -777548, -863948, 52, 0 ]
+}, {
+  "tz" : "America/St_Kitts",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/St_Lucia",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/St_Thomas",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/St_Vincent",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Swift_Current",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 177080, 90680, 4280, -82120, -168520, -254920, -341320, -427720, -514120, -600520, -686920, -773320, -859720, 4280, 0 ]
+}, {
+  "tz" : "America/Tegucigalpa",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172132, 85732, -668, -87068, -173468, -259868, -346268, -432668, -519068, -605468, -691868, -778268, -864668, -668, 0 ]
+}, {
+  "tz" : "America/Thule",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174908, 88508, 2108, -84292, -170692, -257092, -343492, -429892, -516292, -602692, -689092, -775492, -861892, 2108, 0 ]
+}, {
+  "tz" : "America/Thunder_Bay",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366737380, -2208988800 ],
+  "diffs" : [ 176220, 89820, 3420, -82980, -169380, -255780, -342180, -428580, -514980, -601380, -687780, -774180, -860580, 3420, 3600, 0 ]
+}, {
+  "tz" : "America/Tijuana",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 172084, 85684, -716, -87116, -173516, -259916, -346316, -432716, -519116, -605516, -691916, -778316, -864716, -716, 0 ]
+}, {
+  "tz" : "America/Toronto",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736148 ],
+  "diffs" : [ 173852, 87452, 1052, -85348, -171748, -258148, -344548, -430948, -517348, -603748, -690148, -776548, -862948, 1052, 0 ]
+}, {
+  "tz" : "America/Tortola",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Vancouver",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2713880852 ],
+  "diffs" : [ 173548, 87148, 748, -85652, -172052, -258452, -344852, -431252, -517652, -604052, -690452, -776852, -863252, 748, 0 ]
+}, {
+  "tz" : "America/Virgin",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Whitehorse",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 176412, 90012, 3612, -82788, -169188, -255588, -341988, -428388, -514788, -601188, -687588, -773988, -860388, 3612, 0 ]
+}, {
+  "tz" : "America/Winnipeg",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2602258284 ],
+  "diffs" : [ 174516, 88116, 1716, -84684, -171084, -257484, -343884, -430284, -516684, -603084, -689484, -775884, -862284, 1716, 0 ]
+}, {
+  "tz" : "America/Yakutat",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225224862, -2208988800 ],
+  "diffs" : [ 87535, 1135, -85265, -171665, -258065, -344465, -430865, -517265, -603665, -690065, -776465, -862865, -949265, -85265, 1135, 0 ]
+}, {
+  "tz" : "America/Yellowknife",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 147600, 61200, -25200, -111600, -198000, -284400, -370800, -457200, -543600, -630000, -716400, -802800, -889200, -25200, 0 ]
+}, {
+  "tz" : "Antarctica/Casey",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 201600, 115200, 28800, -57600, -144000, -230400, -316800, -403200, -489600, -576000, -662400, -748800, -835200, 28800, 0 ]
+}, {
+  "tz" : "Antarctica/Davis",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 198000, 111600, 25200, -61200, -147600, -234000, -320400, -406800, -493200, -579600, -666000, -752400, -838800, 25200, 0 ]
+}, {
+  "tz" : "Antarctica/DumontDUrville",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2208988800 ],
+  "diffs" : [ 208800, 122400, 36000, -50400, -136800, -223200, -309600, -396000, -482400, -568800, -655200, -741600, -828000, 36000, 0 ]
+}, {
+  "tz" : "Antarctica/Macquarie",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2214262800, -2208988800 ],
+  "diffs" : [ 212400, 126000, 39600, -46800, -133200, -219600, -306000, -392400, -478800, -565200, -651600, -738000, -824400, 39600, 3600, 0 ]
+}, {
+  "tz" : "Antarctica/Mawson",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 190800, 104400, 18000, -68400, -154800, -241200, -327600, -414000, -500400, -586800, -673200, -759600, -846000, 18000, 0 ]
+}, {
+  "tz" : "Antarctica/McMurdo",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -3192437344, -2208988800 ],
+  "diffs" : [ 174056, 87656, 1256, -85144, -171544, -257944, -344344, -430744, -517144, -603544, -689944, -776344, -862744, 1256, 1800, 0 ]
+}, {
+  "tz" : "Antarctica/Palmer",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 162000, 75600, -10800, -97200, -183600, -270000, -356400, -442800, -529200, -615600, -702000, -788400, -874800, -10800, 0 ]
+}, {
+  "tz" : "Antarctica/Rothera",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 162000, 75600, -10800, -97200, -183600, -270000, -356400, -442800, -529200, -615600, -702000, -788400, -874800, -10800, 0 ]
+}, {
+  "tz" : "Antarctica/South_Pole",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -3192437344, -2208988800 ],
+  "diffs" : [ 174056, 87656, 1256, -85144, -171544, -257944, -344344, -430744, -517144, -603544, -689944, -776344, -862744, 1256, 1800, 0 ]
+}, {
+  "tz" : "Antarctica/Syowa",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 183600, 97200, 10800, -75600, -162000, -248400, -334800, -421200, -507600, -594000, -680400, -766800, -853200, 10800, 0 ]
+}, {
+  "tz" : "Antarctica/Troll",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Antarctica/Vostok",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 194400, 108000, 21600, -64800, -151200, -237600, -324000, -410400, -496800, -583200, -669600, -756000, -842400, 21600, 0 ]
+}, {
+  "tz" : "Arctic/Longyearbyen",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2366757780 ],
+  "diffs" : [ 173820, 87420, 1020, -85380, -171780, -258180, -344580, -430980, -517380, -603780, -690180, -776580, -862980, 1020, 0 ]
+}, {
+  "tz" : "Asia/Aden",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 172388, 85988, -412, -86812, -173212, -259612, -346012, -432412, -518812, -605212, -691612, -778012, -864412, -412, 0 ]
+}, {
+  "tz" : "Asia/Almaty",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 175932, 89532, 3132, -83268, -169668, -256068, -342468, -428868, -515268, -601668, -688068, -774468, -860868, 3132, 0 ]
+}, {
+  "tz" : "Asia/Amman",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171376, 84976, -1424, -87824, -174224, -260624, -347024, -433424, -519824, -606224, -692624, -779024, -865424, -1424, 0 ]
+}, {
+  "tz" : "Asia/Anadyr",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 173404, 87004, 604, -85796, -172196, -258596, -344996, -431396, -517796, -604196, -690596, -776996, -863396, 604, 0 ]
+}, {
+  "tz" : "Asia/Aqtau",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 178736, 92336, 5936, -80464, -166864, -253264, -339664, -426064, -512464, -598864, -685264, -771664, -858064, 5936, 0 ]
+}, {
+  "tz" : "Asia/Aqtobe",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 177080, 90680, 4280, -82120, -168520, -254920, -341320, -427720, -514120, -600520, -686920, -773320, -859720, 4280, 0 ]
+}, {
+  "tz" : "Asia/Ashgabat",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 176788, 90388, 3988, -82412, -168812, -255212, -341612, -428012, -514412, -600812, -687212, -773612, -860012, 3988, 0 ]
+}, {
+  "tz" : "Asia/Ashkhabad",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 176788, 90388, 3988, -82412, -168812, -255212, -341612, -428012, -514412, -600812, -687212, -773612, -860012, 3988, 0 ]
+}, {
+  "tz" : "Asia/Atyrau",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 178336, 91936, 5536, -80864, -167264, -253664, -340064, -426464, -512864, -599264, -685664, -772064, -858464, 5536, 0 ]
+}, {
+  "tz" : "Asia/Baghdad",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2524532404, -2208988800 ],
+  "diffs" : [ 172940, 86540, 140, -86260, -172660, -259060, -345460, -431860, -518260, -604660, -691060, -777460, -863860, 140, 144, 0 ]
+}, {
+  "tz" : "Asia/Bahrain",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 171232, 84832, -1568, -87968, -174368, -260768, -347168, -433568, -519968, -606368, -692768, -779168, -865568, -1568, 0 ]
+}, {
+  "tz" : "Asia/Baku",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 175236, 88836, 2436, -83964, -170364, -256764, -343164, -429564, -515964, -602364, -688764, -775164, -861564, 2436, 0 ]
+}, {
+  "tz" : "Asia/Bangkok",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 173876, 87476, 1076, -85324, -171724, -258124, -344524, -430924, -517324, -603724, -690124, -776524, -862924, 1076, 0 ]
+}, {
+  "tz" : "Asia/Barnaul",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 177900, 91500, 5100, -81300, -167700, -254100, -340500, -426900, -513300, -599700, -686100, -772500, -858900, 5100, 0 ]
+}, {
+  "tz" : "Asia/Beirut",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840149320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Asia/Bishkek",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 176496, 90096, 3696, -82704, -169104, -255504, -341904, -428304, -514704, -601104, -687504, -773904, -860304, 3696, 0 ]
+}, {
+  "tz" : "Asia/Brunei",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174020, 87620, 1220, -85180, -171580, -257980, -344380, -430780, -517180, -603580, -689980, -776380, -862780, 1220, 0 ]
+}, {
+  "tz" : "Asia/Calcutta",
+  "switches" : [ -62135789400, -59006467800, -55850707800, -52694947800, -46383427800, -43227667800, -40071907800, -33760387800, -30604627800, -27448867800, -21137347800, -17981587800, -14825827800, -12219312600, -3645235808, -3155695330, -2208988800 ],
+  "diffs" : [ 171392, 84992, -1408, -87808, -174208, -260608, -347008, -433408, -519808, -606208, -692608, -779008, -865408, -1408, -1400, 530, 0 ]
+}, {
+  "tz" : "Asia/Chita",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 177968, 91568, 5168, -81232, -167632, -254032, -340432, -426832, -513232, -599632, -686032, -772432, -858832, 5168, 0 ]
+}, {
+  "tz" : "Asia/Choibalsan",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174120, 87720, 1320, -85080, -171480, -257880, -344280, -430680, -517080, -603480, -689880, -776280, -862680, 1320, 0 ]
+}, {
+  "tz" : "Asia/Chongqing",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172457, 86057, -343, -86743, -173143, -259543, -345943, -432343, -518743, -605143, -691543, -777943, -864343, -343, 0 ]
+}, {
+  "tz" : "Asia/Chungking",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172457, 86057, -343, -86743, -173143, -259543, -345943, -432343, -518743, -605143, -691543, -777943, -864343, -343, 0 ]
+}, {
+  "tz" : "Asia/Colombo",
+  "switches" : [ -62135789400, -59006467800, -55850707800, -52694947800, -46383427800, -43227667800, -40071907800, -33760387800, -30604627800, -27448867800, -21137347800, -17981587800, -14825827800, -12219312600, -2840160592, -2208988800 ],
+  "diffs" : [ 173436, 87036, 636, -85764, -172164, -258564, -344964, -431364, -517764, -604164, -690564, -776964, -863364, 636, 628, 0 ]
+}, {
+  "tz" : "Asia/Dacca",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2524543700, -2208988800 ],
+  "diffs" : [ 172700, 86300, -100, -86500, -172900, -259300, -345700, -432100, -518500, -604900, -691300, -777700, -864100, -100, 400, 0 ]
+}, {
+  "tz" : "Asia/Damascus",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171288, 84888, -1512, -87912, -174312, -260712, -347112, -433512, -519912, -606312, -692712, -779112, -865512, -1512, 0 ]
+}, {
+  "tz" : "Asia/Dhaka",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2524543700, -2208988800 ],
+  "diffs" : [ 172700, 86300, -100, -86500, -172900, -259300, -345700, -432100, -518500, -604900, -691300, -777700, -864100, -100, 400, 0 ]
+}, {
+  "tz" : "Asia/Dili",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 175060, 88660, 2260, -84140, -170540, -256940, -343340, -429740, -516140, -602540, -688940, -775340, -861740, 2260, 0 ]
+}, {
+  "tz" : "Asia/Dubai",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 173928, 87528, 1128, -85272, -171672, -258072, -344472, -430872, -517272, -603672, -690072, -776472, -862872, 1128, 0 ]
+}, {
+  "tz" : "Asia/Dushanbe",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 174288, 87888, 1488, -84912, -171312, -257712, -344112, -430512, -516912, -603312, -689712, -776112, -862512, 1488, 0 ]
+}, {
+  "tz" : "Asia/Famagusta",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171852, 85452, -948, -87348, -173748, -260148, -346548, -432948, -519348, -605748, -692148, -778548, -864948, -948, 0 ]
+}, {
+  "tz" : "Asia/Gaza",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171728, 85328, -1072, -87472, -173872, -260272, -346672, -433072, -519472, -605872, -692272, -778672, -865072, -1072, 0 ]
+}, {
+  "tz" : "Asia/Harbin",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172457, 86057, -343, -86743, -173143, -259543, -345943, -432343, -518743, -605143, -691543, -777943, -864343, -343, 0 ]
+}, {
+  "tz" : "Asia/Hebron",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171577, 85177, -1223, -87623, -174023, -260423, -346823, -433223, -519623, -606023, -692423, -778823, -865223, -1223, 0 ]
+}, {
+  "tz" : "Asia/Ho_Chi_Minh",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 172400, 86000, -400, -86800, -173200, -259600, -346000, -432400, -518800, -605200, -691600, -778000, -864400, -400, 0 ]
+}, {
+  "tz" : "Asia/Hong_Kong",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174198, 87798, 1398, -85002, -171402, -257802, -344202, -430602, -517002, -603402, -689802, -776202, -862602, 1398, 0 ]
+}, {
+  "tz" : "Asia/Hovd",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 176004, 89604, 3204, -83196, -169596, -255996, -342396, -428796, -515196, -601596, -687996, -774396, -860796, 3204, 0 ]
+}, {
+  "tz" : "Asia/Irkutsk",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 176575, 90175, 3775, -82625, -169025, -255425, -341825, -428225, -514625, -601025, -687425, -773825, -860225, 3775, 0 ]
+}, {
+  "tz" : "Asia/Istanbul",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2840151536, -2208988800 ],
+  "diffs" : [ 176648, 90248, 3848, -82552, -168952, -255352, -341752, -428152, -514552, -600952, -687352, -773752, -860152, 3848, 3784, 0 ]
+}, {
+  "tz" : "Asia/Jakarta",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 172368, 85968, -432, -86832, -173232, -259632, -346032, -432432, -518832, -605232, -691632, -778032, -864432, -432, 0 ]
+}, {
+  "tz" : "Asia/Jayapura",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 171432, 85032, -1368, -87768, -174168, -260568, -346968, -433368, -519768, -606168, -692568, -778968, -865368, -1368, 0 ]
+}, {
+  "tz" : "Asia/Jerusalem",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148014, -2208988800 ],
+  "diffs" : [ 171546, 85146, -1254, -87654, -174054, -260454, -346854, -433254, -519654, -606054, -692454, -778854, -865254, -1254, -1240, 0 ]
+}, {
+  "tz" : "Asia/Kabul",
+  "switches" : [ -62135785800, -59006464200, -55850704200, -52694944200, -46383424200, -43227664200, -40071904200, -33760384200, -30604624200, -27448864200, -21137344200, -17981584200, -14825824200, -12219309000, -2524540008, -2208988800 ],
+  "diffs" : [ 172392, 85992, -408, -86808, -173208, -259608, -346008, -432408, -518808, -605208, -691608, -778008, -864408, -408, 1800, 0 ]
+}, {
+  "tz" : "Asia/Kamchatka",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 177924, 91524, 5124, -81276, -167676, -254076, -340476, -426876, -513276, -599676, -686076, -772476, -858876, 5124, 0 ]
+}, {
+  "tz" : "Asia/Karachi",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 174708, 88308, 1908, -84492, -170892, -257292, -343692, -430092, -516492, -602892, -689292, -775692, -862092, 1908, 0 ]
+}, {
+  "tz" : "Asia/Kashgar",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ]
+}, {
+  "tz" : "Asia/Kathmandu",
+  "switches" : [ -62135790300, -59006468700, -55850708700, -52694948700, -46383428700, -43227668700, -40071908700, -33760388700, -30604628700, -27448868700, -21137348700, -17981588700, -14825828700, -12219313500, -2208988800 ],
+  "diffs" : [ 173024, 86624, 224, -86176, -172576, -258976, -345376, -431776, -518176, -604576, -690976, -777376, -863776, 224, 0 ]
+}, {
+  "tz" : "Asia/Katmandu",
+  "switches" : [ -62135790300, -59006468700, -55850708700, -52694948700, -46383428700, -43227668700, -40071908700, -33760388700, -30604628700, -27448868700, -21137348700, -17981588700, -14825828700, -12219313500, -2208988800 ],
+  "diffs" : [ 173024, 86624, 224, -86176, -172576, -258976, -345376, -431776, -518176, -604576, -690976, -777376, -863776, 224, 0 ]
+}, {
+  "tz" : "Asia/Khandyga",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 172667, 86267, -133, -86533, -172933, -259333, -345733, -432133, -518533, -604933, -691333, -777733, -864133, -133, 0 ]
+}, {
+  "tz" : "Asia/Kolkata",
+  "switches" : [ -62135789400, -59006467800, -55850707800, -52694947800, -46383427800, -43227667800, -40071907800, -33760387800, -30604627800, -27448867800, -21137347800, -17981587800, -14825827800, -12219312600, -3645235808, -3155695330, -2208988800 ],
+  "diffs" : [ 171392, 84992, -1408, -87808, -174208, -260608, -347008, -433408, -519808, -606208, -692608, -779008, -865408, -1408, -1400, 530, 0 ]
+}, {
+  "tz" : "Asia/Krasnoyarsk",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 175714, 89314, 2914, -83486, -169886, -256286, -342686, -429086, -515486, -601886, -688286, -774686, -861086, 2914, 0 ]
+}, {
+  "tz" : "Asia/Kuala_Lumpur",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 177194, 90794, 4394, -82006, -168406, -254806, -341206, -427606, -514006, -600406, -686806, -773206, -859606, 4394, 0 ]
+}, {
+  "tz" : "Asia/Kuching",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 175120, 88720, 2320, -84080, -170480, -256880, -343280, -429680, -516080, -602480, -688880, -775280, -861680, 2320, 0 ]
+}, {
+  "tz" : "Asia/Kuwait",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 172388, 85988, -412, -86812, -173212, -259612, -346012, -432412, -518812, -605212, -691612, -778012, -864412, -412, 0 ]
+}, {
+  "tz" : "Asia/Macao",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174350, 87950, 1550, -84850, -171250, -257650, -344050, -430450, -516850, -603250, -689650, -776050, -862450, 1550, 0 ]
+}, {
+  "tz" : "Asia/Macau",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174350, 87950, 1550, -84850, -171250, -257650, -344050, -430450, -516850, -603250, -689650, -776050, -862450, 1550, 0 ]
+}, {
+  "tz" : "Asia/Magadan",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 176208, 89808, 3408, -82992, -169392, -255792, -342192, -428592, -514992, -601392, -687792, -774192, -860592, 3408, 0 ]
+}, {
+  "tz" : "Asia/Makassar",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172944, 86544, 144, -86256, -172656, -259056, -345456, -431856, -518256, -604656, -691056, -777456, -863856, 144, 0 ]
+}, {
+  "tz" : "Asia/Manila",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -3944620800, -2229321840 ],
+  "diffs" : [ 258960, 172560, 86160, -240, -86640, -173040, -259440, -345840, -432240, -518640, -605040, -691440, -777840, 86160, -240, 0 ]
+}, {
+  "tz" : "Asia/Muscat",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 173928, 87528, 1128, -85272, -171672, -258072, -344472, -430872, -517272, -603672, -690072, -776472, -862872, 1128, 0 ]
+}, {
+  "tz" : "Asia/Nicosia",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171992, 85592, -808, -87208, -173608, -260008, -346408, -432808, -519208, -605608, -692008, -778408, -864808, -808, 0 ]
+}, {
+  "tz" : "Asia/Novokuznetsk",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 177072, 90672, 4272, -82128, -168528, -254928, -341328, -427728, -514128, -600528, -686928, -773328, -859728, 4272, 0 ]
+}, {
+  "tz" : "Asia/Novosibirsk",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 178100, 91700, 5300, -81100, -167500, -253900, -340300, -426700, -513100, -599500, -685900, -772300, -858700, 5300, 0 ]
+}, {
+  "tz" : "Asia/Omsk",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 176790, 90390, 3990, -82410, -168810, -255210, -341610, -428010, -514410, -600810, -687210, -773610, -860010, 3990, 0 ]
+}, {
+  "tz" : "Asia/Oral",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 178476, 92076, 5676, -80724, -167124, -253524, -339924, -426324, -512724, -599124, -685524, -771924, -858324, 5676, 0 ]
+}, {
+  "tz" : "Asia/Phnom_Penh",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 173876, 87476, 1076, -85324, -171724, -258124, -344524, -430924, -517324, -603724, -690124, -776524, -862924, 1076, 0 ]
+}, {
+  "tz" : "Asia/Pontianak",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 171760, 85360, -1040, -87440, -173840, -260240, -346640, -433040, -519440, -605840, -692240, -778640, -865040, -1040, 0 ]
+}, {
+  "tz" : "Asia/Pyongyang",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 175020, 88620, 2220, -84180, -170580, -256980, -343380, -429780, -516180, -602580, -688980, -775380, -861780, 2220, 0 ]
+}, {
+  "tz" : "Asia/Qatar",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 171232, 84832, -1568, -87968, -174368, -260768, -347168, -433568, -519968, -606368, -692768, -779168, -865568, -1568, 0 ]
+}, {
+  "tz" : "Asia/Qostanay",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 179132, 92732, 6332, -80068, -166468, -252868, -339268, -425668, -512068, -598468, -684868, -771268, -857668, 6332, 0 ]
+}, {
+  "tz" : "Asia/Qyzylorda",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 175088, 88688, 2288, -84112, -170512, -256912, -343312, -429712, -516112, -602512, -688912, -775312, -861712, 2288, 0 ]
+}, {
+  "tz" : "Asia/Rangoon",
+  "switches" : [ -62135793000, -59006471400, -55850711400, -52694951400, -46383431400, -43227671400, -40071911400, -33760391400, -30604631400, -27448871400, -21137351400, -17981591400, -14825831400, -12219316200, -2208988800 ],
+  "diffs" : [ 173113, 86713, 313, -86087, -172487, -258887, -345287, -431687, -518087, -604487, -690887, -777287, -863687, 313, 0 ]
+}, {
+  "tz" : "Asia/Riyadh",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 172388, 85988, -412, -86812, -173212, -259612, -346012, -432412, -518812, -605212, -691612, -778012, -864412, -412, 0 ]
+}, {
+  "tz" : "Asia/Saigon",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 172400, 86000, -400, -86800, -173200, -259600, -346000, -432400, -518800, -605200, -691600, -778000, -864400, -400, 0 ]
+}, {
+  "tz" : "Asia/Sakhalin",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 178152, 91752, 5352, -81048, -167448, -253848, -340248, -426648, -513048, -599448, -685848, -772248, -858648, 5352, 0 ]
+}, {
+  "tz" : "Asia/Samarkand",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 174727, 88327, 1927, -84473, -170873, -257273, -343673, -430073, -516473, -602873, -689273, -775673, -862073, 1927, 0 ]
+}, {
+  "tz" : "Asia/Seoul",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 174728, 88328, 1928, -84472, -170872, -257272, -343672, -430072, -516472, -602872, -689272, -775672, -862072, 1928, 0 ]
+}, {
+  "tz" : "Asia/Shanghai",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172457, 86057, -343, -86743, -173143, -259543, -345943, -432343, -518743, -605143, -691543, -777943, -864343, -343, 0 ]
+}, {
+  "tz" : "Asia/Singapore",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 176675, 90275, 3875, -82525, -168925, -255325, -341725, -428125, -514525, -600925, -687325, -773725, -860125, 3875, 0 ]
+}, {
+  "tz" : "Asia/Srednekolymsk",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 175508, 89108, 2708, -83692, -170092, -256492, -342892, -429292, -515692, -602092, -688492, -774892, -861292, 2708, 0 ]
+}, {
+  "tz" : "Asia/Taipei",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2335248360 ],
+  "diffs" : [ 172440, 86040, -360, -86760, -173160, -259560, -345960, -432360, -518760, -605160, -691560, -777960, -864360, -360, 0 ]
+}, {
+  "tz" : "Asia/Tashkent",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 174169, 87769, 1369, -85031, -171431, -257831, -344231, -430631, -517031, -603431, -689831, -776231, -862631, 1369, 0 ]
+}, {
+  "tz" : "Asia/Tbilisi",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 176449, 90049, 3649, -82751, -169151, -255551, -341951, -428351, -514751, -601151, -687551, -773951, -860351, 3649, 0 ]
+}, {
+  "tz" : "Asia/Tehran",
+  "switches" : [ -62135782200, -59006460600, -55850700600, -52694940600, -46383420600, -43227660600, -40071900600, -33760380600, -30604620600, -27448860600, -21137340600, -17981580600, -14825820600, -12219305400, -2208988800 ],
+  "diffs" : [ 173056, 86656, 256, -86144, -172544, -258944, -345344, -431744, -518144, -604544, -690944, -777344, -863744, 256, 0 ]
+}, {
+  "tz" : "Asia/Tel_Aviv",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148014, -2208988800 ],
+  "diffs" : [ 171546, 85146, -1254, -87654, -174054, -260454, -346854, -433254, -519654, -606054, -692454, -778854, -865254, -1254, -1240, 0 ]
+}, {
+  "tz" : "Asia/Thimbu",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 172884, 86484, 84, -86316, -172716, -259116, -345516, -431916, -518316, -604716, -691116, -777516, -863916, 84, 0 ]
+}, {
+  "tz" : "Asia/Thimphu",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 172884, 86484, 84, -86316, -172716, -259116, -345516, -431916, -518316, -604716, -691116, -777516, -863916, 84, 0 ]
+}, {
+  "tz" : "Asia/Tokyo",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2587712400 ],
+  "diffs" : [ 171661, 85261, -1139, -87539, -173939, -260339, -346739, -433139, -519539, -605939, -692339, -778739, -865139, -1139, 0 ]
+}, {
+  "tz" : "Asia/Tomsk",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 177609, 91209, 4809, -81591, -167991, -254391, -340791, -427191, -513591, -599991, -686391, -772791, -859191, 4809, 0 ]
+}, {
+  "tz" : "Asia/Ujung_Pandang",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172944, 86544, 144, -86256, -172656, -259056, -345456, -431856, -518256, -604656, -691056, -777456, -863856, 144, 0 ]
+}, {
+  "tz" : "Asia/Ulaanbaatar",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 175948, 89548, 3148, -83252, -169652, -256052, -342452, -428852, -515252, -601652, -688052, -774452, -860852, 3148, 0 ]
+}, {
+  "tz" : "Asia/Ulan_Bator",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 175948, 89548, 3148, -83252, -169652, -256052, -342452, -428852, -515252, -601652, -688052, -774452, -860852, 3148, 0 ]
+}, {
+  "tz" : "Asia/Urumqi",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ]
+}, {
+  "tz" : "Asia/Ust-Nera",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2208988800 ],
+  "diffs" : [ 174426, 88026, 1626, -84774, -171174, -257574, -343974, -430374, -516774, -603174, -689574, -775974, -862374, 1626, 0 ]
+}, {
+  "tz" : "Asia/Vientiane",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 173876, 87476, 1076, -85324, -171724, -258124, -344524, -430924, -517324, -603724, -690124, -776524, -862924, 1076, 0 ]
+}, {
+  "tz" : "Asia/Vladivostok",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2208988800 ],
+  "diffs" : [ 177149, 90749, 4349, -82051, -168451, -254851, -341251, -427651, -514051, -600451, -686851, -773251, -859651, 4349, 0 ]
+}, {
+  "tz" : "Asia/Yakutsk",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 174062, 87662, 1262, -85138, -171538, -257938, -344338, -430738, -517138, -603538, -689938, -776338, -862738, 1262, 0 ]
+}, {
+  "tz" : "Asia/Yangon",
+  "switches" : [ -62135793000, -59006471400, -55850711400, -52694951400, -46383431400, -43227671400, -40071911400, -33760391400, -30604631400, -27448871400, -21137351400, -17981591400, -14825831400, -12219316200, -2208988800 ],
+  "diffs" : [ 173113, 86713, 313, -86087, -172487, -258887, -345287, -431687, -518087, -604487, -690887, -777287, -863687, 313, 0 ]
+}, {
+  "tz" : "Asia/Yekaterinburg",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 176247, 89847, 3447, -82953, -169353, -255753, -342153, -428553, -514953, -601353, -687753, -774153, -860553, 3447, 0 ]
+}, {
+  "tz" : "Asia/Yerevan",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 176520, 90120, 3720, -82680, -169080, -255480, -341880, -428280, -514680, -601080, -687480, -773880, -860280, 3720, 0 ]
+}, {
+  "tz" : "Atlantic/Azores",
+  "switches" : [ -62135766000, -59006444400, -55850684400, -52694924400, -46383404400, -43227644400, -40071884400, -33760364400, -30604604400, -27448844400, -21137324400, -17981564400, -14825804400, -12219289200, -2713907512, -2208988800 ],
+  "diffs" : [ 175360, 88960, 2560, -83840, -170240, -256640, -343040, -429440, -515840, -602240, -688640, -775040, -861440, 2560, 3272, 0 ]
+}, {
+  "tz" : "Atlantic/Bermuda",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173958, 87558, 1158, -85242, -171642, -258042, -344442, -430842, -517242, -603642, -690042, -776442, -862842, 1158, 0 ]
+}, {
+  "tz" : "Atlantic/Canary",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 176496, 90096, 3696, -82704, -169104, -255504, -341904, -428304, -514704, -601104, -687504, -773904, -860304, 3696, 0 ]
+}, {
+  "tz" : "Atlantic/Cape_Verde",
+  "switches" : [ -62135766000, -59006444400, -55850684400, -52694924400, -46383404400, -43227644400, -40071884400, -33760364400, -30604604400, -27448844400, -21137324400, -17981564400, -14825804400, -12219289200, -2208988800 ],
+  "diffs" : [ 174844, 88444, 2044, -84356, -170756, -257156, -343556, -429956, -516356, -602756, -689156, -775556, -861956, 2044, 0 ]
+}, {
+  "tz" : "Atlantic/Faeroe",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 174424, 88024, 1624, -84776, -171176, -257576, -343976, -430376, -516776, -603176, -689576, -775976, -862376, 1624, 0 ]
+}, {
+  "tz" : "Atlantic/Faroe",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 174424, 88024, 1624, -84776, -171176, -257576, -343976, -430376, -516776, -603176, -689576, -775976, -862376, 1624, 0 ]
+}, {
+  "tz" : "Atlantic/Jan_Mayen",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2366757780 ],
+  "diffs" : [ 173820, 87420, 1020, -85380, -171780, -258180, -344580, -430980, -517380, -603780, -690180, -776580, -862980, 1020, 0 ]
+}, {
+  "tz" : "Atlantic/Madeira",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 176856, 90456, 4056, -82344, -168744, -255144, -341544, -427944, -514344, -600744, -687144, -773544, -859944, 4056, 0 ]
+}, {
+  "tz" : "Atlantic/Reykjavik",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 178080, 91680, 5280, -81120, -167520, -253920, -340320, -426720, -513120, -599520, -685920, -772320, -858720, 5280, 0 ]
+}, {
+  "tz" : "Atlantic/South_Georgia",
+  "switches" : [ -62135762400, -59006440800, -55850680800, -52694920800, -46383400800, -43227640800, -40071880800, -33760360800, -30604600800, -27448840800, -21137320800, -17981560800, -14825800800, -12219285600, -2524512832 ],
+  "diffs" : [ 174368, 87968, 1568, -84832, -171232, -257632, -344032, -430432, -516832, -603232, -689632, -776032, -862432, 1568, 0 ]
+}, {
+  "tz" : "Atlantic/St_Helena",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Atlantic/Stanley",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 175884, 89484, 3084, -83316, -169716, -256116, -342516, -428916, -515316, -601716, -688116, -774516, -860916, 3084, 0 ]
+}, {
+  "tz" : "Australia/ACT",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364113092 ],
+  "diffs" : [ 172508, 86108, -292, -86692, -173092, -259492, -345892, -432292, -518692, -605092, -691492, -777892, -864292, -292, 0 ]
+}, {
+  "tz" : "Australia/Adelaide",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364111860, -2230189200 ],
+  "diffs" : [ 173740, 87340, 940, -85460, -171860, -258260, -344660, -431060, -517460, -603860, -690260, -776660, -863060, 940, 1800, 0 ]
+}, {
+  "tz" : "Australia/Brisbane",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2366791928 ],
+  "diffs" : [ 172072, 85672, -728, -87128, -173528, -259928, -346328, -432728, -519128, -605528, -691928, -778328, -864728, -728, 0 ]
+}, {
+  "tz" : "Australia/Broken_Hill",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364108948, -2314953000, -2230189200 ],
+  "diffs" : [ 173052, 86652, 252, -86148, -172548, -258948, -345348, -431748, -518148, -604548, -690948, -777348, -863748, 252, -1800, 1800, 0 ]
+}, {
+  "tz" : "Australia/Canberra",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364113092 ],
+  "diffs" : [ 172508, 86108, -292, -86692, -173092, -259492, -345892, -432292, -518692, -605092, -691492, -777892, -864292, -292, 0 ]
+}, {
+  "tz" : "Australia/Currie",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2345794528 ],
+  "diffs" : [ 174272, 87872, 1472, -84928, -171328, -257728, -344128, -430528, -516928, -603328, -689728, -776128, -862528, 1472, 0 ]
+}, {
+  "tz" : "Australia/Darwin",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364110000, -2230189200 ],
+  "diffs" : [ 175600, 89200, 2800, -83600, -170000, -256400, -342800, -429200, -515600, -602000, -688400, -774800, -861200, 2800, 1800, 0 ]
+}, {
+  "tz" : "Australia/Eucla",
+  "switches" : [ -62135801100, -59006479500, -55850719500, -52694959500, -46383439500, -43227679500, -40071919500, -33760399500, -30604639500, -27448879500, -21137359500, -17981599500, -14825839500, -12219324300, -2337928528 ],
+  "diffs" : [ 173372, 86972, 572, -85828, -172228, -258628, -345028, -431428, -517828, -604228, -690628, -777028, -863428, 572, 0 ]
+}, {
+  "tz" : "Australia/Hobart",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2345795356 ],
+  "diffs" : [ 173444, 87044, 644, -85756, -172156, -258556, -344956, -431356, -517756, -604156, -690556, -776956, -863356, 644, 0 ]
+}, {
+  "tz" : "Australia/LHI",
+  "switches" : [ -62135807400, -59006485800, -55850725800, -52694965800, -46383445800, -43227685800, -40071925800, -33760405800, -30604645800, -27448885800, -21137365800, -17981605800, -14825845800, -12219330600, -2364116780, -2208988800 ],
+  "diffs" : [ 172420, 86020, -380, -86780, -173180, -259580, -345980, -432380, -518780, -605180, -691580, -777980, -864380, -380, 1800, 0 ]
+}, {
+  "tz" : "Australia/Lindeman",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2366790956 ],
+  "diffs" : [ 173044, 86644, 244, -86156, -172556, -258956, -345356, -431756, -518156, -604556, -690956, -777356, -863756, 244, 0 ]
+}, {
+  "tz" : "Australia/Lord_Howe",
+  "switches" : [ -62135807400, -59006485800, -55850725800, -52694965800, -46383445800, -43227685800, -40071925800, -33760405800, -30604645800, -27448885800, -21137365800, -17981605800, -14825845800, -12219330600, -2364116780, -2208988800 ],
+  "diffs" : [ 172420, 86020, -380, -86780, -173180, -259580, -345980, -432380, -518780, -605180, -691580, -777980, -864380, -380, 1800, 0 ]
+}, {
+  "tz" : "Australia/Melbourne",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364111592 ],
+  "diffs" : [ 174008, 87608, 1208, -85192, -171592, -257992, -344392, -430792, -517192, -603592, -689992, -776392, -862792, 1208, 0 ]
+}, {
+  "tz" : "Australia/NSW",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364113092 ],
+  "diffs" : [ 172508, 86108, -292, -86692, -173092, -259492, -345892, -432292, -518692, -605092, -691492, -777892, -864292, -292, 0 ]
+}, {
+  "tz" : "Australia/North",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364110000, -2230189200 ],
+  "diffs" : [ 175600, 89200, 2800, -83600, -170000, -256400, -342800, -429200, -515600, -602000, -688400, -774800, -861200, 2800, 1800, 0 ]
+}, {
+  "tz" : "Australia/Perth",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2337925404 ],
+  "diffs" : [ 173796, 87396, 996, -85404, -171804, -258204, -344604, -431004, -517404, -603804, -690204, -776604, -863004, 996, 0 ]
+}, {
+  "tz" : "Australia/Queensland",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2366791928 ],
+  "diffs" : [ 172072, 85672, -728, -87128, -173528, -259928, -346328, -432728, -519128, -605528, -691928, -778328, -864728, -728, 0 ]
+}, {
+  "tz" : "Australia/South",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364111860, -2230189200 ],
+  "diffs" : [ 173740, 87340, 940, -85460, -171860, -258260, -344660, -431060, -517460, -603860, -690260, -776660, -863060, 940, 1800, 0 ]
+}, {
+  "tz" : "Australia/Sydney",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364113092 ],
+  "diffs" : [ 172508, 86108, -292, -86692, -173092, -259492, -345892, -432292, -518692, -605092, -691492, -777892, -864292, -292, 0 ]
+}, {
+  "tz" : "Australia/Tasmania",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2345795356 ],
+  "diffs" : [ 173444, 87044, 644, -85756, -172156, -258556, -344956, -431356, -517756, -604156, -690556, -776956, -863356, 644, 0 ]
+}, {
+  "tz" : "Australia/Victoria",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364111592 ],
+  "diffs" : [ 174008, 87608, 1208, -85192, -171592, -257992, -344392, -430792, -517192, -603592, -689992, -776392, -862792, 1208, 0 ]
+}, {
+  "tz" : "Australia/West",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2337925404 ],
+  "diffs" : [ 173796, 87396, 996, -85404, -171804, -258204, -344604, -431004, -517404, -603804, -690204, -776604, -863004, 996, 0 ]
+}, {
+  "tz" : "Australia/Yancowinna",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364108948, -2314953000, -2230189200 ],
+  "diffs" : [ 173052, 86652, 252, -86148, -172548, -258948, -345348, -431748, -518148, -604548, -690948, -777348, -863748, 252, -1800, 1800, 0 ]
+}, {
+  "tz" : "Brazil/Acre",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 171072, 84672, -1728, -88128, -174528, -260928, -347328, -433728, -520128, -606528, -692928, -779328, -865728, -1728, 0 ]
+}, {
+  "tz" : "Brazil/DeNoronha",
+  "switches" : [ -62135762400, -59006440800, -55850680800, -52694920800, -46383400800, -43227640800, -40071880800, -33760360800, -30604600800, -27448840800, -21137320800, -17981560800, -14825800800, -12219285600, -2208988800 ],
+  "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ]
+}, {
+  "tz" : "Brazil/East",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 173188, 86788, 388, -86012, -172412, -258812, -345212, -431612, -518012, -604412, -690812, -777212, -863612, 388, 0 ]
+}, {
+  "tz" : "Brazil/West",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172804, 86404, 4, -86396, -172796, -259196, -345596, -431996, -518396, -604796, -691196, -777596, -863996, 4, 0 ]
+}, {
+  "tz" : "CET",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "CST6CDT",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Canada/Atlantic",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173664, 87264, 864, -85536, -171936, -258336, -344736, -431136, -517536, -603936, -690336, -776736, -863136, 864, 0 ]
+}, {
+  "tz" : "Canada/Central",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2602258284 ],
+  "diffs" : [ 174516, 88116, 1716, -84684, -171084, -257484, -343884, -430284, -516684, -603084, -689484, -775884, -862284, 1716, 0 ]
+}, {
+  "tz" : "Canada/Eastern",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736148 ],
+  "diffs" : [ 173852, 87452, 1052, -85348, -171748, -258148, -344548, -430948, -517348, -603748, -690148, -776548, -862948, 1052, 0 ]
+}, {
+  "tz" : "Canada/Mountain",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 174832, 88432, 2032, -84368, -170768, -257168, -343568, -429968, -516368, -602768, -689168, -775568, -861968, 2032, 0 ]
+}, {
+  "tz" : "Canada/Newfoundland",
+  "switches" : [ -62135757000, -59006435400, -55850675400, -52694915400, -46383395400, -43227635400, -40071875400, -33760355400, -30604595400, -27448835400, -21137315400, -17981555400, -14825795400, -12219280200, -2208988800 ],
+  "diffs" : [ 172852, 86452, 52, -86348, -172748, -259148, -345548, -431948, -518348, -604748, -691148, -777548, -863948, 52, 0 ]
+}, {
+  "tz" : "Canada/Pacific",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2713880852 ],
+  "diffs" : [ 173548, 87148, 748, -85652, -172052, -258452, -344852, -431252, -517652, -604052, -690452, -776852, -863252, 748, 0 ]
+}, {
+  "tz" : "Canada/Saskatchewan",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 176316, 89916, 3516, -82884, -169284, -255684, -342084, -428484, -514884, -601284, -687684, -774084, -860484, 3516, 0 ]
+}, {
+  "tz" : "Canada/Yukon",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 176412, 90012, 3612, -82788, -169188, -255588, -341988, -428388, -514788, -601188, -687588, -773988, -860388, 3612, 0 ]
+}, {
+  "tz" : "Chile/Continental",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 175366, 88966, 2566, -83834, -170234, -256634, -343034, -429434, -515834, -602234, -688634, -775034, -861434, 2566, 0 ]
+}, {
+  "tz" : "Chile/EasterIsland",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 177448, 91048, 4648, -81752, -168152, -254552, -340952, -427352, -513752, -600152, -686552, -772952, -859352, 4648, 0 ]
+}, {
+  "tz" : "Cuba",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503608, -2208988800 ],
+  "diffs" : [ 174568, 88168, 1768, -84632, -171032, -257432, -343832, -430232, -516632, -603032, -689432, -775832, -862232, 1768, 1776, 0 ]
+}, {
+  "tz" : "EET",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "EST5EDT",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Egypt",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172491, 86091, -309, -86709, -173109, -259509, -345909, -432309, -518709, -605109, -691509, -777909, -864309, -309, 0 ]
+}, {
+  "tz" : "Eire",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2821651221, -2208988800 ],
+  "diffs" : [ 174300, 87900, 1500, -84900, -171300, -257700, -344100, -430500, -516900, -603300, -689700, -776100, -862500, 1500, 1521, 0 ]
+}, {
+  "tz" : "Etc/GMT",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+0",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+1",
+  "switches" : [ -62135766000, -59006444400, -55850684400, -52694924400, -46383404400, -43227644400, -40071884400, -33760364400, -30604604400, -27448844400, -21137324400, -17981564400, -14825804400, -12219289200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+10",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+11",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+12",
+  "switches" : [ -62135726400, -59006404800, -55850644800, -52694884800, -46383364800, -43227604800, -40071844800, -33760324800, -30604564800, -27448804800, -21137284800, -17981524800, -14825764800, -12219249600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+2",
+  "switches" : [ -62135762400, -59006440800, -55850680800, -52694920800, -46383400800, -43227640800, -40071880800, -33760360800, -30604600800, -27448840800, -21137320800, -17981560800, -14825800800, -12219285600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+3",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+4",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+5",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+6",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+7",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+8",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+9",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-0",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-1",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-10",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-11",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-12",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-13",
+  "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-14",
+  "switches" : [ -62135820000, -59006498400, -55850738400, -52694978400, -46383458400, -43227698400, -40071938400, -33760418400, -30604658400, -27448898400, -21137378400, -17981618400, -14825858400, -12219343200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-2",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-3",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-4",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-5",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-6",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-7",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-8",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-9",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT0",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/Greenwich",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/UCT",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/UTC",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/Universal",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/Zulu",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Europe/Amsterdam",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175228, 88828, 2428, -83972, -170372, -256772, -343172, -429572, -515972, -602372, -688772, -775172, -861572, 2428, 0 ]
+}, {
+  "tz" : "Europe/Andorra",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 176036, 89636, 3236, -83164, -169564, -255964, -342364, -428764, -515164, -601564, -687964, -774364, -860764, 3236, 0 ]
+}, {
+  "tz" : "Europe/Astrakhan",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 175668, 89268, 2868, -83532, -169932, -256332, -342732, -429132, -515532, -601932, -688332, -774732, -861132, 2868, 0 ]
+}, {
+  "tz" : "Europe/Athens",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 174308, 87908, 1508, -84892, -171292, -257692, -344092, -430492, -516892, -603292, -689692, -776092, -862492, 1508, 0 ]
+}, {
+  "tz" : "Europe/Belfast",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "Europe/Belgrade",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/Berlin",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2422054408 ],
+  "diffs" : [ 173192, 86792, 392, -86008, -172408, -258808, -345208, -431608, -518008, -604408, -690808, -777208, -863608, 392, 0 ]
+}, {
+  "tz" : "Europe/Bratislava",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2469401864 ],
+  "diffs" : [ 172936, 86536, 136, -86264, -172664, -259064, -345464, -431864, -518264, -604664, -691064, -777464, -863864, 136, 0 ]
+}, {
+  "tz" : "Europe/Brussels",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2450998800, -2208988800 ],
+  "diffs" : [ 175350, 88950, 2550, -83850, -170250, -256650, -343050, -429450, -515850, -602250, -688650, -775050, -861450, 2550, 3600, 0 ]
+}, {
+  "tz" : "Europe/Bucharest",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 173736, 87336, 936, -85464, -171864, -258264, -344664, -431064, -517464, -603864, -690264, -776664, -863064, 936, 0 ]
+}, {
+  "tz" : "Europe/Budapest",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2500938980 ],
+  "diffs" : [ 171820, 85420, -980, -87380, -173780, -260180, -346580, -432980, -519380, -605780, -692180, -778580, -864980, -980, 0 ]
+}, {
+  "tz" : "Europe/Busingen",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -3675200662, -2385246586 ],
+  "diffs" : [ 174352, 87952, 1552, -84848, -171248, -257648, -344048, -430448, -516848, -603248, -689648, -776048, -862448, 1552, 1814, 0 ]
+}, {
+  "tz" : "Europe/Chisinau",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148020, -2208988800 ],
+  "diffs" : [ 173080, 86680, 280, -86120, -172520, -258920, -345320, -431720, -518120, -604520, -690920, -777320, -863720, 280, 300, 0 ]
+}, {
+  "tz" : "Europe/Copenhagen",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2398294220 ],
+  "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ]
+}, {
+  "tz" : "Europe/Dublin",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2821651221, -2208988800 ],
+  "diffs" : [ 174300, 87900, 1500, -84900, -171300, -257700, -344100, -430500, -516900, -603300, -689700, -776100, -862500, 1500, 1521, 0 ]
+}, {
+  "tz" : "Europe/Gibraltar",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2821653516, -2208988800 ],
+  "diffs" : [ 177684, 91284, 4884, -81516, -167916, -254316, -340716, -427116, -513516, -599916, -686316, -772716, -859116, 4884, 3600, 0 ]
+}, {
+  "tz" : "Europe/Guernsey",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "Europe/Helsinki",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 174011, 87611, 1211, -85189, -171589, -257989, -344389, -430789, -517189, -603589, -689989, -776389, -862789, 1211, 0 ]
+}, {
+  "tz" : "Europe/Isle_of_Man",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "Europe/Istanbul",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2840151536, -2208988800 ],
+  "diffs" : [ 176648, 90248, 3848, -82552, -168952, -255352, -341752, -428152, -514552, -600952, -687352, -773752, -860152, 3848, 3784, 0 ]
+}, {
+  "tz" : "Europe/Jersey",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "Europe/Kaliningrad",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2422059720, -2208988800 ],
+  "diffs" : [ 175080, 88680, 2280, -84120, -170520, -256920, -343320, -429720, -516120, -602520, -688920, -775320, -861720, 2280, 3600, 0 ]
+}, {
+  "tz" : "Europe/Kiev",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172676, 86276, -124, -86524, -172924, -259324, -345724, -432124, -518524, -604924, -691324, -777724, -864124, -124, 0 ]
+}, {
+  "tz" : "Europe/Kirov",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 171672, 85272, -1128, -87528, -173928, -260328, -346728, -433128, -519528, -605928, -692328, -778728, -865128, -1128, 0 ]
+}, {
+  "tz" : "Europe/Lisbon",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 175005, 88605, 2205, -84195, -170595, -256995, -343395, -429795, -516195, -602595, -688995, -775395, -861795, 2205, 0 ]
+}, {
+  "tz" : "Europe/Ljubljana",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/London",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "Europe/Luxembourg",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 174924, 88524, 2124, -84276, -170676, -257076, -343476, -429876, -516276, -602676, -689076, -775476, -861876, 2124, 0 ]
+}, {
+  "tz" : "Europe/Madrid",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 177284, 90884, 4484, -81916, -168316, -254716, -341116, -427516, -513916, -600316, -686716, -773116, -859516, 4484, 0 ]
+}, {
+  "tz" : "Europe/Malta",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2403478684 ],
+  "diffs" : [ 172916, 86516, 116, -86284, -172684, -259084, -345484, -431884, -518284, -604684, -691084, -777484, -863884, 116, 0 ]
+}, {
+  "tz" : "Europe/Mariehamn",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 174011, 87611, 1211, -85189, -171589, -257989, -344389, -430789, -517189, -603589, -689989, -776389, -862789, 1211, 0 ]
+}, {
+  "tz" : "Europe/Minsk",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2840151616, -2208988800 ],
+  "diffs" : [ 176984, 90584, 4184, -82216, -168616, -255016, -341416, -427816, -514216, -600616, -687016, -773416, -859816, 4184, 4200, 0 ]
+}, {
+  "tz" : "Europe/Monaco",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2486683211, -2208988800 ],
+  "diffs" : [ 174628, 88228, 1828, -84572, -170972, -257372, -343772, -430172, -516572, -602972, -689372, -775772, -862172, 1828, 3039, 0 ]
+}, {
+  "tz" : "Europe/Moscow",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174583, 88183, 1783, -84617, -171017, -257417, -343817, -430217, -516617, -603017, -689417, -775817, -862217, 1783, 0 ]
+}, {
+  "tz" : "Europe/Nicosia",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171992, 85592, -808, -87208, -173608, -260008, -346408, -432808, -519208, -605608, -692008, -778408, -864808, -808, 0 ]
+}, {
+  "tz" : "Europe/Oslo",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2366757780 ],
+  "diffs" : [ 173820, 87420, 1020, -85380, -171780, -258180, -344580, -430980, -517380, -603780, -690180, -776580, -862980, 1020, 0 ]
+}, {
+  "tz" : "Europe/Paris",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175839, 89439, 3039, -83361, -169761, -256161, -342561, -428961, -515361, -601761, -688161, -774561, -860961, 3039, 0 ]
+}, {
+  "tz" : "Europe/Podgorica",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/Prague",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2469401864 ],
+  "diffs" : [ 172936, 86536, 136, -86264, -172664, -259064, -345464, -431864, -518264, -604664, -691064, -777464, -863864, 136, 0 ]
+}, {
+  "tz" : "Europe/Riga",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 174206, 87806, 1406, -84994, -171394, -257794, -344194, -430594, -516994, -603394, -689794, -776194, -862594, 1406, 0 ]
+}, {
+  "tz" : "Europe/Rome",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2403565200 ],
+  "diffs" : [ 173404, 87004, 604, -85796, -172196, -258596, -344996, -431396, -517796, -604196, -690596, -776996, -863396, 604, 0 ]
+}, {
+  "tz" : "Europe/Samara",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 175180, 88780, 2380, -84020, -170420, -256820, -343220, -429620, -516020, -602420, -688820, -775220, -861620, 2380, 0 ]
+}, {
+  "tz" : "Europe/San_Marino",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2403565200 ],
+  "diffs" : [ 173404, 87004, 604, -85796, -172196, -258596, -344996, -431396, -517796, -604196, -690596, -776996, -863396, 604, 0 ]
+}, {
+  "tz" : "Europe/Sarajevo",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/Saratov",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 176142, 89742, 3342, -83058, -169458, -255858, -342258, -428658, -515058, -601458, -687858, -774258, -860658, 3342, 0 ]
+}, {
+  "tz" : "Europe/Simferopol",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2840151624, -2208988800 ],
+  "diffs" : [ 175416, 89016, 2616, -83784, -170184, -256584, -342984, -429384, -515784, -602184, -688584, -774984, -861384, 2616, 2640, 0 ]
+}, {
+  "tz" : "Europe/Skopje",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/Sofia",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840146580, -2369527016 ],
+  "diffs" : [ 174404, 88004, 1604, -84796, -171196, -257596, -343996, -430396, -516796, -603196, -689596, -775996, -862396, 1604, 184, 0 ]
+}, {
+  "tz" : "Europe/Stockholm",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2871681118, -2208992414 ],
+  "diffs" : [ 172068, 85668, -732, -87132, -173532, -259932, -346332, -432732, -519132, -605532, -691932, -778332, -864732, -732, -14, 0 ]
+}, {
+  "tz" : "Europe/Tallinn",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 174060, 87660, 1260, -85140, -171540, -257940, -344340, -430740, -517140, -603540, -689940, -776340, -862740, 1260, 0 ]
+}, {
+  "tz" : "Europe/Tirane",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 171640, 85240, -1160, -87560, -173960, -260360, -346760, -433160, -519560, -605960, -692360, -778760, -865160, -1160, 0 ]
+}, {
+  "tz" : "Europe/Tiraspol",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148020, -2208988800 ],
+  "diffs" : [ 173080, 86680, 280, -86120, -172520, -258920, -345320, -431720, -518120, -604520, -690920, -777320, -863720, 280, 300, 0 ]
+}, {
+  "tz" : "Europe/Ulyanovsk",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Europe/Uzhgorod",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2500943352, -2208988800 ],
+  "diffs" : [ 174648, 88248, 1848, -84552, -170952, -257352, -343752, -430152, -516552, -602952, -689352, -775752, -862152, 1848, 3600, 0 ]
+}, {
+  "tz" : "Europe/Vaduz",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -3675200662, -2385246586 ],
+  "diffs" : [ 174352, 87952, 1552, -84848, -171248, -257648, -344048, -430448, -516848, -603248, -689648, -776048, -862448, 1552, 1814, 0 ]
+}, {
+  "tz" : "Europe/Vatican",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2403565200 ],
+  "diffs" : [ 173404, 87004, 604, -85796, -172196, -258596, -344996, -431396, -517796, -604196, -690596, -776996, -863396, 604, 0 ]
+}, {
+  "tz" : "Europe/Vienna",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2422055121 ],
+  "diffs" : [ 172479, 86079, -321, -86721, -173121, -259521, -345921, -432321, -518721, -605121, -691521, -777921, -864321, -321, 0 ]
+}, {
+  "tz" : "Europe/Vilnius",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840149036, -2208988800 ],
+  "diffs" : [ 173924, 87524, 1124, -85276, -171676, -258076, -344476, -430876, -517276, -603676, -690076, -776476, -862876, 1124, 2160, 0 ]
+}, {
+  "tz" : "Europe/Volgograd",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 176540, 90140, 3740, -82660, -169060, -255460, -341860, -428260, -514660, -601060, -687460, -773860, -860260, 3740, 0 ]
+}, {
+  "tz" : "Europe/Warsaw",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 171360, 84960, -1440, -87840, -174240, -260640, -347040, -433440, -519840, -606240, -692640, -779040, -865440, -1440, 0 ]
+}, {
+  "tz" : "Europe/Zagreb",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/Zaporozhye",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148040, -2208988800 ],
+  "diffs" : [ 171560, 85160, -1240, -87640, -174040, -260440, -346840, -433240, -519640, -606040, -692440, -778840, -865240, -1240, -1200, 0 ]
+}, {
+  "tz" : "Europe/Zurich",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -3675200662, -2385246586 ],
+  "diffs" : [ 174352, 87952, 1552, -84848, -171248, -257648, -344048, -430448, -516848, -603248, -689648, -776048, -862448, 1552, 1814, 0 ]
+}, {
+  "tz" : "GB",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "GB-Eire",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "GMT",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "GMT0",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Greenwich",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Hongkong",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174198, 87798, 1398, -85002, -171402, -257802, -344202, -430602, -517002, -603402, -689802, -776202, -862602, 1398, 0 ]
+}, {
+  "tz" : "Iceland",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 178080, 91680, 5280, -81120, -167520, -253920, -340320, -426720, -513120, -599520, -685920, -772320, -858720, 5280, 0 ]
+}, {
+  "tz" : "Indian/Antananarivo",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Indian/Chagos",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 177020, 90620, 4220, -82180, -168580, -254980, -341380, -427780, -514180, -600580, -686980, -773380, -859780, 4220, 0 ]
+}, {
+  "tz" : "Indian/Christmas",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2364102172 ],
+  "diffs" : [ 172628, 86228, -172, -86572, -172972, -259372, -345772, -432172, -518572, -604972, -691372, -777772, -864172, -172, 0 ]
+}, {
+  "tz" : "Indian/Cocos",
+  "switches" : [ -62135793000, -59006471400, -55850711400, -52694951400, -46383431400, -43227671400, -40071911400, -33760391400, -30604631400, -27448871400, -21137351400, -17981591400, -14825831400, -12219316200, -2209012060 ],
+  "diffs" : [ 172940, 86540, 140, -86260, -172660, -259060, -345460, -431860, -518260, -604660, -691060, -777460, -863860, 140, 0 ]
+}, {
+  "tz" : "Indian/Comoro",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Indian/Kerguelen",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 190800, 104400, 18000, -68400, -154800, -241200, -327600, -414000, -500400, -586800, -673200, -759600, -846000, 18000, 0 ]
+}, {
+  "tz" : "Indian/Mahe",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 173892, 87492, 1092, -85308, -171708, -258108, -344508, -430908, -517308, -603708, -690108, -776508, -862908, 1092, 0 ]
+}, {
+  "tz" : "Indian/Maldives",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 173160, 86760, 360, -86040, -172440, -258840, -345240, -431640, -518040, -604440, -690840, -777240, -863640, 360, 0 ]
+}, {
+  "tz" : "Indian/Mauritius",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 173400, 87000, 600, -85800, -172200, -258600, -345000, -431400, -517800, -604200, -690600, -777000, -863400, 600, 0 ]
+}, {
+  "tz" : "Indian/Mayotte",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Indian/Reunion",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 0 ]
+}, {
+  "tz" : "Iran",
+  "switches" : [ -62135782200, -59006460600, -55850700600, -52694940600, -46383420600, -43227660600, -40071900600, -33760380600, -30604620600, -27448860600, -21137340600, -17981580600, -14825820600, -12219305400, -2208988800 ],
+  "diffs" : [ 173056, 86656, 256, -86144, -172544, -258944, -345344, -431744, -518144, -604544, -690944, -777344, -863744, 256, 0 ]
+}, {
+  "tz" : "Israel",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148014, -2208988800 ],
+  "diffs" : [ 171546, 85146, -1254, -87654, -174054, -260454, -346854, -433254, -519654, -606054, -692454, -778854, -865254, -1254, -1240, 0 ]
+}, {
+  "tz" : "Jamaica",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 173230, 86830, 430, -85970, -172370, -258770, -345170, -431570, -517970, -604370, -690770, -777170, -863570, 430, 0 ]
+}, {
+  "tz" : "Japan",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2587712400 ],
+  "diffs" : [ 171661, 85261, -1139, -87539, -173939, -260339, -346739, -433139, -519539, -605939, -692339, -778739, -865139, -1139, 0 ]
+}, {
+  "tz" : "Kwajalein",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 175840, 89440, 3040, -83360, -169760, -256160, -342560, -428960, -515360, -601760, -688160, -774560, -860960, 3040, 0 ]
+}, {
+  "tz" : "Libya",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 176836, 90436, 4036, -82364, -168764, -255164, -341564, -427964, -514364, -600764, -687164, -773564, -859964, 4036, 0 ]
+}, {
+  "tz" : "MET",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "MST7MDT",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Mexico/BajaNorte",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 172084, 85684, -716, -87116, -173516, -259916, -346316, -432716, -519116, -605516, -691916, -778316, -864716, -716, 0 ]
+}, {
+  "tz" : "Mexico/BajaSur",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 173140, 86740, 340, -86060, -172460, -258860, -345260, -431660, -518060, -604460, -690860, -777260, -863660, 340, 0 ]
+}, {
+  "tz" : "Mexico/General",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 174996, 88596, 2196, -84204, -170604, -257004, -343404, -429804, -516204, -602604, -689004, -775404, -861804, 2196, 0 ]
+}, {
+  "tz" : "NZ",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -3192437344, -2208988800 ],
+  "diffs" : [ 174056, 87656, 1256, -85144, -171544, -257944, -344344, -430744, -517144, -603544, -689944, -776344, -862744, 1256, 1800, 0 ]
+}, {
+  "tz" : "NZ-CHAT",
+  "switches" : [ -62135815500, -59006493900, -55850733900, -52694973900, -46383453900, -43227693900, -40071933900, -33760413900, -30604653900, -27448893900, -21137373900, -17981613900, -14825853900, -12219338700, -3192439428, -2208988800 ],
+  "diffs" : [ 174672, 88272, 1872, -84528, -170928, -257328, -343728, -430128, -516528, -602928, -689328, -775728, -862128, 1872, 1800, 0 ]
+}, {
+  "tz" : "Navajo",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 172796, 86396, -4, -86404, -172804, -259204, -345604, -432004, -518404, -604804, -691204, -777604, -864004, -4, 0 ]
+}, {
+  "tz" : "PRC",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172457, 86057, -343, -86743, -173143, -259543, -345943, -432343, -518743, -605143, -691543, -777943, -864343, -343, 0 ]
+}, {
+  "tz" : "PST8PDT",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Pacific/Apia",
+  "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2445512400, -2208988800 ],
+  "diffs" : [ 174416, 88016, 1616, -84784, -171184, -257584, -343984, -430384, -516784, -603184, -689584, -775984, -862384, 1616, 88016, 0 ]
+}, {
+  "tz" : "Pacific/Auckland",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -3192437344, -2208988800 ],
+  "diffs" : [ 174056, 87656, 1256, -85144, -171544, -257944, -344344, -430744, -517144, -603544, -689944, -776344, -862744, 1256, 1800, 0 ]
+}, {
+  "tz" : "Pacific/Bougainville",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2840182424, -2366794112, -2208988800 ],
+  "diffs" : [ 175064, 88664, 2264, -84136, -170536, -256936, -343336, -429736, -516136, -602536, -688936, -775336, -861736, 2264, 4288, 3600, 0 ]
+}, {
+  "tz" : "Pacific/Chatham",
+  "switches" : [ -62135815500, -59006493900, -55850733900, -52694973900, -46383453900, -43227693900, -40071933900, -33760413900, -30604653900, -27448893900, -21137373900, -17981613900, -14825853900, -12219338700, -3192439428, -2208988800 ],
+  "diffs" : [ 174672, 88272, 1872, -84528, -170928, -257328, -343728, -430128, -516528, -602928, -689328, -775728, -862128, 1872, 1800, 0 ]
+}, {
+  "tz" : "Pacific/Chuuk",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ],
+  "diffs" : [ 258772, 172372, 85972, -428, -86828, -173228, -259628, -346028, -432428, -518828, -605228, -691628, -778028, 85972, -428, 0 ]
+}, {
+  "tz" : "Pacific/Easter",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 177448, 91048, 4648, -81752, -168152, -254552, -340952, -427352, -513752, -600152, -686552, -772952, -859352, 4648, 0 ]
+}, {
+  "tz" : "Pacific/Efate",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 172004, 85604, -796, -87196, -173596, -259996, -346396, -432796, -519196, -605596, -691996, -778396, -864796, -796, 0 ]
+}, {
+  "tz" : "Pacific/Enderbury",
+  "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2208988800 ],
+  "diffs" : [ 260660, 174260, 87860, 1460, -84940, -171340, -257740, -344140, -430540, -516940, -603340, -689740, -776140, 87860, 0 ]
+}, {
+  "tz" : "Pacific/Fakaofo",
+  "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2208988800 ],
+  "diffs" : [ 260696, 174296, 87896, 1496, -84904, -171304, -257704, -344104, -430504, -516904, -603304, -689704, -776104, 87896, 0 ]
+}, {
+  "tz" : "Pacific/Fiji",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 173056, 86656, 256, -86144, -172544, -258944, -345344, -431744, -518144, -604544, -690944, -777344, -863744, 256, 0 ]
+}, {
+  "tz" : "Pacific/Funafuti",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 172988, 86588, 188, -86212, -172612, -259012, -345412, -431812, -518212, -604612, -691012, -777412, -863812, 188, 0 ]
+}, {
+  "tz" : "Pacific/Galapagos",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172704, 86304, -96, -86496, -172896, -259296, -345696, -432096, -518496, -604896, -691296, -777696, -864096, -96, 0 ]
+}, {
+  "tz" : "Pacific/Gambier",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -2208988800 ],
+  "diffs" : [ 172788, 86388, -12, -86412, -172812, -259212, -345612, -432012, -518412, -604812, -691212, -777612, -864012, -12, 0 ]
+}, {
+  "tz" : "Pacific/Guadalcanal",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 174012, 87612, 1212, -85188, -171588, -257988, -344388, -430788, -517188, -603588, -689988, -776388, -862788, 1212, 0 ]
+}, {
+  "tz" : "Pacific/Guam",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ],
+  "diffs" : [ 260460, 174060, 87660, 1260, -85140, -171540, -257940, -344340, -430740, -517140, -603540, -689940, -776340, 87660, 1260, 0 ]
+}, {
+  "tz" : "Pacific/Honolulu",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2334103114, -2208988800 ],
+  "diffs" : [ 174686, 88286, 1886, -84514, -170914, -257314, -343714, -430114, -516514, -602914, -689314, -775714, -862114, 1886, 1800, 0 ]
+}, {
+  "tz" : "Pacific/Johnston",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2334103114, -2208988800 ],
+  "diffs" : [ 174686, 88286, 1886, -84514, -170914, -257314, -343714, -430114, -516514, -602914, -689314, -775714, -862114, 1886, 1800, 0 ]
+}, {
+  "tz" : "Pacific/Kiritimati",
+  "switches" : [ -62135820000, -59006498400, -55850738400, -52694978400, -46383458400, -43227698400, -40071938400, -33760418400, -30604658400, -27448898400, -21137378400, -17981618400, -14825858400, -12219343200, -2208988800 ],
+  "diffs" : [ 260960, 174560, 88160, 1760, -84640, -171040, -257440, -343840, -430240, -516640, -603040, -689440, -775840, 88160, 0 ]
+}, {
+  "tz" : "Pacific/Kosrae",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -3944631600, -2208988800 ],
+  "diffs" : [ 259684, 173284, 86884, 484, -85916, -172316, -258716, -345116, -431516, -517916, -604316, -690716, -777116, 86884, 484, 0 ]
+}, {
+  "tz" : "Pacific/Kwajalein",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 175840, 89440, 3040, -83360, -169760, -256160, -342560, -428960, -515360, -601760, -688160, -774560, -860960, 3040, 0 ]
+}, {
+  "tz" : "Pacific/Majuro",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 174912, 88512, 2112, -84288, -170688, -257088, -343488, -429888, -516288, -602688, -689088, -775488, -861888, 2112, 0 ]
+}, {
+  "tz" : "Pacific/Marquesas",
+  "switches" : [ -62135735400, -59006413800, -55850653800, -52694893800, -46383373800, -43227613800, -40071853800, -33760333800, -30604573800, -27448813800, -21137293800, -17981533800, -14825773800, -12219258600, -2208988800 ],
+  "diffs" : [ 172080, 85680, -720, -87120, -173520, -259920, -346320, -432720, -519120, -605520, -691920, -778320, -864720, -720, 0 ]
+}, {
+  "tz" : "Pacific/Midway",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2445426000, -2208988800 ],
+  "diffs" : [ 87768, 1368, -85032, -171432, -257832, -344232, -430632, -517032, -603432, -689832, -776232, -862632, -949032, -85032, 1368, 0 ]
+}, {
+  "tz" : "Pacific/Nauru",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 175940, 89540, 3140, -83260, -169660, -256060, -342460, -428860, -515260, -601660, -688060, -774460, -860860, 3140, 0 ]
+}, {
+  "tz" : "Pacific/Niue",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2208988800 ],
+  "diffs" : [ 173980, 87580, 1180, -85220, -171620, -258020, -344420, -430820, -517220, -603620, -690020, -776420, -862820, 1180, 0 ]
+}, {
+  "tz" : "Pacific/Norfolk",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 172088, 85688, -712, -87112, -173512, -259912, -346312, -432712, -519112, -605512, -691912, -778312, -864712, -712, 0 ]
+}, {
+  "tz" : "Pacific/Noumea",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 172452, 86052, -348, -86748, -173148, -259548, -345948, -432348, -518748, -605148, -691548, -777948, -864348, -348, 0 ]
+}, {
+  "tz" : "Pacific/Pago_Pago",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2445426000, -2208988800 ],
+  "diffs" : [ 87768, 1368, -85032, -171432, -257832, -344232, -430632, -517032, -603432, -689832, -776232, -862632, -949032, -85032, 1368, 0 ]
+}, {
+  "tz" : "Pacific/Palau",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -3944624400, -2208988800 ],
+  "diffs" : [ 259324, 172924, 86524, 124, -86276, -172676, -259076, -345476, -431876, -518276, -604676, -691076, -777476, 86524, 124, 0 ]
+}, {
+  "tz" : "Pacific/Pitcairn",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 175220, 88820, 2420, -83980, -170380, -256780, -343180, -429580, -515980, -602380, -688780, -775180, -861580, 2420, 0 ]
+}, {
+  "tz" : "Pacific/Pohnpei",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -3944631600, -2208988800 ],
+  "diffs" : [ 260828, 174428, 88028, 1628, -84772, -171172, -257572, -343972, -430372, -516772, -603172, -689572, -775972, 88028, 1628, 0 ]
+}, {
+  "tz" : "Pacific/Ponape",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -3944631600, -2208988800 ],
+  "diffs" : [ 260828, 174428, 88028, 1628, -84772, -171172, -257572, -343972, -430372, -516772, -603172, -689572, -775972, 88028, 1628, 0 ]
+}, {
+  "tz" : "Pacific/Port_Moresby",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2840176808, -2366790512 ],
+  "diffs" : [ 173480, 87080, 680, -85720, -172120, -258520, -344920, -431320, -517720, -604120, -690520, -776920, -863320, 680, 688, 0 ]
+}, {
+  "tz" : "Pacific/Rarotonga",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2208988800 ],
+  "diffs" : [ 175144, 88744, 2344, -84056, -170456, -256856, -343256, -429656, -516056, -602456, -688856, -775256, -861656, 2344, 0 ]
+}, {
+  "tz" : "Pacific/Saipan",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ],
+  "diffs" : [ 260460, 174060, 87660, 1260, -85140, -171540, -257940, -344340, -430740, -517140, -603540, -689940, -776340, 87660, 1260, 0 ]
+}, {
+  "tz" : "Pacific/Samoa",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2445426000, -2208988800 ],
+  "diffs" : [ 87768, 1368, -85032, -171432, -257832, -344232, -430632, -517032, -603432, -689832, -776232, -862632, -949032, -85032, 1368, 0 ]
+}, {
+  "tz" : "Pacific/Tahiti",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2208988800 ],
+  "diffs" : [ 172696, 86296, -104, -86504, -172904, -259304, -345704, -432104, -518504, -604904, -691304, -777704, -864104, -104, 0 ]
+}, {
+  "tz" : "Pacific/Tarawa",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 174476, 88076, 1676, -84724, -171124, -257524, -343924, -430324, -516724, -603124, -689524, -775924, -862324, 1676, 0 ]
+}, {
+  "tz" : "Pacific/Tongatapu",
+  "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2208988800 ],
+  "diffs" : [ 175240, 88840, 2440, -83960, -170360, -256760, -343160, -429560, -515960, -602360, -688760, -775160, -861560, 2440, 0 ]
+}, {
+  "tz" : "Pacific/Truk",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ],
+  "diffs" : [ 258772, 172372, 85972, -428, -86828, -173228, -259628, -346028, -432428, -518828, -605228, -691628, -778028, 85972, -428, 0 ]
+}, {
+  "tz" : "Pacific/Wake",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 176012, 89612, 3212, -83188, -169588, -255988, -342388, -428788, -515188, -601588, -687988, -774388, -860788, 3212, 0 ]
+}, {
+  "tz" : "Pacific/Wallis",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 171880, 85480, -920, -87320, -173720, -260120, -346520, -432920, -519320, -605720, -692120, -778520, -864920, -920, 0 ]
+}, {
+  "tz" : "Pacific/Yap",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ],
+  "diffs" : [ 258772, 172372, 85972, -428, -86828, -173228, -259628, -346028, -432428, -518828, -605228, -691628, -778028, 85972, -428, 0 ]
+}, {
+  "tz" : "Poland",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 171360, 84960, -1440, -87840, -174240, -260640, -347040, -433440, -519840, -606240, -692640, -779040, -865440, -1440, 0 ]
+}, {
+  "tz" : "Portugal",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 175005, 88605, 2205, -84195, -170595, -256995, -343395, -429795, -516195, -602595, -688995, -775395, -861795, 2205, 0 ]
+}, {
+  "tz" : "ROK",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 174728, 88328, 1928, -84472, -170872, -257272, -343672, -430072, -516472, -602872, -689272, -775672, -862072, 1928, 0 ]
+}, {
+  "tz" : "Singapore",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 176675, 90275, 3875, -82525, -168925, -255325, -341725, -428125, -514525, -600925, -687325, -773725, -860125, 3875, 0 ]
+}, {
+  "tz" : "SystemV/AST4",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/AST4ADT",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/CST6",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/CST6CDT",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/EST5",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/EST5EDT",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/HST10",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/MST7",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/MST7MDT",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/PST8",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/PST8PDT",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/YST9",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/YST9YDT",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Turkey",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2840151536, -2208988800 ],
+  "diffs" : [ 176648, 90248, 3848, -82552, -168952, -255352, -341752, -428152, -514552, -600952, -687352, -773752, -860152, 3848, 3784, 0 ]
+}, {
+  "tz" : "UCT",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "US/Alaska",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225227303, -2208988800 ],
+  "diffs" : [ 89976, 3576, -82824, -169224, -255624, -342024, -428424, -514824, -601224, -687624, -774024, -860424, -946824, -82824, 3576, 0 ]
+}, {
+  "tz" : "US/Aleutian",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -3225230125, -2208988800 ],
+  "diffs" : [ 92798, 6398, -80002, -166402, -252802, -339202, -425602, -512002, -598402, -684802, -771202, -857602, -944002, -80002, 6398, 0 ]
+}, {
+  "tz" : "US/Arizona",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 174498, 88098, 1698, -84702, -171102, -257502, -343902, -430302, -516702, -603102, -689502, -775902, -862302, 1698, 0 ]
+}, {
+  "tz" : "US/Central",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 172236, 85836, -564, -86964, -173364, -259764, -346164, -432564, -518964, -605364, -691764, -778164, -864564, -564, 0 ]
+}, {
+  "tz" : "US/East-Indiana",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175478, 89078, 2678, -83722, -170122, -256522, -342922, -429322, -515722, -602122, -688522, -774922, -861322, 2678, 3600, 0 ]
+}, {
+  "tz" : "US/Eastern",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800 ],
+  "diffs" : [ 172562, 86162, -238, -86638, -173038, -259438, -345838, -432238, -518638, -605038, -691438, -777838, -864238, -238, 0 ]
+}, {
+  "tz" : "US/Hawaii",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2334103114, -2208988800 ],
+  "diffs" : [ 174686, 88286, 1886, -84514, -170914, -257314, -343714, -430114, -516514, -602914, -689314, -775714, -862114, 1886, 1800, 0 ]
+}, {
+  "tz" : "US/Indiana-Starke",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 171990, 85590, -810, -87210, -173610, -260010, -346410, -432810, -519210, -605610, -692010, -778410, -864810, -810, 0 ]
+}, {
+  "tz" : "US/Michigan",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 174731, 88331, 1931, -84469, -170869, -257269, -343669, -430069, -516469, -602869, -689269, -775669, -862069, 1931, 0 ]
+}, {
+  "tz" : "US/Mountain",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 172796, 86396, -4, -86404, -172804, -259204, -345604, -432004, -518404, -604804, -691204, -777604, -864004, -4, 0 ]
+}, {
+  "tz" : "US/Pacific",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2717640000 ],
+  "diffs" : [ 172378, 85978, -422, -86822, -173222, -259622, -346022, -432422, -518822, -605222, -691622, -778022, -864422, -422, 0 ]
+}, {
+  "tz" : "US/Pacific-New",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2717640000 ],
+  "diffs" : [ 172378, 85978, -422, -86822, -173222, -259622, -346022, -432422, -518822, -605222, -691622, -778022, -864422, -422, 0 ]
+}, {
+  "tz" : "US/Samoa",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2445426000, -2208988800 ],
+  "diffs" : [ 87768, 1368, -85032, -171432, -257832, -344232, -430632, -517032, -603432, -689832, -776232, -862632, -949032, -85032, 1368, 0 ]
+}, {
+  "tz" : "UTC",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Universal",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "W-SU",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174583, 88183, 1783, -84617, -171017, -257417, -343817, -430217, -516617, -603017, -689417, -775817, -862217, 1783, 0 ]
+}, {
+  "tz" : "WET",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Zulu",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+} ]
\ No newline at end of file
diff --git a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
index 75a8bec018a1f..42b92d4593c77 100644
--- a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
+++ b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
@@ -26,6 +26,8 @@ object AttributeMap {
   def apply[A](kvs: Seq[(Attribute, A)]): AttributeMap[A] = {
     new AttributeMap(kvs.map(kv => (kv._1.exprId, kv)).toMap)
   }
+
+  def empty[A]: AttributeMap[A] = new AttributeMap(Map.empty)
 }
 
 class AttributeMap[A](val baseMap: Map[ExprId, (Attribute, A)])
diff --git a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index 699b42bd6d60d..14b8f620017f6 100644
--- a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -39,8 +39,12 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def contains(k: String): Boolean =
     keyLowerCasedMap.contains(k.toLowerCase(Locale.ROOT))
 
-  override def +[B1 >: T](kv: (String, B1)): Map[String, B1] = {
-    new CaseInsensitiveMap(originalMap + kv)
+  override def +[B1 >: T](kv: (String, B1)): CaseInsensitiveMap[B1] = {
+    new CaseInsensitiveMap(originalMap.filter(!_._1.equalsIgnoreCase(kv._1)) + kv)
+  }
+
+  def ++(xs: TraversableOnce[(String, T)]): CaseInsensitiveMap[T] = {
+    xs.foldLeft(this)(_ + _)
   }
 
   override def iterator: Iterator[(String, T)] = keyLowerCasedMap.iterator
@@ -48,6 +52,8 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def -(key: String): Map[String, T] = {
     new CaseInsensitiveMap(originalMap.filter(!_._1.equalsIgnoreCase(key)))
   }
+
+  def toMap: Map[String, T] = originalMap
 }
 
 object CaseInsensitiveMap {
diff --git a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
index 4caa3d0461875..e6b53e3e6548f 100644
--- a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
+++ b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
@@ -26,6 +26,8 @@ object AttributeMap {
   def apply[A](kvs: Seq[(Attribute, A)]): AttributeMap[A] = {
     new AttributeMap(kvs.map(kv => (kv._1.exprId, kv)).toMap)
   }
+
+  def empty[A]: AttributeMap[A] = new AttributeMap(Map.empty)
 }
 
 class AttributeMap[A](val baseMap: Map[ExprId, (Attribute, A)])
diff --git a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index 1f8cc6c0f83a2..1db4b7a4f227b 100644
--- a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -39,8 +39,12 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def contains(k: String): Boolean =
     keyLowerCasedMap.contains(k.toLowerCase(Locale.ROOT))
 
-  override def updated[B1 >: T](key: String, value: B1): Map[String, B1] = {
-    new CaseInsensitiveMap[B1](originalMap + (key -> value))
+  override def updated[B1 >: T](key: String, value: B1): CaseInsensitiveMap[B1] = {
+    new CaseInsensitiveMap[B1](originalMap.filter(!_._1.equalsIgnoreCase(key)) + (key -> value))
+  }
+
+  def ++(xs: IterableOnce[(String, T)]): CaseInsensitiveMap[T] = {
+    xs.iterator.foldLeft(this) { (m, kv) => m.updated(kv._1, kv._2) }
   }
 
   override def iterator: Iterator[(String, T)] = keyLowerCasedMap.iterator
@@ -48,6 +52,8 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def removed(key: String): Map[String, T] = {
     new CaseInsensitiveMap(originalMap.filter(!_._1.equalsIgnoreCase(key)))
   }
+
+  def toMap: Map[String, T] = originalMap
 }
 
 object CaseInsensitiveMap {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
index c43a86ad48ec9..ea760d80541c8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
@@ -58,8 +58,7 @@ import org.apache.spark.sql.types._
  * }}}
  *
  * == Implementation ==
- *  - Encoders are not required to be thread-safe and thus they do not need to use locks to guard
- *    against concurrent access if they reuse internal buffers to improve performance.
+ *  - Encoders should be thread-safe.
  *
  * @since 1.6.0
  */
@@ -76,10 +75,4 @@ trait Encoder[T] extends Serializable {
    * A ClassTag that can be used to construct an Array to contain a collection of `T`.
    */
   def clsTag: ClassTag[T]
-
-  /**
-   * Create a copied [[Encoder]]. The implementation may just copy internal reusable fields to speed
-   * up the [[Encoder]] creation.
-   */
-  def makeCopy: Encoder[T]
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
index 5d31b5bbf12af..24045b5a43a64 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
@@ -146,7 +146,7 @@ object Encoders {
    *  - String
    *  - java.math.BigDecimal, java.math.BigInteger
    *  - time related: java.sql.Date, java.sql.Timestamp, java.time.LocalDate, java.time.Instant
-   *  - collection types: only array and java.util.List currently, map support is in progress
+   *  - collection types: array, java.util.List, and map
    *  - nested java bean.
    *
    * @since 1.6.0
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
index 9a7e077b658df..3928a8c95eee4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
@@ -160,13 +160,17 @@ trait Row extends Serializable {
    *   ByteType -> java.lang.Byte
    *   ShortType -> java.lang.Short
    *   IntegerType -> java.lang.Integer
+   *   LongType -> java.lang.Long
    *   FloatType -> java.lang.Float
    *   DoubleType -> java.lang.Double
    *   StringType -> String
    *   DecimalType -> java.math.BigDecimal
    *
-   *   DateType -> java.sql.Date
-   *   TimestampType -> java.sql.Timestamp
+   *   DateType -> java.sql.Date if spark.sql.datetime.java8API.enabled is false
+   *   DateType -> java.time.LocalDate if spark.sql.datetime.java8API.enabled is true
+   *
+   *   TimestampType -> java.sql.Timestamp if spark.sql.datetime.java8API.enabled is false
+   *   TimestampType -> java.time.Instant if spark.sql.datetime.java8API.enabled is true
    *
    *   BinaryType -> byte array
    *   ArrayType -> scala.collection.Seq (use getList for java.util.List)
@@ -185,13 +189,17 @@ trait Row extends Serializable {
    *   ByteType -> java.lang.Byte
    *   ShortType -> java.lang.Short
    *   IntegerType -> java.lang.Integer
+   *   LongType -> java.lang.Long
    *   FloatType -> java.lang.Float
    *   DoubleType -> java.lang.Double
    *   StringType -> String
    *   DecimalType -> java.math.BigDecimal
    *
-   *   DateType -> java.sql.Date
-   *   TimestampType -> java.sql.Timestamp
+   *   DateType -> java.sql.Date if spark.sql.datetime.java8API.enabled is false
+   *   DateType -> java.time.LocalDate if spark.sql.datetime.java8API.enabled is true
+   *
+   *   TimestampType -> java.sql.Timestamp if spark.sql.datetime.java8API.enabled is false
+   *   TimestampType -> java.time.Instant if spark.sql.datetime.java8API.enabled is true
    *
    *   BinaryType -> byte array
    *   ArrayType -> scala.collection.Seq (use getList for java.util.List)
@@ -565,14 +573,10 @@ trait Row extends Serializable {
       case (s: String, _) => JString(s)
       case (b: Array[Byte], BinaryType) =>
         JString(Base64.getEncoder.encodeToString(b))
-      case (d: LocalDate, _) =>
-        JString(dateFormatter.format(DateTimeUtils.localDateToDays(d)))
-      case (d: Date, _) =>
-        JString(dateFormatter.format(DateTimeUtils.fromJavaDate(d)))
-      case (i: Instant, _) =>
-        JString(timestampFormatter.format(DateTimeUtils.instantToMicros(i)))
-      case (t: Timestamp, _) =>
-        JString(timestampFormatter.format(DateTimeUtils.fromJavaTimestamp(t)))
+      case (d: LocalDate, _) => JString(dateFormatter.format(d))
+      case (d: Date, _) => JString(dateFormatter.format(d))
+      case (i: Instant, _) => JString(timestampFormatter.format(i))
+      case (t: Timestamp, _) => JString(timestampFormatter.format(t))
       case (i: CalendarInterval, _) => JString(i.toString)
       case (a: Array[_], ArrayType(elementType, _)) =>
         iteratorToJsonArray(a.iterator, elementType)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
index e55c25c4b0c54..701e4e3483c0f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
@@ -161,6 +161,10 @@ object DeserializerBuildHelper {
     case _: StructType => expr
     case _: ArrayType => expr
     case _: MapType => expr
+    case _: DecimalType =>
+      // For Scala/Java `BigDecimal`, we accept decimal types of any valid precision/scale.
+      // Here we use the `DecimalType` object to indicate it.
+      UpCast(expr, DecimalType, walkedTypePath.getPaths)
     case _ => UpCast(expr, expected, walkedTypePath.getPaths)
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala
index cd75407c7ee7a..35551d8ba77dc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala
@@ -28,7 +28,7 @@ import org.apache.spark.util.BoundedPriorityQueue
  * There are two separate concepts we track:
  *
  * 1. Phases: These are broad scope phases in query planning, as listed below, i.e. analysis,
- * optimizationm and physical planning (just planning).
+ * optimization and physical planning (just planning).
  *
  * 2. Rules: These are the individual Catalyst rules that we track. In addition to time, we also
  * track the number of invocations and effective invocations.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 1f7634bafa420..05de21b9eafec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.catalyst
 
+import javax.lang.model.SourceVersion
+
 import org.apache.commons.lang3.reflect.ConstructorUtils
 
 import org.apache.spark.internal.Logging
@@ -124,6 +126,9 @@ object ScalaReflection extends ScalaReflection {
       case t if isSubtype(t, definitions.ShortTpe) => classOf[Array[Short]]
       case t if isSubtype(t, definitions.ByteTpe) => classOf[Array[Byte]]
       case t if isSubtype(t, definitions.BooleanTpe) => classOf[Array[Boolean]]
+      case t if isSubtype(t, localTypeOf[Array[Byte]]) => classOf[Array[Array[Byte]]]
+      case t if isSubtype(t, localTypeOf[CalendarInterval]) => classOf[Array[CalendarInterval]]
+      case t if isSubtype(t, localTypeOf[Decimal]) => classOf[Array[Decimal]]
       case other =>
         // There is probably a better way to do this, but I couldn't find it...
         val elementType = dataTypeFor(other).asInstanceOf[ObjectType].cls
@@ -539,9 +544,10 @@ object ScalaReflection extends ScalaReflection {
 
         val params = getConstructorParameters(t)
         val fields = params.map { case (fieldName, fieldType) =>
-          if (javaKeywords.contains(fieldName)) {
-            throw new UnsupportedOperationException(s"`$fieldName` is a reserved keyword and " +
-              "cannot be used as field name\n" + walkedTypePath)
+          if (SourceVersion.isKeyword(fieldName) ||
+              !SourceVersion.isIdentifier(encodeFieldNameToIdentifier(fieldName))) {
+            throw new UnsupportedOperationException(s"`$fieldName` is not a valid identifier of " +
+              "Java and cannot be used as field name\n" + walkedTypePath)
           }
 
           // SPARK-26730 inputObject won't be null with If's guard below. And KnownNotNul
@@ -611,10 +617,39 @@ object ScalaReflection extends ScalaReflection {
     }
   }
 
+  private def erasure(tpe: Type): Type = {
+    // For user-defined AnyVal classes, we should not erasure it. Otherwise, it will
+    // resolve to underlying type which wrapped by this class, e.g erasure
+    // `case class Foo(i: Int) extends AnyVal` will return type `Int` instead of `Foo`.
+    // But, for other types, we do need to erasure it. For example, we need to erasure
+    // `scala.Any` to `java.lang.Object` in order to load it from Java ClassLoader.
+    // Please see SPARK-17368 & SPARK-31190 for more details.
+    if (isSubtype(tpe, localTypeOf[AnyVal]) && !tpe.toString.startsWith("scala")) {
+      tpe
+    } else {
+      tpe.erasure
+    }
+  }
+
+  /**
+   * Returns the full class name for a type. The returned name is the canonical
+   * Scala name, where each component is separated by a period. It is NOT the
+   * Java-equivalent runtime name (no dollar signs).
+   *
+   * In simple cases, both the Scala and Java names are the same, however when Scala
+   * generates constructs that do not map to a Java equivalent, such as singleton objects
+   * or nested classes in package objects, it uses the dollar sign ($) to create
+   * synthetic classes, emulating behaviour in Java bytecode.
+   */
+  def getClassNameFromType(tpe: `Type`): String = {
+    erasure(tpe).dealias.typeSymbol.asClass.fullName
+  }
+
   /*
    * Retrieves the runtime class corresponding to the provided type.
    */
-  def getClassFromType(tpe: Type): Class[_] = mirror.runtimeClass(tpe.dealias.typeSymbol.asClass)
+  def getClassFromType(tpe: Type): Class[_] =
+    mirror.runtimeClass(erasure(tpe).dealias.typeSymbol.asClass)
 
   case class Schema(dataType: DataType, nullable: Boolean)
 
@@ -671,6 +706,8 @@ object ScalaReflection extends ScalaReflection {
         Schema(TimestampType, nullable = true)
       case t if isSubtype(t, localTypeOf[java.time.LocalDate]) => Schema(DateType, nullable = true)
       case t if isSubtype(t, localTypeOf[java.sql.Date]) => Schema(DateType, nullable = true)
+      case t if isSubtype(t, localTypeOf[CalendarInterval]) =>
+        Schema(CalendarIntervalType, nullable = true)
       case t if isSubtype(t, localTypeOf[BigDecimal]) =>
         Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
       case t if isSubtype(t, localTypeOf[java.math.BigDecimal]) =>
@@ -753,13 +790,6 @@ object ScalaReflection extends ScalaReflection {
     }
   }
 
-  private val javaKeywords = Set("abstract", "assert", "boolean", "break", "byte", "case", "catch",
-    "char", "class", "const", "continue", "default", "do", "double", "else", "extends", "false",
-    "final", "finally", "float", "for", "goto", "if", "implements", "import", "instanceof", "int",
-    "interface", "long", "native", "new", "null", "package", "private", "protected", "public",
-    "return", "short", "static", "strictfp", "super", "switch", "synchronized", "this", "throw",
-    "throws", "transient", "true", "try", "void", "volatile", "while")
-
   val typeJavaMapping = Map[DataType, Class[_]](
     BooleanType -> classOf[Boolean],
     ByteType -> classOf[Byte],
@@ -818,6 +848,10 @@ object ScalaReflection extends ScalaReflection {
       Seq.empty
     }
   }
+
+  def encodeFieldNameToIdentifier(fieldName: String): String = {
+    TermName(fieldName).encodedName.toString
+  }
 }
 
 /**
@@ -863,20 +897,6 @@ trait ScalaReflection extends Logging {
     tag.in(mirror).tpe.dealias
   }
 
-  /**
-   * Returns the full class name for a type. The returned name is the canonical
-   * Scala name, where each component is separated by a period. It is NOT the
-   * Java-equivalent runtime name (no dollar signs).
-   *
-   * In simple cases, both the Scala and Java names are the same, however when Scala
-   * generates constructs that do not map to a Java equivalent, such as singleton objects
-   * or nested classes in package objects, it uses the dollar sign ($) to create
-   * synthetic classes, emulating behaviour in Java bytecode.
-   */
-  def getClassNameFromType(tpe: `Type`): String = {
-    tpe.dealias.erasure.typeSymbol.asClass.fullName
-  }
-
   /**
    * Returns the parameter names and types for the primary constructor of this type.
    *
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 3fd5039a4f116..fbe60412b2f39 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.analysis
 
 import java.util
 import java.util.Locale
+import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
@@ -128,8 +129,7 @@ object AnalysisContext {
  */
 class Analyzer(
     override val catalogManager: CatalogManager,
-    conf: SQLConf,
-    maxIterations: Int)
+    conf: SQLConf)
   extends RuleExecutor[LogicalPlan] with CheckAnalysis with LookupCatalog {
 
   private val v1SessionCatalog: SessionCatalog = catalogManager.v1SessionCatalog
@@ -140,15 +140,11 @@ class Analyzer(
   def this(catalog: SessionCatalog, conf: SQLConf) = {
     this(
       new CatalogManager(conf, FakeV2SessionCatalog, catalog),
-      conf,
-      conf.analyzerMaxIterations)
-  }
-
-  def this(catalogManager: CatalogManager, conf: SQLConf) = {
-    this(catalogManager, conf, conf.analyzerMaxIterations)
+      conf)
   }
 
   def executeAndCheck(plan: LogicalPlan, tracker: QueryPlanningTracker): LogicalPlan = {
+    if (plan.analyzed) return plan
     AnalysisHelper.markInAnalyzer {
       val analyzed = executeAndTrack(plan, tracker)
       try {
@@ -176,7 +172,15 @@ class Analyzer(
 
   def resolver: Resolver = conf.resolver
 
-  protected val fixedPoint = FixedPoint(maxIterations)
+  /**
+   * If the plan cannot be resolved within maxIterations, analyzer will throw exception to inform
+   * user to increase the value of SQLConf.ANALYZER_MAX_ITERATIONS.
+   */
+  protected def fixedPoint =
+    FixedPoint(
+      conf.analyzerMaxIterations,
+      errorOnExceed = true,
+      maxIterationsSetting = SQLConf.ANALYZER_MAX_ITERATIONS.key)
 
   /**
    * Override to provide additional rules for the "Resolution" batch.
@@ -190,17 +194,17 @@ class Analyzer(
    */
   val postHocResolutionRules: Seq[Rule[LogicalPlan]] = Nil
 
-  lazy val batches: Seq[Batch] = Seq(
-    Batch("Hints", fixedPoint,
-      new ResolveHints.ResolveJoinStrategyHints(conf),
-      new ResolveHints.ResolveCoalesceHints(conf)),
-    Batch("Simple Sanity Check", Once,
-      LookupFunctions),
+  override def batches: Seq[Batch] = Seq(
     Batch("Substitution", fixedPoint,
       CTESubstitution,
       WindowsSubstitution,
       EliminateUnions,
       new SubstituteUnresolvedOrdinals(conf)),
+    Batch("Hints", fixedPoint,
+      new ResolveHints.ResolveJoinStrategyHints(conf),
+      new ResolveHints.ResolveCoalesceHints(conf)),
+    Batch("Simple Sanity Check", Once,
+      LookupFunctions),
     Batch("Resolution", fixedPoint,
       ResolveTableValuedFunctions ::
       ResolveNamespace(catalogManager) ::
@@ -237,7 +241,7 @@ class Analyzer(
       ResolveLambdaVariables(conf) ::
       ResolveTimeZone(conf) ::
       ResolveRandomSeed ::
-      ResolveBinaryArithmetic(conf) ::
+      ResolveBinaryArithmetic ::
       TypeCoercion.typeCoercionRules(conf) ++
       extendedResolutionRules : _*),
     Batch("Post-Hoc Resolution", Once, postHocResolutionRules: _*),
@@ -259,17 +263,21 @@ class Analyzer(
   /**
    * For [[Add]]:
    * 1. if both side are interval, stays the same;
-   * 2. else if one side is interval, turns it to [[TimeAdd]];
-   * 3. else if one side is date, turns it to [[DateAdd]] ;
-   * 4. else stays the same.
+   * 2. else if one side is date and the other is interval,
+   *    turns it to [[DateAddInterval]];
+   * 3. else if one side is interval, turns it to [[TimeAdd]];
+   * 4. else if one side is date, turns it to [[DateAdd]] ;
+   * 5. else stays the same.
    *
    * For [[Subtract]]:
    * 1. if both side are interval, stays the same;
-   * 2. else if the right side is an interval, turns it to [[TimeSub]];
-   * 3. else if one side is timestamp, turns it to [[SubtractTimestamps]];
-   * 4. else if the right side is date, turns it to [[DateDiff]]/[[SubtractDates]];
-   * 5. else if the left side is date, turns it to [[DateSub]];
-   * 6. else turns it to stays the same.
+   * 2. else if the left side is date and the right side is interval,
+   *    turns it to [[DateAddInterval(l, -r)]];
+   * 3. else if the right side is an interval, turns it to [[TimeSub]];
+   * 4. else if one side is timestamp, turns it to [[SubtractTimestamps]];
+   * 5. else if the right side is date, turns it to [[DateDiff]]/[[SubtractDates]];
+   * 6. else if the left side is date, turns it to [[DateSub]];
+   * 7. else turns it to stays the same.
    *
    * For [[Multiply]]:
    * 1. If one side is interval, turns it to [[MultiplyInterval]];
@@ -279,24 +287,27 @@ class Analyzer(
    * 1. If the left side is interval, turns it to [[DivideInterval]];
    * 2. otherwise, stays the same.
    */
-  case class ResolveBinaryArithmetic(conf: SQLConf) extends Rule[LogicalPlan] {
+  object ResolveBinaryArithmetic extends Rule[LogicalPlan] {
     override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
       case p: LogicalPlan => p.transformExpressionsUp {
         case a @ Add(l, r) if a.childrenResolved => (l.dataType, r.dataType) match {
           case (CalendarIntervalType, CalendarIntervalType) => a
+          case (DateType, CalendarIntervalType) => DateAddInterval(l, r)
           case (_, CalendarIntervalType) => Cast(TimeAdd(l, r), l.dataType)
+          case (CalendarIntervalType, DateType) => DateAddInterval(r, l)
           case (CalendarIntervalType, _) => Cast(TimeAdd(r, l), r.dataType)
-          case (DateType, _) => DateAdd(l, r)
-          case (_, DateType) => DateAdd(r, l)
+          case (DateType, dt) if dt != StringType => DateAdd(l, r)
+          case (dt, DateType) if dt != StringType => DateAdd(r, l)
           case _ => a
         }
         case s @ Subtract(l, r) if s.childrenResolved => (l.dataType, r.dataType) match {
           case (CalendarIntervalType, CalendarIntervalType) => s
+          case (DateType, CalendarIntervalType) => DateAddInterval(l, UnaryMinus(r))
           case (_, CalendarIntervalType) => Cast(TimeSub(l, r), l.dataType)
           case (TimestampType, _) => SubtractTimestamps(l, r)
           case (_, TimestampType) => SubtractTimestamps(l, r)
           case (_, DateType) => SubtractDates(l, r)
-          case (DateType, _) => DateSub(l, r)
+          case (DateType, dt) if dt != StringType => DateSub(l, r)
           case _ => s
         }
         case m @ Multiply(l, r) if m.childrenResolved => (l.dataType, r.dataType) match {
@@ -441,7 +452,7 @@ class Analyzer(
      */
     private def constructGroupByAlias(groupByExprs: Seq[Expression]): Seq[Alias] = {
       groupByExprs.map {
-        case e: NamedExpression => Alias(e, e.name)()
+        case e: NamedExpression => Alias(e, e.name)(qualifier = e.qualifier)
         case other => Alias(other, other.toString)()
       }
     }
@@ -514,20 +525,13 @@ class Analyzer(
       }.asInstanceOf[NamedExpression]
     }
 
-    /*
-     * Construct [[Aggregate]] operator from Cube/Rollup/GroupingSets.
-     */
-    private def constructAggregate(
+    private def getFinalGroupByExpressions(
         selectedGroupByExprs: Seq[Seq[Expression]],
-        groupByExprs: Seq[Expression],
-        aggregationExprs: Seq[NamedExpression],
-        child: LogicalPlan): LogicalPlan = {
-      val gid = AttributeReference(VirtualColumn.groupingIdName, IntegerType, false)()
-
+        groupByExprs: Seq[Expression]): Seq[Expression] = {
       // In case of ANSI-SQL compliant syntax for GROUPING SETS, groupByExprs is optional and
       // can be null. In such case, we derive the groupByExprs from the user supplied values for
       // grouping sets.
-      val finalGroupByExpressions = if (groupByExprs == Nil) {
+      if (groupByExprs == Nil) {
         selectedGroupByExprs.flatten.foldLeft(Seq.empty[Expression]) { (result, currentExpr) =>
           // Only unique expressions are included in the group by expressions and is determined
           // based on their semantic equality. Example. grouping sets ((a * b), (b * a)) results
@@ -541,6 +545,18 @@ class Analyzer(
       } else {
         groupByExprs
       }
+    }
+
+    /*
+     * Construct [[Aggregate]] operator from Cube/Rollup/GroupingSets.
+     */
+    private def constructAggregate(
+        selectedGroupByExprs: Seq[Seq[Expression]],
+        groupByExprs: Seq[Expression],
+        aggregationExprs: Seq[NamedExpression],
+        child: LogicalPlan): LogicalPlan = {
+      val gid = AttributeReference(VirtualColumn.groupingIdName, IntegerType, false)()
+      val finalGroupByExpressions = getFinalGroupByExpressions(selectedGroupByExprs, groupByExprs)
 
       // Expand works by setting grouping expressions to null as determined by the
       // `selectedGroupByExprs`. To prevent these null values from being used in an aggregate
@@ -572,8 +588,70 @@ class Analyzer(
       }
     }
 
-    // This require transformUp to replace grouping()/grouping_id() in resolved Filter/Sort
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
+    private def tryResolveHavingCondition(h: UnresolvedHaving): LogicalPlan = {
+      val aggForResolving = h.child match {
+        // For CUBE/ROLLUP expressions, to avoid resolving repeatedly, here we delete them from
+        // groupingExpressions for condition resolving.
+        case a @ Aggregate(Seq(c @ Cube(groupByExprs)), _, _) =>
+          a.copy(groupingExpressions = groupByExprs)
+        case a @ Aggregate(Seq(r @ Rollup(groupByExprs)), _, _) =>
+          a.copy(groupingExpressions = groupByExprs)
+        case g: GroupingSets =>
+          Aggregate(
+            getFinalGroupByExpressions(g.selectedGroupByExprs, g.groupByExprs),
+            g.aggregations, g.child)
+      }
+      // Try resolving the condition of the filter as though it is in the aggregate clause
+      val resolvedInfo =
+        ResolveAggregateFunctions.resolveFilterCondInAggregate(h.havingCondition, aggForResolving)
+
+      // Push the aggregate expressions into the aggregate (if any).
+      if (resolvedInfo.nonEmpty) {
+        val (extraAggExprs, resolvedHavingCond) = resolvedInfo.get
+        val newChild = h.child match {
+          case Aggregate(Seq(c @ Cube(groupByExprs)), aggregateExpressions, child) =>
+            constructAggregate(
+              cubeExprs(groupByExprs), groupByExprs, aggregateExpressions ++ extraAggExprs, child)
+          case Aggregate(Seq(r @ Rollup(groupByExprs)), aggregateExpressions, child) =>
+            constructAggregate(
+              rollupExprs(groupByExprs), groupByExprs, aggregateExpressions ++ extraAggExprs, child)
+          case x: GroupingSets =>
+            constructAggregate(
+              x.selectedGroupByExprs, x.groupByExprs, x.aggregations ++ extraAggExprs, x.child)
+        }
+
+        // Since the exprId of extraAggExprs will be changed in the constructed aggregate, and the
+        // aggregateExpressions keeps the input order. So here we build an exprMap to resolve the
+        // condition again.
+        val exprMap = extraAggExprs.zip(
+          newChild.asInstanceOf[Aggregate].aggregateExpressions.takeRight(
+            extraAggExprs.length)).toMap
+        val newCond = resolvedHavingCond.transform {
+          case ne: NamedExpression if exprMap.contains(ne) => exprMap(ne)
+        }
+        Project(newChild.output.dropRight(extraAggExprs.length),
+          Filter(newCond, newChild))
+      } else {
+        h
+      }
+    }
+
+    // This require transformDown to resolve having condition when generating aggregate node for
+    // CUBE/ROLLUP/GROUPING SETS. This also replace grouping()/grouping_id() in resolved
+    // Filter/Sort.
+    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsDown {
+      case h @ UnresolvedHaving(
+          _, agg @ Aggregate(Seq(c @ Cube(groupByExprs)), aggregateExpressions, _))
+          if agg.childrenResolved && (groupByExprs ++ aggregateExpressions).forall(_.resolved) =>
+        tryResolveHavingCondition(h)
+      case h @ UnresolvedHaving(
+          _, agg @ Aggregate(Seq(r @ Rollup(groupByExprs)), aggregateExpressions, _))
+          if agg.childrenResolved && (groupByExprs ++ aggregateExpressions).forall(_.resolved) =>
+        tryResolveHavingCondition(h)
+      case h @ UnresolvedHaving(_, g: GroupingSets)
+          if g.childrenResolved && g.expressions.forall(_.resolved) =>
+        tryResolveHavingCondition(h)
+
       case a if !a.childrenResolved => a // be sure all of the children are resolved.
 
       // Ensure group by expressions and aggregate expressions have been resolved.
@@ -697,9 +775,9 @@ class Analyzer(
                 // AggregateFunction's with the exception of First and Last in their default mode
                 // (which we handle) and possibly some Hive UDAF's.
                 case First(expr, _) =>
-                  First(ifExpr(expr), Literal(true))
+                  First(ifExpr(expr), true)
                 case Last(expr, _) =>
-                  Last(ifExpr(expr), Literal(true))
+                  Last(ifExpr(expr), true)
                 case a: AggregateFunction =>
                   a.withNewChildren(a.children.map(ifExpr))
               }.transform {
@@ -734,6 +812,8 @@ class Analyzer(
     def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
       case s @ ShowTables(UnresolvedNamespace(Seq()), _) =>
         s.copy(namespace = ResolvedNamespace(currentCatalog, catalogManager.currentNamespace))
+      case s @ ShowViews(UnresolvedNamespace(Seq()), _) =>
+        s.copy(namespace = ResolvedNamespace(currentCatalog, catalogManager.currentNamespace))
       case UnresolvedNamespace(Seq()) =>
         ResolvedNamespace(currentCatalog, Seq.empty[String])
       case UnresolvedNamespace(CatalogAndNamespace(catalog, ns)) =>
@@ -799,7 +879,10 @@ class Analyzer(
     def apply(plan: LogicalPlan): LogicalPlan = ResolveTempViews(plan).resolveOperatorsUp {
       case u: UnresolvedRelation =>
         lookupV2Relation(u.multipartIdentifier)
-          .getOrElse(u)
+          .map { rel =>
+            val ident = rel.identifier.get
+            SubqueryAlias(rel.catalog.get.name +: ident.namespace :+ ident.name, rel)
+          }.getOrElse(u)
 
       case u @ UnresolvedTable(NonSessionCatalogAndIdentifier(catalog, ident)) =>
         CatalogV2Util.loadTable(catalog, ident)
@@ -919,14 +1002,21 @@ class Analyzer(
     private def lookupRelation(identifier: Seq[String]): Option[LogicalPlan] = {
       expandRelationName(identifier) match {
         case SessionCatalogAndIdentifier(catalog, ident) =>
-          def loaded = CatalogV2Util.loadTable(catalog, ident).map {
+          lazy val loaded = CatalogV2Util.loadTable(catalog, ident).map {
             case v1Table: V1Table =>
               v1SessionCatalog.getRelation(v1Table.v1Table)
             case table =>
-              DataSourceV2Relation.create(table, Some(catalog), Some(ident))
+              SubqueryAlias(
+                catalog.name +: ident.asMultipartIdentifier,
+                DataSourceV2Relation.create(table, Some(catalog), Some(ident)))
           }
           val key = catalog.name +: ident.namespace :+ ident.name
-          Option(AnalysisContext.get.relationCache.getOrElseUpdate(key, loaded.orNull))
+          AnalysisContext.get.relationCache.get(key).map(_.transform {
+            case multi: MultiInstanceRelation => multi.newInstance()
+          }).orElse {
+            loaded.foreach(AnalysisContext.get.relationCache.update(key, _))
+            loaded
+          }
         case _ => None
       }
     }
@@ -946,12 +1036,10 @@ class Analyzer(
 
         val staticPartitions = i.partitionSpec.filter(_._2.isDefined).mapValues(_.get)
         val query = addStaticPartitionColumns(r, i.query, staticPartitions)
-        val dynamicPartitionOverwrite = partCols.size > staticPartitions.size &&
-          conf.partitionOverwriteMode == PartitionOverwriteMode.DYNAMIC
 
         if (!i.overwrite) {
           AppendData.byPosition(r, query)
-        } else if (dynamicPartitionOverwrite) {
+        } else if (conf.partitionOverwriteMode == PartitionOverwriteMode.DYNAMIC) {
           OverwritePartitionsDynamic.byPosition(r, query)
         } else {
           OverwriteByExpression.byPosition(r, query, staticDeleteExpression(r, staticPartitions))
@@ -1088,11 +1176,24 @@ class Analyzer(
             if findAliases(projectList).intersect(conflictingAttributes).nonEmpty =>
           Seq((oldVersion, oldVersion.copy(projectList = newAliases(projectList))))
 
+        // We don't need to search child plan recursively if the projectList of a Project
+        // is only composed of Alias and doesn't contain any conflicting attributes.
+        // Because, even if the child plan has some conflicting attributes, the attributes
+        // will be aliased to non-conflicting attributes by the Project at the end.
+        case _ @ Project(projectList, _)
+          if findAliases(projectList).size == projectList.size =>
+          Nil
+
         case oldVersion @ Aggregate(_, aggregateExpressions, _)
             if findAliases(aggregateExpressions).intersect(conflictingAttributes).nonEmpty =>
           Seq((oldVersion, oldVersion.copy(
             aggregateExpressions = newAliases(aggregateExpressions))))
 
+        // We don't search the child plan recursively for the same reason as the above Project.
+        case _ @ Aggregate(_, aggregateExpressions, _)
+          if findAliases(aggregateExpressions).size == aggregateExpressions.size =>
+          Nil
+
         case oldVersion @ FlatMapGroupsInPandas(_, _, output, _)
             if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
           Seq((oldVersion, oldVersion.copy(output = output.map(_.newInstance()))))
@@ -1133,78 +1234,13 @@ class Analyzer(
       if (conflictPlans.isEmpty) {
         right
       } else {
-        val attributeRewrites = AttributeMap(conflictPlans.flatMap {
-          case (oldRelation, newRelation) => oldRelation.output.zip(newRelation.output)})
-        val conflictPlanMap = conflictPlans.toMap
-        // transformDown so that we can replace all the old Relations in one turn due to
-        // the reason that `conflictPlans` are also collected in pre-order.
-        right transformDown {
-          case r => conflictPlanMap.getOrElse(r, r)
-        } transformUp {
-          case other => other transformExpressions {
-            case a: Attribute =>
-              dedupAttr(a, attributeRewrites)
-            case s: SubqueryExpression =>
-              s.withNewPlan(dedupOuterReferencesInSubquery(s.plan, attributeRewrites))
-          }
-        }
-      }
-    }
-
-    private def dedupAttr(attr: Attribute, attrMap: AttributeMap[Attribute]): Attribute = {
-      val exprId = attrMap.getOrElse(attr, attr).exprId
-      attr.withExprId(exprId)
-    }
-
-    /**
-     * The outer plan may have been de-duplicated and the function below updates the
-     * outer references to refer to the de-duplicated attributes.
-     *
-     * For example (SQL):
-     * {{{
-     *   SELECT * FROM t1
-     *   INTERSECT
-     *   SELECT * FROM t1
-     *   WHERE EXISTS (SELECT 1
-     *                 FROM t2
-     *                 WHERE t1.c1 = t2.c1)
-     * }}}
-     * Plan before resolveReference rule.
-     *    'Intersect
-     *    :- Project [c1#245, c2#246]
-     *    :  +- SubqueryAlias t1
-     *    :     +- Relation[c1#245,c2#246] parquet
-     *    +- 'Project [*]
-     *       +- Filter exists#257 [c1#245]
-     *       :  +- Project [1 AS 1#258]
-     *       :     +- Filter (outer(c1#245) = c1#251)
-     *       :        +- SubqueryAlias t2
-     *       :           +- Relation[c1#251,c2#252] parquet
-     *       +- SubqueryAlias t1
-     *          +- Relation[c1#245,c2#246] parquet
-     * Plan after the resolveReference rule.
-     *    Intersect
-     *    :- Project [c1#245, c2#246]
-     *    :  +- SubqueryAlias t1
-     *    :     +- Relation[c1#245,c2#246] parquet
-     *    +- Project [c1#259, c2#260]
-     *       +- Filter exists#257 [c1#259]
-     *       :  +- Project [1 AS 1#258]
-     *       :     +- Filter (outer(c1#259) = c1#251) => Updated
-     *       :        +- SubqueryAlias t2
-     *       :           +- Relation[c1#251,c2#252] parquet
-     *       +- SubqueryAlias t1
-     *          +- Relation[c1#259,c2#260] parquet  => Outer plan's attributes are de-duplicated.
-     */
-    private def dedupOuterReferencesInSubquery(
-        plan: LogicalPlan,
-        attrMap: AttributeMap[Attribute]): LogicalPlan = {
-      plan transformDown { case currentFragment =>
-        currentFragment transformExpressions {
-          case OuterReference(a: Attribute) =>
-            OuterReference(dedupAttr(a, attrMap))
-          case s: SubqueryExpression =>
-            s.withNewPlan(dedupOuterReferencesInSubquery(s.plan, attrMap))
+        val planMapping = conflictPlans.toMap
+        right.transformUpWithNewOutput {
+          case oldPlan =>
+            val newPlanOpt = planMapping.get(oldPlan)
+            newPlanOpt.map { newPlan =>
+              newPlan -> oldPlan.output.zip(newPlan.output)
+            }.getOrElse(oldPlan -> Nil)
         }
       }
     }
@@ -1212,13 +1248,13 @@ class Analyzer(
     /**
      * Resolves the attribute and extract value expressions(s) by traversing the
      * input expression in top down manner. The traversal is done in top-down manner as
-     * we need to skip over unbound lamda function expression. The lamda expressions are
+     * we need to skip over unbound lambda function expression. The lambda expressions are
      * resolved in a different rule [[ResolveLambdaVariables]]
      *
      * Example :
      * SELECT transform(array(1, 2, 3), (x, i) -> x + i)"
      *
-     * In the case above, x and i are resolved as lamda variables in [[ResolveLambdaVariables]]
+     * In the case above, x and i are resolved as lambda variables in [[ResolveLambdaVariables]]
      *
      * Note : In this routine, the unresolved attributes are resolved from the input plan's
      * children attributes.
@@ -1328,10 +1364,10 @@ class Analyzer(
       case a @ Aggregate(groupingExprs, aggExprs, appendColumns: AppendColumns) =>
         a.mapExpressions(resolveExpressionTopDown(_, appendColumns))
 
-      case o: OverwriteByExpression if !o.outputResolved =>
-        // do not resolve expression attributes until the query attributes are resolved against the
-        // table by ResolveOutputRelation. that rule will alias the attributes to the table's names.
-        o
+      case o: OverwriteByExpression if o.table.resolved =>
+        // The delete condition of `OverwriteByExpression` will be passed to the table
+        // implementation and should be resolved based on the table schema.
+        o.copy(deleteExpr = resolveExpressionBottomUp(o.deleteExpr, o.table))
 
       case m @ MergeIntoTable(targetTable, sourceTable, _, _, _)
         if !m.resolved && targetTable.resolved && sourceTable.resolved =>
@@ -1373,6 +1409,9 @@ class Analyzer(
               notMatchedActions = newNotMatchedActions)
         }
 
+      // Skip the having clause here, this will be handled in ResolveAggregateFunctions.
+      case h: UnresolvedHaving => h
+
       case q: LogicalPlan =>
         logTrace(s"Attempting to resolve ${q.simpleString(SQLConf.get.maxToStringFields)}")
         q.mapExpressions(resolveExpressionTopDown(_, q))
@@ -1782,6 +1821,7 @@ class Analyzer(
    * Replaces [[UnresolvedFunction]]s with concrete [[Expression]]s.
    */
   object ResolveFunctions extends Rule[LogicalPlan] {
+    val trimWarningEnabled = new AtomicBoolean(true)
     def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
       case q: LogicalPlan =>
         q transformExpressions {
@@ -1826,13 +1866,19 @@ class Analyzer(
                   }
                   AggregateExpression(agg, Complete, isDistinct, filter)
                 // This function is not an aggregate function, just return the resolved one.
-                case other =>
-                  if (isDistinct || filter.isDefined) {
-                    failAnalysis("DISTINCT or FILTER specified, " +
-                      s"but ${other.prettyName} is not an aggregate function")
-                  } else {
-                    other
+                case other if (isDistinct || filter.isDefined) =>
+                  failAnalysis("DISTINCT or FILTER specified, " +
+                    s"but ${other.prettyName} is not an aggregate function")
+                case e: String2TrimExpression if arguments.size == 2 =>
+                  if (trimWarningEnabled.get) {
+                    log.warn("Two-parameter TRIM/LTRIM/RTRIM function signatures are deprecated." +
+                      " Use SQL syntax `TRIM((BOTH | LEADING | TRAILING)? trimStr FROM str)`" +
+                      " instead.")
+                    trimWarningEnabled.set(false)
                   }
+                  e
+                case other =>
+                  other
               }
             }
         }
@@ -2006,62 +2052,14 @@ class Analyzer(
    */
   object ResolveAggregateFunctions extends Rule[LogicalPlan] {
     def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
-      case f @ Filter(cond, agg @ Aggregate(grouping, originalAggExprs, child)) if agg.resolved =>
+      // Resolve aggregate with having clause to Filter(..., Aggregate()). Note, to avoid wrongly
+      // resolve the having condition expression, here we skip resolving it in ResolveReferences
+      // and transform it to Filter after aggregate is resolved. See more details in SPARK-31519.
+      case UnresolvedHaving(cond, agg: Aggregate) if agg.resolved =>
+        resolveHaving(Filter(cond, agg), agg)
 
-        // Try resolving the condition of the filter as though it is in the aggregate clause
-        try {
-          val aggregatedCondition =
-            Aggregate(
-              grouping,
-              Alias(cond, "havingCondition")() :: Nil,
-              child)
-          val resolvedOperator = executeSameContext(aggregatedCondition)
-          def resolvedAggregateFilter =
-            resolvedOperator
-              .asInstanceOf[Aggregate]
-              .aggregateExpressions.head
-
-          // If resolution was successful and we see the filter has an aggregate in it, add it to
-          // the original aggregate operator.
-          if (resolvedOperator.resolved) {
-            // Try to replace all aggregate expressions in the filter by an alias.
-            val aggregateExpressions = ArrayBuffer.empty[NamedExpression]
-            val transformedAggregateFilter = resolvedAggregateFilter.transform {
-              case ae: AggregateExpression =>
-                val alias = Alias(ae, ae.toString)()
-                aggregateExpressions += alias
-                alias.toAttribute
-              // Grouping functions are handled in the rule [[ResolveGroupingAnalytics]].
-              case e: Expression if grouping.exists(_.semanticEquals(e)) &&
-                  !ResolveGroupingAnalytics.hasGroupingFunction(e) &&
-                  !agg.output.exists(_.semanticEquals(e)) =>
-                e match {
-                  case ne: NamedExpression =>
-                    aggregateExpressions += ne
-                    ne.toAttribute
-                  case _ =>
-                    val alias = Alias(e, e.toString)()
-                    aggregateExpressions += alias
-                    alias.toAttribute
-                }
-            }
-
-            // Push the aggregate expressions into the aggregate (if any).
-            if (aggregateExpressions.nonEmpty) {
-              Project(agg.output,
-                Filter(transformedAggregateFilter,
-                  agg.copy(aggregateExpressions = originalAggExprs ++ aggregateExpressions)))
-            } else {
-              f
-            }
-          } else {
-            f
-          }
-        } catch {
-          // Attempting to resolve in the aggregate can result in ambiguity.  When this happens,
-          // just return the original plan.
-          case ae: AnalysisException => f
-        }
+      case f @ Filter(_, agg: Aggregate) if agg.resolved =>
+        resolveHaving(f, agg)
 
       case sort @ Sort(sortOrder, global, aggregate: Aggregate) if aggregate.resolved =>
 
@@ -2132,6 +2130,74 @@ class Analyzer(
     def containsAggregate(condition: Expression): Boolean = {
       condition.find(_.isInstanceOf[AggregateExpression]).isDefined
     }
+
+    def resolveFilterCondInAggregate(
+        filterCond: Expression, agg: Aggregate): Option[(Seq[NamedExpression], Expression)] = {
+      try {
+        val aggregatedCondition =
+          Aggregate(
+            agg.groupingExpressions,
+            Alias(filterCond, "havingCondition")() :: Nil,
+            agg.child)
+        val resolvedOperator = executeSameContext(aggregatedCondition)
+        def resolvedAggregateFilter =
+          resolvedOperator
+            .asInstanceOf[Aggregate]
+            .aggregateExpressions.head
+
+        // If resolution was successful and we see the filter has an aggregate in it, add it to
+        // the original aggregate operator.
+        if (resolvedOperator.resolved) {
+          // Try to replace all aggregate expressions in the filter by an alias.
+          val aggregateExpressions = ArrayBuffer.empty[NamedExpression]
+          val transformedAggregateFilter = resolvedAggregateFilter.transform {
+            case ae: AggregateExpression =>
+              val alias = Alias(ae, ae.toString)()
+              aggregateExpressions += alias
+              alias.toAttribute
+            // Grouping functions are handled in the rule [[ResolveGroupingAnalytics]].
+            case e: Expression if agg.groupingExpressions.exists(_.semanticEquals(e)) &&
+                !ResolveGroupingAnalytics.hasGroupingFunction(e) &&
+                !agg.output.exists(_.semanticEquals(e)) =>
+              e match {
+                case ne: NamedExpression =>
+                  aggregateExpressions += ne
+                  ne.toAttribute
+                case _ =>
+                  val alias = Alias(e, e.toString)()
+                  aggregateExpressions += alias
+                  alias.toAttribute
+              }
+          }
+          if (aggregateExpressions.nonEmpty) {
+            Some(aggregateExpressions, transformedAggregateFilter)
+          } else {
+            None
+          }
+        } else {
+          None
+        }
+      } catch {
+        // Attempting to resolve in the aggregate can result in ambiguity. When this happens,
+        // just return None and the caller side will return the original plan.
+        case ae: AnalysisException => None
+      }
+    }
+
+    def resolveHaving(filter: Filter, agg: Aggregate): LogicalPlan = {
+      // Try resolving the condition of the filter as though it is in the aggregate clause
+      val resolvedInfo = resolveFilterCondInAggregate(filter.condition, agg)
+
+      // Push the aggregate expressions into the aggregate (if any).
+      if (resolvedInfo.nonEmpty) {
+        val (aggregateExpressions, resolvedHavingCond) = resolvedInfo.get
+        Project(agg.output,
+          Filter(resolvedHavingCond,
+            agg.copy(aggregateExpressions = agg.aggregateExpressions ++ aggregateExpressions)))
+      } else {
+        filter
+      }
+    }
   }
 
   /**
@@ -2151,14 +2217,33 @@ class Analyzer(
     }
 
     private def hasNestedGenerator(expr: NamedExpression): Boolean = {
+      def hasInnerGenerator(g: Generator): Boolean = g match {
+        // Since `GeneratorOuter` is just a wrapper of generators, we skip it here
+        case go: GeneratorOuter =>
+          hasInnerGenerator(go.child)
+        case _ =>
+          g.children.exists { _.find {
+            case _: Generator => true
+            case _ => false
+          }.isDefined }
+      }
       CleanupAliases.trimNonTopLevelAliases(expr) match {
-        case UnresolvedAlias(_: Generator, _) => false
-        case Alias(_: Generator, _) => false
-        case MultiAlias(_: Generator, _) => false
+        case UnresolvedAlias(g: Generator, _) => hasInnerGenerator(g)
+        case Alias(g: Generator, _) => hasInnerGenerator(g)
+        case MultiAlias(g: Generator, _) => hasInnerGenerator(g)
         case other => hasGenerator(other)
       }
     }
 
+    private def hasAggFunctionInGenerator(ne: Seq[NamedExpression]): Boolean = {
+      ne.exists(_.find {
+        case g: Generator =>
+          g.children.exists(_.find(_.isInstanceOf[AggregateFunction]).isDefined)
+        case _ =>
+          false
+      }.nonEmpty)
+    }
+
     private def trimAlias(expr: NamedExpression): Expression = expr match {
       case UnresolvedAlias(child, _) => child
       case Alias(child, _) => child
@@ -2245,6 +2330,11 @@ class Analyzer(
         val newAgg = Aggregate(groupList, newAggList, child)
         Project(projectExprs.toList, newAgg)
 
+      case p @ Project(projectList, _) if hasAggFunctionInGenerator(projectList) =>
+        // If a generator has any aggregate function, we need to apply the `GlobalAggregates` rule
+        // first for replacing `Project` with `Aggregate`.
+        p
+
       case p @ Project(projectList, child) =>
         // Holds the resolved generator, if one exists in the project list.
         var resolvedGenerator: Generate = null
@@ -2425,6 +2515,10 @@ class Analyzer(
             }
             wsc.copy(partitionSpec = newPartitionSpec, orderSpec = newOrderSpec)
 
+          case WindowExpression(ae: AggregateExpression, _) if ae.filter.isDefined =>
+            failAnalysis(
+              "window aggregate function with filter predicate is not supported yet.")
+
           // Extract Windowed AggregateExpression
           case we @ WindowExpression(
               ae @ AggregateExpression(function, _, _, _, _),
@@ -2528,11 +2622,14 @@ class Analyzer(
     def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsDown {
 
       case Filter(condition, _) if hasWindowFunction(condition) =>
-        failAnalysis("It is not allowed to use window functions inside WHERE and HAVING clauses")
+        failAnalysis("It is not allowed to use window functions inside WHERE clause")
+
+      case UnresolvedHaving(condition, _) if hasWindowFunction(condition) =>
+        failAnalysis("It is not allowed to use window functions inside HAVING clause")
 
       // Aggregate with Having clause. This rule works with an unresolved Aggregate because
       // a resolved Aggregate will not have Window Functions.
-      case f @ Filter(condition, a @ Aggregate(groupingExprs, aggregateExprs, child))
+      case f @ UnresolvedHaving(condition, a @ Aggregate(groupingExprs, aggregateExprs, child))
         if child.resolved &&
            hasWindowFunction(aggregateExprs) &&
            a.expressions.forall(_.resolved) =>
@@ -2615,7 +2712,10 @@ class Analyzer(
 
     private def getNondeterToAttr(exprs: Seq[Expression]): Map[Expression, NamedExpression] = {
       exprs.filterNot(_.deterministic).flatMap { expr =>
-        val leafNondeterministic = expr.collect { case n: Nondeterministic => n }
+        val leafNondeterministic = expr.collect {
+          case n: Nondeterministic => n
+          case udf: UserDefinedExpression if !udf.deterministic => udf
+        }
         leafNondeterministic.distinct.map { e =>
           val ne = e match {
             case n: NamedExpression => n
@@ -2654,13 +2754,13 @@ class Analyzer(
 
       case p => p transformExpressionsUp {
 
-        case udf @ ScalaUDF(_, _, inputs, inputPrimitives, _, _, _, _)
-            if inputPrimitives.contains(true) =>
+        case udf @ ScalaUDF(_, _, inputs, _, _, _, _)
+            if udf.inputPrimitives.contains(true) =>
           // Otherwise, add special handling of null for fields that can't accept null.
           // The result of operations like this, when passed null, is generally to return null.
-          assert(inputPrimitives.length == inputs.length)
+          assert(udf.inputPrimitives.length == inputs.length)
 
-          val inputPrimitivesPair = inputPrimitives.zip(inputs)
+          val inputPrimitivesPair = udf.inputPrimitives.zip(inputs)
           val inputNullCheck = inputPrimitivesPair.collect {
             case (isPrimitive, input) if isPrimitive && input.nullable =>
               IsNull(input)
@@ -2991,15 +3091,29 @@ class Analyzer(
       case p => p transformExpressions {
         case u @ UpCast(child, _, _) if !child.resolved => u
 
-        case UpCast(child, dt: AtomicType, _)
+        case UpCast(_, target, _) if target != DecimalType && !target.isInstanceOf[DataType] =>
+          throw new AnalysisException(
+            s"UpCast only support DecimalType as AbstractDataType yet, but got: $target")
+
+        case UpCast(child, target, walkedTypePath) if target == DecimalType
+          && child.dataType.isInstanceOf[DecimalType] =>
+          assert(walkedTypePath.nonEmpty,
+            "object DecimalType should only be used inside ExpressionEncoder")
+
+          // SPARK-31750: if we want to upcast to the general decimal type, and the `child` is
+          // already decimal type, we can remove the `Upcast` and accept any precision/scale.
+          // This can happen for cases like `spark.read.parquet("/tmp/file").as[BigDecimal]`.
+          child
+
+        case UpCast(child, target: AtomicType, _)
             if SQLConf.get.getConf(SQLConf.LEGACY_LOOSE_UPCAST) &&
               child.dataType == StringType =>
-          Cast(child, dt.asNullable)
+          Cast(child, target.asNullable)
 
-        case UpCast(child, dataType, walkedTypePath) if !Cast.canUpCast(child.dataType, dataType) =>
-          fail(child, dataType, walkedTypePath)
+        case u @ UpCast(child, _, walkedTypePath) if !Cast.canUpCast(child.dataType, u.dataType) =>
+          fail(child, u.dataType, walkedTypePath)
 
-        case UpCast(child, dataType, _) => Cast(child, dataType.asNullable)
+        case u @ UpCast(child, _, _) => Cast(child, u.dataType.asNullable)
       }
     }
   }
@@ -3008,9 +3122,29 @@ class Analyzer(
   object ResolveAlterTableChanges extends Rule[LogicalPlan] {
     def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
       case a @ AlterTable(_, _, t: NamedRelation, changes) if t.resolved =>
+        // 'colsToAdd' keeps track of new columns being added. It stores a mapping from a
+        // normalized parent name of fields to field names that belong to the parent.
+        // For example, if we add columns "a.b.c", "a.b.d", and "a.c", 'colsToAdd' will become
+        // Map(Seq("a", "b") -> Seq("c", "d"), Seq("a") -> Seq("c")).
+        val colsToAdd = mutable.Map.empty[Seq[String], Seq[String]]
         val schema = t.schema
         val normalizedChanges = changes.flatMap {
           case add: AddColumn =>
+            def addColumn(
+                parentSchema: StructType,
+                parentName: String,
+                normalizedParentName: Seq[String]): TableChange = {
+              val fieldsAdded = colsToAdd.getOrElse(normalizedParentName, Nil)
+              val pos = findColumnPosition(add.position(), parentName, parentSchema, fieldsAdded)
+              val field = add.fieldNames().last
+              colsToAdd(normalizedParentName) = fieldsAdded :+ field
+              TableChange.addColumn(
+                (normalizedParentName :+ field).toArray,
+                add.dataType(),
+                add.isNullable,
+                add.comment,
+                pos)
+            }
             val parent = add.fieldNames().init
             if (parent.nonEmpty) {
               // Adding a nested field, need to normalize the parent column and position
@@ -3022,27 +3156,14 @@ class Analyzer(
                 val (normalizedName, sf) = target.get
                 sf.dataType match {
                   case struct: StructType =>
-                    val pos = findColumnPosition(add.position(), parent.quoted, struct)
-                    Some(TableChange.addColumn(
-                      (normalizedName ++ Seq(sf.name, add.fieldNames().last)).toArray,
-                      add.dataType(),
-                      add.isNullable,
-                      add.comment,
-                      pos))
-
+                    Some(addColumn(struct, parent.quoted, normalizedName :+ sf.name))
                   case other =>
                     Some(add)
                 }
               }
             } else {
               // Adding to the root. Just need to normalize position
-              val pos = findColumnPosition(add.position(), "root", schema)
-              Some(TableChange.addColumn(
-                add.fieldNames(),
-                add.dataType(),
-                add.isNullable,
-                add.comment,
-                pos))
+              Some(addColumn(schema, "root", Nil))
             }
 
           case typeChange: UpdateColumnType =>
@@ -3141,17 +3262,18 @@ class Analyzer(
 
     private def findColumnPosition(
         position: ColumnPosition,
-        field: String,
-        struct: StructType): ColumnPosition = {
+        parentName: String,
+        struct: StructType,
+        fieldsAdded: Seq[String]): ColumnPosition = {
       position match {
         case null => null
         case after: After =>
-          struct.fieldNames.find(n => conf.resolver(n, after.column())) match {
+          (struct.fieldNames ++ fieldsAdded).find(n => conf.resolver(n, after.column())) match {
             case Some(colName) =>
               ColumnPosition.after(colName)
             case None =>
               throw new AnalysisException("Couldn't find the reference column for " +
-                s"$after at $field")
+                s"$after at $parentName")
           }
         case other => other
       }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
index 60e6bf8db06d7..41e4c1fc39303 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
@@ -17,21 +17,70 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
+import scala.collection.mutable
+
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, With}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias, With}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LEGACY_CTE_PRECEDENCE_ENABLED
+import org.apache.spark.sql.internal.SQLConf.{LEGACY_CTE_PRECEDENCE_POLICY, LegacyBehaviorPolicy}
 
 /**
  * Analyze WITH nodes and substitute child plan with CTE definitions.
  */
 object CTESubstitution extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = {
-    if (SQLConf.get.getConf(LEGACY_CTE_PRECEDENCE_ENABLED)) {
-      legacyTraverseAndSubstituteCTE(plan)
-    } else {
-      traverseAndSubstituteCTE(plan, false)
+    LegacyBehaviorPolicy.withName(SQLConf.get.getConf(LEGACY_CTE_PRECEDENCE_POLICY)) match {
+      case LegacyBehaviorPolicy.EXCEPTION =>
+        assertNoNameConflictsInCTE(plan)
+        traverseAndSubstituteCTE(plan)
+      case LegacyBehaviorPolicy.LEGACY =>
+        legacyTraverseAndSubstituteCTE(plan)
+      case LegacyBehaviorPolicy.CORRECTED =>
+        traverseAndSubstituteCTE(plan)
+    }
+  }
+
+  /**
+   * Spark 3.0 changes the CTE relations resolution, and inner relations take precedence. This is
+   * correct but we need to warn users about this behavior change under EXCEPTION mode, when we see
+   * CTE relations with conflicting names.
+   *
+   * Note that, before Spark 3.0 the parser didn't support CTE in the FROM clause. For example,
+   * `WITH ... SELECT * FROM (WITH ... SELECT ...)` was not supported. We should not fail for this
+   * case, as Spark versions before 3.0 can't run it anyway. The parameter `startOfQuery` is used
+   * to indicate where we can define CTE relations before Spark 3.0, and we should only check
+   * name conflicts when `startOfQuery` is true.
+   */
+  private def assertNoNameConflictsInCTE(
+      plan: LogicalPlan,
+      outerCTERelationNames: Seq[String] = Nil,
+      startOfQuery: Boolean = true): Unit = {
+    val resolver = SQLConf.get.resolver
+    plan match {
+      case With(child, relations) =>
+        val newNames = mutable.ArrayBuffer.empty[String]
+        newNames ++= outerCTERelationNames
+        relations.foreach {
+          case (name, relation) =>
+            if (startOfQuery && outerCTERelationNames.exists(resolver(_, name))) {
+              throw new AnalysisException(s"Name $name is ambiguous in nested CTE. " +
+                s"Please set ${LEGACY_CTE_PRECEDENCE_POLICY.key} to CORRECTED so that name " +
+                "defined in inner CTE takes precedence. If set it to LEGACY, outer CTE " +
+                "definitions will take precedence. See more details in SPARK-28228.")
+            }
+            // CTE relation is defined as `SubqueryAlias`. Here we skip it and check the child
+            // directly, so that `startOfQuery` is set correctly.
+            assertNoNameConflictsInCTE(relation.child, newNames)
+            newNames += name
+        }
+        assertNoNameConflictsInCTE(child, newNames, startOfQuery = false)
+
+      case other =>
+        other.subqueries.foreach(assertNoNameConflictsInCTE(_, outerCTERelationNames))
+        other.children.foreach(
+          assertNoNameConflictsInCTE(_, outerCTERelationNames, startOfQuery = false))
     }
   }
 
@@ -85,37 +134,27 @@ object CTESubstitution extends Rule[LogicalPlan] {
    *     SELECT * FROM t
    *   )
    * @param plan the plan to be traversed
-   * @param inTraverse whether the current traverse is called from another traverse, only in this
-   *                   case name collision can occur
    * @return the plan where CTE substitution is applied
    */
-  private def traverseAndSubstituteCTE(plan: LogicalPlan, inTraverse: Boolean): LogicalPlan = {
+  private def traverseAndSubstituteCTE(plan: LogicalPlan): LogicalPlan = {
     plan.resolveOperatorsUp {
       case With(child: LogicalPlan, relations) =>
-        // child might contain an inner CTE that has priority so traverse and substitute inner CTEs
-        // in child first
-        val traversedChild: LogicalPlan = child transformExpressions {
-          case e: SubqueryExpression => e.withNewPlan(traverseAndSubstituteCTE(e.plan, true))
-        }
-
         // Substitute CTE definitions from last to first as a CTE definition can reference a
         // previous one
-        relations.foldRight(traversedChild) {
+        relations.foldRight(child) {
           case ((cteName, ctePlan), currentPlan) =>
             // A CTE definition might contain an inner CTE that has priority, so traverse and
             // substitute CTE defined in ctePlan.
             // A CTE definition might not be used at all or might be used multiple times. To avoid
             // computation if it is not used and to avoid multiple recomputation if it is used
             // multiple times we use a lazy construct with call-by-name parameter passing.
-            lazy val substitutedCTEPlan = traverseAndSubstituteCTE(ctePlan, true)
+            lazy val substitutedCTEPlan = traverseAndSubstituteCTE(ctePlan)
             substituteCTE(currentPlan, cteName, substitutedCTEPlan)
         }
 
-      // CTE name collision can occur only when inTraverse is true, it helps to avoid eager CTE
-      // substitution in a subquery expression.
-      case other if inTraverse =>
+      case other =>
         other.transformExpressions {
-          case e: SubqueryExpression => e.withNewPlan(traverseAndSubstituteCTE(e.plan, true))
+          case e: SubqueryExpression => e.withNewPlan(traverseAndSubstituteCTE(e.plan))
         }
     }
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 4ec737fd9b70d..e5b8ae7b22292 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -308,6 +308,9 @@ trait CheckAnalysis extends PredicateHelper {
                 case a: AggregateExpression if a.isDistinct =>
                   e.failAnalysis(
                     "distinct aggregates are not allowed in observed metrics, but found: " + s.sql)
+                case a: AggregateExpression if a.filter.isDefined =>
+                  e.failAnalysis("aggregates with filter predicate are not allowed in " +
+                    "observed metrics, but found: " + s.sql)
                 case _: Attribute if !seenAggregate =>
                   e.failAnalysis (s"attribute ${s.sql} can only be used as an argument to an " +
                     "aggregate function.")
@@ -338,7 +341,8 @@ trait CheckAnalysis extends PredicateHelper {
             def ordinalNumber(i: Int): String = i match {
               case 0 => "first"
               case 1 => "second"
-              case i => s"${i}th"
+              case 2 => "third"
+              case i => s"${i + 1}th"
             }
             val ref = dataTypes(operator.children.head)
             operator.children.tail.zipWithIndex.foreach { case (child, ti) =>
@@ -437,12 +441,16 @@ trait CheckAnalysis extends PredicateHelper {
               }
               field.get._2
             }
-            def positionArgumentExists(position: ColumnPosition, struct: StructType): Unit = {
+            def positionArgumentExists(
+                position: ColumnPosition,
+                struct: StructType,
+                fieldsAdded: Seq[String]): Unit = {
               position match {
                 case after: After =>
-                  if (!struct.fieldNames.contains(after.column())) {
+                  val allFields = struct.fieldNames ++ fieldsAdded
+                  if (!allFields.contains(after.column())) {
                     alter.failAnalysis(s"Couldn't resolve positional argument $position amongst " +
-                      s"${struct.fieldNames.mkString("[", ", ", "]")}")
+                      s"${allFields.mkString("[", ", ", "]")}")
                   }
                 case _ =>
               }
@@ -470,12 +478,26 @@ trait CheckAnalysis extends PredicateHelper {
               }
             }
 
+            val colsToDelete = mutable.Set.empty[Seq[String]]
+            // 'colsToAdd' keeps track of new columns being added. It stores a mapping from a parent
+            // name of fields to field names that belong to the parent. For example, if we add
+            // columns "a.b.c", "a.b.d", and "a.c", 'colsToAdd' will become
+            // Map(Seq("a", "b") -> Seq("c", "d"), Seq("a") -> Seq("c")).
+            val colsToAdd = mutable.Map.empty[Seq[String], Seq[String]]
+
             alter.changes.foreach {
               case add: AddColumn =>
-                checkColumnNotExists("add", add.fieldNames(), table.schema)
+                // If a column to add is a part of columns to delete, we don't need to check
+                // if column already exists - applies to REPLACE COLUMNS scenario.
+                if (!colsToDelete.contains(add.fieldNames())) {
+                  checkColumnNotExists("add", add.fieldNames(), table.schema)
+                }
                 val parent = findParentStruct("add", add.fieldNames())
-                positionArgumentExists(add.position(), parent)
+                val parentName = add.fieldNames().init
+                val fieldsAdded = colsToAdd.getOrElse(parentName, Nil)
+                positionArgumentExists(add.position(), parent, fieldsAdded)
                 TypeUtils.failWithIntervalType(add.dataType())
+                colsToAdd(parentName) = fieldsAdded :+ add.fieldNames().last
               case update: UpdateColumnType =>
                 val field = findField("update", update.fieldNames)
                 val fieldName = update.fieldNames.quoted
@@ -514,7 +536,11 @@ trait CheckAnalysis extends PredicateHelper {
               case updatePos: UpdateColumnPosition =>
                 findField("update", updatePos.fieldNames)
                 val parent = findParentStruct("update", updatePos.fieldNames())
-                positionArgumentExists(updatePos.position(), parent)
+                val parentName = updatePos.fieldNames().init
+                positionArgumentExists(
+                  updatePos.position(),
+                  parent,
+                  colsToAdd.getOrElse(parentName, Nil))
               case rename: RenameColumn =>
                 findField("rename", rename.fieldNames)
                 checkColumnNotExists(
@@ -523,6 +549,10 @@ trait CheckAnalysis extends PredicateHelper {
                 findField("update", update.fieldNames)
               case delete: DeleteColumn =>
                 findField("delete", delete.fieldNames)
+                // REPLACE COLUMNS has deletes followed by adds. Remember the deleted columns
+                // so that add operations do not fail when the columns to add exist and they
+                // are to be deleted.
+                colsToDelete += delete.fieldNames
               case _ =>
               // no validation needed for set and remove property
             }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
index a64befecb68d6..f2d607e5b737c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
@@ -177,8 +177,9 @@ object DecimalPrecision extends TypeCoercionRule {
     case expr @ IntegralDivide(
         e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2)) =>
       val widerType = widerDecimalType(p1, s1, p2, s2)
-      val promotedExpr =
-        IntegralDivide(promotePrecision(e1, widerType), promotePrecision(e2, widerType))
+      val promotedExpr = IntegralDivide(
+        promotePrecision(e1, widerType),
+        promotePrecision(e2, widerType))
       if (expr.dataType.isInstanceOf[DecimalType]) {
         // This follows division rule
         val intDig = p1 - s1 + s2
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 6c4aee4f58b75..8608558a4cfe9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -218,7 +218,7 @@ object FunctionRegistry {
     expression[PosExplode]("posexplode"),
     expressionGeneratorOuter[PosExplode]("posexplode_outer"),
     expression[Rand]("rand"),
-    expression[Rand]("random"),
+    expression[Rand]("random", true),
     expression[Randn]("randn"),
     expression[Stack]("stack"),
     expression[CaseWhen]("when"),
@@ -235,7 +235,7 @@ object FunctionRegistry {
     expression[BRound]("bround"),
     expression[Cbrt]("cbrt"),
     expression[Ceil]("ceil"),
-    expression[Ceil]("ceiling"),
+    expression[Ceil]("ceiling", true),
     expression[Cos]("cos"),
     expression[Cosh]("cosh"),
     expression[Conv]("conv"),
@@ -252,12 +252,12 @@ object FunctionRegistry {
     expression[Log1p]("log1p"),
     expression[Log2]("log2"),
     expression[Log]("ln"),
-    expression[Remainder]("mod"),
-    expression[UnaryMinus]("negative"),
+    expression[Remainder]("mod", true),
+    expression[UnaryMinus]("negative", true),
     expression[Pi]("pi"),
     expression[Pmod]("pmod"),
     expression[UnaryPositive]("positive"),
-    expression[Pow]("pow"),
+    expression[Pow]("pow", true),
     expression[Pow]("power"),
     expression[ToRadians]("radians"),
     expression[Rint]("rint"),
@@ -265,7 +265,7 @@ object FunctionRegistry {
     expression[ShiftLeft]("shiftleft"),
     expression[ShiftRight]("shiftright"),
     expression[ShiftRightUnsigned]("shiftrightunsigned"),
-    expression[Signum]("sign"),
+    expression[Signum]("sign", true),
     expression[Signum]("signum"),
     expression[Sin]("sin"),
     expression[Sinh]("sinh"),
@@ -323,12 +323,12 @@ object FunctionRegistry {
 
     // string functions
     expression[Ascii]("ascii"),
-    expression[Chr]("char"),
+    expression[Chr]("char", true),
     expression[Chr]("chr"),
     expression[Base64]("base64"),
     expression[BitLength]("bit_length"),
-    expression[Length]("char_length"),
-    expression[Length]("character_length"),
+    expression[Length]("char_length", true),
+    expression[Length]("character_length", true),
     expression[ConcatWs]("concat_ws"),
     expression[Decode]("decode"),
     expression[Elt]("elt"),
@@ -339,7 +339,7 @@ object FunctionRegistry {
     expression[GetJsonObject]("get_json_object"),
     expression[InitCap]("initcap"),
     expression[StringInstr]("instr"),
-    expression[Lower]("lcase"),
+    expression[Lower]("lcase", true),
     expression[Length]("length"),
     expression[Levenshtein]("levenshtein"),
     expression[Like]("like"),
@@ -350,8 +350,8 @@ object FunctionRegistry {
     expression[StringTrimLeft]("ltrim"),
     expression[JsonTuple]("json_tuple"),
     expression[ParseUrl]("parse_url"),
-    expression[StringLocate]("position"),
-    expression[FormatString]("printf"),
+    expression[StringLocate]("position", true),
+    expression[FormatString]("printf", true),
     expression[RegExpExtract]("regexp_extract"),
     expression[RegExpReplace]("regexp_replace"),
     expression[StringRepeat]("repeat"),
@@ -364,21 +364,21 @@ object FunctionRegistry {
     expression[SoundEx]("soundex"),
     expression[StringSpace]("space"),
     expression[StringSplit]("split"),
-    expression[Substring]("substr"),
+    expression[Substring]("substr", true),
     expression[Substring]("substring"),
     expression[Left]("left"),
     expression[Right]("right"),
     expression[SubstringIndex]("substring_index"),
     expression[StringTranslate]("translate"),
     expression[StringTrim]("trim"),
-    expression[Upper]("ucase"),
+    expression[Upper]("ucase", true),
     expression[UnBase64]("unbase64"),
     expression[Unhex]("unhex"),
     expression[Upper]("upper"),
     expression[XPathList]("xpath"),
     expression[XPathBoolean]("xpath_boolean"),
     expression[XPathDouble]("xpath_double"),
-    expression[XPathDouble]("xpath_number"),
+    expression[XPathDouble]("xpath_number", true),
     expression[XPathFloat]("xpath_float"),
     expression[XPathInt]("xpath_int"),
     expression[XPathLong]("xpath_long"),
@@ -393,7 +393,7 @@ object FunctionRegistry {
     expression[DateAdd]("date_add"),
     expression[DateFormatClass]("date_format"),
     expression[DateSub]("date_sub"),
-    expression[DayOfMonth]("day"),
+    expression[DayOfMonth]("day", true),
     expression[DayOfYear]("dayofyear"),
     expression[DayOfMonth]("dayofmonth"),
     expression[FromUnixTime]("from_unixtime"),
@@ -404,7 +404,7 @@ object FunctionRegistry {
     expression[Month]("month"),
     expression[MonthsBetween]("months_between"),
     expression[NextDay]("next_day"),
-    expression[CurrentTimestamp]("now"),
+    expression[Now]("now"),
     expression[Quarter]("quarter"),
     expression[Second]("second"),
     expression[ParseToTimestamp]("to_timestamp"),
@@ -423,6 +423,7 @@ object FunctionRegistry {
     expression[MakeTimestamp]("make_timestamp"),
     expression[MakeInterval]("make_interval"),
     expression[DatePart]("date_part"),
+    expression[Extract]("extract"),
 
     // collection functions
     expression[CreateArray]("array"),
@@ -445,7 +446,7 @@ object FunctionRegistry {
     expression[MapConcat]("map_concat"),
     expression[Size]("size"),
     expression[Slice]("slice"),
-    expression[Size]("cardinality"),
+    expression[Size]("cardinality", true),
     expression[ArraysZip]("arrays_zip"),
     expression[SortArray]("sort_array"),
     expression[Shuffle]("shuffle"),
@@ -478,7 +479,7 @@ object FunctionRegistry {
     expression[Uuid]("uuid"),
     expression[Murmur3Hash]("hash"),
     expression[XxHash64]("xxhash64"),
-    expression[Sha1]("sha"),
+    expression[Sha1]("sha", true),
     expression[Sha1]("sha1"),
     expression[Sha2]("sha2"),
     expression[SparkPartitionID]("spark_partition_id"),
@@ -488,7 +489,7 @@ object FunctionRegistry {
     expression[MonotonicallyIncreasingID]("monotonically_increasing_id"),
     expression[CurrentDatabase]("current_database"),
     expression[CallMethodViaReflection]("reflect"),
-    expression[CallMethodViaReflection]("java_method"),
+    expression[CallMethodViaReflection]("java_method", true),
     expression[SparkVersion]("version"),
     expression[TypeOf]("typeof"),
 
@@ -590,7 +591,9 @@ object FunctionRegistry {
       if (varargCtor.isDefined) {
         // If there is an apply method that accepts Seq[Expression], use that one.
         try {
-          varargCtor.get.newInstance(expressions).asInstanceOf[Expression]
+          val exp = varargCtor.get.newInstance(expressions).asInstanceOf[Expression]
+          if (setAlias) exp.setTagValue(FUNC_ALIAS, name)
+          exp
         } catch {
           // the exception is an invocation exception. To get a meaningful message, we need the
           // cause.
@@ -650,7 +653,7 @@ object FunctionRegistry {
     val clazz = scala.reflect.classTag[Cast].runtimeClass
     val usage = "_FUNC_(expr) - Casts the value `expr` to the target data type `_FUNC_`."
     val expressionInfo =
-      new ExpressionInfo(clazz.getCanonicalName, null, name, usage, "", "", "", "", "")
+      new ExpressionInfo(clazz.getCanonicalName, null, name, usage, "", "", "", "", "", "")
     (name, (expressionInfo, builder))
   }
 
@@ -670,6 +673,7 @@ object FunctionRegistry {
           df.arguments(),
           df.examples(),
           df.note(),
+          df.group(),
           df.since(),
           df.deprecated())
       } else {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HintErrorLogger.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HintErrorLogger.scala
index c6e0c74527b8f..71c6d40dfba16 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HintErrorLogger.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HintErrorLogger.scala
@@ -24,15 +24,16 @@ import org.apache.spark.sql.catalyst.plans.logical.{HintErrorHandler, HintInfo}
  * The hint error handler that logs warnings for each hint error.
  */
 object HintErrorLogger extends HintErrorHandler with Logging {
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 
   override def hintNotRecognized(name: String, parameters: Seq[Any]): Unit = {
     logWarning(s"Unrecognized hint: ${hintToPrettyString(name, parameters)}")
   }
 
   override def hintRelationsNotFound(
-      name: String, parameters: Seq[Any], invalidRelations: Set[String]): Unit = {
-    invalidRelations.foreach { n =>
-      logWarning(s"Count not find relation '$n' specified in hint " +
+      name: String, parameters: Seq[Any], invalidRelations: Set[Seq[String]]): Unit = {
+    invalidRelations.foreach { ident =>
+      logWarning(s"Count not find relation '${ident.quoted}' specified in hint " +
         s"'${hintToPrettyString(name, parameters)}'.")
     }
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
index 88a3c0a73a10b..403e4e8758e72 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
@@ -34,6 +34,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
   override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
     case AlterTableAddColumnsStatement(
          nameParts @ NonSessionCatalogAndTable(catalog, tbl), cols) =>
+      cols.foreach(c => failCharType(c.dataType))
       val changes = cols.map { col =>
         TableChange.addColumn(
           col.name.toArray,
@@ -46,6 +47,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
 
     case a @ AlterTableAlterColumnStatement(
          nameParts @ NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _) =>
+      a.dataType.foreach(failCharType)
       val colName = a.column.toArray
       val typeChange = a.dataType.map { newDataType =>
         TableChange.updateColumnType(colName, newDataType)
@@ -121,6 +123,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
 
     case c @ CreateTableStatement(
          NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+      assertNoCharTypeInSchema(c.tableSchema)
       CreateV2Table(
         catalog.asTableCatalog,
         tbl.asIdentifier,
@@ -131,7 +134,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         ignoreIfExists = c.ifNotExists)
 
     case c @ CreateTableAsSelectStatement(
-         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
       CreateTableAsSelect(
         catalog.asTableCatalog,
         tbl.asIdentifier,
@@ -139,7 +142,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
         c.asSelect,
         convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
-        writeOptions = c.options,
+        writeOptions = c.writeOptions,
         ignoreIfExists = c.ifNotExists)
 
     case RefreshTableStatement(NonSessionCatalogAndTable(catalog, tbl)) =>
@@ -147,6 +150,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
 
     case c @ ReplaceTableStatement(
          NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+      assertNoCharTypeInSchema(c.tableSchema)
       ReplaceTable(
         catalog.asTableCatalog,
         tbl.asIdentifier,
@@ -157,7 +161,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         orCreate = c.orCreate)
 
     case c @ ReplaceTableAsSelectStatement(
-         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
       ReplaceTableAsSelect(
         catalog.asTableCatalog,
         tbl.asIdentifier,
@@ -165,7 +169,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
         c.asSelect,
         convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
-        writeOptions = c.options,
+        writeOptions = c.writeOptions,
         orCreate = c.orCreate)
 
     case DropTableStatement(NonSessionCatalogAndTable(catalog, tbl), ifExists, _) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala
index 5b77d67bd1340..81de086e78f91 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala
@@ -64,31 +64,59 @@ object ResolveHints {
             _.toUpperCase(Locale.ROOT)).contains(hintName.toUpperCase(Locale.ROOT))))
     }
 
+    // This method checks if given multi-part identifiers are matched with each other.
+    // The [[ResolveJoinStrategyHints]] rule is applied before the resolution batch
+    // in the analyzer and we cannot semantically compare them at this stage.
+    // Therefore, we follow a simple rule; they match if an identifier in a hint
+    // is a tail of an identifier in a relation. This process is independent of a session
+    // catalog (`currentDb` in [[SessionCatalog]]) and it just compares them literally.
+    //
+    // For example,
+    //  * in a query `SELECT /*+ BROADCAST(t) */ * FROM db1.t JOIN t`,
+    //    the broadcast hint will match both tables, `db1.t` and `t`,
+    //    even when the current db is `db2`.
+    //  * in a query `SELECT /*+ BROADCAST(default.t) */ * FROM default.t JOIN t`,
+    //    the broadcast hint will match the left-side table only, `default.t`.
+    private def matchedIdentifier(identInHint: Seq[String], identInQuery: Seq[String]): Boolean = {
+      if (identInHint.length <= identInQuery.length) {
+        identInHint.zip(identInQuery.takeRight(identInHint.length))
+          .forall { case (i1, i2) => resolver(i1, i2) }
+      } else {
+        false
+      }
+    }
+
+    private def extractIdentifier(r: SubqueryAlias): Seq[String] = {
+      r.identifier.qualifier :+ r.identifier.name
+    }
+
     private def applyJoinStrategyHint(
         plan: LogicalPlan,
-        relations: mutable.HashSet[String],
+        relationsInHint: Set[Seq[String]],
+        relationsInHintWithMatch: mutable.HashSet[Seq[String]],
         hintName: String): LogicalPlan = {
       // Whether to continue recursing down the tree
       var recurse = true
 
+      def matchedIdentifierInHint(identInQuery: Seq[String]): Boolean = {
+        relationsInHint.find(matchedIdentifier(_, identInQuery))
+          .map(relationsInHintWithMatch.add).nonEmpty
+      }
+
       val newNode = CurrentOrigin.withOrigin(plan.origin) {
         plan match {
           case ResolvedHint(u @ UnresolvedRelation(ident), hint)
-              if relations.exists(resolver(_, ident.last)) =>
-            relations.remove(ident.last)
+              if matchedIdentifierInHint(ident) =>
             ResolvedHint(u, createHintInfo(hintName).merge(hint, hintErrorHandler))
 
           case ResolvedHint(r: SubqueryAlias, hint)
-              if relations.exists(resolver(_, r.alias)) =>
-            relations.remove(r.alias)
+              if matchedIdentifierInHint(extractIdentifier(r)) =>
             ResolvedHint(r, createHintInfo(hintName).merge(hint, hintErrorHandler))
 
-          case u @ UnresolvedRelation(ident) if relations.exists(resolver(_, ident.last)) =>
-            relations.remove(ident.last)
+          case UnresolvedRelation(ident) if matchedIdentifierInHint(ident) =>
             ResolvedHint(plan, createHintInfo(hintName))
 
-          case r: SubqueryAlias if relations.exists(resolver(_, r.alias)) =>
-            relations.remove(r.alias)
+          case r: SubqueryAlias if matchedIdentifierInHint(extractIdentifier(r)) =>
             ResolvedHint(plan, createHintInfo(hintName))
 
           case _: ResolvedHint | _: View | _: With | _: SubqueryAlias =>
@@ -107,7 +135,9 @@ object ResolveHints {
       }
 
       if ((plan fastEquals newNode) && recurse) {
-        newNode.mapChildren(child => applyJoinStrategyHint(child, relations, hintName))
+        newNode.mapChildren { child =>
+          applyJoinStrategyHint(child, relationsInHint, relationsInHintWithMatch, hintName)
+        }
       } else {
         newNode
       }
@@ -120,17 +150,19 @@ object ResolveHints {
           ResolvedHint(h.child, createHintInfo(h.name))
         } else {
           // Otherwise, find within the subtree query plans to apply the hint.
-          val relationNames = h.parameters.map {
-            case tableName: String => tableName
-            case tableId: UnresolvedAttribute => tableId.name
+          val relationNamesInHint = h.parameters.map {
+            case tableName: String => UnresolvedAttribute.parseAttributeName(tableName)
+            case tableId: UnresolvedAttribute => tableId.nameParts
             case unsupported => throw new AnalysisException("Join strategy hint parameter " +
               s"should be an identifier or string but was $unsupported (${unsupported.getClass}")
-          }
-          val relationNameSet = new mutable.HashSet[String]
-          relationNames.foreach(relationNameSet.add)
-
-          val applied = applyJoinStrategyHint(h.child, relationNameSet, h.name)
-          hintErrorHandler.hintRelationsNotFound(h.name, h.parameters, relationNameSet.toSet)
+          }.toSet
+          val relationsInHintWithMatch = new mutable.HashSet[Seq[String]]
+          val applied = applyJoinStrategyHint(
+            h.child, relationNamesInHint, relationsInHintWithMatch, h.name)
+
+          // Filters unmatched relation identifiers in the hint
+          val unmatchedIdents = relationNamesInHint -- relationsInHintWithMatch
+          hintErrorHandler.hintRelationsNotFound(h.name, h.parameters, unmatchedIdents)
           applied
         }
     }
@@ -246,5 +278,4 @@ object ResolveHints {
         h.child
     }
   }
-
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index f416e8efe17ab..df8835dd915ad 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -23,6 +23,7 @@ import scala.annotation.tailrec
 import scala.collection.mutable
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.logical._
@@ -60,9 +61,11 @@ object TypeCoercion {
       IfCoercion ::
       StackCoercion ::
       Division ::
+      IntegralDivision ::
       ImplicitTypeCasts ::
       DateTimeOperations ::
       WindowFrameCoercion ::
+      StringLiteralCoercion ::
       Nil
 
   // See https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types.
@@ -323,25 +326,42 @@ object TypeCoercion {
    *
    * This rule is only applied to Union/Except/Intersect
    */
-  object WidenSetOperationTypes extends Rule[LogicalPlan] {
-
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
-      case s @ Except(left, right, isAll) if s.childrenResolved &&
-        left.output.length == right.output.length && !s.resolved =>
-        val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
-        assert(newChildren.length == 2)
-        Except(newChildren.head, newChildren.last, isAll)
-
-      case s @ Intersect(left, right, isAll) if s.childrenResolved &&
-        left.output.length == right.output.length && !s.resolved =>
-        val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
-        assert(newChildren.length == 2)
-        Intersect(newChildren.head, newChildren.last, isAll)
-
-      case s: Union if s.childrenResolved &&
+  object WidenSetOperationTypes extends TypeCoercionRule {
+
+    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = {
+      plan resolveOperatorsUpWithNewOutput {
+        case s @ Except(left, right, isAll) if s.childrenResolved &&
+          left.output.length == right.output.length && !s.resolved =>
+          val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
+          if (newChildren.isEmpty) {
+            s -> Nil
+          } else {
+            assert(newChildren.length == 2)
+            val attrMapping = left.output.zip(newChildren.head.output)
+            Except(newChildren.head, newChildren.last, isAll) -> attrMapping
+          }
+
+        case s @ Intersect(left, right, isAll) if s.childrenResolved &&
+          left.output.length == right.output.length && !s.resolved =>
+          val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
+          if (newChildren.isEmpty) {
+            s -> Nil
+          } else {
+            assert(newChildren.length == 2)
+            val attrMapping = left.output.zip(newChildren.head.output)
+            Intersect(newChildren.head, newChildren.last, isAll) -> attrMapping
+          }
+
+        case s: Union if s.childrenResolved &&
           s.children.forall(_.output.length == s.children.head.output.length) && !s.resolved =>
-        val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(s.children)
-        s.makeCopy(Array(newChildren))
+          val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(s.children)
+          if (newChildren.isEmpty) {
+            s -> Nil
+          } else {
+            val attrMapping = s.children.head.output.zip(newChildren.head.output)
+            s.copy(children = newChildren) -> attrMapping
+          }
+      }
     }
 
     /** Build new children with the widest types for each attribute among all the children */
@@ -357,8 +377,7 @@ object TypeCoercion {
         // Add an extra Project if the targetTypes are different from the original types.
         children.map(widenTypes(_, targetTypes))
       } else {
-        // Unable to find a target type to widen, then just return the original set.
-        children
+        Nil
       }
     }
 
@@ -384,7 +403,8 @@ object TypeCoercion {
     /** Given a plan, add an extra project on top to widen some columns' data types. */
     private def widenTypes(plan: LogicalPlan, targetTypes: Seq[DataType]): LogicalPlan = {
       val casted = plan.output.zip(targetTypes).map {
-        case (e, dt) if e.dataType != dt => Alias(Cast(e, dt), e.name)()
+        case (e, dt) if e.dataType != dt =>
+          Alias(Cast(e, dt, Some(SQLConf.get.sessionLocalTimeZone)), e.name)()
         case (e, _) => e
       }
       Project(casted, plan)
@@ -553,10 +573,10 @@ object TypeCoercion {
       // Skip nodes who's children have not been resolved yet.
       case e if !e.childrenResolved => e
 
-      case a @ CreateArray(children) if !haveSameType(children.map(_.dataType)) =>
+      case a @ CreateArray(children, _) if !haveSameType(children.map(_.dataType)) =>
         val types = children.map(_.dataType)
         findWiderCommonType(types) match {
-          case Some(finalDataType) => CreateArray(children.map(castIfNotSameType(_, finalDataType)))
+          case Some(finalDataType) => a.copy(children.map(castIfNotSameType(_, finalDataType)))
           case None => a
         }
 
@@ -592,7 +612,7 @@ object TypeCoercion {
           case None => m
         }
 
-      case m @ CreateMap(children) if m.keys.length == m.values.length &&
+      case m @ CreateMap(children, _) if m.keys.length == m.values.length &&
           (!haveSameType(m.keys.map(_.dataType)) || !haveSameType(m.values.map(_.dataType))) =>
         val keyTypes = m.keys.map(_.dataType)
         val newKeys = findWiderCommonType(keyTypes) match {
@@ -606,7 +626,7 @@ object TypeCoercion {
           case None => m.values
         }
 
-        CreateMap(newKeys.zip(newValues).flatMap { case (k, v) => Seq(k, v) })
+        m.copy(newKeys.zip(newValues).flatMap { case (k, v) => Seq(k, v) })
 
       // Promote SUM, SUM DISTINCT and AVERAGE to largest types to prevent overflows.
       case s @ Sum(e @ DecimalType()) => s // Decimal is already the biggest.
@@ -682,6 +702,23 @@ object TypeCoercion {
     }
   }
 
+  /**
+   * The DIV operator always returns long-type value.
+   * This rule cast the integral inputs to long type, to avoid overflow during calculation.
+   */
+  object IntegralDivision extends TypeCoercionRule {
+    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+      case e if !e.childrenResolved => e
+      case d @ IntegralDivide(left, right) =>
+        IntegralDivide(mayCastToLong(left), mayCastToLong(right))
+    }
+
+    private def mayCastToLong(expr: Expression): Expression = expr.dataType match {
+      case _: ByteType | _: ShortType | _: IntegerType => Cast(expr, LongType)
+      case _ => expr
+    }
+  }
+
   /**
    * Coerces the type of different branches of a CASE WHEN statement to a common type.
    */
@@ -840,15 +877,26 @@ object TypeCoercion {
    * Casts types according to the expected input types for [[Expression]]s.
    */
   object ImplicitTypeCasts extends TypeCoercionRule {
+
+    private def canHandleTypeCoercion(leftType: DataType, rightType: DataType): Boolean = {
+      (leftType, rightType) match {
+        case (_: DecimalType, NullType) => true
+        case (NullType, _: DecimalType) => true
+        case _ =>
+          // If DecimalType operands are involved except for the two cases above,
+          // DecimalPrecision will handle it.
+          !leftType.isInstanceOf[DecimalType] && !rightType.isInstanceOf[DecimalType] &&
+            leftType != rightType
+      }
+    }
+
     override protected def coerceTypes(
         plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
       // Skip nodes who's children have not been resolved yet.
       case e if !e.childrenResolved => e
 
-      // If DecimalType operands are involved, DecimalPrecision will handle it
-      case b @ BinaryOperator(left, right) if !left.dataType.isInstanceOf[DecimalType] &&
-          !right.dataType.isInstanceOf[DecimalType] &&
-          left.dataType != right.dataType =>
+      case b @ BinaryOperator(left, right)
+          if canHandleTypeCoercion(left.dataType, right.dataType) =>
         findTightestCommonType(left.dataType, right.dataType).map { commonType =>
           if (b.inputType.acceptsType(commonType)) {
             // If the expression accepts the tightest common type, cast to that.
@@ -1043,6 +1091,34 @@ object TypeCoercion {
       }
     }
   }
+
+  /**
+   * A special rule to support string literal as the second argument of date_add/date_sub functions,
+   * to keep backward compatibility as a temporary workaround.
+   * TODO(SPARK-28589): implement ANSI type type coercion and handle string literals.
+   */
+  object StringLiteralCoercion extends TypeCoercionRule {
+    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+      // Skip nodes who's children have not been resolved yet.
+      case e if !e.childrenResolved => e
+      case DateAdd(l, r) if r.dataType == StringType && r.foldable =>
+        val days = try {
+          AnsiCast(r, IntegerType).eval().asInstanceOf[Int]
+        } catch {
+          case e: NumberFormatException => throw new AnalysisException(
+            "The second argument of 'date_add' function needs to be an integer.", cause = Some(e))
+        }
+        DateAdd(l, Literal(days))
+      case DateSub(l, r) if r.dataType == StringType && r.foldable =>
+        val days = try {
+          AnsiCast(r, IntegerType).eval().asInstanceOf[Int]
+        } catch {
+          case e: NumberFormatException => throw new AnalysisException(
+            "The second argument of 'date_sub' function needs to be an integer.", cause = Some(e))
+        }
+        DateSub(l, Literal(days))
+    }
+  }
 }
 
 trait TypeCoercionRule extends Rule[LogicalPlan] with Logging {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
index 2f8cb26ffaa9c..676e28f7d4a80 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
@@ -19,13 +19,11 @@ package org.apache.spark.sql.catalyst.analysis
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, CurrentDate, CurrentTimestamp, MonotonicallyIncreasingID}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, CurrentDate, CurrentTimestamp, MonotonicallyIncreasingID, Now}
 import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
-import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.OutputMode
 
 /**
@@ -93,7 +91,13 @@ object UnsupportedOperationChecker extends Logging {
 
     /** Collect all the streaming aggregates in a sub plan */
     def collectStreamingAggregates(subplan: LogicalPlan): Seq[Aggregate] = {
-      subplan.collect { case a: Aggregate if a.isStreaming => a }
+      subplan.collect {
+        case a: Aggregate if a.isStreaming => a
+        // Since the Distinct node will be replaced to Aggregate in the optimizer rule
+        // [[ReplaceDistinctWithAggregate]], here we also need to check all Distinct node by
+        // assuming it as Aggregate.
+        case d @ Distinct(c: LogicalPlan) if d.isStreaming => Aggregate(c.output, c.output, c)
+      }
     }
 
     val mapGroupsWithStates = plan.collect {
@@ -412,7 +416,7 @@ object UnsupportedOperationChecker extends Logging {
 
       subPlan.expressions.foreach { e =>
         if (e.collectLeaves().exists {
-          case (_: CurrentTimestamp | _: CurrentDate) => true
+          case (_: CurrentTimestamp | _: Now | _: CurrentDate) => true
           case _ => false
         }) {
           throwError(s"Continuous processing does not support current time operations.")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index 608f39c2d86fd..b28be042c43f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.parser.ParserUtils
-import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, UnaryNode}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LeafNode, LogicalPlan, UnaryNode}
 import org.apache.spark.sql.catalyst.trees.TreeNode
 import org.apache.spark.sql.catalyst.util.quoteIdentifier
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
@@ -298,35 +298,26 @@ abstract class Star extends LeafExpression with NamedExpression {
 case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevaluable {
 
   /**
-   * Returns true if the nameParts match the qualifier of the attribute
+   * Returns true if the nameParts is a subset of the last elements of qualifier of the attribute.
    *
-   * There are two checks: i) Check if the nameParts match the qualifier fully.
-   * E.g. SELECT db.t1.* FROM db1.t1   In this case, the nameParts is Seq("db1", "t1") and
-   * qualifier of the attribute is Seq("db1","t1")
-   * ii) If (i) is not true, then check if nameParts is only a single element and it
-   * matches the table portion of the qualifier
-   *
-   * E.g. SELECT t1.* FROM db1.t1  In this case nameParts is Seq("t1") and
-   * qualifier is Seq("db1","t1")
-   * SELECT a.* FROM db1.t1 AS a
-   * In this case nameParts is Seq("a") and qualifier for
-   * attribute is Seq("a")
+   * For example, the following should all return true:
+   *   - `SELECT ns1.ns2.t.* FROM ns1.n2.t` where nameParts is Seq("ns1", "ns2", "t") and
+   *     qualifier is Seq("ns1", "ns2", "t").
+   *   - `SELECT ns2.t.* FROM ns1.n2.t` where nameParts is Seq("ns2", "t") and
+   *     qualifier is Seq("ns1", "ns2", "t").
+   *   - `SELECT t.* FROM ns1.n2.t` where nameParts is Seq("t") and
+   *     qualifier is Seq("ns1", "ns2", "t").
    */
   private def matchedQualifier(
       attribute: Attribute,
       nameParts: Seq[String],
       resolver: Resolver): Boolean = {
-    val qualifierList = attribute.qualifier
-
-    val matched = nameParts.corresponds(qualifierList)(resolver) || {
-      // check if it matches the table portion of the qualifier
-      if (nameParts.length == 1 && qualifierList.nonEmpty) {
-        resolver(nameParts.head, qualifierList.last)
-      } else {
-        false
-      }
+    val qualifierList = if (nameParts.length == attribute.qualifier.length) {
+      attribute.qualifier
+    } else {
+      attribute.qualifier.takeRight(nameParts.length)
     }
-    matched
+    nameParts.corresponds(qualifierList)(resolver)
   }
 
   override def expand(
@@ -547,3 +538,15 @@ case class UnresolvedOrdinal(ordinal: Int)
   override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
   override lazy val resolved = false
 }
+
+/**
+ * Represents unresolved having clause, the child for it can be Aggregate, GroupingSets, Rollup
+ * and Cube. It is turned by the analyzer into a Filter.
+ */
+case class UnresolvedHaving(
+    havingCondition: Expression,
+    child: LogicalPlan)
+  extends UnaryNode {
+  override lazy val resolved: Boolean = false
+  override def output: Seq[Attribute] = child.output
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala
index dcc143982a4aa..db930cf7890e6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala
@@ -136,6 +136,8 @@ trait ExternalCatalog {
 
   def listTables(db: String, pattern: String): Seq[String]
 
+  def listViews(db: String, pattern: String): Seq[String]
+
   /**
    * Loads data into a table.
    *
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
index ae3b75dc3334b..9cb71bb51294c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
@@ -159,6 +159,10 @@ object ExternalCatalogUtils {
     }
   }
 
+  private def isNullPartitionValue(value: String): Boolean = {
+    value == null || value == DEFAULT_PARTITION_NAME
+  }
+
   /**
    * Returns true if `spec1` is a partial partition spec w.r.t. `spec2`, e.g. PARTITION (a=1) is a
    * partial partition spec w.r.t. PARTITION (a=1,b=2).
@@ -167,9 +171,15 @@ object ExternalCatalogUtils {
       spec1: TablePartitionSpec,
       spec2: TablePartitionSpec): Boolean = {
     spec1.forall {
+      case (partitionColumn, value) if isNullPartitionValue(value) =>
+        isNullPartitionValue(spec2(partitionColumn))
       case (partitionColumn, value) => spec2(partitionColumn) == value
     }
   }
+
+  def convertNullPartitionValues(spec: TablePartitionSpec): TablePartitionSpec = {
+    spec.mapValues(v => if (v == null) DEFAULT_PARTITION_NAME else v).toMap
+  }
 }
 
 object CatalogUtils {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogWithListener.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogWithListener.scala
index 86113d3ec3ead..c2613ff74da4a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogWithListener.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogWithListener.scala
@@ -154,6 +154,10 @@ class ExternalCatalogWithListener(delegate: ExternalCatalog)
     delegate.listTables(db, pattern)
   }
 
+  override def listViews(db: String, pattern: String): Seq[String] = {
+    delegate.listViews(db, pattern)
+  }
+
   override def loadTable(
       db: String,
       table: String,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
index abf69939dea13..c05950300279f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
@@ -346,6 +346,12 @@ class InMemoryCatalog(
     StringUtils.filterPattern(listTables(db), pattern)
   }
 
+  override def listViews(db: String, pattern: String): Seq[String] = synchronized {
+    requireDbExists(db)
+    val views = catalog(db).tables.filter(_._2.table.tableType == CatalogTableType.VIEW).keySet
+    StringUtils.filterPattern(views.toSeq.sorted, pattern)
+  }
+
   override def loadTable(
       db: String,
       table: String,
@@ -535,7 +541,12 @@ class InMemoryCatalog(
 
     listPartitions(db, table, partialSpec).map { partition =>
       partitionColumnNames.map { name =>
-        escapePathName(name) + "=" + escapePathName(partition.spec(name))
+        val partValue = if (partition.spec(name) == null) {
+          DEFAULT_PARTITION_NAME
+        } else {
+          escapePathName(partition.spec(name))
+        }
+        escapePathName(name) + "=" + partValue
       }.mkString("/")
     }.sorted
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index 45f0ef6c97e70..389e410234154 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo, Im
 import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParserInterface}
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias, View}
 import org.apache.spark.sql.catalyst.util.StringUtils
+import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
 import org.apache.spark.sql.types.StructType
@@ -758,6 +759,7 @@ class SessionCatalog(
     val name = metadata.identifier
     val db = formatDatabaseName(name.database.getOrElse(currentDb))
     val table = formatTableName(name.table)
+    val multiParts = Seq(CatalogManager.SESSION_CATALOG_NAME, db, table)
 
     if (metadata.tableType == CatalogTableType.VIEW) {
       val viewText = metadata.viewText.getOrElse(sys.error("Invalid view without text."))
@@ -769,9 +771,9 @@ class SessionCatalog(
         desc = metadata,
         output = metadata.schema.toAttributes,
         child = parser.parsePlan(viewText))
-      SubqueryAlias(table, db, child)
+      SubqueryAlias(multiParts, child)
     } else {
-      SubqueryAlias(table, db, UnresolvedCatalogRelation(metadata))
+      SubqueryAlias(multiParts, UnresolvedCatalogRelation(metadata))
     }
   }
 
@@ -826,6 +828,8 @@ class SessionCatalog(
         getTempViewOrPermanentTableMetadata(ident).tableType == CatalogTableType.VIEW
       } catch {
         case _: NoSuchTableException => false
+        case _: NoSuchDatabaseException => false
+        case _: NoSuchNamespaceException => false
       }
     }
   }
@@ -876,6 +880,25 @@ class SessionCatalog(
     }
   }
 
+  /**
+   * List all matching views in the specified database, including local temporary views.
+   */
+  def listViews(db: String, pattern: String): Seq[TableIdentifier] = {
+    val dbName = formatDatabaseName(db)
+    val dbViews = if (dbName == globalTempViewManager.database) {
+      globalTempViewManager.listViewNames(pattern).map { name =>
+        TableIdentifier(name, Some(globalTempViewManager.database))
+      }
+    } else {
+      requireDbExists(dbName)
+      externalCatalog.listViews(dbName, pattern).map { name =>
+        TableIdentifier(name, Some(dbName))
+      }
+    }
+
+    dbViews ++ listLocalTempViews(pattern)
+  }
+
   /**
    * List all matching local temporary views.
    */
@@ -1081,7 +1104,7 @@ class SessionCatalog(
    */
   private def requireNonEmptyValueInPartitionSpec(specs: Seq[TablePartitionSpec]): Unit = {
     specs.foreach { s =>
-      if (s.values.exists(_.isEmpty)) {
+      if (s.values.exists(v => v != null && v.isEmpty)) {
         val spec = s.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")
         throw new AnalysisException(
           s"Partition spec is invalid. The spec ($spec) contains an empty partition column value")
@@ -1342,6 +1365,10 @@ class SessionCatalog(
       !hiveFunctions.contains(name.funcName.toLowerCase(Locale.ROOT))
   }
 
+  def isTempFunction(name: String): Boolean = {
+    isTemporaryFunction(FunctionIdentifier(name))
+  }
+
   /**
    * Return whether this function has been registered in the function registry of the current
    * session. If not existed, return false.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index 81561c53f83c1..6d658326369b5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -413,6 +413,33 @@ object CatalogTable {
   val VIEW_QUERY_OUTPUT_PREFIX = VIEW_PREFIX + "query.out."
   val VIEW_QUERY_OUTPUT_NUM_COLUMNS = VIEW_QUERY_OUTPUT_PREFIX + "numCols"
   val VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX = VIEW_QUERY_OUTPUT_PREFIX + "col."
+
+  def normalize(table: CatalogTable): CatalogTable = {
+    val nondeterministicProps = Set(
+      "CreateTime",
+      "transient_lastDdlTime",
+      "grantTime",
+      "lastUpdateTime",
+      "last_modified_by",
+      "last_modified_time",
+      "Owner:",
+      // The following are hive specific schema parameters which we do not need to match exactly.
+      "totalNumberFiles",
+      "maxFileSize",
+      "minFileSize"
+    )
+
+    table.copy(
+      createTime = 0L,
+      lastAccessTime = 0L,
+      properties = table.properties
+        .filterKeys(!nondeterministicProps.contains(_))
+        .map(identity)
+        .toMap,
+      stats = None,
+      ignoredProperties = Map.empty
+    )
+  }
 }
 
 /**
@@ -521,8 +548,11 @@ object CatalogColumnStat extends Logging {
 
   val VERSION = 2
 
-  private def getTimestampFormatter(): TimestampFormatter = {
-    TimestampFormatter(format = "uuuu-MM-dd HH:mm:ss.SSSSSS", zoneId = ZoneOffset.UTC)
+  private def getTimestampFormatter(isParsing: Boolean): TimestampFormatter = {
+    TimestampFormatter(
+      format = "yyyy-MM-dd HH:mm:ss.SSSSSS",
+      zoneId = ZoneOffset.UTC,
+      isParsing = isParsing)
   }
 
   /**
@@ -535,7 +565,7 @@ object CatalogColumnStat extends Logging {
       case DateType => DateFormatter(ZoneOffset.UTC).parse(s)
       case TimestampType if version == 1 =>
         DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf(s))
-      case TimestampType => getTimestampFormatter().parse(s)
+      case TimestampType => getTimestampFormatter(isParsing = true).parse(s)
       case ByteType => s.toByte
       case ShortType => s.toShort
       case IntegerType => s.toInt
@@ -558,7 +588,7 @@ object CatalogColumnStat extends Logging {
   def toExternalString(v: Any, colName: String, dataType: DataType): String = {
     val externalValue = dataType match {
       case DateType => DateFormatter(ZoneOffset.UTC).format(v.asInstanceOf[Int])
-      case TimestampType => getTimestampFormatter().format(v.asInstanceOf[Long])
+      case TimestampType => getTimestampFormatter(isParsing = false).format(v.asInstanceOf[Long])
       case BooleanType | _: IntegralType | FloatType | DoubleType => v
       case _: DecimalType => v.asInstanceOf[Decimal].toJavaBigDecimal
       // This version of Spark does not use min/max for binary/string types so we ignore it.
@@ -664,16 +694,14 @@ case class HiveTableRelation(
   def isPartitioned: Boolean = partitionCols.nonEmpty
 
   override def doCanonicalize(): HiveTableRelation = copy(
-    tableMeta = tableMeta.copy(
-      storage = CatalogStorageFormat.empty,
-      createTime = -1
-    ),
+    tableMeta = CatalogTable.normalize(tableMeta),
     dataCols = dataCols.zipWithIndex.map {
       case (attr, index) => attr.withExprId(ExprId(index))
     },
     partitionCols = partitionCols.zipWithIndex.map {
       case (attr, index) => attr.withExprId(ExprId(index + dataCols.length))
-    }
+    },
+    tableStats = None
   )
 
   override def computeStats(): Statistics = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
index 3e83c1dcb4758..efe4188377142 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
@@ -25,8 +25,13 @@ object CSVExprUtils {
    * This is currently being used in CSV reading path and CSV schema inference.
    */
   def filterCommentAndEmpty(iter: Iterator[String], options: CSVOptions): Iterator[String] = {
-    iter.filter { line =>
-      line.trim.nonEmpty && !line.startsWith(options.comment.toString)
+    if (options.isCommentSet) {
+      val commentPrefix = options.comment.toString
+      iter.filter { line =>
+        line.trim.nonEmpty && !line.startsWith(commentPrefix)
+      }
+    } else {
+      iter.filter(_.trim.nonEmpty)
     }
   }
 
@@ -34,7 +39,7 @@ object CSVExprUtils {
     if (options.isCommentSet) {
       val commentPrefix = options.comment.toString
       iter.dropWhile { line =>
-        line.trim.isEmpty || line.trim.startsWith(commentPrefix)
+        line.trim.isEmpty || line.startsWith(commentPrefix)
       }
     } else {
       iter.dropWhile(_.trim.isEmpty)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
index 03cc3cbdf790a..f0df18da8eed6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
@@ -24,6 +24,7 @@ import scala.util.control.Exception.allCatch
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.analysis.TypeCoercion
 import org.apache.spark.sql.catalyst.expressions.ExprUtils
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.catalyst.util.TimestampFormatter
 import org.apache.spark.sql.types._
 
@@ -32,7 +33,9 @@ class CSVInferSchema(val options: CSVOptions) extends Serializable {
   private val timestampParser = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = true)
 
   private val decimalParser = if (options.locale == Locale.US) {
     // Special handling the default locale for backward compatibility
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
index 5e40d74e54f11..f2191fcf35f1a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
@@ -26,6 +26,7 @@ import com.univocity.parsers.csv.{CsvParserSettings, CsvWriterSettings, Unescape
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 
 class CSVOptions(
     @transient val parameters: CaseInsensitiveMap[String],
@@ -146,10 +147,14 @@ class CSVOptions(
   // A language tag in IETF BCP 47 format
   val locale: Locale = parameters.get("locale").map(Locale.forLanguageTag).getOrElse(Locale.US)
 
-  val dateFormat: String = parameters.getOrElse("dateFormat", "uuuu-MM-dd")
+  val dateFormat: String = parameters.getOrElse("dateFormat", DateFormatter.defaultPattern)
 
-  val timestampFormat: String =
-    parameters.getOrElse("timestampFormat", "uuuu-MM-dd'T'HH:mm:ss.SSSXXX")
+  val timestampFormat: String = parameters.getOrElse("timestampFormat",
+    if (SQLConf.get.legacyTimeParserPolicy == LegacyBehaviorPolicy.LEGACY) {
+      s"${DateFormatter.defaultPattern}'T'HH:mm:ss.SSSXXX"
+    } else {
+      s"${DateFormatter.defaultPattern}'T'HH:mm:ss[.SSS][XXX]"
+    })
 
   val multiLine = parameters.get("multiLine").map(_.toBoolean).getOrElse(false)
 
@@ -215,7 +220,9 @@ class CSVOptions(
     format.setQuote(quote)
     format.setQuoteEscape(escape)
     charToEscapeQuoteEscaping.foreach(format.setCharToEscapeQuoteEscaping)
-    format.setComment(comment)
+    if (isCommentSet) {
+      format.setComment(comment)
+    }
     lineSeparatorInWrite.foreach(format.setLineSeparator)
 
     writerSettings.setIgnoreLeadingWhitespaces(ignoreLeadingWhiteSpaceFlagInWrite)
@@ -237,7 +244,11 @@ class CSVOptions(
     format.setQuoteEscape(escape)
     lineSeparator.foreach(format.setLineSeparator)
     charToEscapeQuoteEscaping.foreach(format.setCharToEscapeQuoteEscaping)
-    format.setComment(comment)
+    if (isCommentSet) {
+      format.setComment(comment)
+    } else {
+      settings.setCommentProcessingEnabled(false)
+    }
 
     settings.setIgnoreLeadingWhitespaces(ignoreLeadingWhiteSpaceInRead)
     settings.setIgnoreTrailingWhitespaces(ignoreTrailingWhiteSpaceInRead)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
index 05cb91d10868e..a3ee129cd6d64 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
@@ -23,6 +23,7 @@ import com.univocity.parsers.csv.CsvWriter
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.util.{DateFormatter, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.types._
 
 class UnivocityGenerator(
@@ -44,11 +45,15 @@ class UnivocityGenerator(
   private val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = false)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = false)
 
   private def makeConverter(dataType: DataType): ValueConverter = dataType match {
     case DateType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
index 5510953804025..3898eca79478e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
@@ -23,10 +23,12 @@ import scala.util.control.NonFatal
 
 import com.univocity.parsers.csv.CsvParser
 
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{ExprUtils, GenericInternalRow}
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -83,20 +85,25 @@ class UnivocityParser(
   // We preallocate it avoid unnecessary allocations.
   private val noRows = None
 
-  private val timestampFormatter = TimestampFormatter(
+  private lazy val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
-  private val dateFormatter = DateFormatter(
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = true)
+  private lazy val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = true)
 
   private val csvFilters = new CSVFilters(filters, requiredSchema)
 
   // Retrieve the raw record string.
   private def getCurrentInput: UTF8String = {
-    UTF8String.fromString(tokenizer.getContext.currentParsedContent().stripLineEnd)
+    val currentContent = tokenizer.getContext.currentParsedContent()
+    if (currentContent == null) null else UTF8String.fromString(currentContent.stripLineEnd)
   }
 
   // This parser first picks some tokens from the input tokens, according to the required schema,
@@ -172,10 +179,30 @@ class UnivocityParser(
       }
 
     case _: TimestampType => (d: String) =>
-      nullSafeDatum(d, name, nullable, options)(timestampFormatter.parse)
+      nullSafeDatum(d, name, nullable, options) { datum =>
+        try {
+          timestampFormatter.parse(datum)
+        } catch {
+          case NonFatal(e) =>
+            // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
+            // compatibility.
+            val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(datum))
+            DateTimeUtils.stringToTimestamp(str, options.zoneId).getOrElse(throw e)
+        }
+      }
 
     case _: DateType => (d: String) =>
-      nullSafeDatum(d, name, nullable, options)(dateFormatter.parse)
+      nullSafeDatum(d, name, nullable, options) { datum =>
+        try {
+          dateFormatter.parse(datum)
+        } catch {
+          case NonFatal(e) =>
+            // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
+            // compatibility.
+            val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(datum))
+            DateTimeUtils.stringToDate(str, options.zoneId).getOrElse(throw e)
+        }
+      }
 
     case _: StringType => (d: String) =>
       nullSafeDatum(d, name, nullable, options)(UTF8String.fromString)
@@ -262,6 +289,7 @@ class UnivocityParser(
           }
         }
       } catch {
+        case e: SparkUpgradeException => throw e
         case NonFatal(e) =>
           badRecordException = badRecordException.orElse(Some(e))
           row.setNullAt(i)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index 40998080bc4e3..141bcfaa739e0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.catalyst
 
 import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate}
 
 import scala.language.implicitConversions
 
@@ -99,7 +100,7 @@ package object dsl {
     }
 
     def like(other: Expression, escapeChar: Char = '\\'): Expression =
-      Like(expr, other, Literal(escapeChar.toString))
+      Like(expr, other, escapeChar)
     def rlike(other: Expression): Expression = RLike(expr, other)
     def contains(other: Expression): Expression = Contains(expr, other)
     def startsWith(other: Expression): Expression = StartsWith(expr, other)
@@ -146,10 +147,12 @@ package object dsl {
     implicit def doubleToLiteral(d: Double): Literal = Literal(d)
     implicit def stringToLiteral(s: String): Literal = Literal.create(s, StringType)
     implicit def dateToLiteral(d: Date): Literal = Literal(d)
+    implicit def localDateToLiteral(d: LocalDate): Literal = Literal(d)
     implicit def bigDecimalToLiteral(d: BigDecimal): Literal = Literal(d.underlying())
     implicit def bigDecimalToLiteral(d: java.math.BigDecimal): Literal = Literal(d)
     implicit def decimalToLiteral(d: Decimal): Literal = Literal(d)
     implicit def timestampToLiteral(t: Timestamp): Literal = Literal(t)
+    implicit def instantToLiteral(i: Instant): Literal = Literal(i)
     implicit def binaryToLiteral(a: Array[Byte]): Literal = Literal(a)
 
     implicit def symbolToUnresolvedAttribute(s: Symbol): analysis.UnresolvedAttribute =
@@ -362,6 +365,14 @@ package object dsl {
         Aggregate(groupingExprs, aliasedExprs, logicalPlan)
       }
 
+      def having(
+          groupingExprs: Expression*)(
+          aggregateExprs: Expression*)(
+          havingCondition: Expression): LogicalPlan = {
+        UnresolvedHaving(havingCondition,
+          groupBy(groupingExprs: _*)(aggregateExprs: _*).asInstanceOf[Aggregate])
+      }
+
       def window(
           windowExpressions: Seq[NamedExpression],
           partitionSpec: Seq[Expression],
@@ -407,8 +418,11 @@ package object dsl {
       def distribute(exprs: Expression*)(n: Int): LogicalPlan =
         RepartitionByExpression(exprs, logicalPlan, numPartitions = n)
 
-      def analyze: LogicalPlan =
-        EliminateSubqueryAliases(analysis.SimpleAnalyzer.execute(logicalPlan))
+      def analyze: LogicalPlan = {
+        val analyzed = analysis.SimpleAnalyzer.execute(logicalPlan)
+        analysis.SimpleAnalyzer.checkAnalysis(analyzed)
+        EliminateSubqueryAliases(analyzed)
+      }
 
       def hint(name: String, parameters: Any*): LogicalPlan =
         UnresolvedHint(name, parameters, logicalPlan)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
index bd499671d6441..213fbcc989e7c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
@@ -17,12 +17,15 @@
 
 package org.apache.spark.sql.catalyst.encoders
 
+import java.io.ObjectInputStream
+
 import scala.reflect.ClassTag
 import scala.reflect.runtime.universe.{typeTag, TypeTag}
 
 import org.apache.spark.sql.Encoder
 import org.apache.spark.sql.catalyst.{InternalRow, JavaTypeInference, ScalaReflection}
 import org.apache.spark.sql.catalyst.analysis.{Analyzer, GetColumnByOrdinal, SimpleAnalyzer, UnresolvedAttribute, UnresolvedExtractValue}
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.{Deserializer, Serializer}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, InitializeJavaBean, Invoke, NewInstance}
@@ -162,6 +165,56 @@ object ExpressionEncoder {
       e4: ExpressionEncoder[T4],
       e5: ExpressionEncoder[T5]): ExpressionEncoder[(T1, T2, T3, T4, T5)] =
     tuple(Seq(e1, e2, e3, e4, e5)).asInstanceOf[ExpressionEncoder[(T1, T2, T3, T4, T5)]]
+
+  private val anyObjectType = ObjectType(classOf[Any])
+
+  /**
+   * Function that deserializes an [[InternalRow]] into an object of type `T`. This class is not
+   * thread-safe.
+   */
+  class Deserializer[T](private val expressions: Seq[Expression])
+    extends (InternalRow => T) with Serializable {
+    @transient
+    private[this] var constructProjection: Projection = _
+
+    override def apply(row: InternalRow): T = try {
+      if (constructProjection == null) {
+        constructProjection = SafeProjection.create(expressions)
+      }
+      constructProjection(row).get(0, anyObjectType).asInstanceOf[T]
+    } catch {
+      case e: Exception =>
+        throw new RuntimeException(s"Error while decoding: $e\n" +
+          s"${expressions.map(_.simpleString(SQLConf.get.maxToStringFields)).mkString("\n")}", e)
+    }
+  }
+
+  /**
+   * Function that serializesa an object of type `T` to an [[InternalRow]]. This class is not
+   * thread-safe. Note that multiple calls to `apply(..)` return the same actual [[InternalRow]]
+   * object.  Thus, the caller should copy the result before making another call if required.
+   */
+  class Serializer[T](private val expressions: Seq[Expression])
+    extends (T => InternalRow) with Serializable {
+    @transient
+    private[this] var inputRow: GenericInternalRow = _
+
+    @transient
+    private[this] var extractProjection: UnsafeProjection = _
+
+    override def apply(t: T): InternalRow = try {
+      if (extractProjection == null) {
+        inputRow = new GenericInternalRow(1)
+        extractProjection = GenerateUnsafeProjection.generate(expressions)
+      }
+      inputRow(0) = t
+      extractProjection(inputRow)
+    } catch {
+      case e: Exception =>
+        throw new RuntimeException(s"Error while encoding: $e\n" +
+          s"${expressions.map(_.simpleString(SQLConf.get.maxToStringFields)).mkString("\n")}", e)
+    }
+  }
 }
 
 /**
@@ -301,25 +354,22 @@ case class ExpressionEncoder[T](
   }
 
   @transient
-  private lazy val extractProjection = GenerateUnsafeProjection.generate({
+  private lazy val optimizedDeserializer: Seq[Expression] = {
     // When using `ExpressionEncoder` directly, we will skip the normal query processing steps
     // (analyzer, optimizer, etc.). Here we apply the ReassignLambdaVariableID rule, as it's
     // important to codegen performance.
-    val optimizedPlan = ReassignLambdaVariableID.apply(DummyExpressionHolder(serializer))
+    val optimizedPlan = ReassignLambdaVariableID.apply(DummyExpressionHolder(Seq(deserializer)))
     optimizedPlan.asInstanceOf[DummyExpressionHolder].exprs
-  })
-
-  @transient
-  private lazy val inputRow = new GenericInternalRow(1)
+  }
 
   @transient
-  private lazy val constructProjection = SafeProjection.create({
+  private lazy val optimizedSerializer = {
     // When using `ExpressionEncoder` directly, we will skip the normal query processing steps
     // (analyzer, optimizer, etc.). Here we apply the ReassignLambdaVariableID rule, as it's
     // important to codegen performance.
-    val optimizedPlan = ReassignLambdaVariableID.apply(DummyExpressionHolder(Seq(deserializer)))
+    val optimizedPlan = ReassignLambdaVariableID.apply(DummyExpressionHolder(serializer))
     optimizedPlan.asInstanceOf[DummyExpressionHolder].exprs
-  })
+  }
 
   /**
    * Returns a new set (with unique ids) of [[NamedExpression]] that represent the serialized form
@@ -331,31 +381,21 @@ case class ExpressionEncoder[T](
   }
 
   /**
-   * Returns an encoded version of `t` as a Spark SQL row.  Note that multiple calls to
-   * toRow are allowed to return the same actual [[InternalRow]] object.  Thus, the caller should
-   * copy the result before making another call if required.
+   * Create a serializer that can convert an object of type `T` to a Spark SQL Row.
+   *
+   * Note that the returned [[Serializer]] is not thread safe. Multiple calls to
+   * `serializer.apply(..)` are allowed to return the same actual [[InternalRow]] object.  Thus,
+   *  the caller should copy the result before making another call if required.
    */
-  def toRow(t: T): InternalRow = try {
-    inputRow(0) = t
-    extractProjection(inputRow)
-  } catch {
-    case e: Exception =>
-      throw new RuntimeException(s"Error while encoding: $e\n" +
-          s"${serializer.map(_.simpleString(SQLConf.get.maxToStringFields)).mkString("\n")}", e)
-  }
+  def createSerializer(): Serializer[T] = new Serializer[T](optimizedSerializer)
 
   /**
-   * Returns an object of type `T`, extracting the required values from the provided row.  Note that
-   * you must `resolveAndBind` an encoder to a specific schema before you can call this
-   * function.
+   * Create a deserializer that can convert a Spark SQL Row into an object of type `T`.
+   *
+   * Note that you must `resolveAndBind` an encoder to a specific schema before you can create a
+   * deserializer.
    */
-  def fromRow(row: InternalRow): T = try {
-    constructProjection(row).get(0, ObjectType(clsTag.runtimeClass)).asInstanceOf[T]
-  } catch {
-    case e: Exception =>
-      throw new RuntimeException(s"Error while decoding: $e\n" +
-        s"${deserializer.simpleString(SQLConf.get.maxToStringFields)}", e)
-  }
+  def createDeserializer(): Deserializer[T] = new Deserializer[T](optimizedDeserializer)
 
   /**
    * The process of resolution to a given schema throws away information about where a given field
@@ -382,8 +422,6 @@ case class ExpressionEncoder[T](
       .map { case(f, a) => s"${f.name}$a: ${f.dataType.simpleString}"}.mkString(", ")
 
   override def toString: String = s"class[$schemaString]"
-
-  override def makeCopy: ExpressionEncoder[T] = copy()
 }
 
 // A dummy logical plan that can hold expressions and go through optimizer rules.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
index 65bb9a8c642b6..6d711af192ad3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import java.lang.reflect.{Method, Modifier}
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.types._
@@ -53,9 +53,9 @@ import org.apache.spark.util.Utils
        a5cf6c42-0c85-418f-af6c-3e4e5b1328f2
   """)
 case class CallMethodViaReflection(children: Seq[Expression])
-  extends Expression with CodegenFallback {
+  extends Nondeterministic with CodegenFallback {
 
-  override def prettyName: String = "reflect"
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("reflect")
 
   override def checkInputDataTypes(): TypeCheckResult = {
     if (children.size < 2) {
@@ -76,11 +76,11 @@ case class CallMethodViaReflection(children: Seq[Expression])
     }
   }
 
-  override lazy val deterministic: Boolean = false
   override def nullable: Boolean = true
   override val dataType: DataType = StringType
+  override protected def initializeInternal(partitionIndex: Int): Unit = {}
 
-  override def eval(input: InternalRow): Any = {
+  override protected def evalInternal(input: InternalRow): Any = {
     var i = 0
     while (i < argExprs.length) {
       buffer(i) = argExprs(i).eval(input).asInstanceOf[Object]
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
index 4d218b936b3a2..a8031086d82f7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
@@ -27,6 +27,7 @@ package org.apache.spark.sql.catalyst.expressions
  * The following rules are applied:
  *  - Names and nullability hints for [[org.apache.spark.sql.types.DataType]]s are stripped.
  *  - Names for [[GetStructField]] are stripped.
+ *  - TimeZoneId for [[Cast]] and [[AnsiCast]] are stripped if `needsTimeZone` is false.
  *  - Commutative and associative operations ([[Add]] and [[Multiply]]) have their children ordered
  *    by `hashCode`.
  *  - [[EqualTo]] and [[EqualNullSafe]] are reordered by `hashCode`.
@@ -35,7 +36,7 @@ package org.apache.spark.sql.catalyst.expressions
  */
 object Canonicalize {
   def execute(e: Expression): Expression = {
-    expressionReorder(ignoreNamesTypes(e))
+    expressionReorder(ignoreTimeZone(ignoreNamesTypes(e)))
   }
 
   /** Remove names and nullability from types, and names from `GetStructField`. */
@@ -46,6 +47,13 @@ object Canonicalize {
     case _ => e
   }
 
+  /** Remove TimeZoneId for Cast if needsTimeZone return false. */
+  private[expressions] def ignoreTimeZone(e: Expression): Expression = e match {
+    case c: CastBase if c.timeZoneId.nonEmpty && !c.needsTimeZone =>
+      c.withTimeZone(null)
+    case _ => e
+  }
+
   /** Collects adjacent commutative operations. */
   private def gatherCommutative(
       e: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 05b4fbef2b697..388fb231f3ce3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -108,6 +108,7 @@ object Cast {
    */
   def needsTimeZone(from: DataType, to: DataType): Boolean = (from, to) match {
     case (StringType, TimestampType | DateType) => true
+    case (DateType, StringType) => true
     case (DateType, TimestampType) => true
     case (TimestampType, StringType) => true
     case (TimestampType, DateType) => true
@@ -206,8 +207,13 @@ object Cast {
     case _ => false  // overflow
   }
 
+  /**
+   * Returns `true` if casting non-nullable values from `from` type to `to` type
+   * may return null. Note that the caller side should take care of input nullability
+   * first and only call this method if the input is not nullable.
+   */
   def forceNullable(from: DataType, to: DataType): Boolean = (from, to) match {
-    case (NullType, _) => true
+    case (NullType, _) => false // empty array or map case
     case (_, _) if from == to => false
 
     case (StringType, BinaryType) => false
@@ -251,7 +257,10 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
 
   def dataType: DataType
 
-  override def toString: String = s"cast($child as ${dataType.simpleString})"
+  override def toString: String = {
+    val ansi = if (ansiEnabled) "ansi_" else ""
+    s"${ansi}cast($child as ${dataType.simpleString})"
+  }
 
   override def checkInputDataTypes(): TypeCheckResult = {
     if (Cast.canCast(child.dataType, dataType)) {
@@ -262,7 +271,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     }
   }
 
-  override def nullable: Boolean = Cast.forceNullable(child.dataType, dataType) || child.nullable
+  override def nullable: Boolean = child.nullable || Cast.forceNullable(child.dataType, dataType)
 
   protected def ansiEnabled: Boolean
 
@@ -271,7 +280,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
   override lazy val resolved: Boolean =
     childrenResolved && checkInputDataTypes().isSuccess && (!needsTimeZone || timeZoneId.isDefined)
 
-  private[this] def needsTimeZone: Boolean = Cast.needsTimeZone(child.dataType, dataType)
+  def needsTimeZone: Boolean = Cast.needsTimeZone(child.dataType, dataType)
 
   // [[func]] assumes the input is no longer null because eval already does the null check.
   @inline private[this] def buildCast[T](a: Any, func: T => Any): Any = func(a.asInstanceOf[T])
@@ -503,7 +512,14 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     case DateType =>
       buildCast[Int](_, d => null)
     case TimestampType if ansiEnabled =>
-      buildCast[Long](_, t => LongExactNumeric.toInt(timestampToLong(t)))
+      buildCast[Long](_, t => {
+        val longValue = timestampToLong(t)
+        if (longValue == longValue.toInt) {
+          longValue.toInt
+        } else {
+          throw new ArithmeticException(s"Casting $t to int causes overflow")
+        }
+      })
     case TimestampType =>
       buildCast[Long](_, t => timestampToLong(t).toInt)
     case x: NumericType if ansiEnabled =>
@@ -1138,7 +1154,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       case StringType =>
         (c, evPrim, evNull) =>
           val handleException = if (ansiEnabled) {
-            s"""throw new NumberFormatException("invalid input syntax for type numeric: $c");"""
+            s"""throw new NumberFormatException("invalid input syntax for type numeric: " + $c);"""
           } else {
             s"$evNull =true;"
           }
@@ -1498,7 +1514,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         val floatStr = ctx.freshVariable("floatStr", StringType)
         (c, evPrim, evNull) =>
           val handleNull = if (ansiEnabled) {
-            s"""throw new NumberFormatException("invalid input syntax for type numeric: $c");"""
+            s"""throw new NumberFormatException("invalid input syntax for type numeric: " + $c);"""
           } else {
             s"$evNull = true;"
           }
@@ -1534,7 +1550,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         val doubleStr = ctx.freshVariable("doubleStr", StringType)
         (c, evPrim, evNull) =>
           val handleNull = if (ansiEnabled) {
-            s"""throw new NumberFormatException("invalid input syntax for type numeric: $c");"""
+            s"""throw new NumberFormatException("invalid input syntax for type numeric: " + $c);"""
           } else {
             s"$evNull = true;"
           }
@@ -1700,6 +1716,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
   """)
 case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String] = None)
   extends CastBase {
+
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
 
@@ -1716,6 +1733,7 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
  */
 case class AnsiCast(child: Expression, dataType: DataType, timeZoneId: Option[String] = None)
   extends CastBase {
+
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
 
@@ -1725,8 +1743,16 @@ case class AnsiCast(child: Expression, dataType: DataType, timeZoneId: Option[St
 /**
  * Cast the child expression to the target data type, but will throw error if the cast might
  * truncate, e.g. long -> int, timestamp -> data.
+ *
+ * Note: `target` is `AbstractDataType`, so that we can put `object DecimalType`, which means
+ * we accept `DecimalType` with any valid precision/scale.
  */
-case class UpCast(child: Expression, dataType: DataType, walkedTypePath: Seq[String] = Nil)
+case class UpCast(child: Expression, target: AbstractDataType, walkedTypePath: Seq[String] = Nil)
   extends UnaryExpression with Unevaluable {
   override lazy val resolved = false
+
+  def dataType: DataType = target match {
+    case DecimalType => DecimalType.SYSTEM_DEFAULT
+    case _ => target.asInstanceOf[DataType]
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala
index 07fa813a98922..3b7219477bd69 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala
@@ -51,9 +51,8 @@ abstract class CodeGeneratorWithInterpretedFallback[IN, OUT] extends Logging {
         try {
           createCodeGeneratedObject(in)
         } catch {
-          case NonFatal(_) =>
-            // We should have already seen the error message in `CodeGenerator`
-            logWarning("Expr codegen error and falling back to interpreter mode")
+          case NonFatal(e) =>
+            logWarning("Expr codegen error and falling back to interpreter mode", e)
             createInterpretedObject(in)
         }
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index 4632957e7afdd..f29ece2e03b08 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import java.util.Locale
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult, TypeCoercion}
 import org.apache.spark.sql.catalyst.expressions.aggregate.DeclarativeAggregate
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
@@ -258,9 +258,10 @@ abstract class Expression extends TreeNode[Expression] {
    * Returns a user-facing string representation of this expression's name.
    * This should usually match the name of the function in SQL.
    */
-  def prettyName: String = nodeName.toLowerCase(Locale.ROOT)
+  def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse(nodeName.toLowerCase(Locale.ROOT))
 
-  protected def flatArguments: Iterator[Any] = productIterator.flatMap {
+  protected def flatArguments: Iterator[Any] = stringArgs.flatMap {
     case t: Iterable[_] => t
     case single => single :: Nil
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index 10f8ec9617d1b..58a9f68bff240 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -17,11 +17,15 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import scala.collection.mutable
+
 import org.apache.spark.SparkException
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, ScalaReflection}
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.types.{AbstractDataType, DataType}
+import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType, UserDefinedType}
+import org.apache.spark.util.Utils
 
 /**
  * User-defined function.
@@ -31,14 +35,9 @@ import org.apache.spark.sql.types.{AbstractDataType, DataType}
  *                  null. Use boxed type or [[Option]] if you wanna do the null-handling yourself.
  * @param dataType  Return type of function.
  * @param children  The input expressions of this UDF.
- * @param inputPrimitives The analyzer should be aware of Scala primitive types so as to make the
- *                        UDF return null if there is any null input value of these types. On the
- *                        other hand, Java UDFs can only have boxed types, thus this parameter will
- *                        always be all false.
- * @param inputTypes  The expected input types of this UDF, used to perform type coercion. If we do
- *                    not want to perform coercion, simply use "Nil". Note that it would've been
- *                    better to use Option of Seq[DataType] so we can use "None" as the case for no
- *                    type coercion. However, that would require more refactoring of the codebase.
+ * @param inputEncoders ExpressionEncoder for each input parameters. For a input parameter which
+ *                      serialized as struct will use encoder instead of CatalystTypeConverters to
+ *                      convert internal value to Scala value.
  * @param udfName  The user-specified name of this UDF.
  * @param nullable  True if the UDF can return null value.
  * @param udfDeterministic  True if the UDF is deterministic. Deterministic UDF returns same result
@@ -48,8 +47,7 @@ case class ScalaUDF(
     function: AnyRef,
     dataType: DataType,
     children: Seq[Expression],
-    inputPrimitives: Seq[Boolean],
-    inputTypes: Seq[AbstractDataType] = Nil,
+    inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil,
     udfName: Option[String] = None,
     nullable: Boolean = true,
     udfDeterministic: Boolean = true)
@@ -59,6 +57,68 @@ case class ScalaUDF(
 
   override def toString: String = s"${udfName.getOrElse("UDF")}(${children.mkString(", ")})"
 
+  /**
+   * The analyzer should be aware of Scala primitive types so as to make the
+   * UDF return null if there is any null input value of these types. On the
+   * other hand, Java UDFs can only have boxed types, thus this will return
+   * Nil(has same effect with all false) and analyzer will skip null-handling
+   * on them.
+   */
+  def inputPrimitives: Seq[Boolean] = {
+    inputEncoders.map { encoderOpt =>
+      // It's possible that some of the inputs don't have a specific encoder(e.g. `Any`)
+      if (encoderOpt.isDefined) {
+        val encoder = encoderOpt.get
+        if (encoder.isSerializedAsStruct) {
+          // struct type is not primitive
+          false
+        } else {
+          // `nullable` is false iff the type is primitive
+          !encoder.schema.head.nullable
+        }
+      } else {
+        // Any type is not primitive
+        false
+      }
+    }
+  }
+
+  /**
+   * The expected input types of this UDF, used to perform type coercion. If we do
+   * not want to perform coercion, simply use "Nil". Note that it would've been
+   * better to use Option of Seq[DataType] so we can use "None" as the case for no
+   * type coercion. However, that would require more refactoring of the codebase.
+   */
+  def inputTypes: Seq[AbstractDataType] = {
+    inputEncoders.map { encoderOpt =>
+      if (encoderOpt.isDefined) {
+        val encoder = encoderOpt.get
+        if (encoder.isSerializedAsStruct) {
+          encoder.schema
+        } else {
+          encoder.schema.head.dataType
+        }
+      } else {
+        AnyDataType
+      }
+    }
+  }
+
+  private def createToScalaConverter(i: Int, dataType: DataType): Any => Any = {
+    if (inputEncoders.isEmpty) {
+      // for untyped Scala UDF
+      CatalystTypeConverters.createToScalaConverter(dataType)
+    } else {
+      val encoder = inputEncoders(i)
+      if (encoder.isDefined && encoder.get.isSerializedAsStructForTopLevel) {
+        val fromRow = encoder.get.resolveAndBind().createDeserializer()
+        row: Any => fromRow(row.asInstanceOf[InternalRow])
+      } else {
+        CatalystTypeConverters.createToScalaConverter(dataType)
+      }
+    }
+  }
+
   // scalastyle:off line.size.limit
 
   /** This method has been generated by this script
@@ -66,7 +126,7 @@ case class ScalaUDF(
     (1 to 22).map { x =>
       val anys = (1 to x).map(x => "Any").reduce(_ + ", " + _)
       val childs = (0 to x - 1).map(x => s"val child$x = children($x)").reduce(_ + "\n  " + _)
-      val converters = (0 to x - 1).map(x => s"lazy val converter$x = CatalystTypeConverters.createToScalaConverter(child$x.dataType)").reduce(_ + "\n  " + _)
+      val converters = (0 to x - 1).map(x => s"lazy val converter$x = createToScalaConverter($x, child$x.dataType)").reduce(_ + "\n  " + _)
       val evals = (0 to x - 1).map(x => s"converter$x(child$x.eval(input))").reduce(_ + ",\n      " + _)
 
       s"""case $x =>
@@ -91,7 +151,7 @@ case class ScalaUDF(
     case 1 =>
       val func = function.asInstanceOf[(Any) => Any]
       val child0 = children(0)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)))
@@ -101,8 +161,8 @@ case class ScalaUDF(
       val func = function.asInstanceOf[(Any, Any) => Any]
       val child0 = children(0)
       val child1 = children(1)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -114,9 +174,9 @@ case class ScalaUDF(
       val child0 = children(0)
       val child1 = children(1)
       val child2 = children(2)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -130,10 +190,10 @@ case class ScalaUDF(
       val child1 = children(1)
       val child2 = children(2)
       val child3 = children(3)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -149,11 +209,11 @@ case class ScalaUDF(
       val child2 = children(2)
       val child3 = children(3)
       val child4 = children(4)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -171,12 +231,12 @@ case class ScalaUDF(
       val child3 = children(3)
       val child4 = children(4)
       val child5 = children(5)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -196,13 +256,13 @@ case class ScalaUDF(
       val child4 = children(4)
       val child5 = children(5)
       val child6 = children(6)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -224,14 +284,14 @@ case class ScalaUDF(
       val child5 = children(5)
       val child6 = children(6)
       val child7 = children(7)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -255,15 +315,15 @@ case class ScalaUDF(
       val child6 = children(6)
       val child7 = children(7)
       val child8 = children(8)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -289,16 +349,16 @@ case class ScalaUDF(
       val child7 = children(7)
       val child8 = children(8)
       val child9 = children(9)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -326,17 +386,17 @@ case class ScalaUDF(
       val child8 = children(8)
       val child9 = children(9)
       val child10 = children(10)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -366,18 +426,18 @@ case class ScalaUDF(
       val child9 = children(9)
       val child10 = children(10)
       val child11 = children(11)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -409,19 +469,19 @@ case class ScalaUDF(
       val child10 = children(10)
       val child11 = children(11)
       val child12 = children(12)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -455,20 +515,20 @@ case class ScalaUDF(
       val child11 = children(11)
       val child12 = children(12)
       val child13 = children(13)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -504,21 +564,21 @@ case class ScalaUDF(
       val child12 = children(12)
       val child13 = children(13)
       val child14 = children(14)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -556,22 +616,22 @@ case class ScalaUDF(
       val child13 = children(13)
       val child14 = children(14)
       val child15 = children(15)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -611,23 +671,23 @@ case class ScalaUDF(
       val child14 = children(14)
       val child15 = children(15)
       val child16 = children(16)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -669,24 +729,24 @@ case class ScalaUDF(
       val child15 = children(15)
       val child16 = children(16)
       val child17 = children(17)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
-      lazy val converter17 = CatalystTypeConverters.createToScalaConverter(child17.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
+      lazy val converter17 = createToScalaConverter(17, child17.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -730,25 +790,25 @@ case class ScalaUDF(
       val child16 = children(16)
       val child17 = children(17)
       val child18 = children(18)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
-      lazy val converter17 = CatalystTypeConverters.createToScalaConverter(child17.dataType)
-      lazy val converter18 = CatalystTypeConverters.createToScalaConverter(child18.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
+      lazy val converter17 = createToScalaConverter(17, child17.dataType)
+      lazy val converter18 = createToScalaConverter(18, child18.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -794,26 +854,26 @@ case class ScalaUDF(
       val child17 = children(17)
       val child18 = children(18)
       val child19 = children(19)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
-      lazy val converter17 = CatalystTypeConverters.createToScalaConverter(child17.dataType)
-      lazy val converter18 = CatalystTypeConverters.createToScalaConverter(child18.dataType)
-      lazy val converter19 = CatalystTypeConverters.createToScalaConverter(child19.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
+      lazy val converter17 = createToScalaConverter(17, child17.dataType)
+      lazy val converter18 = createToScalaConverter(18, child18.dataType)
+      lazy val converter19 = createToScalaConverter(19, child19.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -861,27 +921,27 @@ case class ScalaUDF(
       val child18 = children(18)
       val child19 = children(19)
       val child20 = children(20)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
-      lazy val converter17 = CatalystTypeConverters.createToScalaConverter(child17.dataType)
-      lazy val converter18 = CatalystTypeConverters.createToScalaConverter(child18.dataType)
-      lazy val converter19 = CatalystTypeConverters.createToScalaConverter(child19.dataType)
-      lazy val converter20 = CatalystTypeConverters.createToScalaConverter(child20.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
+      lazy val converter17 = createToScalaConverter(17, child17.dataType)
+      lazy val converter18 = createToScalaConverter(18, child18.dataType)
+      lazy val converter19 = createToScalaConverter(19, child19.dataType)
+      lazy val converter20 = createToScalaConverter(20, child20.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -931,28 +991,28 @@ case class ScalaUDF(
       val child19 = children(19)
       val child20 = children(20)
       val child21 = children(21)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
-      lazy val converter17 = CatalystTypeConverters.createToScalaConverter(child17.dataType)
-      lazy val converter18 = CatalystTypeConverters.createToScalaConverter(child18.dataType)
-      lazy val converter19 = CatalystTypeConverters.createToScalaConverter(child19.dataType)
-      lazy val converter20 = CatalystTypeConverters.createToScalaConverter(child20.dataType)
-      lazy val converter21 = CatalystTypeConverters.createToScalaConverter(child21.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
+      lazy val converter17 = createToScalaConverter(17, child17.dataType)
+      lazy val converter18 = createToScalaConverter(18, child18.dataType)
+      lazy val converter19 = createToScalaConverter(19, child19.dataType)
+      lazy val converter20 = createToScalaConverter(20, child20.dataType)
+      lazy val converter21 = createToScalaConverter(21, child21.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -987,8 +1047,8 @@ case class ScalaUDF(
     val converterClassName = classOf[Any => Any].getName
 
     // The type converters for inputs and the result.
-    val converters: Array[Any => Any] = children.map { c =>
-      CatalystTypeConverters.createToScalaConverter(c.dataType)
+    val converters: Array[Any => Any] = children.zipWithIndex.map { case (c, i) =>
+      createToScalaConverter(i, c.dataType)
     }.toArray :+ CatalystTypeConverters.createToCatalystConverter(dataType)
     val convertersTerm = ctx.addReferenceObj("converters", converters, s"$converterClassName[]")
     val errorMsgTerm = ctx.addReferenceObj("errMsg", udfErrorMessage)
@@ -1056,7 +1116,7 @@ case class ScalaUDF(
   private[this] val resultConverter = CatalystTypeConverters.createToCatalystConverter(dataType)
 
   lazy val udfErrorMessage = {
-    val funcCls = function.getClass.getSimpleName
+    val funcCls = Utils.getSimpleName(function.getClass)
     val inputTypes = children.map(_.dataType.catalogString).mkString(", ")
     val outputType = dataType.catalogString
     s"Failed to execute user defined function($funcCls: ($inputTypes) => $outputType)"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
index caacb71814f17..82d689477080d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
@@ -63,6 +63,7 @@ case class TimeWindow(
   override def dataType: DataType = new StructType()
     .add(StructField("start", TimestampType))
     .add(StructField("end", TimestampType))
+  override def prettyName: String = "window"
 
   // This expression is replaced in the analyzer.
   override lazy val resolved = false
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala
index f96a087972f1f..103f55e58febd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala
@@ -39,7 +39,8 @@ import org.apache.spark.unsafe.Platform
  *                            and its elements should be sorted into ascending order.
  *                            Duplicate endpoints are allowed, e.g. (1, 5, 5, 10), and ndv for
  *                            interval (5, 5] would be 1.
- * @param relativeSD The maximum estimation error allowed in the HyperLogLogPlusPlus algorithm.
+ * @param relativeSD The maximum relative standard deviation allowed
+ *                   in the HyperLogLogPlusPlus algorithm.
  */
 case class ApproxCountDistinctForIntervals(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
index f8060956df875..3327f4ccf4461 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
@@ -49,22 +49,25 @@ import org.apache.spark.sql.types._
  */
 @ExpressionDescription(
   usage = """
-    _FUNC_(col, percentage [, accuracy]) - Returns the approximate percentile value of numeric
-      column `col` at the given percentage. The value of percentage must be between 0.0
-      and 1.0. The `accuracy` parameter (default: 10000) is a positive numeric literal which
-      controls approximation accuracy at the cost of memory. Higher value of `accuracy` yields
-      better accuracy, `1.0/accuracy` is the relative error of the approximation.
+    _FUNC_(col, percentage [, accuracy]) - Returns the approximate `percentile` of the numeric
+      column `col` which is the smallest value in the ordered `col` values (sorted from least to
+      greatest) such that no more than `percentage` of `col` values is less than the value
+      or equal to that value. The value of percentage must be between 0.0 and 1.0. The `accuracy`
+      parameter (default: 10000) is a positive numeric literal which controls approximation accuracy
+      at the cost of memory. Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is
+      the relative error of the approximation.
       When `percentage` is an array, each value of the percentage array must be between 0.0 and 1.0.
       In this case, returns the approximate percentile array of column `col` at the given
       percentage array.
   """,
   examples = """
     Examples:
-      > SELECT _FUNC_(10.0, array(0.5, 0.4, 0.1), 100);
-       [10.0,10.0,10.0]
-      > SELECT _FUNC_(10.0, 0.5, 100);
-       10.0
+      > SELECT _FUNC_(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col);
+       [1,1,0]
+      > SELECT _FUNC_(col, 0.5, 100) FROM VALUES (0), (6), (7), (9), (10) AS tab(col);
+       7
   """,
+  group = "agg_funcs",
   since = "2.1.0")
 case class ApproximatePercentile(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
index 996c548e1329c..d3ce1f8d331ab 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
@@ -17,9 +17,10 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
-import org.apache.spark.sql.catalyst.analysis.{DecimalPrecision, FunctionRegistry}
+import org.apache.spark.sql.catalyst.analysis.{DecimalPrecision, FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.types._
 
 @ExpressionDescription(
@@ -30,9 +31,8 @@ import org.apache.spark.sql.types._
        2.0
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (NULL) AS tab(col);
        1.5
-      > SELECT _FUNC_(cast(v as interval)) FROM VALUES ('-1 weeks'), ('2 seconds'), (null) t(v);
-       -3 days -11 hours -59 minutes -59 seconds
   """,
+  group = "agg_funcs",
   since = "1.0.0")
 case class Average(child: Expression) extends DeclarativeAggregate with ImplicitCastInputTypes {
 
@@ -40,7 +40,10 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
 
   override def children: Seq[Expression] = child :: Nil
 
-  override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection.NumericAndInterval)
+  override def inputTypes: Seq[AbstractDataType] = Seq(NumericType)
+
+  override def checkInputDataTypes(): TypeCheckResult =
+    TypeUtils.checkForNumericExpr(child.dataType, "function average")
 
   override def nullable: Boolean = true
 
@@ -50,13 +53,11 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
   private lazy val resultType = child.dataType match {
     case DecimalType.Fixed(p, s) =>
       DecimalType.bounded(p + 4, s + 4)
-    case interval: CalendarIntervalType => interval
     case _ => DoubleType
   }
 
   private lazy val sumDataType = child.dataType match {
     case _ @ DecimalType.Fixed(p, s) => DecimalType.bounded(p + 10, s)
-    case interval: CalendarIntervalType => interval
     case _ => DoubleType
   }
 
@@ -79,9 +80,6 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
   override lazy val evaluateExpression = child.dataType match {
     case _: DecimalType =>
       DecimalPrecision.decimalAndDecimal(sum / count.cast(DecimalType.LongDecimal)).cast(resultType)
-    case CalendarIntervalType =>
-      val newCount = If(EqualTo(count, Literal(0L)), Literal(null, LongType), count)
-      DivideInterval(sum.cast(resultType), newCount.cast(DoubleType))
     case _ =>
       sum.cast(resultType) / count.cast(resultType)
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala
index bf402807d62d3..53759ca3d9165 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala
@@ -142,6 +142,7 @@ abstract class CentralMomentAgg(child: Expression)
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col);
        0.816496580927726
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 // scalastyle:on line.size.limit
 case class StddevPop(child: Expression) extends CentralMomentAgg(child) {
@@ -164,6 +165,7 @@ case class StddevPop(child: Expression) extends CentralMomentAgg(child) {
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col);
        1.0
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 // scalastyle:on line.size.limit
 case class StddevSamp(child: Expression) extends CentralMomentAgg(child) {
@@ -187,6 +189,7 @@ case class StddevSamp(child: Expression) extends CentralMomentAgg(child) {
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col);
        0.6666666666666666
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 case class VariancePop(child: Expression) extends CentralMomentAgg(child) {
 
@@ -207,6 +210,7 @@ case class VariancePop(child: Expression) extends CentralMomentAgg(child) {
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col);
        1.0
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) {
 
@@ -229,6 +233,7 @@ case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) {
       > SELECT _FUNC_(col) FROM VALUES (-1000), (-100), (10), (20) AS tab(col);
        -1.1135657469022011
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 case class Skewness(child: Expression) extends CentralMomentAgg(child) {
 
@@ -251,6 +256,7 @@ case class Skewness(child: Expression) extends CentralMomentAgg(child) {
       > SELECT _FUNC_(col) FROM VALUES (1), (10), (100), (10), (1) as tab(col);
        0.19432323191699075
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 case class Kurtosis(child: Expression) extends CentralMomentAgg(child) {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala
index 91446e05d853d..9ef05bb5d4fec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala
@@ -99,6 +99,7 @@ abstract class PearsonCorrelation(x: Expression, y: Expression)
       > SELECT _FUNC_(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2);
        0.8660254037844387
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 // scalastyle:on line.size.limit
 case class Corr(x: Expression, y: Expression)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
index 2a8edac502c0f..e043c81975066 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
@@ -39,6 +39,7 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(DISTINCT col) FROM VALUES (NULL), (5), (5), (10) AS tab(col);
        2
   """,
+  group = "agg_funcs",
   since = "1.0.0")
 // scalastyle:on line.size.limit
 case class Count(children: Seq[Expression]) extends DeclarativeAggregate {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala
index d31355cd022fa..5bb95ead3f715 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.types.{AbstractDataType, BooleanType, DataType, Long
       > SELECT _FUNC_(col IS NULL) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col);
        1
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class CountIf(predicate: Expression) extends UnevaluableAggregate with ImplicitCastInputTypes {
   override def prettyName: String = "count_if"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala
index 4bd13cf284935..787b21859c6da 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala
@@ -44,6 +44,7 @@ import org.apache.spark.util.sketch.CountMinSketch
       `CountMinSketch` before usage. Count-min sketch is a probabilistic data structure used for
       cardinality estimation using sub-linear space.
   """,
+  group = "agg_funcs",
   since = "2.2.0")
 case class CountMinSketchAgg(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala
index ac99fa8049f93..f03c2f2710a04 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala
@@ -86,6 +86,7 @@ abstract class Covariance(x: Expression, y: Expression)
       > SELECT _FUNC_(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2);
        0.6666666666666666
   """,
+  group = "agg_funcs",
   since = "2.0.0")
 case class CovPopulation(left: Expression, right: Expression) extends Covariance(left, right) {
   override val evaluateExpression: Expression = {
@@ -102,6 +103,7 @@ case class CovPopulation(left: Expression, right: Expression) extends Covariance
       > SELECT _FUNC_(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2);
        1.0
   """,
+  group = "agg_funcs",
   since = "2.0.0")
 case class CovSample(left: Expression, right: Expression) extends Covariance(left, right) {
   override val evaluateExpression: Expression = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
index 210acf33fc431..65fd43c924d08 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
@@ -17,8 +17,9 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
-import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckSuccess
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.types._
@@ -47,13 +48,18 @@ import org.apache.spark.sql.types._
     The function is non-deterministic because its results depends on the order of the rows
     which may be non-deterministic after a shuffle.
   """,
+  group = "agg_funcs",
   since = "2.0.0")
-case class First(child: Expression, ignoreNullsExpr: Expression)
+case class First(child: Expression, ignoreNulls: Boolean)
   extends DeclarativeAggregate with ExpectsInputTypes {
 
-  def this(child: Expression) = this(child, Literal.create(false, BooleanType))
+  def this(child: Expression) = this(child, false)
 
-  override def children: Seq[Expression] = child :: ignoreNullsExpr :: Nil
+  def this(child: Expression, ignoreNullsExpr: Expression) = {
+    this(child, FirstLast.validateIgnoreNullExpr(ignoreNullsExpr, "first"))
+  }
+
+  override def children: Seq[Expression] = child :: Nil
 
   override def nullable: Boolean = true
 
@@ -70,16 +76,11 @@ case class First(child: Expression, ignoreNullsExpr: Expression)
     val defaultCheck = super.checkInputDataTypes()
     if (defaultCheck.isFailure) {
       defaultCheck
-    } else if (!ignoreNullsExpr.foldable) {
-      TypeCheckFailure(
-        s"The second argument of First must be a boolean literal, but got: ${ignoreNullsExpr.sql}")
     } else {
       TypeCheckSuccess
     }
   }
 
-  private def ignoreNulls: Boolean = ignoreNullsExpr.eval().asInstanceOf[Boolean]
-
   private lazy val first = AttributeReference("first", child.dataType)()
 
   private lazy val valueSet = AttributeReference("valueSet", BooleanType)()
@@ -117,7 +118,13 @@ case class First(child: Expression, ignoreNullsExpr: Expression)
 
   override lazy val evaluateExpression: AttributeReference = first
 
-  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("first")
-
   override def toString: String = s"$prettyName($child)${if (ignoreNulls) " ignore nulls"}"
 }
+
+object FirstLast {
+  def validateIgnoreNullExpr(exp: Expression, funcName: String): Boolean = exp match {
+    case Literal(b: Boolean, BooleanType) => b
+    case _ => throw new AnalysisException(
+      s"The second argument in $funcName should be a boolean literal.")
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
index e3c628e70d11b..b3cc9a31fec2f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
@@ -41,18 +41,19 @@ import org.apache.spark.sql.types._
  * https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen#
  *
  * @param child to estimate the cardinality of.
- * @param relativeSD the maximum estimation error allowed.
+ * @param relativeSD the maximum relative standard deviation allowed.
  */
 // scalastyle:on
 @ExpressionDescription(
   usage = """
     _FUNC_(expr[, relativeSD]) - Returns the estimated cardinality by HyperLogLog++.
-      `relativeSD` defines the maximum estimation error allowed.""",
+      `relativeSD` defines the maximum relative standard deviation allowed.""",
   examples = """
     Examples:
       > SELECT _FUNC_(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1);
        3
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 case class HyperLogLogPlusPlus(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
index 2c89a4b973a7e..8d17a48a69f6f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
@@ -17,8 +17,8 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
-import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckSuccess
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.types._
@@ -47,13 +47,18 @@ import org.apache.spark.sql.types._
     The function is non-deterministic because its results depends on the order of the rows
     which may be non-deterministic after a shuffle.
   """,
+  group = "agg_funcs",
   since = "2.0.0")
-case class Last(child: Expression, ignoreNullsExpr: Expression)
+case class Last(child: Expression, ignoreNulls: Boolean)
   extends DeclarativeAggregate with ExpectsInputTypes {
 
-  def this(child: Expression) = this(child, Literal.create(false, BooleanType))
+  def this(child: Expression) = this(child, false)
 
-  override def children: Seq[Expression] = child :: ignoreNullsExpr :: Nil
+  def this(child: Expression, ignoreNullsExpr: Expression) = {
+    this(child, FirstLast.validateIgnoreNullExpr(ignoreNullsExpr, "last"))
+  }
+
+  override def children: Seq[Expression] = child :: Nil
 
   override def nullable: Boolean = true
 
@@ -70,16 +75,11 @@ case class Last(child: Expression, ignoreNullsExpr: Expression)
     val defaultCheck = super.checkInputDataTypes()
     if (defaultCheck.isFailure) {
       defaultCheck
-    } else if (!ignoreNullsExpr.foldable) {
-      TypeCheckFailure(
-        s"The second argument of Last must be a boolean literal, but got: ${ignoreNullsExpr.sql}")
     } else {
       TypeCheckSuccess
     }
   }
 
-  private def ignoreNulls: Boolean = ignoreNullsExpr.eval().asInstanceOf[Boolean]
-
   private lazy val last = AttributeReference("last", child.dataType)()
 
   private lazy val valueSet = AttributeReference("valueSet", BooleanType)()
@@ -115,7 +115,5 @@ case class Last(child: Expression, ignoreNullsExpr: Expression)
 
   override lazy val evaluateExpression: AttributeReference = last
 
-  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("last")
-
   override def toString: String = s"$prettyName($child)${if (ignoreNulls) " ignore nulls"}"
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala
index 7520db146ba6a..9bba6604c84ac 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(col) FROM VALUES (10), (50), (20) AS tab(col);
        50
   """,
+  group = "agg_funcs",
   since = "1.0.0")
 case class Max(child: Expression) extends DeclarativeAggregate {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
index b69b341b0ee3e..6d3d3dafe16e4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
@@ -31,7 +31,6 @@ abstract class MaxMinBy extends DeclarativeAggregate {
   def valueExpr: Expression
   def orderingExpr: Expression
 
-  protected def funcName: String
   // The predicate compares two ordering values.
   protected def predicate(oldExpr: Expression, newExpr: Expression): Expression
   // The arithmetic expression returns greatest/least value of all parameters.
@@ -46,7 +45,7 @@ abstract class MaxMinBy extends DeclarativeAggregate {
   override def dataType: DataType = valueExpr.dataType
 
   override def checkInputDataTypes(): TypeCheckResult =
-    TypeUtils.checkForOrderingExpr(orderingExpr.dataType, s"function $funcName")
+    TypeUtils.checkForOrderingExpr(orderingExpr.dataType, s"function $prettyName")
 
   // The attributes used to keep extremum (max or min) and associated aggregated values.
   private lazy val extremumOrdering =
@@ -98,9 +97,11 @@ abstract class MaxMinBy extends DeclarativeAggregate {
       > SELECT _FUNC_(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y);
        b
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class MaxBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMinBy {
-  override protected def funcName: String = "max_by"
+
+  override def prettyName: String = "max_by"
 
   override protected def predicate(oldExpr: Expression, newExpr: Expression): Expression =
     oldExpr > newExpr
@@ -116,9 +117,11 @@ case class MaxBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMin
       > SELECT _FUNC_(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y);
        a
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class MinBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMinBy {
-  override protected def funcName: String = "min_by"
+
+  override def prettyName: String = "min_by"
 
   override protected def predicate(oldExpr: Expression, newExpr: Expression): Expression =
     oldExpr < newExpr
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala
index 106eb968e3917..1d861aa0dd8cf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(col) FROM VALUES (10), (-1), (20) AS tab(col);
        -1
   """,
+  group = "agg_funcs",
   since = "1.0.0")
 case class Min(child: Expression) extends DeclarativeAggregate {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
index 0f1c0fb5fcb69..0eba61c741133 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
@@ -62,6 +62,7 @@ import org.apache.spark.util.collection.OpenHashMap
       > SELECT _FUNC_(col, array(0.25, 0.75)) FROM VALUES (0), (10) AS tab(col);
        [2.5,7.5]
   """,
+  group = "agg_funcs",
   since = "2.1.0")
 case class Percentile(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
index 87f1a4f02e4fc..d2daaac72fc85 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
@@ -17,12 +17,13 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
-// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns the sum calculated from values of a group.",
   examples = """
@@ -33,11 +34,9 @@ import org.apache.spark.sql.types._
        25
       > SELECT _FUNC_(col) FROM VALUES (NULL), (NULL) AS tab(col);
        NULL
-      > SELECT _FUNC_(cast(col as interval)) FROM VALUES ('1 seconds'), ('2 seconds'), (null) tab(col);
-       3 seconds
   """,
+  group = "agg_funcs",
   since = "1.0.0")
-// scalastyle:on line.size.limit
 case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCastInputTypes {
 
   override def children: Seq[Expression] = child :: Nil
@@ -47,12 +46,14 @@ case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCast
   // Return data type.
   override def dataType: DataType = resultType
 
-  override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection.NumericAndInterval)
+  override def inputTypes: Seq[AbstractDataType] = Seq(NumericType)
+
+  override def checkInputDataTypes(): TypeCheckResult =
+    TypeUtils.checkForNumericExpr(child.dataType, "function sum")
 
   private lazy val resultType = child.dataType match {
     case DecimalType.Fixed(precision, scale) =>
       DecimalType.bounded(precision + 10, scale)
-    case _: CalendarIntervalType => CalendarIntervalType
     case _: IntegralType => LongType
     case _ => DoubleType
   }
@@ -61,7 +62,7 @@ case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCast
 
   private lazy val sum = AttributeReference("sum", sumDataType)()
 
-  private lazy val zero = Literal.default(resultType)
+  private lazy val zero = Literal.default(sumDataType)
 
   override lazy val aggBufferAttributes = sum :: Nil
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala
index a1cd4a77d0445..cb77ded3372a2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala
@@ -51,6 +51,7 @@ abstract class UnevaluableBooleanAggBase(arg: Expression)
       > SELECT _FUNC_(col) FROM VALUES (true), (false), (true) AS tab(col);
        false
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class BoolAnd(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
   override def nodeName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("bool_and")
@@ -67,6 +68,7 @@ case class BoolAnd(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
       > SELECT _FUNC_(col) FROM VALUES (false), (false), (NULL) AS tab(col);
        false
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class BoolOr(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
   override def nodeName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("bool_or")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala
index b77c3bd9cbde4..b4c1b2c708fb2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala
@@ -77,6 +77,7 @@ case class BitAndAgg(child: Expression) extends BitAggregate {
       > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col);
        7
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class BitOrAgg(child: Expression) extends BitAggregate {
 
@@ -94,6 +95,7 @@ case class BitOrAgg(child: Expression) extends BitAggregate {
       > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col);
        6
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class BitXorAgg(child: Expression) extends BitAggregate {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
index 29f89989b4961..0a3d87623be8b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
@@ -23,6 +23,7 @@ import scala.collection.mutable
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.util.ArrayData
 import org.apache.spark.sql.catalyst.util.GenericArrayData
 import org.apache.spark.sql.types._
 
@@ -46,13 +47,15 @@ abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImper
   // actual order of input rows.
   override lazy val deterministic: Boolean = false
 
+  protected def convertToBufferElement(value: Any): Any
+
   override def update(buffer: T, input: InternalRow): T = {
     val value = child.eval(input)
 
     // Do not allow null values. We follow the semantics of Hive's collect_list/collect_set here.
     // See: org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMkCollectionEvaluator
     if (value != null) {
-      buffer += InternalRow.copyValue(value)
+      buffer += convertToBufferElement(value)
     }
     buffer
   }
@@ -61,12 +64,10 @@ abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImper
     buffer ++= other
   }
 
-  override def eval(buffer: T): Any = {
-    new GenericArrayData(buffer.toArray)
-  }
+  protected val bufferElementType: DataType
 
   private lazy val projection = UnsafeProjection.create(
-    Array[DataType](ArrayType(elementType = child.dataType, containsNull = false)))
+    Array[DataType](ArrayType(elementType = bufferElementType, containsNull = false)))
   private lazy val row = new UnsafeRow(1)
 
   override def serialize(obj: T): Array[Byte] = {
@@ -77,7 +78,7 @@ abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImper
   override def deserialize(bytes: Array[Byte]): T = {
     val buffer = createAggregationBuffer()
     row.pointTo(bytes, bytes.length)
-    row.getArray(0).foreach(child.dataType, (_, x: Any) => buffer += x)
+    row.getArray(0).foreach(bufferElementType, (_, x: Any) => buffer += x)
     buffer
   }
 }
@@ -96,6 +97,7 @@ abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImper
     The function is non-deterministic because the order of collected results depends
     on the order of the rows which may be non-deterministic after a shuffle.
   """,
+  group = "agg_funcs",
   since = "2.0.0")
 case class CollectList(
     child: Expression,
@@ -104,6 +106,10 @@ case class CollectList(
 
   def this(child: Expression) = this(child, 0, 0)
 
+  override lazy val bufferElementType = child.dataType
+
+  override def convertToBufferElement(value: Any): Any = InternalRow.copyValue(value)
+
   override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
     copy(mutableAggBufferOffset = newMutableAggBufferOffset)
 
@@ -113,6 +119,10 @@ case class CollectList(
   override def createAggregationBuffer(): mutable.ArrayBuffer[Any] = mutable.ArrayBuffer.empty
 
   override def prettyName: String = "collect_list"
+
+  override def eval(buffer: mutable.ArrayBuffer[Any]): Any = {
+    new GenericArrayData(buffer.toArray)
+  }
 }
 
 /**
@@ -129,6 +139,7 @@ case class CollectList(
     The function is non-deterministic because the order of collected results depends
     on the order of the rows which may be non-deterministic after a shuffle.
   """,
+  group = "agg_funcs",
   since = "2.0.0")
 case class CollectSet(
     child: Expression,
@@ -137,6 +148,30 @@ case class CollectSet(
 
   def this(child: Expression) = this(child, 0, 0)
 
+  override lazy val bufferElementType = child.dataType match {
+    case BinaryType => ArrayType(ByteType)
+    case other => other
+  }
+
+  override def convertToBufferElement(value: Any): Any = child.dataType match {
+    /*
+     * collect_set() of BinaryType should not return duplicate elements,
+     * Java byte arrays use referential equality and identity hash codes
+     * so we need to use a different catalyst value for arrays
+     */
+    case BinaryType => UnsafeArrayData.fromPrimitiveArray(value.asInstanceOf[Array[Byte]])
+    case _ => InternalRow.copyValue(value)
+  }
+
+  override def eval(buffer: mutable.HashSet[Any]): Any = {
+    val array = child.dataType match {
+      case BinaryType =>
+        buffer.iterator.map(_.asInstanceOf[ArrayData].toByteArray).toArray
+      case _ => buffer.toArray
+    }
+    new GenericArrayData(array)
+  }
+
   override def checkInputDataTypes(): TypeCheckResult = {
     if (!child.dataType.existsRecursively(_.isInstanceOf[MapType])) {
       TypeCheckResult.TypeCheckSuccess
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index debd7c89adb9f..7c521838447d3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult, TypeCoercion}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{IntervalUtils, TypeUtils}
@@ -86,7 +86,12 @@ case class UnaryMinus(child: Expression) extends UnaryExpression
     case _ => numeric.negate(input)
   }
 
-  override def sql: String = s"(- ${child.sql})"
+  override def sql: String = {
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("-") match {
+      case "-" => s"(- ${child.sql})"
+      case funcName => s"$funcName(${child.sql})"
+    }
+  }
 }
 
 @ExpressionDescription(
@@ -395,7 +400,7 @@ case class Divide(left: Expression, right: Expression) extends DivModLike {
 
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "expr1 _FUNC_ expr2 - Divide `expr1` by `expr2`. It returns NULL if an operand is NULL or `expr2` is 0. The result is casted to long if spark.sql.legacy.integralDivide.returnBigint is true, otherwise the data type of the operands is returned.",
+  usage = "expr1 _FUNC_ expr2 - Divide `expr1` by `expr2`. It returns NULL if an operand is NULL or `expr2` is 0. The result is casted to long.",
   examples = """
     Examples:
       > SELECT 3 _FUNC_ 2;
@@ -403,26 +408,17 @@ case class Divide(left: Expression, right: Expression) extends DivModLike {
   """,
   since = "3.0.0")
 // scalastyle:on line.size.limit
-case class IntegralDivide(left: Expression, right: Expression) extends DivModLike {
+case class IntegralDivide(
+    left: Expression,
+    right: Expression) extends DivModLike {
 
-  override def inputType: AbstractDataType = TypeCollection(IntegralType, DecimalType)
+  override def inputType: AbstractDataType = TypeCollection(LongType, DecimalType)
 
-  override def dataType: DataType = if (SQLConf.get.integralDivideReturnLong) {
-    LongType
-  } else {
-    left.dataType
-  }
+  override def dataType: DataType = LongType
 
   override def symbol: String = "/"
   override def decimalMethod: String = "quot"
-  override def decimalToDataTypeCodeGen(decimalResult: String): String = {
-    if (SQLConf.get.integralDivideReturnLong) {
-      s"$decimalResult.toLong()"
-    } else {
-      decimalResult
-    }
-  }
-
+  override def decimalToDataTypeCodeGen(decimalResult: String): String = s"$decimalResult.toLong()"
   override def sqlOperator: String = "div"
 
   private lazy val div: (Any, Any) => Any = {
@@ -432,25 +428,25 @@ case class IntegralDivide(left: Expression, right: Expression) extends DivModLik
       case d: DecimalType =>
         d.asIntegral.asInstanceOf[Integral[Any]]
     }
-    val divide = integral.quot _
-    if (SQLConf.get.integralDivideReturnLong) {
-      val toLong = integral.asInstanceOf[Integral[Any]].toLong _
-      (x, y) => {
-        val res = divide(x, y)
-        if (res == null) {
-          null
-        } else {
-          toLong(res)
-        }
+    (x, y) => {
+      val res = integral.quot(x, y)
+      if (res == null) {
+        null
+      } else {
+        integral.asInstanceOf[Integral[Any]].toLong(res)
       }
-    } else {
-      divide
     }
   }
 
   override def evalOperation(left: Any, right: Any): Any = div(left, right)
 }
 
+object IntegralDivide {
+  def apply(left: Expression, right: Expression): IntegralDivide = {
+    new IntegralDivide(left, right)
+  }
+}
+
 @ExpressionDescription(
   usage = "expr1 _FUNC_ expr2 - Returns the remainder after `expr1`/`expr2`.",
   examples = """
@@ -466,6 +462,18 @@ case class Remainder(left: Expression, right: Expression) extends DivModLike {
 
   override def symbol: String = "%"
   override def decimalMethod: String = "remainder"
+  override def toString: String = {
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse(sqlOperator) match {
+      case operator if operator == sqlOperator => s"($left $sqlOperator $right)"
+      case funcName => s"$funcName($left, $right)"
+    }
+  }
+  override def sql: String = {
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse(sqlOperator) match {
+      case operator if operator == sqlOperator => s"(${left.sql} $sqlOperator ${right.sql})"
+      case funcName => s"$funcName(${left.sql}, ${right.sql})"
+    }
+  }
 
   private lazy val mod: (Any, Any) => Any = dataType match {
     // special cases to make float/double primitive types faster
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
index 56f170cd77073..7b819db32e425 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
@@ -162,7 +162,8 @@ case class BitwiseNot(child: Expression) extends UnaryExpression with ExpectsInp
     Examples:
       > SELECT _FUNC_(0);
        0
-  """)
+  """,
+  since = "3.0.0")
 case class BitwiseCount(child: Expression) extends UnaryExpression with ExpectsInputTypes {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(IntegralType, BooleanType))
@@ -171,6 +172,8 @@ case class BitwiseCount(child: Expression) extends UnaryExpression with ExpectsI
 
   override def toString: String = s"bit_count($child)"
 
+  override def prettyName: String = "bit_count"
+
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = child.dataType match {
     case BooleanType => defineCodeGen(ctx, ev, c => s"if ($c) 1 else 0")
     case _ => defineCodeGen(ctx, ev, c => s"java.lang.Long.bitCount($c)")
@@ -183,6 +186,4 @@ case class BitwiseCount(child: Expression) extends UnaryExpression with ExpectsI
     case IntegerType => java.lang.Long.bitCount(input.asInstanceOf[Int])
     case LongType => java.lang.Long.bitCount(input.asInstanceOf[Long])
   }
-
-  override def sql: String = s"bit_count(${child.sql})"
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index 58c95c94ba198..510696ea845f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -38,7 +38,7 @@ import org.apache.spark.metrics.source.CodegenMetrics
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData, MapData}
+import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData, MapData, SQLOrderingUtil}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_MILLIS
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
@@ -133,7 +133,7 @@ class CodegenContext extends Logging {
   def addReferenceObj(objName: String, obj: Any, className: String = null): String = {
     val idx = references.length
     references += obj
-    val clsName = Option(className).getOrElse(obj.getClass.getName)
+    val clsName = Option(className).getOrElse(CodeGenerator.typeName(obj.getClass))
     s"(($clsName) references[$idx] /* $objName */)"
   }
 
@@ -624,8 +624,12 @@ class CodegenContext extends Logging {
   def genComp(dataType: DataType, c1: String, c2: String): String = dataType match {
     // java boolean doesn't support > or < operator
     case BooleanType => s"($c1 == $c2 ? 0 : ($c1 ? 1 : -1))"
-    case DoubleType => s"java.lang.Double.compare($c1, $c2)"
-    case FloatType => s"java.lang.Float.compare($c1, $c2)"
+    case DoubleType =>
+      val clsName = SQLOrderingUtil.getClass.getName.stripSuffix("$")
+      s"$clsName.compareDoubles($c1, $c2)"
+    case FloatType =>
+      val clsName = SQLOrderingUtil.getClass.getName.stripSuffix("$")
+      s"$clsName.compareFloats($c1, $c2)"
     // use c1 - c2 may overflow
     case dt: DataType if isPrimitiveType(dt) => s"($c1 > $c2 ? 1 : $c1 < $c2 ? -1 : 0)"
     case BinaryType => s"org.apache.spark.sql.catalyst.util.TypeUtils.compareBinary($c1, $c2)"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
index 63bd59e7628b2..5d00519d27c53 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
@@ -71,7 +71,9 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], BaseOrdering] with
     ctx.INPUT_ROW = row
     // to use INPUT_ROW we must make sure currentVars is null
     ctx.currentVars = null
-    ordering.map(_.child.genCode(ctx))
+    // SPARK-33260: To avoid unpredictable modifications to `ctx` when `ordering` is a Stream, we
+    // use `toIndexedSeq` to make the transformation eager.
+    ordering.toIndexedSeq.map(_.child.genCode(ctx))
   }
 
   /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
index d9393b9df6bbd..1c59c3c2ad667 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
@@ -143,10 +143,10 @@ trait Block extends TreeNode[Block] with JavaCode {
     case _ => code.trim
   }
 
-  def length: Int = {
-    // Returns a code length without comments
-    CodeFormatter.stripExtraNewLinesAndComments(toString).length
-  }
+  // We could remove comments, extra whitespaces and newlines when calculating length as it is used
+  // only for codegen method splitting, but SPARK-30564 showed that this is a performance critical
+  // function so we decided not to do so.
+  def length: Int = toString.length
 
   def isEmpty: Boolean = toString.isEmpty
 
@@ -223,6 +223,11 @@ object Block {
   implicit def blocksToBlock(blocks: Seq[Block]): Block = blocks.reduceLeft(_ + _)
 
   implicit class BlockHelper(val sc: StringContext) extends AnyVal {
+    /**
+     * A string interpolator that retains references to the `JavaCode` inputs, and behaves like
+     * the Scala builtin StringContext.s() interpolator otherwise, i.e. it will treat escapes in
+     * the code parts, and will not treat escapes in the input arguments.
+     */
     def code(args: Any*): Block = {
       sc.checkLengths(args)
       if (sc.parts.length == 0) {
@@ -250,7 +255,7 @@ object Block {
     val inputs = args.iterator
     val buf = new StringBuilder(Block.CODE_BLOCK_BUFFER_LENGTH)
 
-    buf.append(strings.next)
+    buf.append(StringContext.treatEscapes(strings.next))
     while (strings.hasNext) {
       val input = inputs.next
       input match {
@@ -262,7 +267,7 @@ object Block {
         case _ =>
           buf.append(input)
       }
-      buf.append(strings.next)
+      buf.append(StringContext.treatEscapes(strings.next))
     }
     codeParts += buf.toString
 
@@ -286,10 +291,10 @@ case class CodeBlock(codeParts: Seq[String], blockInputs: Seq[JavaCode]) extends
     val strings = codeParts.iterator
     val inputs = blockInputs.iterator
     val buf = new StringBuilder(Block.CODE_BLOCK_BUFFER_LENGTH)
-    buf.append(StringContext.treatEscapes(strings.next))
+    buf.append(strings.next)
     while (strings.hasNext) {
       buf.append(inputs.next)
-      buf.append(StringContext.treatEscapes(strings.next))
+      buf.append(strings.next)
     }
     buf.toString
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 6ed68e47ce7a1..26b3003790e48 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -77,9 +77,9 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
 @ExpressionDescription(
   usage = """
     _FUNC_(expr) - Returns the size of an array or a map.
-    The function returns -1 if its input is null and spark.sql.legacy.sizeOfNull is set to true.
-    If spark.sql.legacy.sizeOfNull is set to false, the function returns null for null input.
-    By default, the spark.sql.legacy.sizeOfNull parameter is set to false.
+    The function returns null for null input if spark.sql.legacy.sizeOfNull is set to false or
+    spark.sql.ansi.enabled is set to true. Otherwise, the function returns -1 for null input.
+    With the default settings, the function returns -1 for null input.
   """,
   examples = """
     Examples:
@@ -88,11 +88,12 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
       > SELECT _FUNC_(map('a', 1, 'b', 2));
        2
       > SELECT _FUNC_(NULL);
-       NULL
+       -1
   """)
-case class Size(child: Expression) extends UnaryExpression with ExpectsInputTypes {
+case class Size(child: Expression, legacySizeOfNull: Boolean)
+  extends UnaryExpression with ExpectsInputTypes {
 
-  val legacySizeOfNull = SQLConf.get.legacySizeOfNull
+  def this(child: Expression) = this(child, SQLConf.get.legacySizeOfNull)
 
   override def dataType: DataType = IntegerType
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(ArrayType, MapType))
@@ -124,6 +125,10 @@ case class Size(child: Expression) extends UnaryExpression with ExpectsInputType
   }
 }
 
+object Size {
+  def apply(child: Expression): Size = new Size(child)
+}
+
 /**
  * Returns an unordered array containing the keys of the map.
  */
@@ -133,7 +138,8 @@ case class Size(child: Expression) extends UnaryExpression with ExpectsInputType
     Examples:
       > SELECT _FUNC_(map(1, 'a', 2, 'b'));
        [1,2]
-  """)
+  """,
+  group = "map_funcs")
 case class MapKeys(child: Expression)
   extends UnaryExpression with ExpectsInputTypes {
 
@@ -164,6 +170,7 @@ case class MapKeys(child: Expression)
       > SELECT _FUNC_(array(1, 2), array(2, 3), array(3, 4));
        [{"0":1,"1":2,"2":3},{"0":2,"1":3,"2":4}]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsInputTypes {
 
@@ -322,7 +329,8 @@ case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsI
     Examples:
       > SELECT _FUNC_(map(1, 'a', 2, 'b'));
        ["a","b"]
-  """)
+  """,
+  group = "map_funcs")
 case class MapValues(child: Expression)
   extends UnaryExpression with ExpectsInputTypes {
 
@@ -351,6 +359,7 @@ case class MapValues(child: Expression)
       > SELECT _FUNC_(map(1, 'a', 2, 'b'));
        [{"key":1,"value":"a"},{"key":2,"value":"b"}]
   """,
+  group = "map_funcs",
   since = "3.0.0")
 case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInputTypes {
 
@@ -516,9 +525,11 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp
   usage = "_FUNC_(map, ...) - Returns the union of all the given maps",
   examples = """
     Examples:
-      > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(2, 'c', 3, 'd'));
-       {1:"a",2:"c",3:"d"}
-  """, since = "2.4.0")
+      > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(3, 'c'));
+       {1:"a",2:"b",3:"c"}
+  """,
+  group = "map_funcs",
+  since = "2.4.0")
 case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
   override def checkInputDataTypes(): TypeCheckResult = {
@@ -636,6 +647,7 @@ case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpres
       > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b')));
        {1:"a",2:"b"}
   """,
+  group = "map_funcs",
   since = "2.4.0")
 case class MapFromEntries(child: Expression) extends UnaryExpression {
 
@@ -857,7 +869,8 @@ object ArraySortLike {
     Examples:
       > SELECT _FUNC_(array('b', 'd', null, 'c', 'a'), true);
        [null,"a","b","c","d"]
-  """)
+  """,
+  group = "array_funcs")
 // scalastyle:on line.size.limit
 case class SortArray(base: Expression, ascendingOrder: Expression)
   extends BinaryExpression with ArraySortLike {
@@ -915,6 +928,7 @@ case class SortArray(base: Expression, ascendingOrder: Expression)
   note = """
     The function is non-deterministic.
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class Shuffle(child: Expression, randomSeed: Option[Long] = None)
   extends UnaryExpression with ExpectsInputTypes with Stateful with ExpressionWithRandomSeed {
@@ -997,6 +1011,7 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None)
       > SELECT _FUNC_(array(2, 1, 4, 3));
        [3,4,1,2]
   """,
+  group = "array_funcs",
   since = "1.5.0",
   note = """
     Reverse logic for arrays is available since 2.4.0.
@@ -1068,7 +1083,8 @@ case class Reverse(child: Expression) extends UnaryExpression with ImplicitCastI
     Examples:
       > SELECT _FUNC_(array(1, 2, 3), 2);
        true
-  """)
+  """,
+  group = "array_funcs")
 case class ArrayContains(left: Expression, right: Expression)
   extends BinaryExpression with ImplicitCastInputTypes {
 
@@ -1164,7 +1180,9 @@ case class ArrayContains(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_(array(1, 2, 3), array(3, 4, 5));
        true
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 // scalastyle:off line.size.limit
 case class ArraysOverlap(left: Expression, right: Expression)
   extends BinaryArrayExpressionWithImplicitCast {
@@ -1387,7 +1405,9 @@ case class ArraysOverlap(left: Expression, right: Expression)
        [2,3]
       > SELECT _FUNC_(array(1, 2, 3, 4), -2, 2);
        [3,4]
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 // scalastyle:on line.size.limit
 case class Slice(x: Expression, start: Expression, length: Expression)
   extends TernaryExpression with ImplicitCastInputTypes {
@@ -1500,7 +1520,9 @@ case class Slice(x: Expression, start: Expression, length: Expression)
        hello world
       > SELECT _FUNC_(array('hello', null ,'world'), ' ', ',');
        hello , world
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 case class ArrayJoin(
     array: Expression,
     delimiter: Expression,
@@ -1663,7 +1685,9 @@ case class ArrayJoin(
     Examples:
       > SELECT _FUNC_(array(1, 20, null, 3));
        1
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 case class ArrayMin(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
   override def nullable: Boolean = true
@@ -1728,7 +1752,9 @@ case class ArrayMin(child: Expression) extends UnaryExpression with ImplicitCast
     Examples:
       > SELECT _FUNC_(array(1, 20, null, 3));
        20
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 case class ArrayMax(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
   override def nullable: Boolean = true
@@ -1802,6 +1828,7 @@ case class ArrayMax(child: Expression) extends UnaryExpression with ImplicitCast
       > SELECT _FUNC_(array(3, 2, 1), 1);
        3
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArrayPosition(left: Expression, right: Expression)
   extends BinaryExpression with ImplicitCastInputTypes {
@@ -1927,6 +1954,36 @@ case class ElementAt(left: Expression, right: Expression)
     }
   }
 
+  private def nullability(elements: Seq[Expression], ordinal: Int): Boolean = {
+    if (ordinal == 0) {
+      false
+    } else if (elements.length < math.abs(ordinal)) {
+      true
+    } else {
+      if (ordinal < 0) {
+        elements(elements.length + ordinal).nullable
+      } else {
+        elements(ordinal - 1).nullable
+      }
+    }
+  }
+
+  override def computeNullabilityFromArray(child: Expression, ordinal: Expression): Boolean = {
+    if (ordinal.foldable && !ordinal.nullable) {
+      val intOrdinal = ordinal.eval().asInstanceOf[Number].intValue()
+      child match {
+        case CreateArray(ar, _) =>
+          nullability(ar, intOrdinal)
+        case GetArrayStructFields(CreateArray(elements, _), field, _, _, _) =>
+          nullability(elements, intOrdinal) || field.nullable
+        case _ =>
+          true
+      }
+    } else {
+      true
+    }
+  }
+
   override def nullable: Boolean = left.dataType match {
     case _: ArrayType => computeNullabilityFromArray(left, right)
     case _: MapType => true
@@ -2016,7 +2073,8 @@ case class ElementAt(left: Expression, right: Expression)
   """,
   note = """
     Concat logic for arrays is available since 2.4.0.
-  """)
+  """,
+  group = "array_funcs")
 case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
   private def allowedTypes: Seq[AbstractDataType] = Seq(StringType, BinaryType, ArrayType)
@@ -2215,6 +2273,7 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio
       > SELECT _FUNC_(array(array(1, 2), array(3, 4)));
        [1,2,3,4]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class Flatten(child: Expression) extends UnaryExpression {
 
@@ -2347,6 +2406,7 @@ case class Flatten(child: Expression) extends UnaryExpression {
       > SELECT _FUNC_(to_date('2018-01-01'), to_date('2018-03-01'), interval 1 month);
        [2018-01-01,2018-02-01,2018-03-01]
   """,
+  group = "array_funcs",
   since = "2.4.0"
 )
 case class Sequence(
@@ -2594,7 +2654,7 @@ object Sequence {
         val maxEstimatedArrayLength =
           getSequenceLength(startMicros, stopMicros, intervalStepInMicros)
 
-        val stepSign = if (stopMicros > startMicros) +1 else -1
+        val stepSign = if (stopMicros >= startMicros) +1 else -1
         val exclusiveItem = stopMicros + stepSign
         val arr = new Array[T](maxEstimatedArrayLength)
         var t = startMicros
@@ -2656,7 +2716,7 @@ object Sequence {
          |
          |  $sequenceLengthCode
          |
-         |  final int $stepSign = $stopMicros > $startMicros ? +1 : -1;
+         |  final int $stepSign = $stopMicros >= $startMicros ? +1 : -1;
          |  final long $exclusiveItem = $stopMicros + $stepSign;
          |
          |  $arr = new $elemType[$arrLength];
@@ -2729,6 +2789,7 @@ object Sequence {
       > SELECT _FUNC_('123', 2);
        ["123","123"]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArrayRepeat(left: Expression, right: Expression)
   extends BinaryExpression with ExpectsInputTypes {
@@ -2849,7 +2910,9 @@ case class ArrayRepeat(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_(array(1, 2, 3, null, 3), 3);
        [1,2,null]
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 case class ArrayRemove(left: Expression, right: Expression)
   extends BinaryExpression with ImplicitCastInputTypes {
 
@@ -3044,7 +3107,9 @@ trait ArraySetLike {
     Examples:
       > SELECT _FUNC_(array(1, 2, 3, null, 3));
        [1,2,3,null]
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 case class ArrayDistinct(child: Expression)
   extends UnaryExpression with ArraySetLike with ExpectsInputTypes {
 
@@ -3221,6 +3286,7 @@ object ArrayBinaryLike {
       > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
        [1,2,3,5]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLike
   with ComplexTypeMergingExpression {
@@ -3432,6 +3498,7 @@ object ArrayUnion {
       > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
        [1,3]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBinaryLike
   with ComplexTypeMergingExpression {
@@ -3673,6 +3740,7 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
       > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
        [2]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryLike
   with ComplexTypeMergingExpression {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 9ce87a4922c01..5212ef3930bc9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -19,10 +19,11 @@ package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
+import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.{FUNC_ALIAS, FunctionBuilder}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -36,18 +37,33 @@ import org.apache.spark.unsafe.types.UTF8String
       > SELECT _FUNC_(1, 2, 3);
        [1,2,3]
   """)
-case class CreateArray(children: Seq[Expression]) extends Expression {
+case class CreateArray(children: Seq[Expression], useStringTypeWhenEmpty: Boolean)
+  extends Expression {
+
+  def this(children: Seq[Expression]) = {
+    this(children, SQLConf.get.getConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE))
+  }
 
   override def foldable: Boolean = children.forall(_.foldable)
 
+  override def stringArgs: Iterator[Any] = super.stringArgs.take(1)
+
   override def checkInputDataTypes(): TypeCheckResult = {
     TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), s"function $prettyName")
   }
 
+  private val defaultElementType: DataType = {
+    if (useStringTypeWhenEmpty) {
+      StringType
+    } else {
+      NullType
+    }
+  }
+
   override def dataType: ArrayType = {
     ArrayType(
       TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(children.map(_.dataType))
-        .getOrElse(StringType),
+        .getOrElse(defaultElementType),
       containsNull = children.exists(_.nullable))
   }
 
@@ -70,6 +86,12 @@ case class CreateArray(children: Seq[Expression]) extends Expression {
   override def prettyName: String = "array"
 }
 
+object CreateArray {
+  def apply(children: Seq[Expression]): CreateArray = {
+    new CreateArray(children)
+  }
+}
+
 private [sql] object GenArrayData {
   /**
    * Return Java code pieces based on DataType and array size to allocate ArrayData class
@@ -132,12 +154,28 @@ private [sql] object GenArrayData {
       > SELECT _FUNC_(1.0, '2', 3.0, '4');
        {1.0:"2",3.0:"4"}
   """)
-case class CreateMap(children: Seq[Expression]) extends Expression {
+case class CreateMap(children: Seq[Expression], useStringTypeWhenEmpty: Boolean)
+  extends Expression {
+
+  def this(children: Seq[Expression]) = {
+    this(children, SQLConf.get.getConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE))
+  }
+
   lazy val keys = children.indices.filter(_ % 2 == 0).map(children)
   lazy val values = children.indices.filter(_ % 2 != 0).map(children)
 
+  private val defaultElementType: DataType = {
+    if (useStringTypeWhenEmpty) {
+      StringType
+    } else {
+      NullType
+    }
+  }
+
   override def foldable: Boolean = children.forall(_.foldable)
 
+  override def stringArgs: Iterator[Any] = super.stringArgs.take(1)
+
   override def checkInputDataTypes(): TypeCheckResult = {
     if (children.size % 2 != 0) {
       TypeCheckResult.TypeCheckFailure(
@@ -158,9 +196,9 @@ case class CreateMap(children: Seq[Expression]) extends Expression {
   override lazy val dataType: MapType = {
     MapType(
       keyType = TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(keys.map(_.dataType))
-        .getOrElse(StringType),
+        .getOrElse(defaultElementType),
       valueType = TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(values.map(_.dataType))
-        .getOrElse(StringType),
+        .getOrElse(defaultElementType),
       valueContainsNull = values.exists(_.nullable))
   }
 
@@ -198,6 +236,12 @@ case class CreateMap(children: Seq[Expression]) extends Expression {
   override def prettyName: String = "map"
 }
 
+object CreateMap {
+  def apply(children: Seq[Expression]): CreateMap = {
+    new CreateMap(children)
+  }
+}
+
 /**
  * Returns a catalyst Map containing the two arrays in children expressions as keys and values.
  */
@@ -267,7 +311,12 @@ case object NamePlaceholder extends LeafExpression with Unevaluable {
 /**
  * Returns a Row containing the evaluation of all children expressions.
  */
-object CreateStruct extends FunctionBuilder {
+object CreateStruct {
+  /**
+   * Returns a named struct with generated names or using the names when available.
+   * It should not be used for `struct` expressions or functions explicitly called
+   * by users.
+   */
   def apply(children: Seq[Expression]): CreateNamedStruct = {
     CreateNamedStruct(children.zipWithIndex.flatMap {
       case (e: NamedExpression, _) if e.resolved => Seq(Literal(e.name), e)
@@ -276,12 +325,23 @@ object CreateStruct extends FunctionBuilder {
     })
   }
 
+  /**
+   * Returns a named struct with a pretty SQL name. It will show the pretty SQL string
+   * in its output column name as if `struct(...)` was called. Should be
+   * used for `struct` expressions or functions explicitly called by users.
+   */
+  def create(children: Seq[Expression]): CreateNamedStruct = {
+    val expr = CreateStruct(children)
+    expr.setTagValue(FUNC_ALIAS, "struct")
+    expr
+  }
+
   /**
    * Entry to use in the function registry.
    */
   val registryEntry: (String, (ExpressionInfo, FunctionBuilder)) = {
     val info: ExpressionInfo = new ExpressionInfo(
-      "org.apache.spark.sql.catalyst.expressions.NamedStruct",
+      classOf[CreateNamedStruct].getCanonicalName,
       null,
       "struct",
       "_FUNC_(col1, col2, col3, ...) - Creates a struct with the given field values.",
@@ -289,8 +349,9 @@ object CreateStruct extends FunctionBuilder {
       "",
       "",
       "",
+      "",
       "")
-    ("struct", (info, this))
+    ("struct", (info, this.create))
   }
 }
 
@@ -388,7 +449,15 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression {
        """.stripMargin, isNull = FalseLiteral)
   }
 
-  override def prettyName: String = "named_struct"
+  // There is an alias set at `CreateStruct.create`. If there is an alias,
+  // this is the struct function explicitly called by a user and we should
+  // respect it in the SQL string as `struct(...)`.
+  override def prettyName: String = getTagValue(FUNC_ALIAS).getOrElse("named_struct")
+
+  override def sql: String = getTagValue(FUNC_ALIAS).map { alias =>
+    val childrenSQL = children.indices.filter(_ % 2 == 1).map(children(_).sql).mkString(", ")
+    s"$alias($childrenSQL)"
+  }.getOrElse(super.sql)
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index e9d60ed3a481f..89ff4facd25a9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -57,7 +57,7 @@ object ExtractValue {
         val fieldName = v.toString
         val ordinal = findField(fields, fieldName, resolver)
         GetArrayStructFields(child, fields(ordinal).copy(name = fieldName),
-          ordinal, fields.length, containsNull)
+          ordinal, fields.length, containsNull || fields(ordinal).nullable)
 
       case (_: ArrayType, _) => GetArrayItem(child, extraction)
 
@@ -275,9 +275,9 @@ trait GetArrayItemUtil {
     if (ordinal.foldable && !ordinal.nullable) {
       val intOrdinal = ordinal.eval().asInstanceOf[Number].intValue()
       child match {
-        case CreateArray(ar) if intOrdinal < ar.length =>
+        case CreateArray(ar, _) if intOrdinal < ar.length =>
           ar(intOrdinal).nullable
-        case GetArrayStructFields(CreateArray(elements), field, _, _, _)
+        case GetArrayStructFields(CreateArray(elements, _), field, _, _, _)
           if intOrdinal < elements.length =>
           elements(intOrdinal).nullable || field.nullable
         case _ =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 28f1d34267224..9d6705822d551 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -20,19 +20,22 @@ package org.apache.spark.sql.catalyst.expressions
 import java.sql.Timestamp
 import java.time.{DateTimeException, LocalDate, LocalDateTime, ZoneId}
 import java.time.temporal.IsoFields
-import java.util.{Locale, TimeZone}
+import java.util.Locale
 
 import scala.util.control.NonFatal
 
 import org.apache.commons.text.StringEscapeUtils
 
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.{DateTimeUtils, LegacyDateFormats, TimestampFormatter}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.SIMPLE_DATE_FORMAT
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
@@ -50,19 +53,33 @@ trait TimeZoneAwareExpression extends Expression {
   /** Returns a copy of this expression with the specified timeZoneId. */
   def withTimeZone(timeZoneId: String): TimeZoneAwareExpression
 
-  @transient lazy val timeZone: TimeZone = DateTimeUtils.getTimeZone(timeZoneId.get)
   @transient lazy val zoneId: ZoneId = DateTimeUtils.getZoneId(timeZoneId.get)
 }
 
 /**
  * Returns the current date at the start of query evaluation.
- * All calls of current_date within the same query return the same value.
- *
  * There is no code generation since this expression should get constant folded by the optimizer.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_() - Returns the current date at the start of query evaluation.",
+  usage = """
+    _FUNC_() - Returns the current date at the start of query evaluation. All calls of current_date within the same query return the same value.
+
+    _FUNC_ - Returns the current date at the start of query evaluation.
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       2020-04-25
+      > SELECT _FUNC_;
+       2020-04-25
+  """,
+  note = """
+    The syntax without braces has been supported since 2.0.1.
+  """,
+  group = "datetime_funcs",
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class CurrentDate(timeZoneId: Option[String] = None)
   extends LeafExpression with TimeZoneAwareExpression with CodegenFallback {
 
@@ -81,26 +98,54 @@ case class CurrentDate(timeZoneId: Option[String] = None)
   override def prettyName: String = "current_date"
 }
 
+abstract class CurrentTimestampLike() extends LeafExpression with CodegenFallback {
+  override def foldable: Boolean = true
+  override def nullable: Boolean = false
+  override def dataType: DataType = TimestampType
+  override def eval(input: InternalRow): Any = currentTimestamp()
+}
+
 /**
  * Returns the current timestamp at the start of query evaluation.
- * All calls of current_timestamp within the same query return the same value.
- *
  * There is no code generation since this expression should get constant folded by the optimizer.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_() - Returns the current timestamp at the start of query evaluation.",
-  since = "1.5.0")
-case class CurrentTimestamp() extends LeafExpression with CodegenFallback {
-  override def foldable: Boolean = true
-  override def nullable: Boolean = false
-
-  override def dataType: DataType = TimestampType
-
-  override def eval(input: InternalRow): Any = currentTimestamp()
+  usage = """
+    _FUNC_() - Returns the current timestamp at the start of query evaluation. All calls of current_timestamp within the same query return the same value.
 
+    _FUNC_ - Returns the current timestamp at the start of query evaluation.
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       2020-04-25 15:49:11.914
+      > SELECT _FUNC_;
+       2020-04-25 15:49:11.914
+  """,
+  note = """
+    The syntax without braces has been supported since 2.0.1.
+  """,
+  group = "datetime_funcs",
+  since = "1.5.0")
+// scalastyle:on line.size.limit
+case class CurrentTimestamp() extends CurrentTimestampLike {
   override def prettyName: String = "current_timestamp"
 }
 
+@ExpressionDescription(
+  usage = "_FUNC_() - Returns the current timestamp at the start of query evaluation.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       2020-04-25 15:49:11.914
+  """,
+  group = "datetime_funcs",
+  since = "1.6.0")
+case class Now() extends CurrentTimestampLike {
+  override def prettyName: String = "now"
+}
+
 /**
  * Expression representing the current batch time, which is used by StreamExecution to
  * 1. prevent optimizer from pushing this expression below a stateful operator
@@ -134,7 +179,7 @@ case class CurrentBatchTimestamp(
   def toLiteral: Literal = dataType match {
     case _: TimestampType =>
       Literal(DateTimeUtils.fromJavaTimestamp(new Timestamp(timestampMs)), TimestampType)
-    case _: DateType => Literal(DateTimeUtils.millisToDays(timestampMs, timeZone), DateType)
+    case _: DateType => Literal(DateTimeUtils.millisToDays(timestampMs, zoneId), DateType)
   }
 }
 
@@ -148,6 +193,7 @@ case class CurrentBatchTimestamp(
       > SELECT _FUNC_('2016-07-30', 1);
        2016-07-31
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class DateAdd(startDate: Expression, days: Expression)
   extends BinaryExpression with ExpectsInputTypes {
@@ -183,6 +229,7 @@ case class DateAdd(startDate: Expression, days: Expression)
       > SELECT _FUNC_('2016-07-30', 1);
        2016-07-29
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class DateSub(startDate: Expression, days: Expression)
   extends BinaryExpression with ExpectsInputTypes {
@@ -214,6 +261,7 @@ case class DateSub(startDate: Expression, days: Expression)
       > SELECT _FUNC_('2009-07-30 12:58:59');
        12
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Hour(child: Expression, timeZoneId: Option[String] = None)
   extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
@@ -228,13 +276,13 @@ case class Hour(child: Expression, timeZoneId: Option[String] = None)
     copy(timeZoneId = Option(timeZoneId))
 
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getHours(timestamp.asInstanceOf[Long], timeZone)
+    DateTimeUtils.getHours(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getHours($c, $tz)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getHours($c, $zid)")
   }
 }
 
@@ -245,6 +293,7 @@ case class Hour(child: Expression, timeZoneId: Option[String] = None)
       > SELECT _FUNC_('2009-07-30 12:58:59');
        58
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Minute(child: Expression, timeZoneId: Option[String] = None)
   extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
@@ -259,13 +308,13 @@ case class Minute(child: Expression, timeZoneId: Option[String] = None)
     copy(timeZoneId = Option(timeZoneId))
 
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getMinutes(timestamp.asInstanceOf[Long], timeZone)
+    DateTimeUtils.getMinutes(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMinutes($c, $tz)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getMinutes($c, $zid)")
   }
 }
 
@@ -276,6 +325,7 @@ case class Minute(child: Expression, timeZoneId: Option[String] = None)
       > SELECT _FUNC_('2009-07-30 12:58:59');
        59
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Second(child: Expression, timeZoneId: Option[String] = None)
   extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
@@ -290,13 +340,13 @@ case class Second(child: Expression, timeZoneId: Option[String] = None)
     copy(timeZoneId = Option(timeZoneId))
 
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getSeconds(timestamp.asInstanceOf[Long], timeZone)
+    DateTimeUtils.getSeconds(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getSeconds($c, $tz)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getSeconds($c, $zid)")
   }
 }
 
@@ -314,55 +364,13 @@ case class SecondWithFraction(child: Expression, timeZoneId: Option[String] = No
     copy(timeZoneId = Option(timeZoneId))
 
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getSecondsWithFraction(timestamp.asInstanceOf[Long], timeZone)
+    DateTimeUtils.getSecondsWithFraction(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getSecondsWithFraction($c, $tz)")
-  }
-}
-
-case class Milliseconds(child: Expression, timeZoneId: Option[String] = None)
-  extends UnaryExpression with ImplicitCastInputTypes with TimeZoneAwareExpression {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
-  // DecimalType is used here to not lose precision while converting microseconds to
-  // the fractional part of milliseconds. Scale 3 is taken to have all microseconds as
-  // the fraction. The precision 8 should cover 2 digits for seconds, 3 digits for
-  // milliseconds and 3 digits for microseconds.
-  override def dataType: DataType = DecimalType(8, 3)
-  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
-    copy(timeZoneId = Option(timeZoneId))
-
-  override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getMilliseconds(timestamp.asInstanceOf[Long], timeZone)
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMilliseconds($c, $tz)")
-  }
-}
-
-case class Microseconds(child: Expression, timeZoneId: Option[String] = None)
-  extends UnaryExpression with ImplicitCastInputTypes with TimeZoneAwareExpression {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
-  override def dataType: DataType = IntegerType
-  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
-    copy(timeZoneId = Option(timeZoneId))
-
-  override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getMicroseconds(timestamp.asInstanceOf[Long], timeZone)
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMicroseconds($c, $tz)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getSecondsWithFraction($c, $zid)")
   }
 }
 
@@ -373,6 +381,7 @@ case class Microseconds(child: Expression, timeZoneId: Option[String] = None)
       > SELECT _FUNC_('2016-04-09');
        100
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
@@ -397,6 +406,7 @@ case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCas
       > SELECT _FUNC_('2016-07-30');
        2016
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Year(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
@@ -414,19 +424,19 @@ case class Year(child: Expression) extends UnaryExpression with ImplicitCastInpu
   }
 }
 
-case class IsoYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+case class YearOfWeek(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
 
   override def dataType: DataType = IntegerType
 
   override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getIsoYear(date.asInstanceOf[Int])
+    DateTimeUtils.getWeekBasedYear(date.asInstanceOf[Int])
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getIsoYear($c)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getWeekBasedYear($c)")
   }
 }
 
@@ -437,6 +447,7 @@ case class IsoYear(child: Expression) extends UnaryExpression with ImplicitCastI
       > SELECT _FUNC_('2016-08-31');
        3
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
@@ -461,6 +472,7 @@ case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastI
       > SELECT _FUNC_('2016-07-30');
        7
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Month(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
@@ -510,6 +522,7 @@ case class DayOfMonth(child: Expression) extends UnaryExpression with ImplicitCa
       > SELECT _FUNC_('2009-07-30');
        5
   """,
+  group = "datetime_funcs",
   since = "2.3.0")
 // scalastyle:on line.size.limit
 case class DayOfWeek(child: Expression) extends DayWeek {
@@ -536,6 +549,7 @@ case class DayOfWeek(child: Expression) extends DayWeek {
       > SELECT _FUNC_('2009-07-30');
        3
   """,
+  group = "datetime_funcs",
   since = "2.4.0")
 // scalastyle:on line.size.limit
 case class WeekDay(child: Expression) extends DayWeek {
@@ -569,6 +583,7 @@ abstract class DayWeek extends UnaryExpression with ImplicitCastInputTypes {
       > SELECT _FUNC_('2008-02-20');
        8
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
@@ -598,7 +613,7 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa
   arguments = """
     Arguments:
       * timestamp - A date/timestamp or string to be converted to the given format.
-      * fmt - Date/time format pattern to follow. See `java.time.format.DateTimeFormatter` for valid date
+      * fmt - Date/time format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a> for valid date
               and time format patterns.
   """,
   examples = """
@@ -606,6 +621,7 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa
       > SELECT _FUNC_('2016-04-08', 'y');
        2016
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Option[String] = None)
@@ -622,13 +638,23 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
 
   @transient private lazy val formatter: Option[TimestampFormatter] = {
     if (right.foldable) {
-      Option(right.eval()).map(format => TimestampFormatter(format.toString, zoneId))
+      Option(right.eval()).map { format =>
+        TimestampFormatter(
+          format.toString,
+          zoneId,
+          legacyFormat = SIMPLE_DATE_FORMAT,
+          isParsing = false)
+      }
     } else None
   }
 
   override protected def nullSafeEval(timestamp: Any, format: Any): Any = {
     val tf = if (formatter.isEmpty) {
-      TimestampFormatter(format.toString, zoneId)
+      TimestampFormatter(
+        format.toString,
+        zoneId,
+        legacyFormat = SIMPLE_DATE_FORMAT,
+        isParsing = false)
     } else {
       formatter.get
     }
@@ -643,10 +669,15 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
       })
     }.getOrElse {
       val tf = TimestampFormatter.getClass.getName.stripSuffix("$")
+      val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$")
       val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
       defineCodeGen(ctx, ev, (timestamp, format) => {
-        s"""UTF8String.fromString($tf$$.MODULE$$.apply($format.toString(), $zid)
-          .format($timestamp))"""
+        s"""|UTF8String.fromString($tf$$.MODULE$$.apply(
+            |  $format.toString(),
+            |  $zid,
+            |  $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(),
+            |  false)
+            |.format($timestamp))""".stripMargin
       })
     }
   }
@@ -658,13 +689,14 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
  * Converts time string with given pattern.
  * Deterministic version of [[UnixTimestamp]], must have at least one parameter.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = "_FUNC_(timeExp[, format]) - Returns the UNIX timestamp of the given time.",
   arguments = """
     Arguments:
       * timeExp - A date/timestamp or string which is returned as a UNIX timestamp.
       * format - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
-                 Default value is "uuuu-MM-dd HH:mm:ss". See `java.time.format.DateTimeFormatter`
+                 Default value is "yyyy-MM-dd HH:mm:ss". See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>
                  for valid date and time format patterns.
   """,
   examples = """
@@ -672,7 +704,9 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
       > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd');
        1460098800
   """,
+  group = "datetime_funcs",
   since = "1.6.0")
+// scalastyle:on line.size.limit
 case class ToUnixTimestamp(
     timeExp: Expression,
     format: Expression,
@@ -688,17 +722,18 @@ case class ToUnixTimestamp(
     copy(timeZoneId = Option(timeZoneId))
 
   def this(time: Expression) = {
-    this(time, Literal("uuuu-MM-dd HH:mm:ss"))
+    this(time, Literal(TimestampFormatter.defaultPattern))
   }
 
   override def prettyName: String = "to_unix_timestamp"
 }
 
+// scalastyle:off line.size.limit
 /**
  * Converts time string with given pattern to Unix time stamp (in seconds), returns null if fail.
- * See [https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html].
+ * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>.
  * Note that hive Language Manual says it returns 0 if fail, but in fact it returns null.
- * If the second parameter is missing, use "uuuu-MM-dd HH:mm:ss".
+ * If the second parameter is missing, use "yyyy-MM-dd HH:mm:ss".
  * If no parameters provided, the first parameter will be current_timestamp.
  * If the first parameter is a Date or Timestamp instead of String, we will ignore the
  * second parameter.
@@ -709,7 +744,7 @@ case class ToUnixTimestamp(
     Arguments:
       * timeExp - A date/timestamp or string. If not provided, this defaults to current time.
       * format - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
-                 Default value is "uuuu-MM-dd HH:mm:ss". See `java.time.format.DateTimeFormatter`
+                 Default value is "yyyy-MM-dd HH:mm:ss". See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html"> Datetime Patterns</a>
                  for valid date and time format patterns.
   """,
   examples = """
@@ -719,7 +754,9 @@ case class ToUnixTimestamp(
       > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd');
        1460041200
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class UnixTimestamp(timeExp: Expression, format: Expression, timeZoneId: Option[String] = None)
   extends UnixTime {
 
@@ -732,7 +769,7 @@ case class UnixTimestamp(timeExp: Expression, format: Expression, timeZoneId: Op
     copy(timeZoneId = Option(timeZoneId))
 
   def this(time: Expression) = {
-    this(time, Literal("uuuu-MM-dd HH:mm:ss"))
+    this(time, Literal(TimestampFormatter.defaultPattern))
   }
 
   def this() = {
@@ -758,8 +795,13 @@ abstract class ToTimestamp
   private lazy val constFormat: UTF8String = right.eval().asInstanceOf[UTF8String]
   private lazy val formatter: TimestampFormatter =
     try {
-      TimestampFormatter(constFormat.toString, zoneId)
+      TimestampFormatter(
+        constFormat.toString,
+        zoneId,
+        legacyFormat = SIMPLE_DATE_FORMAT,
+        isParsing = true)
     } catch {
+      case e: SparkUpgradeException => throw e
       case NonFatal(_) => null
     }
 
@@ -781,6 +823,7 @@ abstract class ToTimestamp
               formatter.parse(
                 t.asInstanceOf[UTF8String].toString) / downScaleFactor
             } catch {
+              case e: SparkUpgradeException => throw e
               case NonFatal(_) => null
             }
           }
@@ -791,9 +834,14 @@ abstract class ToTimestamp
           } else {
             val formatString = f.asInstanceOf[UTF8String].toString
             try {
-              TimestampFormatter(formatString, zoneId).parse(
-                t.asInstanceOf[UTF8String].toString) / downScaleFactor
+              TimestampFormatter(
+                formatString,
+                zoneId,
+                legacyFormat = SIMPLE_DATE_FORMAT,
+                isParsing = true)
+                .parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor
             } catch {
+              case e: SparkUpgradeException => throw e
               case NonFatal(_) => null
             }
           }
@@ -831,13 +879,17 @@ abstract class ToTimestamp
         }
       case StringType =>
         val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
-        val locale = ctx.addReferenceObj("locale", Locale.US)
         val tf = TimestampFormatter.getClass.getName.stripSuffix("$")
+        val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$")
         nullSafeCodeGen(ctx, ev, (string, format) => {
           s"""
             try {
-              ${ev.value} = $tf$$.MODULE$$.apply($format.toString(), $zid, $locale)
-                .parse($string.toString()) / $downScaleFactor;
+              ${ev.value} = $tf$$.MODULE$$.apply(
+                $format.toString(),
+                $zid,
+                $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(),
+                true)
+              .parse($string.toString()) / $downScaleFactor;
             } catch (java.lang.IllegalArgumentException e) {
               ${ev.isNull} = true;
             } catch (java.text.ParseException e) {
@@ -883,20 +935,26 @@ abstract class UnixTime extends ToTimestamp {
  * format. If the format is missing, using format like "1970-01-01 00:00:00".
  * Note that hive Language Manual says it returns 0 if fail, but in fact it returns null.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(unix_time, format) - Returns `unix_time` in the specified `format`.",
+  usage = "_FUNC_(unix_time[, format]) - Returns `unix_time` in the specified `format`.",
   arguments = """
     Arguments:
       * unix_time - UNIX Timestamp to be converted to the provided format.
-      * format - Date/time format pattern to follow. See `java.time.format.DateTimeFormatter`
-                 for valid date and time format patterns.
+      * format - Date/time format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>
+                 for valid date and time format patterns. The 'yyyy-MM-dd HH:mm:ss' pattern is used if omitted.
   """,
   examples = """
     Examples:
       > SELECT _FUNC_(0, 'yyyy-MM-dd HH:mm:ss');
        1969-12-31 16:00:00
+
+      > SELECT _FUNC_(0);
+       1969-12-31 16:00:00
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[String] = None)
   extends BinaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
 
@@ -908,7 +966,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
   override def prettyName: String = "from_unixtime"
 
   def this(unix: Expression) = {
-    this(unix, Literal("uuuu-MM-dd HH:mm:ss"))
+    this(unix, Literal(TimestampFormatter.defaultPattern))
   }
 
   override def dataType: DataType = StringType
@@ -922,8 +980,13 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
   private lazy val constFormat: UTF8String = right.eval().asInstanceOf[UTF8String]
   private lazy val formatter: TimestampFormatter =
     try {
-      TimestampFormatter(constFormat.toString, zoneId)
+      TimestampFormatter(
+        constFormat.toString,
+        zoneId,
+        legacyFormat = SIMPLE_DATE_FORMAT,
+        isParsing = false)
     } catch {
+      case e: SparkUpgradeException => throw e
       case NonFatal(_) => null
     }
 
@@ -939,6 +1002,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
           try {
             UTF8String.fromString(formatter.format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
           } catch {
+            case e: SparkUpgradeException => throw e
             case NonFatal(_) => null
           }
         }
@@ -948,9 +1012,15 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
           null
         } else {
           try {
-            UTF8String.fromString(TimestampFormatter(f.toString, zoneId)
-              .format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
+            UTF8String.fromString(
+              TimestampFormatter(
+                f.toString,
+                zoneId,
+                legacyFormat = SIMPLE_DATE_FORMAT,
+                isParsing = false)
+                .format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
           } catch {
+            case e: SparkUpgradeException => throw e
             case NonFatal(_) => null
           }
         }
@@ -980,13 +1050,14 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
       }
     } else {
       val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
-      val locale = ctx.addReferenceObj("locale", Locale.US)
       val tf = TimestampFormatter.getClass.getName.stripSuffix("$")
+      val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$")
       nullSafeCodeGen(ctx, ev, (seconds, f) => {
         s"""
         try {
-          ${ev.value} = UTF8String.fromString($tf$$.MODULE$$.apply($f.toString(), $zid, $locale).
-            format($seconds * 1000000L));
+          ${ev.value} = UTF8String.fromString(
+            $tf$$.MODULE$$.apply($f.toString(), $zid, $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(), false)
+              .format($seconds * 1000000L));
         } catch (java.lang.IllegalArgumentException e) {
           ${ev.isNull} = true;
         }"""
@@ -1005,6 +1076,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
       > SELECT _FUNC_('2009-01-12');
        2009-01-31
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitCastInputTypes {
   override def child: Expression = startDate
@@ -1040,6 +1112,7 @@ case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitC
       > SELECT _FUNC_('2015-01-14', 'TU');
        2015-01-20
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class NextDay(startDate: Expression, dayOfWeek: Expression)
@@ -1130,6 +1203,68 @@ case class TimeAdd(start: Expression, interval: Expression, timeZoneId: Option[S
   }
 }
 
+/**
+ * Adds date and an interval.
+ *
+ * When ansi mode is on, the microseconds part of interval needs to be 0, otherwise a runtime
+ * [[IllegalArgumentException]] will be raised.
+ * When ansi mode is off, if the microseconds part of interval is 0, we perform date + interval
+ * for better performance. if the microseconds part is not 0, then the date will be converted to a
+ * timestamp to add with the whole interval parts.
+ */
+case class DateAddInterval(
+    start: Expression,
+    interval: Expression,
+    timeZoneId: Option[String] = None,
+    ansiEnabled: Boolean = SQLConf.get.ansiEnabled)
+  extends BinaryExpression with ExpectsInputTypes with TimeZoneAwareExpression {
+
+  override def left: Expression = start
+  override def right: Expression = interval
+
+  override def toString: String = s"$left + $right"
+  override def sql: String = s"${left.sql} + ${right.sql}"
+  override def inputTypes: Seq[AbstractDataType] = Seq(DateType, CalendarIntervalType)
+
+  override def dataType: DataType = DateType
+
+  override def nullSafeEval(start: Any, interval: Any): Any = {
+    val itvl = interval.asInstanceOf[CalendarInterval]
+    if (ansiEnabled || itvl.microseconds == 0) {
+      DateTimeUtils.dateAddInterval(start.asInstanceOf[Int], itvl)
+    } else {
+      val startTs = DateTimeUtils.epochDaysToMicros(start.asInstanceOf[Int], zoneId)
+      val resultTs = DateTimeUtils.timestampAddInterval(
+        startTs, itvl.months, itvl.days, itvl.microseconds, zoneId)
+      DateTimeUtils.microsToEpochDays(resultTs, zoneId)
+    }
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+    nullSafeCodeGen(ctx, ev, (sd, i) => if (ansiEnabled) {
+      s"""${ev.value} = $dtu.dateAddInterval($sd, $i);"""
+    } else {
+      val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
+      val startTs = ctx.freshName("startTs")
+      val resultTs = ctx.freshName("resultTs")
+      s"""
+         |if ($i.microseconds == 0) {
+         |  ${ev.value} = $dtu.dateAddInterval($sd, $i);
+         |} else {
+         |  long $startTs = $dtu.epochDaysToMicros($sd, $zid);
+         |  long $resultTs =
+         |    $dtu.timestampAddInterval($startTs, $i.months, $i.days, $i.microseconds, $zid);
+         |  ${ev.value} = $dtu.microsToEpochDays($resultTs, $zid);
+         |}
+         |""".stripMargin
+    })
+  }
+
+  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
+    copy(timeZoneId = Option(timeZoneId))
+}
+
 /**
  * This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
  * takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in UTC, and
@@ -1152,6 +1287,7 @@ case class TimeAdd(start: Expression, interval: Expression, timeZoneId: Option[S
       > SELECT _FUNC_('2016-08-31', 'Asia/Seoul');
        2016-08-31 09:00:00
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class FromUTCTimestamp(left: Expression, right: Expression)
@@ -1176,14 +1312,12 @@ case class FromUTCTimestamp(left: Expression, right: Expression)
            |long ${ev.value} = 0;
          """.stripMargin)
       } else {
-        val tzClass = classOf[TimeZone].getName
+        val tzClass = classOf[ZoneId].getName
         val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
         val escapedTz = StringEscapeUtils.escapeJava(tz.toString)
         val tzTerm = ctx.addMutableState(tzClass, "tz",
-          v => s"""$v = $dtu.getTimeZone("$escapedTz");""")
-        val utcTerm = "tzUTC"
-        ctx.addImmutableStateIfNotExists(tzClass, utcTerm,
-          v => s"""$v = $dtu.getTimeZone("UTC");""")
+          v => s"""$v = $dtu.getZoneId("$escapedTz");""")
+        val utcTerm = "java.time.ZoneOffset.UTC"
         val eval = left.genCode(ctx)
         ev.copy(code = code"""
            |${eval.code}
@@ -1248,6 +1382,7 @@ case class TimeSub(start: Expression, interval: Expression, timeZoneId: Option[S
       > SELECT _FUNC_('2016-08-31', 1);
        2016-09-30
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class AddMonths(startDate: Expression, numMonths: Expression)
@@ -1297,6 +1432,7 @@ case class AddMonths(startDate: Expression, numMonths: Expression)
       > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30', false);
        3.9495967741935485
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class MonthsBetween(
@@ -1322,14 +1458,14 @@ case class MonthsBetween(
 
   override def nullSafeEval(t1: Any, t2: Any, roundOff: Any): Any = {
     DateTimeUtils.monthsBetween(
-      t1.asInstanceOf[Long], t2.asInstanceOf[Long], roundOff.asInstanceOf[Boolean], timeZone)
+      t1.asInstanceOf[Long], t2.asInstanceOf[Long], roundOff.asInstanceOf[Boolean], zoneId)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
     defineCodeGen(ctx, ev, (d1, d2, roundOff) => {
-      s"""$dtu.monthsBetween($d1, $d2, $roundOff, $tz)"""
+      s"""$dtu.monthsBetween($d1, $d2, $roundOff, $zid)"""
     })
   }
 
@@ -1358,6 +1494,7 @@ case class MonthsBetween(
       > SELECT _FUNC_('2016-08-31', 'Asia/Seoul');
        2016-08-30 15:00:00
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class ToUTCTimestamp(left: Expression, right: Expression)
@@ -1382,14 +1519,12 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
            |long ${ev.value} = 0;
          """.stripMargin)
       } else {
-        val tzClass = classOf[TimeZone].getName
+        val tzClass = classOf[ZoneId].getName
         val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
         val escapedTz = StringEscapeUtils.escapeJava(tz.toString)
         val tzTerm = ctx.addMutableState(tzClass, "tz",
-          v => s"""$v = $dtu.getTimeZone("$escapedTz");""")
-        val utcTerm = "tzUTC"
-        ctx.addImmutableStateIfNotExists(tzClass, utcTerm,
-          v => s"""$v = $dtu.getTimeZone("UTC");""")
+          v => s"""$v = $dtu.getZoneId("$escapedTz");""")
+        val utcTerm = "java.time.ZoneOffset.UTC"
         val eval = left.genCode(ctx)
         ev.copy(code = code"""
            |${eval.code}
@@ -1411,6 +1546,7 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
 /**
  * Parses a column to a date based on the given format.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """
     _FUNC_(date_str[, fmt]) - Parses the `date_str` expression with the `fmt` expression to
@@ -1420,7 +1556,7 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
   arguments = """
     Arguments:
       * date_str - A string to be parsed to date.
-      * fmt - Date format pattern to follow. See `java.time.format.DateTimeFormatter` for valid
+      * fmt - Date format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a> for valid
               date and time format patterns.
   """,
   examples = """
@@ -1430,7 +1566,9 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
       > SELECT _FUNC_('2016-12-31', 'yyyy-MM-dd');
        2016-12-31
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class ParseToDate(left: Expression, format: Option[Expression], child: Expression)
   extends RuntimeReplaceable {
 
@@ -1459,6 +1597,7 @@ case class ParseToDate(left: Expression, format: Option[Expression], child: Expr
 /**
  * Parses a column to a timestamp based on the supplied format.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """
     _FUNC_(timestamp_str[, fmt]) - Parses the `timestamp_str` expression with the `fmt` expression
@@ -1468,7 +1607,7 @@ case class ParseToDate(left: Expression, format: Option[Expression], child: Expr
   arguments = """
     Arguments:
       * timestamp_str - A string to be parsed to timestamp.
-      * fmt - Timestamp format pattern to follow. See `java.time.format.DateTimeFormatter` for valid
+      * fmt - Timestamp format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a> for valid
               date and time format patterns.
   """,
   examples = """
@@ -1478,7 +1617,9 @@ case class ParseToDate(left: Expression, format: Option[Expression], child: Expr
       > SELECT _FUNC_('2016-12-31', 'yyyy-MM-dd');
        2016-12-31 00:00:00
   """,
+  group = "datetime_funcs",
   since = "2.2.0")
+// scalastyle:on line.size.limit
 case class ParseToTimestamp(left: Expression, format: Option[Expression], child: Expression)
   extends RuntimeReplaceable {
 
@@ -1589,7 +1730,15 @@ trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes {
 @ExpressionDescription(
   usage = """
     _FUNC_(date, fmt) - Returns `date` with the time portion of the day truncated to the unit specified by the format model `fmt`.
-    `fmt` should be one of ["week", "mon", "month", "mm", "quarter", "year", "yyyy", "yy", "decade", "century", "millennium"]
+  """,
+  arguments = """
+     Arguments:
+       * date - date value or valid date string
+       * fmt - the format representing the unit to be truncated to
+           - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `date` falls in
+           - "QUARTER" - truncate to the first date of the quarter that the `date` falls in
+           - "MONTH", "MM", "MON" - truncate to the first date of the month that the `date` falls in
+           - "WEEK" - truncate to the Monday of the week that the `date` falls in
   """,
   examples = """
     Examples:
@@ -1601,13 +1750,8 @@ trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes {
        2009-02-01
       > SELECT _FUNC_('2015-10-27', 'YEAR');
        2015-01-01
-      > SELECT _FUNC_('2015-10-27', 'DECADE');
-       2010-01-01
-      > SELECT _FUNC_('1981-01-19', 'century');
-       1901-01-01
-      > SELECT _FUNC_('1981-01-19', 'millennium');
-       1001-01-01
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class TruncDate(date: Expression, format: Expression)
@@ -1640,9 +1784,21 @@ case class TruncDate(date: Expression, format: Expression)
 @ExpressionDescription(
   usage = """
     _FUNC_(fmt, ts) - Returns timestamp `ts` truncated to the unit specified by the format model `fmt`.
-    `fmt` should be one of ["MILLENNIUM", "CENTURY", "DECADE", "YEAR", "YYYY", "YY",
-                            "QUARTER", "MON", "MONTH", "MM", "WEEK", "DAY", "DD",
-                            "HOUR", "MINUTE", "SECOND", "MILLISECOND", "MICROSECOND"]
+  """,
+  arguments = """
+     Arguments:
+       * fmt - the format representing the unit to be truncated to
+           - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `ts` falls in, the time part will be zero out
+           - "QUARTER" - truncate to the first date of the quarter that the `ts` falls in, the time part will be zero out
+           - "MONTH", "MM", "MON" - truncate to the first date of the month that the `ts` falls in, the time part will be zero out
+           - "WEEK" - truncate to the Monday of the week that the `ts` falls in, the time part will be zero out
+           - "DAY", "DD" - zero out the time part
+           - "HOUR" - zero out the minute and second with fraction part
+           - "MINUTE"- zero out the second with fraction part
+           - "SECOND" -  zero out the second fraction part
+           - "MILLISECOND" - zero out the microseconds
+           - "MICROSECOND" - everything remains
+       * ts - datetime value or valid timestamp string
   """,
   examples = """
     Examples:
@@ -1656,11 +1812,8 @@ case class TruncDate(date: Expression, format: Expression)
        2015-03-05 09:00:00
       > SELECT _FUNC_('MILLISECOND', '2015-03-05T09:32:05.123456');
        2015-03-05 09:32:05.123
-      > SELECT _FUNC_('DECADE', '2015-03-05T09:32:05.123456');
-       2010-01-01 00:00:00
-      > SELECT _FUNC_('CENTURY', '2015-03-05T09:32:05.123456');
-       2001-01-01 00:00:00
   """,
+  group = "datetime_funcs",
   since = "2.3.0")
 // scalastyle:on line.size.limit
 case class TruncTimestamp(
@@ -1682,15 +1835,15 @@ case class TruncTimestamp(
 
   override def eval(input: InternalRow): Any = {
     evalHelper(input, minLevel = MIN_LEVEL_OF_TIMESTAMP_TRUNC) { (t: Any, level: Int) =>
-      DateTimeUtils.truncTimestamp(t.asInstanceOf[Long], level, timeZone)
+      DateTimeUtils.truncTimestamp(t.asInstanceOf[Long], level, zoneId)
     }
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     codeGenHelper(ctx, ev, minLevel = MIN_LEVEL_OF_TIMESTAMP_TRUNC, true) {
       (date: String, fmt: String) =>
-        s"truncTimestamp($date, $fmt, $tz);"
+        s"truncTimestamp($date, $fmt, $zid);"
     }
   }
 }
@@ -1708,6 +1861,7 @@ case class TruncTimestamp(
       > SELECT _FUNC_('2009-07-30', '2009-07-31');
        -1
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class DateDiff(endDate: Expression, startDate: Expression)
   extends BinaryExpression with ImplicitCastInputTypes {
@@ -1761,6 +1915,7 @@ private case class GetTimestamp(
       > SELECT _FUNC_(2019, 2, 30);
        NULL
   """,
+  group = "datetime_funcs",
   since = "3.0.0")
 case class MakeDate(year: Expression, month: Expression, day: Expression)
   extends TernaryExpression with ImplicitCastInputTypes {
@@ -1822,6 +1977,7 @@ case class MakeDate(year: Expression, month: Expression, day: Expression)
       > SELECT _FUNC_(null, 7, 22, 15, 30, 0);
        NULL
   """,
+  group = "datetime_funcs",
   since = "3.0.0")
 // scalastyle:on line.size.limit
 case class MakeTimestamp(
@@ -1953,147 +2109,56 @@ case class MakeTimestamp(
   override def prettyName: String = "make_timestamp"
 }
 
-case class Millennium(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
-
-  override def dataType: DataType = IntegerType
-
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getMillennium(date.asInstanceOf[Int])
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMillennium($c)")
-  }
-}
-
-case class Century(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
-
-  override def dataType: DataType = IntegerType
-
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getCentury(date.asInstanceOf[Int])
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getCentury($c)")
-  }
-}
-
-case class Decade(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
-
-  override def dataType: DataType = IntegerType
-
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getDecade(date.asInstanceOf[Int])
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getDecade($c)")
-  }
-}
-
-case class Epoch(child: Expression, timeZoneId: Option[String] = None)
-    extends UnaryExpression with ImplicitCastInputTypes with TimeZoneAwareExpression {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
-  // DecimalType is used to not lose precision while converting microseconds to
-  // the fractional part of seconds. Scale 6 is taken to have all microseconds as
-  // the fraction. The precision 20 should cover whole valid range of years [1, 9999]
-  // plus negative years that can be used in some cases though are not officially supported.
-  override def dataType: DataType = DecimalType(20, 6)
-  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
-    copy(timeZoneId = Option(timeZoneId))
-
-  override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getEpoch(timestamp.asInstanceOf[Long], zoneId)
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
-    defineCodeGen(ctx, ev, c => s"$dtu.getEpoch($c, $zid)")
-  }
-}
-
 object DatePart {
 
   def parseExtractField(
       extractField: String,
       source: Expression,
       errorHandleFunc: => Nothing): Expression = extractField.toUpperCase(Locale.ROOT) match {
-    case "MILLENNIUM" | "MILLENNIA" | "MIL" | "MILS" => Millennium(source)
-    case "CENTURY" | "CENTURIES" | "C" | "CENT" => Century(source)
-    case "DECADE" | "DECADES" | "DEC" | "DECS" => Decade(source)
     case "YEAR" | "Y" | "YEARS" | "YR" | "YRS" => Year(source)
-    case "ISOYEAR" => IsoYear(source)
+    case "YEAROFWEEK" => YearOfWeek(source)
     case "QUARTER" | "QTR" => Quarter(source)
     case "MONTH" | "MON" | "MONS" | "MONTHS" => Month(source)
     case "WEEK" | "W" | "WEEKS" => WeekOfYear(source)
     case "DAY" | "D" | "DAYS" => DayOfMonth(source)
-    case "DAYOFWEEK" => DayOfWeek(source)
-    case "DOW" => Subtract(DayOfWeek(source), Literal(1))
-    case "ISODOW" => Add(WeekDay(source), Literal(1))
+    case "DAYOFWEEK" | "DOW" => DayOfWeek(source)
+    case "DAYOFWEEK_ISO" | "DOW_ISO" => Add(WeekDay(source), Literal(1))
     case "DOY" => DayOfYear(source)
     case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => Hour(source)
     case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => Minute(source)
     case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" => SecondWithFraction(source)
-    case "MILLISECONDS" | "MSEC" | "MSECS" | "MILLISECON" | "MSECONDS" | "MS" =>
-      Milliseconds(source)
-    case "MICROSECONDS" | "USEC" | "USECS" | "USECONDS" | "MICROSECON" | "US" =>
-      Microseconds(source)
-    case "EPOCH" => Epoch(source)
     case _ => errorHandleFunc
   }
+
+  def toEquivalentExpr(field: Expression, source: Expression): Expression = {
+    if (!field.foldable) {
+      throw new AnalysisException("The field parameter needs to be a foldable string value.")
+    }
+    val fieldEval = field.eval()
+    if (fieldEval == null) {
+      Literal(null, DoubleType)
+    } else {
+      val fieldStr = fieldEval.asInstanceOf[UTF8String].toString
+      val errMsg = s"Literals of type '$fieldStr' are currently not supported " +
+        s"for the ${source.dataType.catalogString} type."
+      if (source.dataType == CalendarIntervalType) {
+        ExtractIntervalPart.parseExtractField(
+          fieldStr,
+          source,
+          throw new AnalysisException(errMsg))
+      } else {
+        DatePart.parseExtractField(fieldStr, source, throw new AnalysisException(errMsg))
+      }
+    }
+  }
 }
 
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = "_FUNC_(field, source) - Extracts a part of the date/timestamp or interval source.",
   arguments = """
     Arguments:
-      * field - selects which part of the source should be extracted.
-               Supported string values of `field` for dates and timestamps are:
-                ["MILLENNIUM", ("MILLENNIA", "MIL", "MILS"),
-                 "CENTURY", ("CENTURIES", "C", "CENT"),
-                 "DECADE", ("DECADES", "DEC", "DECS"),
-                 "YEAR", ("Y", "YEARS", "YR", "YRS"),
-                 "ISOYEAR",
-                 "QUARTER", ("QTR"),
-                 "MONTH", ("MON", "MONS", "MONTHS"),
-                 "WEEK", ("W", "WEEKS"),
-                 "DAY", ("D", "DAYS"),
-                 "DAYOFWEEK",
-                 "DOW",
-                 "ISODOW",
-                 "DOY",
-                 "HOUR", ("H", "HOURS", "HR", "HRS"),
-                 "MINUTE", ("M", "MIN", "MINS", "MINUTES"),
-                 "SECOND", ("S", "SEC", "SECONDS", "SECS"),
-                 "MILLISECONDS", ("MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS"),
-                 "MICROSECONDS", ("USEC", "USECS", "USECONDS", "MICROSECON", "US"),
-                 "EPOCH"]
-                Supported string values of `field` for intervals are:
-                 ["MILLENNIUM", ("MILLENNIA", "MIL", "MILS"),
-                   "CENTURY", ("CENTURIES", "C", "CENT"),
-                   "DECADE", ("DECADES", "DEC", "DECS"),
-                   "YEAR", ("Y", "YEARS", "YR", "YRS"),
-                   "QUARTER", ("QTR"),
-                   "MONTH", ("MON", "MONS", "MONTHS"),
-                   "DAY", ("D", "DAYS"),
-                   "HOUR", ("H", "HOURS", "HR", "HRS"),
-                   "MINUTE", ("M", "MIN", "MINS", "MINUTES"),
-                   "SECOND", ("S", "SEC", "SECONDS", "SECS"),
-                   "MILLISECONDS", ("MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS"),
-                   "MICROSECONDS", ("USEC", "USECS", "USECONDS", "MICROSECON", "US"),
-                   "EPOCH"]
+      * field - selects which part of the source should be extracted, and supported string values are as same as the fields of the equivalent function `EXTRACT`.
       * source - a date/timestamp or interval column from where `field` should be extracted
   """,
   examples = """
@@ -2111,32 +2176,17 @@ object DatePart {
       > SELECT _FUNC_('seconds', interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
        30.001001
   """,
+  note = """
+    The _FUNC_ function is equivalent to the SQL-standard function `EXTRACT(field FROM source)`
+  """,
+  group = "datetime_funcs",
   since = "3.0.0")
+// scalastyle:on line.size.limit
 case class DatePart(field: Expression, source: Expression, child: Expression)
   extends RuntimeReplaceable {
 
-  def this(field: Expression, source: Expression) {
-    this(field, source, {
-      if (!field.foldable) {
-        throw new AnalysisException("The field parameter needs to be a foldable string value.")
-      }
-      val fieldEval = field.eval()
-      if (fieldEval == null) {
-        Literal(null, DoubleType)
-      } else {
-        val fieldStr = fieldEval.asInstanceOf[UTF8String].toString
-        val errMsg = s"Literals of type '$fieldStr' are currently not supported " +
-          s"for the ${source.dataType.catalogString} type."
-        if (source.dataType == CalendarIntervalType) {
-          ExtractIntervalPart.parseExtractField(
-            fieldStr,
-            source,
-            throw new AnalysisException(errMsg))
-        } else {
-          DatePart.parseExtractField(fieldStr, source, throw new AnalysisException(errMsg))
-        }
-      }
-    })
+  def this(field: Expression, source: Expression) = {
+    this(field, source, DatePart.toEquivalentExpr(field, source))
   }
 
   override def flatArguments: Iterator[Any] = Iterator(field, source)
@@ -2144,6 +2194,66 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
   override def prettyName: String = "date_part"
 }
 
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+  usage = "_FUNC_(field FROM source) - Extracts a part of the date/timestamp or interval source.",
+  arguments = """
+    Arguments:
+      * field - selects which part of the source should be extracted
+          - Supported string values of `field` for dates and timestamps are(case insensitive):
+              - "YEAR", ("Y", "YEARS", "YR", "YRS") - the year field
+              - "YEAROFWEEK" - the ISO 8601 week-numbering year that the datetime falls in. For example, 2005-01-02 is part of the 53rd week of year 2004, so the result is 2004
+              - "QUARTER", ("QTR") - the quarter (1 - 4) of the year that the datetime falls in
+              - "MONTH", ("MON", "MONS", "MONTHS") - the month field (1 - 12)
+              - "WEEK", ("W", "WEEKS") - the number of the ISO 8601 week-of-week-based-year. A week is considered to start on a Monday and week 1 is the first week with >3 days. In the ISO week-numbering system, it is possible for early-January dates to be part of the 52nd or 53rd week of the previous year, and for late-December dates to be part of the first week of the next year. For example, 2005-01-02 is part of the 53rd week of year 2004, while 2012-12-31 is part of the first week of 2013
+              - "DAY", ("D", "DAYS") - the day of the month field (1 - 31)
+              - "DAYOFWEEK",("DOW") - the day of the week for datetime as Sunday(1) to Saturday(7)
+              - "DAYOFWEEK_ISO",("DOW_ISO") - ISO 8601 based day of the week for datetime as Monday(1) to Sunday(7)
+              - "DOY" - the day of the year (1 - 365/366)
+              - "HOUR", ("H", "HOURS", "HR", "HRS") - The hour field (0 - 23)
+              - "MINUTE", ("M", "MIN", "MINS", "MINUTES") - the minutes field (0 - 59)
+              - "SECOND", ("S", "SEC", "SECONDS", "SECS") - the seconds field, including fractional parts
+          - Supported string values of `field` for interval(which consists of `months`, `days`, `microseconds`) are(case insensitive):
+              - "YEAR", ("Y", "YEARS", "YR", "YRS") - the total `months` / 12
+              - "MONTH", ("MON", "MONS", "MONTHS") - the total `months` % 12
+              - "DAY", ("D", "DAYS") - the `days` part of interval
+              - "HOUR", ("H", "HOURS", "HR", "HRS") - how many hours the `microseconds` contains
+              - "MINUTE", ("M", "MIN", "MINS", "MINUTES") - how many minutes left after taking hours from `microseconds`
+              - "SECOND", ("S", "SEC", "SECONDS", "SECS") - how many second with fractions left after taking hours and minutes from `microseconds`
+      * source - a date/timestamp or interval column from where `field` should be extracted
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456');
+       2019
+      > SELECT _FUNC_(week FROM timestamp'2019-08-12 01:00:00.123456');
+       33
+      > SELECT _FUNC_(doy FROM DATE'2019-08-12');
+       224
+      > SELECT _FUNC_(SECONDS FROM timestamp'2019-10-01 00:00:01.000001');
+       1.000001
+      > SELECT _FUNC_(days FROM interval 1 year 10 months 5 days);
+       5
+      > SELECT _FUNC_(seconds FROM interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
+       30.001001
+  """,
+  note = """
+    The _FUNC_ function is equivalent to `date_part(field, source)`.
+  """,
+  since = "3.0.0")
+// scalastyle:on line.size.limit
+case class Extract(field: Expression, source: Expression, child: Expression)
+  extends RuntimeReplaceable {
+
+  def this(field: Expression, source: Expression) = {
+    this(field, source, DatePart.toEquivalentExpr(field, source))
+  }
+
+  override def flatArguments: Iterator[Any] = Iterator(field, source)
+  override def sql: String = s"$prettyName(${field.sql} FROM ${source.sql})"
+  override def prettyName: String = "extract"
+}
+
 /**
  * Returns the interval from startTimestamp to endTimestamp in which the `months` and `day` field
  * is set to 0 and the `microseconds` field is initialized to the microsecond difference
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala
index 7b2489e682661..9014ebfe2f96a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala
@@ -45,9 +45,15 @@ case class UnscaledValue(child: Expression) extends UnaryExpression {
  * Note: this expression is internal and created only by the optimizer,
  * we don't need to do type check for it.
  */
-case class MakeDecimal(child: Expression, precision: Int, scale: Int) extends UnaryExpression {
+case class MakeDecimal(
+    child: Expression,
+    precision: Int,
+    scale: Int,
+    nullOnOverflow: Boolean) extends UnaryExpression {
 
-  private val nullOnOverflow = !SQLConf.get.ansiEnabled
+  def this(child: Expression, precision: Int, scale: Int) = {
+    this(child, precision, scale, !SQLConf.get.ansiEnabled)
+  }
 
   override def dataType: DataType = DecimalType(precision, scale)
   override def nullable: Boolean = child.nullable || nullOnOverflow
@@ -83,6 +89,12 @@ case class MakeDecimal(child: Expression, precision: Int, scale: Int) extends Un
   }
 }
 
+object MakeDecimal {
+  def apply(child: Expression, precision: Int, scale: Int): MakeDecimal = {
+    new MakeDecimal(child, precision, scale)
+  }
+}
+
 /**
  * An expression used to wrap the children when promote the precision of DecimalType to avoid
  * promote multiple times.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
index 52429a63b306d..4c8c58ae232f4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{ArrayData, MapData}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.Platform
 import org.apache.spark.unsafe.hash.Murmur3_x86_32
@@ -232,9 +233,6 @@ case class Crc32(child: Expression) extends UnaryExpression with ImplicitCastInp
  *  - array:              The `result` starts with seed, then use `result` as seed, recursively
  *                        calculate hash value for each element, and assign the element hash value
  *                        to `result`.
- *  - map:                The `result` starts with seed, then use `result` as seed, recursively
- *                        calculate hash value for each key-value, and assign the key-value hash
- *                        value to `result`.
  *  - struct:             The `result` starts with seed, then use `result` as seed, recursively
  *                        calculate hash value for each field, and assign the field hash value to
  *                        `result`.
@@ -249,10 +247,21 @@ abstract class HashExpression[E] extends Expression {
 
   override def nullable: Boolean = false
 
+  private def hasMapType(dt: DataType): Boolean = {
+    dt.existsRecursively(_.isInstanceOf[MapType])
+  }
+
   override def checkInputDataTypes(): TypeCheckResult = {
     if (children.length < 1) {
       TypeCheckResult.TypeCheckFailure(
         s"input to function $prettyName requires at least one argument")
+    } else if (children.exists(child => hasMapType(child.dataType)) &&
+        !SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE)) {
+      TypeCheckResult.TypeCheckFailure(
+        s"input to function $prettyName cannot contain elements of MapType. In Spark, same maps " +
+          "may have different hashcode, thus hash expressions are prohibited on MapType elements." +
+          s" To restore previous behavior set ${SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE.key} " +
+          "to true.")
     } else {
       TypeCheckResult.TypeCheckSuccess
     }
@@ -601,7 +610,8 @@ object Murmur3HashFunction extends InterpretedHashFunction {
     Examples:
       > SELECT _FUNC_('Spark', array(123), 2);
        5602566077635097486
-  """)
+  """,
+  since = "3.0.0")
 case class XxHash64(children: Seq[Expression], seed: Long) extends HashExpression[Long] {
   def this(arguments: Seq[Expression]) = this(arguments, 42L)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index f8142d6b993cb..e5cf8c0a023d9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -454,6 +454,8 @@ case class MapFilter(
        [1,3]
       > SELECT _FUNC_(array(0, 2, 3), (x, i) -> x > i);
        [2,3]
+      > SELECT _FUNC_(array(0, null, 2, 3, null), x -> x IS NOT NULL);
+       [0,2,3]
   """,
   since = "2.4.0",
   note = """
@@ -518,15 +520,26 @@ case class ArrayFilter(
        false
       > SELECT _FUNC_(array(1, null, 3), x -> x % 2 == 0);
        NULL
+      > SELECT _FUNC_(array(0, null, 2, 3, null), x -> x IS NULL);
+       true
+      > SELECT _FUNC_(array(1, 2, 3), x -> x IS NULL);
+       false
   """,
   since = "2.4.0")
 case class ArrayExists(
     argument: Expression,
-    function: Expression)
+    function: Expression,
+    followThreeValuedLogic: Boolean)
   extends ArrayBasedSimpleHigherOrderFunction with CodegenFallback {
 
-  private val followThreeValuedLogic =
-    SQLConf.get.getConf(SQLConf.LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC)
+  def this(argument: Expression, function: Expression) = {
+    this(
+      argument,
+      function,
+      SQLConf.get.getConf(SQLConf.LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC))
+  }
+
+  override def stringArgs: Iterator[Any] = super.stringArgs.take(2)
 
   override def nullable: Boolean =
     if (followThreeValuedLogic) {
@@ -574,6 +587,12 @@ case class ArrayExists(
   override def prettyName: String = "exists"
 }
 
+object ArrayExists {
+  def apply(argument: Expression, function: Expression): ArrayExists = {
+    new ArrayExists(argument, function)
+  }
+}
+
 /**
  * Tests whether a predicate holds for all elements in the array.
  */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
index 831510e7f0f3e..8cfea1e6f6cb5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
@@ -22,6 +22,7 @@ import java.util.Locale
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.util.IntervalUtils
 import org.apache.spark.sql.catalyst.util.IntervalUtils._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
 
@@ -44,21 +45,9 @@ abstract class ExtractIntervalPart(
   }
 }
 
-case class ExtractIntervalMillenniums(child: Expression)
-  extends ExtractIntervalPart(child, IntegerType, getMillenniums, "getMillenniums")
-
-case class ExtractIntervalCenturies(child: Expression)
-  extends ExtractIntervalPart(child, IntegerType, getCenturies, "getCenturies")
-
-case class ExtractIntervalDecades(child: Expression)
-  extends ExtractIntervalPart(child, IntegerType, getDecades, "getDecades")
-
 case class ExtractIntervalYears(child: Expression)
   extends ExtractIntervalPart(child, IntegerType, getYears, "getYears")
 
-case class ExtractIntervalQuarters(child: Expression)
-  extends ExtractIntervalPart(child, ByteType, getQuarters, "getQuarters")
-
 case class ExtractIntervalMonths(child: Expression)
   extends ExtractIntervalPart(child, ByteType, getMonths, "getMonths")
 
@@ -74,51 +63,32 @@ case class ExtractIntervalMinutes(child: Expression)
 case class ExtractIntervalSeconds(child: Expression)
   extends ExtractIntervalPart(child, DecimalType(8, 6), getSeconds, "getSeconds")
 
-case class ExtractIntervalMilliseconds(child: Expression)
-  extends ExtractIntervalPart(child, DecimalType(8, 3), getMilliseconds, "getMilliseconds")
-
-case class ExtractIntervalMicroseconds(child: Expression)
-  extends ExtractIntervalPart(child, LongType, getMicroseconds, "getMicroseconds")
-
-// Number of seconds in 10000 years is 315576000001 (30 days per one month)
-// which is 12 digits + 6 digits for the fractional part of seconds.
-case class ExtractIntervalEpoch(child: Expression)
-  extends ExtractIntervalPart(child, DecimalType(18, 6), getEpoch, "getEpoch")
-
 object ExtractIntervalPart {
 
   def parseExtractField(
       extractField: String,
       source: Expression,
       errorHandleFunc: => Nothing): Expression = extractField.toUpperCase(Locale.ROOT) match {
-    case "MILLENNIUM" | "MILLENNIA" | "MIL" | "MILS" => ExtractIntervalMillenniums(source)
-    case "CENTURY" | "CENTURIES" | "C" | "CENT" => ExtractIntervalCenturies(source)
-    case "DECADE" | "DECADES" | "DEC" | "DECS" => ExtractIntervalDecades(source)
     case "YEAR" | "Y" | "YEARS" | "YR" | "YRS" => ExtractIntervalYears(source)
-    case "QUARTER" | "QTR" => ExtractIntervalQuarters(source)
     case "MONTH" | "MON" | "MONS" | "MONTHS" => ExtractIntervalMonths(source)
     case "DAY" | "D" | "DAYS" => ExtractIntervalDays(source)
     case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => ExtractIntervalHours(source)
     case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => ExtractIntervalMinutes(source)
     case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" => ExtractIntervalSeconds(source)
-    case "MILLISECONDS" | "MSEC" | "MSECS" | "MILLISECON" | "MSECONDS" | "MS" =>
-      ExtractIntervalMilliseconds(source)
-    case "MICROSECONDS" | "USEC" | "USECS" | "USECONDS" | "MICROSECON" | "US" =>
-      ExtractIntervalMicroseconds(source)
-    case "EPOCH" => ExtractIntervalEpoch(source)
     case _ => errorHandleFunc
   }
 }
 
 abstract class IntervalNumOperation(
     interval: Expression,
-    num: Expression,
-    operation: (CalendarInterval, Double) => CalendarInterval,
-    operationName: String)
+    num: Expression)
   extends BinaryExpression with ImplicitCastInputTypes with Serializable {
   override def left: Expression = interval
   override def right: Expression = num
 
+  protected val operation: (CalendarInterval, Double) => CalendarInterval
+  protected def operationName: String
+
   override def inputTypes: Seq[AbstractDataType] = Seq(CalendarIntervalType, DoubleType)
   override def dataType: DataType = CalendarIntervalType
 
@@ -136,11 +106,29 @@ abstract class IntervalNumOperation(
   override def prettyName: String = operationName.stripSuffix("Exact") + "_interval"
 }
 
-case class MultiplyInterval(interval: Expression, num: Expression)
-  extends IntervalNumOperation(interval, num, multiplyExact, "multiplyExact")
+case class MultiplyInterval(
+    interval: Expression,
+    num: Expression,
+    checkOverflow: Boolean = SQLConf.get.ansiEnabled)
+  extends IntervalNumOperation(interval, num) {
+
+  override protected val operation: (CalendarInterval, Double) => CalendarInterval =
+    if (checkOverflow) multiplyExact else multiply
 
-case class DivideInterval(interval: Expression, num: Expression)
-  extends IntervalNumOperation(interval, num, divideExact, "divideExact")
+  override protected def operationName: String = if (checkOverflow) "multiplyExact" else "multiply"
+}
+
+case class DivideInterval(
+    interval: Expression,
+    num: Expression,
+    checkOverflow: Boolean = SQLConf.get.ansiEnabled)
+  extends IntervalNumOperation(interval, num) {
+
+  override protected val operation: (CalendarInterval, Double) => CalendarInterval =
+    if (checkOverflow) divideExact else divide
+
+  override protected def operationName: String = if (checkOverflow) "divideExact" else "divide"
+}
 
 // scalastyle:off line.size.limit
 @ExpressionDescription(
@@ -161,6 +149,8 @@ case class DivideInterval(interval: Expression, num: Expression)
        100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds
       > SELECT _FUNC_(100, null, 3);
        NULL
+      > SELECT _FUNC_(0, 1, 0, 1, 0, 0, 100.000001);
+       1 months 1 days 1 minutes 40.000001 seconds
   """,
   since = "3.0.0")
 // scalastyle:on line.size.limit
@@ -181,7 +171,7 @@ case class MakeInterval(
       days: Expression,
       hours: Expression,
       mins: Expression) = {
-    this(years, months, weeks, days, hours, mins, Literal(Decimal(0, 8, 6)))
+    this(years, months, weeks, days, hours, mins, Literal(Decimal(0, Decimal.MAX_LONG_DIGITS, 6)))
   }
   def this(
       years: Expression,
@@ -203,7 +193,7 @@ case class MakeInterval(
   // Accept `secs` as DecimalType to avoid loosing precision of microseconds while converting
   // them to the fractional part of `secs`.
   override def inputTypes: Seq[AbstractDataType] = Seq(IntegerType, IntegerType, IntegerType,
-    IntegerType, IntegerType, IntegerType, DecimalType(8, 6))
+    IntegerType, IntegerType, IntegerType, DecimalType(Decimal.MAX_LONG_DIGITS, 6))
   override def dataType: DataType = CalendarIntervalType
   override def nullable: Boolean = true
 
@@ -223,7 +213,7 @@ case class MakeInterval(
         day.asInstanceOf[Int],
         hour.asInstanceOf[Int],
         min.asInstanceOf[Int],
-        sec.map(_.asInstanceOf[Decimal]).getOrElse(Decimal(0, 8, 6)))
+        sec.map(_.asInstanceOf[Decimal]).getOrElse(Decimal(0, Decimal.MAX_LONG_DIGITS, 6)))
     } catch {
       case _: ArithmeticException => null
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index 61afdb6c9492f..38faaf500b9b4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -117,7 +117,8 @@ private[this] object SharedFactory {
     Examples:
       > SELECT _FUNC_('{"a":"b"}', '$.a');
        b
-  """)
+  """,
+  group = "json_funcs")
 case class GetJsonObject(json: Expression, path: Expression)
   extends BinaryExpression with ExpectsInputTypes with CodegenFallback {
 
@@ -340,7 +341,8 @@ case class GetJsonObject(json: Expression, path: Expression)
     Examples:
       > SELECT _FUNC_('{"a":1, "b":2}', 'a', 'b');
        1	2
-  """)
+  """,
+  group = "json_funcs")
 // scalastyle:on line.size.limit line.contains.tab
 case class JsonTuple(children: Seq[Expression])
   extends Generator with CodegenFallback {
@@ -508,6 +510,7 @@ case class JsonTuple(children: Seq[Expression])
       > SELECT _FUNC_('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'));
        {"time":2015-08-26 00:00:00}
   """,
+  group = "json_funcs",
   since = "2.2.0")
 // scalastyle:on line.size.limit
 case class JsonToStructs(
@@ -627,6 +630,7 @@ case class JsonToStructs(
       > SELECT _FUNC_(array((map('a', 1))));
        [{"a":1}]
   """,
+  group = "json_funcs",
   since = "2.2.0")
 // scalastyle:on line.size.limit
 case class StructsToJson(
@@ -736,6 +740,7 @@ case class StructsToJson(
       > SELECT _FUNC_('[{"col":01}]', map('allowNumericLeadingZeros', 'true'));
        array<struct<col:bigint>>
   """,
+  group = "json_funcs",
   since = "2.4.0")
 case class SchemaOfJson(
     child: Expression,
@@ -773,7 +778,18 @@ case class SchemaOfJson(
   override def eval(v: InternalRow): Any = {
     val dt = Utils.tryWithResource(CreateJacksonParser.utf8String(jsonFactory, json)) { parser =>
       parser.nextToken()
-      jsonInferSchema.inferField(parser)
+      // To match with schema inference from JSON datasource.
+      jsonInferSchema.inferField(parser) match {
+        case st: StructType =>
+          jsonInferSchema.canonicalizeType(st, jsonOptions).getOrElse(StructType(Nil))
+        case at: ArrayType if at.elementType.isInstanceOf[StructType] =>
+          jsonInferSchema
+            .canonicalizeType(at.elementType, jsonOptions)
+            .map(ArrayType(_, containsNull = at.containsNull))
+            .getOrElse(ArrayType(StructType(Nil), containsNull = at.containsNull))
+        case other: DataType =>
+          jsonInferSchema.canonicalizeType(other, jsonOptions).getOrElse(StringType)
+      }
     }
 
     UTF8String.fromString(dt.catalogString)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 213a58a3244e2..413d0af61a05c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -297,6 +297,7 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
   override def toString: String = value match {
     case null => "null"
     case binary: Array[Byte] => s"0x" + DatatypeConverter.printHexBinary(binary)
+    case d: ArrayBasedMapData => s"map(${d.toString})"
     case other => other.toString
   }
 
@@ -315,6 +316,8 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
       (value, o.value) match {
         case (null, null) => true
         case (a: Array[Byte], b: Array[Byte]) => util.Arrays.equals(a, b)
+        case (a: ArrayBasedMapData, b: ArrayBasedMapData) =>
+          a.keyArray == b.keyArray && a.valueArray == b.valueArray
         case (a, b) => a != null && a.equals(b)
       }
     case _ => false
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
index d5b959b91c23d..8c6fbc0fc8e44 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
@@ -21,7 +21,7 @@ import java.{lang => jl}
 import java.util.Locale
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
@@ -62,8 +62,8 @@ abstract class UnaryMathExpression(val f: Double => Double, name: String)
   override def inputTypes: Seq[AbstractDataType] = Seq(DoubleType)
   override def dataType: DataType = DoubleType
   override def nullable: Boolean = true
-  override def toString: String = s"$name($child)"
-  override def prettyName: String = name
+  override def toString: String = s"$prettyName($child)"
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse(name)
 
   protected override def nullSafeEval(input: Any): Any = {
     f(input.asInstanceOf[Double])
@@ -115,9 +115,9 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
 
   override def inputTypes: Seq[DataType] = Seq(DoubleType, DoubleType)
 
-  override def toString: String = s"$name($left, $right)"
+  override def toString: String = s"$prettyName($left, $right)"
 
-  override def prettyName: String = name
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse(name)
 
   override def dataType: DataType = DoubleType
 
@@ -491,7 +491,9 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas
       > SELECT _FUNC_(1);
        0.0
   """)
-case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LOG")
+case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LOG") {
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("ln")
+}
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 2.",
@@ -546,6 +548,7 @@ case class Log1p(child: Expression) extends UnaryLogExpression(StrictMath.log1p,
 // scalastyle:on line.size.limit
 case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND") {
   override def funcName: String = "rint"
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("rint")
 }
 
 @ExpressionDescription(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index f576873829f27..8ce3ddd30a69e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -173,6 +173,7 @@ case class SparkVersion() extends LeafExpression with CodegenFallback {
   override def nullable: Boolean = false
   override def foldable: Boolean = true
   override def dataType: DataType = StringType
+  override def prettyName: String = "version"
   override def eval(input: InternalRow): Any = {
     UTF8String.fromString(SPARK_VERSION_SHORT + " " + SPARK_REVISION)
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 3362353e2662a..77b4cecc263c7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -236,8 +236,6 @@ case class AttributeReference(
     val qualifier: Seq[String] = Seq.empty[String])
   extends Attribute with Unevaluable {
 
-  // currently can only handle qualifier of length 2
-  require(qualifier.length <= 2)
   /**
    * Returns true iff the expression id is the same for both attributes.
    */
@@ -384,6 +382,7 @@ case class OuterReference(e: NamedExpression)
   override def nullable: Boolean = e.nullable
   override def prettyName: String = "outer"
 
+  override def sql: String = s"$prettyName(${e.sql})"
   override def name: String = e.name
   override def qualifier: Seq[String] = e.qualifier
   override def exprId: ExprId = e.exprId
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index 54abd09d89ddb..ab2f66b1a53e4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -28,7 +28,6 @@ import org.apache.spark.{SparkConf, SparkEnv}
 import org.apache.spark.serializer._
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, ScalaReflection}
-import org.apache.spark.sql.catalyst.ScalaReflection.universe.TermName
 import org.apache.spark.sql.catalyst.encoders.RowEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen._
@@ -311,7 +310,7 @@ case class Invoke(
   override def nullable: Boolean = targetObject.nullable || needNullCheck || returnNullable
   override def children: Seq[Expression] = targetObject +: arguments
 
-  private lazy val encodedFunctionName = TermName(functionName).encodedName.toString
+  private lazy val encodedFunctionName = ScalaReflection.encodeFieldNameToIdentifier(functionName)
 
   @transient lazy val method = targetObject.dataType match {
     case ObjectType(cls) =>
@@ -679,6 +678,13 @@ object MapObjects {
       elementType: DataType,
       elementNullable: Boolean = true,
       customCollectionCls: Option[Class[_]] = None): MapObjects = {
+    // UnresolvedMapObjects does not serialize its 'function' field.
+    // If an array expression or array Encoder is not correctly resolved before
+    // serialization, this exception condition may occur.
+    require(function != null,
+      "MapObjects applied with a null function. " +
+      "Likely cause is failure to resolve an array expression or encoder. " +
+      "(See UnresolvedMapObjects)")
     val loopVar = LambdaVariable("MapObject", elementType, elementNullable)
     MapObjects(loopVar, function(loopVar), inputData, customCollectionCls)
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
index 7164b6b82adbc..8bf1f19844556 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
@@ -23,7 +23,6 @@ import com.google.common.collect.Maps
 
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute}
-import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.types.{StructField, StructType}
 
 /**
@@ -143,44 +142,102 @@ package object expressions  {
     }
 
     /** Map to use for qualified case insensitive attribute lookups with 3 part key */
-    @transient private val qualified3Part: Map[(String, String, String), Seq[Attribute]] = {
+    @transient private lazy val qualified3Part: Map[(String, String, String), Seq[Attribute]] = {
       // key is 3 part: database name, table name and name
-      val grouped = attrs.filter(_.qualifier.length == 2).groupBy { a =>
-        (a.qualifier.head.toLowerCase(Locale.ROOT),
-          a.qualifier.last.toLowerCase(Locale.ROOT),
-          a.name.toLowerCase(Locale.ROOT))
+      val grouped = attrs.filter(a => a.qualifier.length >= 2 && a.qualifier.length <= 3)
+        .groupBy { a =>
+          val qualifier = if (a.qualifier.length == 2) {
+            a.qualifier
+          } else {
+            a.qualifier.takeRight(2)
+          }
+          (qualifier.head.toLowerCase(Locale.ROOT),
+            qualifier.last.toLowerCase(Locale.ROOT),
+            a.name.toLowerCase(Locale.ROOT))
+        }
+      unique(grouped)
+    }
+
+    /** Map to use for qualified case insensitive attribute lookups with 4 part key */
+    @transient
+    private lazy val qualified4Part: Map[(String, String, String, String), Seq[Attribute]] = {
+      // key is 4 part: catalog name, database name, table name and name
+      val grouped = attrs.filter(_.qualifier.length == 3).groupBy { a =>
+        a.qualifier match {
+          case Seq(catalog, db, tbl) =>
+            (catalog.toLowerCase(Locale.ROOT),
+              db.toLowerCase(Locale.ROOT),
+              tbl.toLowerCase(Locale.ROOT),
+              a.name.toLowerCase(Locale.ROOT))
+        }
       }
       unique(grouped)
     }
 
-    /** Perform attribute resolution given a name and a resolver. */
-    def resolve(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression] = {
+    /** Returns true if all qualifiers in `attrs` have 3 or less parts. */
+    @transient private val hasThreeOrLessQualifierParts: Boolean =
+      attrs.forall(_.qualifier.length <= 3)
+
+    /** Match attributes for the case where all qualifiers in `attrs` have 3 or less parts. */
+    private def matchWithThreeOrLessQualifierParts(
+        nameParts: Seq[String],
+        resolver: Resolver): (Seq[Attribute], Seq[String]) = {
       // Collect matching attributes given a name and a lookup.
       def collectMatches(name: String, candidates: Option[Seq[Attribute]]): Seq[Attribute] = {
-        candidates.toSeq.flatMap(_.collect {
+        candidates.getOrElse(Nil).collect {
           case a if resolver(a.name, name) => a.withName(name)
-        })
+        }
       }
 
-      // Find matches for the given name assuming that the 1st two parts are qualifier
-      // (i.e. database name and table name) and the 3rd part is the actual column name.
+      // Find matches for the given name assuming that the 1st three parts are qualifier
+      // (i.e. catalog name, database name and table name) and the 4th part is the actual
+      // column name.
       //
-      // For example, consider an example where "db1" is the database name, "a" is the table name
-      // and "b" is the column name and "c" is the struct field name.
-      // If the name parts is db1.a.b.c, then Attribute will match
-      // Attribute(b, qualifier("db1,"a")) and List("c") will be the second element
+      // For example, consider an example where "cat" is the catalog name, "db1" is the database
+      // name, "a" is the table name and "b" is the column name and "c" is the struct field name.
+      // If the name parts is cat.db1.a.b.c, then Attribute will match
+      // Attribute(b, qualifier("cat", "db1", "a")) and List("c") will be the second element
       var matches: (Seq[Attribute], Seq[String]) = nameParts match {
-        case dbPart +: tblPart +: name +: nestedFields =>
-          val key = (dbPart.toLowerCase(Locale.ROOT),
+        case catalogPart +: dbPart +: tblPart +: name +: nestedFields =>
+          val key = (catalogPart.toLowerCase(Locale.ROOT), dbPart.toLowerCase(Locale.ROOT),
             tblPart.toLowerCase(Locale.ROOT), name.toLowerCase(Locale.ROOT))
-          val attributes = collectMatches(name, qualified3Part.get(key)).filter {
-            a => (resolver(dbPart, a.qualifier.head) && resolver(tblPart, a.qualifier.last))
+          val attributes = collectMatches(name, qualified4Part.get(key)).filter { a =>
+            assert(a.qualifier.length == 3)
+            resolver(catalogPart, a.qualifier(0)) && resolver(dbPart, a.qualifier(1)) &&
+              resolver(tblPart, a.qualifier(2))
           }
           (attributes, nestedFields)
         case _ =>
           (Seq.empty, Seq.empty)
       }
 
+      // Find matches for the given name assuming that the 1st two parts are qualifier
+      // (i.e. database name and table name) and the 3rd part is the actual column name.
+      //
+      // For example, consider an example where "db1" is the database name, "a" is the table name
+      // and "b" is the column name and "c" is the struct field name.
+      // If the name parts is db1.a.b.c, then it can match both
+      // Attribute(b, qualifier("cat", "db1, "a")) and Attribute(b, qualifier("db1, "a")),
+      // and List("c") will be the second element
+      if (matches._1.isEmpty) {
+        matches = nameParts match {
+          case dbPart +: tblPart +: name +: nestedFields =>
+            val key = (dbPart.toLowerCase(Locale.ROOT),
+              tblPart.toLowerCase(Locale.ROOT), name.toLowerCase(Locale.ROOT))
+            val attributes = collectMatches(name, qualified3Part.get(key)).filter { a =>
+              val qualifier = if (a.qualifier.length == 2) {
+                a.qualifier
+              } else {
+                a.qualifier.takeRight(2)
+              }
+              resolver(dbPart, qualifier.head) && resolver(tblPart, qualifier.last)
+            }
+            (attributes, nestedFields)
+          case _ =>
+            (Seq.empty, Seq.empty)
+        }
+      }
+
       // If there are no matches, then find matches for the given name assuming that
       // the 1st part is a qualifier (i.e. table name, alias, or subquery alias) and the
       // 2nd part is the actual name. This returns a tuple of
@@ -204,13 +261,79 @@ package object expressions  {
 
       // If none of attributes match database.table.column pattern or
       // `table.column` pattern, we try to resolve it as a column.
-      val (candidates, nestedFields) = matches match {
+      matches match {
         case (Seq(), _) =>
           val name = nameParts.head
           val attributes = collectMatches(name, direct.get(name.toLowerCase(Locale.ROOT)))
           (attributes, nameParts.tail)
         case _ => matches
       }
+    }
+
+    /**
+     * Match attributes for the case where at least one qualifier in `attrs` has more than 3 parts.
+     */
+    private def matchWithFourOrMoreQualifierParts(
+        nameParts: Seq[String],
+        resolver: Resolver): (Seq[Attribute], Seq[String]) = {
+      // Returns true if the `short` qualifier is a subset of the last elements of
+      // `long` qualifier. For example, Seq("a", "b") is a subset of Seq("a", "a", "b"),
+      // but not a subset of Seq("a", "b", "b").
+      def matchQualifier(short: Seq[String], long: Seq[String]): Boolean = {
+        (long.length >= short.length) &&
+          long.takeRight(short.length)
+            .zip(short)
+            .forall(x => resolver(x._1, x._2))
+      }
+
+      // Collect attributes that match the given name and qualifier.
+      // A match occurs if
+      //   1) the given name matches the attribute's name according to the resolver.
+      //   2) the given qualifier is a subset of the attribute's qualifier.
+      def collectMatches(
+          name: String,
+          qualifier: Seq[String],
+          candidates: Option[Seq[Attribute]]): Seq[Attribute] = {
+        candidates.getOrElse(Nil).collect {
+          case a if resolver(name, a.name) && matchQualifier(qualifier, a.qualifier) =>
+            a.withName(name)
+        }
+      }
+
+      // Iterate each string in `nameParts` in a reverse order and try to match the attributes
+      // considering the current string as the attribute name. For example, if `nameParts` is
+      // Seq("a", "b", "c"), the match will be performed in the following order:
+      // 1) name = "c", qualifier = Seq("a", "b")
+      // 2) name = "b", qualifier = Seq("a")
+      // 3) name = "a", qualifier = Seq()
+      // Note that the match is performed in the reverse order in order to match the longest
+      // qualifier as possible. If a match is found, the remaining portion of `nameParts`
+      // is also returned as nested fields.
+      var candidates: Seq[Attribute] = Nil
+      var nestedFields: Seq[String] = Nil
+      var i = nameParts.length - 1
+      while (i >= 0 && candidates.isEmpty) {
+        val name = nameParts(i)
+        candidates = collectMatches(
+          name,
+          nameParts.take(i),
+          direct.get(name.toLowerCase(Locale.ROOT)))
+        if (candidates.nonEmpty) {
+          nestedFields = nameParts.takeRight(nameParts.length - i - 1)
+        }
+        i -= 1
+      }
+
+      (candidates, nestedFields)
+    }
+
+    /** Perform attribute resolution given a name and a resolver. */
+    def resolve(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression] = {
+      val (candidates, nestedFields) = if (hasThreeOrLessQualifierParts) {
+        matchWithThreeOrLessQualifierParts(nameParts, resolver)
+      } else {
+        matchWithFourOrMoreQualifierParts(nameParts, resolver)
+      }
 
       def name = UnresolvedAttribute(nameParts).name
       candidates match {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index bcd442ad3cc35..2c4f41f98ac20 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.expressions
 
 import scala.collection.immutable.TreeSet
 
+import org.apache.spark.sql.catalyst.CatalystTypeConverters.convertToScala
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReference
@@ -519,7 +520,9 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
 
   override def sql: String = {
     val valueSQL = child.sql
-    val listSQL = hset.toSeq.map(Literal(_).sql).mkString(", ")
+    val listSQL = hset.toSeq
+      .map(elem => Literal(elem, child.dataType).sql)
+      .mkString(", ")
     s"($valueSQL IN ($listSQL))"
   }
 }
@@ -927,66 +930,6 @@ case class GreaterThanOrEqual(left: Expression, right: Expression)
   protected override def nullSafeEval(input1: Any, input2: Any): Any = ordering.gteq(input1, input2)
 }
 
-trait BooleanTest extends UnaryExpression with Predicate with ExpectsInputTypes {
-
-  def boolValueForComparison: Boolean
-  def boolValueWhenNull: Boolean
-
-  override def nullable: Boolean = false
-  override def inputTypes: Seq[DataType] = Seq(BooleanType)
-
-  override def eval(input: InternalRow): Any = {
-    val value = child.eval(input)
-    Option(value) match {
-      case None => boolValueWhenNull
-      case other => if (boolValueWhenNull) {
-        value == !boolValueForComparison
-      } else {
-        value == boolValueForComparison
-      }
-    }
-  }
-
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val eval = child.genCode(ctx)
-    ev.copy(code = code"""
-      ${eval.code}
-      ${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
-      if (${eval.isNull}) {
-        ${ev.value} = $boolValueWhenNull;
-      } else if ($boolValueWhenNull) {
-        ${ev.value} = ${eval.value} == !$boolValueForComparison;
-      } else {
-        ${ev.value} = ${eval.value} == $boolValueForComparison;
-      }
-      """, isNull = FalseLiteral)
-  }
-}
-
-case class IsTrue(child: Expression) extends BooleanTest {
-  override def boolValueForComparison: Boolean = true
-  override def boolValueWhenNull: Boolean = false
-  override def sql: String = s"(${child.sql} IS TRUE)"
-}
-
-case class IsNotTrue(child: Expression) extends BooleanTest {
-  override def boolValueForComparison: Boolean = true
-  override def boolValueWhenNull: Boolean = true
-  override def sql: String = s"(${child.sql} IS NOT TRUE)"
-}
-
-case class IsFalse(child: Expression) extends BooleanTest {
-  override def boolValueForComparison: Boolean = false
-  override def boolValueWhenNull: Boolean = false
-  override def sql: String = s"(${child.sql} IS FALSE)"
-}
-
-case class IsNotFalse(child: Expression) extends BooleanTest {
-  override def boolValueForComparison: Boolean = false
-  override def boolValueWhenNull: Boolean = true
-  override def sql: String = s"(${child.sql} IS NOT FALSE)"
-}
-
 /**
  * IS UNKNOWN and IS NOT UNKNOWN are the same as IS NULL and IS NOT NULL, respectively,
  * except that the input expression must be of a boolean type.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index c9ddc70bf5bc6..933c77ac4f681 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -22,7 +22,6 @@ import java.util.regex.{MatchResult, Pattern}
 
 import org.apache.commons.text.StringEscapeUtils
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{GenericArrayData, StringUtils}
@@ -30,20 +29,18 @@ import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
 
-trait StringRegexExpression extends Expression
+abstract class StringRegexExpression extends BinaryExpression
   with ImplicitCastInputTypes with NullIntolerant {
 
-  def str: Expression
-  def pattern: Expression
-
   def escape(v: String): String
   def matches(regex: Pattern, str: String): Boolean
 
   override def dataType: DataType = BooleanType
+  override def inputTypes: Seq[DataType] = Seq(StringType, StringType)
 
   // try cache the pattern for Literal
-  private lazy val cache: Pattern = pattern match {
-    case Literal(value: String, StringType) => compile(value)
+  private lazy val cache: Pattern = right match {
+    case Literal(value: UTF8String, StringType) => compile(value.toString)
     case _ => null
   }
 
@@ -54,9 +51,10 @@ trait StringRegexExpression extends Expression
     Pattern.compile(escape(str))
   }
 
-  def nullSafeMatch(input1: Any, input2: Any): Any = {
-    val s = input2.asInstanceOf[UTF8String].toString
-    val regex = if (cache == null) compile(s) else cache
+  protected def pattern(str: String) = if (cache == null) compile(str) else cache
+
+  protected override def nullSafeEval(input1: Any, input2: Any): Any = {
+    val regex = pattern(input2.asInstanceOf[UTF8String].toString)
     if(regex == null) {
       null
     } else {
@@ -64,7 +62,7 @@ trait StringRegexExpression extends Expression
     }
   }
 
-  override def sql: String = s"${str.sql} ${prettyName.toUpperCase(Locale.ROOT)} ${pattern.sql}"
+  override def sql: String = s"${left.sql} ${prettyName.toUpperCase(Locale.ROOT)} ${right.sql}"
 }
 
 // scalastyle:off line.contains.tab
@@ -109,65 +107,46 @@ trait StringRegexExpression extends Expression
       true
       > SELECT '%SystemDrive%/Users/John' _FUNC_ '/%SystemDrive/%//Users%' ESCAPE '/';
       true
-      > SELECT _FUNC_('_Apache Spark_', '__%Spark__', '_');
-      true
   """,
   note = """
     Use RLIKE to match with standard regular expressions.
   """,
   since = "1.0.0")
 // scalastyle:on line.contains.tab
-case class Like(str: Expression, pattern: Expression, escape: Expression)
-  extends TernaryExpression with StringRegexExpression {
-
-  def this(str: Expression, pattern: Expression) = this(str, pattern, Literal("\\"))
+case class Like(left: Expression, right: Expression, escapeChar: Char)
+  extends StringRegexExpression {
 
-  override def inputTypes: Seq[DataType] = Seq(StringType, StringType, StringType)
-  override def children: Seq[Expression] = Seq(str, pattern, escape)
-
-  private lazy val escapeChar: Char = if (escape.foldable) {
-    escape.eval() match {
-      case s: UTF8String if s != null && s.numChars() == 1 => s.toString.charAt(0)
-      case s => throw new AnalysisException(
-        s"The 'escape' parameter must be a string literal of one char but it is $s.")
-    }
-  } else {
-    throw new AnalysisException("The 'escape' parameter must be a string literal.")
-  }
+  def this(left: Expression, right: Expression) = this(left, right, '\\')
 
   override def escape(v: String): String = StringUtils.escapeLikeRegex(v, escapeChar)
 
   override def matches(regex: Pattern, str: String): Boolean = regex.matcher(str).matches()
 
   override def toString: String = escapeChar match {
-    case '\\' => s"$str LIKE $pattern"
-    case c => s"$str LIKE $pattern ESCAPE '$c'"
-  }
-
-  protected override def nullSafeEval(input1: Any, input2: Any, input3: Any): Any = {
-    nullSafeMatch(input1, input2)
+    case '\\' => s"$left LIKE $right"
+    case c => s"$left LIKE $right ESCAPE '$c'"
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val patternClass = classOf[Pattern].getName
     val escapeFunc = StringUtils.getClass.getName.stripSuffix("$") + ".escapeLikeRegex"
 
-    if (pattern.foldable) {
-      val patternVal = pattern.eval()
-      if (patternVal != null) {
+    if (right.foldable) {
+      val rVal = right.eval()
+      if (rVal != null) {
         val regexStr =
-          StringEscapeUtils.escapeJava(escape(patternVal.asInstanceOf[UTF8String].toString()))
-        val compiledPattern = ctx.addMutableState(patternClass, "compiledPattern",
+          StringEscapeUtils.escapeJava(escape(rVal.asInstanceOf[UTF8String].toString()))
+        val pattern = ctx.addMutableState(patternClass, "patternLike",
           v => s"""$v = $patternClass.compile("$regexStr");""")
 
         // We don't use nullSafeCodeGen here because we don't want to re-evaluate right again.
-        val eval = str.genCode(ctx)
+        val eval = left.genCode(ctx)
         ev.copy(code = code"""
           ${eval.code}
           boolean ${ev.isNull} = ${eval.isNull};
           ${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
           if (!${ev.isNull}) {
-            ${ev.value} = $compiledPattern.matcher(${eval.value}.toString()).matches();
+            ${ev.value} = $pattern.matcher(${eval.value}.toString()).matches();
           }
         """)
       } else {
@@ -177,26 +156,17 @@ case class Like(str: Expression, pattern: Expression, escape: Expression)
         """)
       }
     } else {
-      // We need double escape to avoid org.codehaus.commons.compiler.CompileException.
-      // '\\' will cause exception 'Single quote must be backslash-escaped in character literal'.
-      // '\"' will cause exception 'Line break in literal not allowed'.
-      val newEscapeChar = if (escapeChar == '\"' || escapeChar == '\\') {
-        s"""\\\\\\$escapeChar"""
-      } else {
-        escapeChar
-      }
-      val patternStr = ctx.freshName("patternStr")
-      val compiledPattern = ctx.addMutableState(patternClass, "compiledPattern")
-      val lastPatternStr = ctx.addMutableState(classOf[String].getName, "lastPatternStr")
-
-      nullSafeCodeGen(ctx, ev, (eval1, eval2, _) => {
+      val pattern = ctx.freshName("pattern")
+      val rightStr = ctx.freshName("rightStr")
+      // We need to escape the escapeChar to make sure the generated code is valid.
+      // Otherwise we'll hit org.codehaus.commons.compiler.CompileException.
+      val escapedEscapeChar = StringEscapeUtils.escapeJava(escapeChar.toString)
+      nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
         s"""
-          String $patternStr = $eval2.toString();
-          if (!$patternStr.equals($lastPatternStr)) {
-            $compiledPattern = $patternClass.compile($escapeFunc($patternStr, '$newEscapeChar'));
-            $lastPatternStr = $patternStr;
-          }
-          ${ev.value} = $compiledPattern.matcher($eval1.toString()).matches();
+          String $rightStr = $eval2.toString();
+          $patternClass $pattern = $patternClass.compile(
+            $escapeFunc($rightStr, '$escapedEscapeChar'));
+          ${ev.value} = $pattern.matcher($eval1.toString()).matches();
         """
       })
     }
@@ -235,20 +205,12 @@ case class Like(str: Expression, pattern: Expression, escape: Expression)
   """,
   since = "1.0.0")
 // scalastyle:on line.contains.tab
-case class RLike(left: Expression, right: Expression)
-  extends BinaryExpression with StringRegexExpression {
-
-  override def inputTypes: Seq[DataType] = Seq(StringType, StringType)
-
-  override def str: Expression = left
-  override def pattern: Expression = right
+case class RLike(left: Expression, right: Expression) extends StringRegexExpression {
 
   override def escape(v: String): String = v
   override def matches(regex: Pattern, str: String): Boolean = regex.matcher(str).find(0)
   override def toString: String = s"$left RLIKE $right"
 
-  protected override def nullSafeEval(input1: Any, input2: Any): Any = nullSafeMatch(input1, input2)
-
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val patternClass = classOf[Pattern].getName
 
@@ -278,16 +240,11 @@ case class RLike(left: Expression, right: Expression)
       }
     } else {
       val rightStr = ctx.freshName("rightStr")
-      val pattern = ctx.addMutableState(patternClass, "pattern")
-      val lastRightStr = ctx.addMutableState(classOf[String].getName, "lastRightStr")
-
+      val pattern = ctx.freshName("pattern")
       nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
         s"""
           String $rightStr = $eval2.toString();
-          if (!$rightStr.equals($lastRightStr)) {
-            $pattern = $patternClass.compile($rightStr);
-            $lastRightStr = $rightStr;
-          }
+          $patternClass $pattern = $patternClass.compile($rightStr);
           ${ev.value} = $pattern.matcher($eval1.toString()).find(0);
         """
       })
@@ -452,6 +409,17 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
   }
 }
 
+object RegExpExtract {
+  def checkGroupIndex(groupCount: Int, groupIndex: Int): Unit = {
+    if (groupIndex < 0) {
+      throw new IllegalArgumentException("The specified group index cannot be less than zero")
+    } else if (groupCount < groupIndex) {
+      throw new IllegalArgumentException(
+        s"Regex group count is $groupCount, but the specified group index is $groupIndex")
+    }
+  }
+}
+
 /**
  * Extract a specific(idx) group identified by a Java regex.
  *
@@ -459,6 +427,14 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
  */
 @ExpressionDescription(
   usage = "_FUNC_(str, regexp[, idx]) - Extracts a group that matches `regexp`.",
+  arguments = """
+    Arguments:
+      * str - a string expression.
+      * regexp - a string representing a regular expression.
+          The regex string should be a Java regular expression.
+      * idx - an integer expression that representing the group index. The group index should be
+          non-negative. If `idx` is not specified, the default group index value is 1.
+  """,
   examples = """
     Examples:
       > SELECT _FUNC_('100-200', '(\\d+)-(\\d+)', 1);
@@ -483,7 +459,9 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio
     val m = pattern.matcher(s.toString)
     if (m.find) {
       val mr: MatchResult = m.toMatchResult
-      val group = mr.group(r.asInstanceOf[Int])
+      val index = r.asInstanceOf[Int]
+      RegExpExtract.checkGroupIndex(mr.groupCount, index)
+      val group = mr.group(index)
       if (group == null) { // Pattern matched, but not optional group
         UTF8String.EMPTY_UTF8
       } else {
@@ -501,6 +479,7 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val classNamePattern = classOf[Pattern].getCanonicalName
+    val classNameRegExpExtract = classOf[RegExpExtract].getCanonicalName
     val matcher = ctx.freshName("matcher")
     val matchResult = ctx.freshName("matchResult")
 
@@ -524,6 +503,7 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio
         $termPattern.matcher($subject.toString());
       if ($matcher.find()) {
         java.util.regex.MatchResult $matchResult = $matcher.toMatchResult();
+        $classNameRegExpExtract.checkGroupIndex($matchResult.groupCount(), $idx);
         if ($matchResult.group($idx) == null) {
           ${ev.value} = UTF8String.EMPTY_UTF8;
         } else {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 211ae3f02a0d8..2cd92d7826c64 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -27,7 +27,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.commons.codec.binary.{Base64 => CommonsBase64}
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData, TypeUtils}
@@ -365,6 +365,9 @@ case class Lower(child: Expression) extends UnaryExpression with String2StringEx
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, c => s"($c).toLowerCase()")
   }
+
+  override def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("lower")
 }
 
 /** A base trait for functions that compare two strings, returning a boolean. */
@@ -683,11 +686,22 @@ case class FindInSet(left: Expression, right: Expression) extends BinaryExpressi
 
 trait String2TrimExpression extends Expression with ImplicitCastInputTypes {
 
+  protected def srcStr: Expression
+  protected def trimStr: Option[Expression]
+  protected def direction: String
+
+  override def children: Seq[Expression] = srcStr +: trimStr.toSeq
   override def dataType: DataType = StringType
   override def inputTypes: Seq[AbstractDataType] = Seq.fill(children.size)(StringType)
 
   override def nullable: Boolean = children.exists(_.nullable)
   override def foldable: Boolean = children.forall(_.foldable)
+
+  override def sql: String = if (trimStr.isDefined) {
+    s"TRIM($direction ${trimStr.get.sql} FROM ${srcStr.sql})"
+  } else {
+    super.sql
+  }
 }
 
 object StringTrim {
@@ -719,8 +733,6 @@ object StringTrim {
 
     _FUNC_(TRAILING FROM str) - Removes the trailing space characters from `str`.
 
-    _FUNC_(str, trimStr) - Remove the leading and trailing `trimStr` characters from `str`.
-
     _FUNC_(trimStr FROM str) - Remove the leading and trailing `trimStr` characters from `str`.
 
     _FUNC_(BOTH trimStr FROM str) - Remove the leading and trailing `trimStr` characters from `str`.
@@ -750,8 +762,6 @@ object StringTrim {
        SparkSQL
       > SELECT _FUNC_(TRAILING FROM '    SparkSQL   ');
            SparkSQL
-      > SELECT _FUNC_('SSparkSQLS', 'SL');
-       parkSQ
       > SELECT _FUNC_('SL' FROM 'SSparkSQLS');
        parkSQ
       > SELECT _FUNC_(BOTH 'SL' FROM 'SSparkSQLS');
@@ -767,17 +777,14 @@ case class StringTrim(
     trimStr: Option[Expression] = None)
   extends String2TrimExpression {
 
-  def this(srcStr: Expression, trimStr: Expression) = this(srcStr, Option(trimStr))
+  def this(trimStr: Expression, srcStr: Expression) = this(srcStr, Option(trimStr))
 
   def this(srcStr: Expression) = this(srcStr, None)
 
   override def prettyName: String = "trim"
 
-  override def children: Seq[Expression] = if (trimStr.isDefined) {
-    srcStr :: trimStr.get :: Nil
-  } else {
-    srcStr :: Nil
-  }
+  override protected def direction: String = "BOTH"
+
   override def eval(input: InternalRow): Any = {
     val srcString = srcStr.eval(input).asInstanceOf[UTF8String]
     if (srcString == null) {
@@ -846,8 +853,6 @@ object StringTrimLeft {
 @ExpressionDescription(
   usage = """
     _FUNC_(str) - Removes the leading space characters from `str`.
-
-    _FUNC_(str, trimStr) - Removes the leading string contains the characters from the trim string
   """,
   arguments = """
     Arguments:
@@ -858,8 +863,6 @@ object StringTrimLeft {
     Examples:
       > SELECT _FUNC_('    SparkSQL   ');
        SparkSQL
-      > SELECT _FUNC_('SparkSQLS', 'Sp');
-       arkSQLS
   """,
   since = "1.5.0")
 case class StringTrimLeft(
@@ -867,17 +870,13 @@ case class StringTrimLeft(
     trimStr: Option[Expression] = None)
   extends String2TrimExpression {
 
-  def this(srcStr: Expression, trimStr: Expression) = this(srcStr, Option(trimStr))
+  def this(trimStr: Expression, srcStr: Expression) = this(srcStr, Option(trimStr))
 
   def this(srcStr: Expression) = this(srcStr, None)
 
   override def prettyName: String = "ltrim"
 
-  override def children: Seq[Expression] = if (trimStr.isDefined) {
-    srcStr :: trimStr.get :: Nil
-  } else {
-    srcStr :: Nil
-  }
+  override protected def direction: String = "LEADING"
 
   override def eval(input: InternalRow): Any = {
     val srcString = srcStr.eval(input).asInstanceOf[UTF8String]
@@ -948,8 +947,6 @@ object StringTrimRight {
 @ExpressionDescription(
   usage = """
     _FUNC_(str) - Removes the trailing space characters from `str`.
-
-    _FUNC_(str, trimStr) - Removes the trailing string which contains the characters from the trim string from the `str`
   """,
   arguments = """
     Arguments:
@@ -960,8 +957,6 @@ object StringTrimRight {
     Examples:
       > SELECT _FUNC_('    SparkSQL   ');
        SparkSQL
-      > SELECT _FUNC_('SSparkSQLS', 'SQLS');
-       SSpark
   """,
   since = "1.5.0")
 // scalastyle:on line.size.limit
@@ -970,17 +965,13 @@ case class StringTrimRight(
     trimStr: Option[Expression] = None)
   extends String2TrimExpression {
 
-  def this(srcStr: Expression, trimStr: Expression) = this(srcStr, Option(trimStr))
+  def this(trimStr: Expression, srcStr: Expression) = this(srcStr, Option(trimStr))
 
   def this(srcStr: Expression) = this(srcStr, None)
 
   override def prettyName: String = "rtrim"
 
-  override def children: Seq[Expression] = if (trimStr.isDefined) {
-    srcStr :: trimStr.get :: Nil
-  } else {
-    srcStr :: Nil
-  }
+  override protected def direction: String = "TRAILING"
 
   override def eval(input: InternalRow): Any = {
     val srcString = srcStr.eval(input).asInstanceOf[UTF8String]
@@ -1194,7 +1185,8 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression)
      """)
   }
 
-  override def prettyName: String = "locate"
+  override def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("locate")
 }
 
 /**
@@ -1462,7 +1454,7 @@ case class ParseUrl(children: Seq[Expression])
 // scalastyle:on line.size.limit
 case class FormatString(children: Expression*) extends Expression with ImplicitCastInputTypes {
 
-  require(children.nonEmpty, "format_string() should take at least 1 argument")
+  require(children.nonEmpty, s"$prettyName() should take at least 1 argument")
 
   override def foldable: Boolean = children.forall(_.foldable)
   override def nullable: Boolean = children(0).nullable
@@ -1529,7 +1521,8 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC
       }""")
   }
 
-  override def prettyName: String = "format_string"
+  override def prettyName: String = getTagValue(
+    FunctionRegistry.FUNC_ALIAS).getOrElse("format_string")
 }
 
 /**
@@ -1630,7 +1623,11 @@ case class StringSpace(child: Expression)
  */
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(str, pos[, len]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.",
+  usage = """
+    _FUNC_(str, pos[, len]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.
+
+    _FUNC_(str FROM pos[ FOR len]]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.
+  """,
   examples = """
     Examples:
       > SELECT _FUNC_('Spark SQL', 5);
@@ -1639,6 +1636,12 @@ case class StringSpace(child: Expression)
        SQL
       > SELECT _FUNC_('Spark SQL', 5, 1);
        k
+      > SELECT _FUNC_('Spark SQL' FROM 5);
+       k SQL
+      > SELECT _FUNC_('Spark SQL' FROM -3);
+       SQL
+      > SELECT _FUNC_('Spark SQL' FROM 5 FOR 1);
+       k
   """,
   since = "1.5.0")
 // scalastyle:on line.size.limit
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
index 74df1e7b8e8c3..c8b6433207355 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
@@ -410,10 +410,6 @@ abstract class OffsetWindowFunction
  * the window. Offsets start at 0, which is the current row. The offset must be constant
  * integer value. The default offset is 1. When the value of `input` is null at the `offset`th row,
  * null is returned. If there is no such offset row, the `default` expression is evaluated.
- *
- * @param input expression to evaluate `offset` rows after the current row.
- * @param offset rows to jump ahead in the partition.
- * @param default to use when the offset is larger than the window. The default value is null.
  */
 @ExpressionDescription(
   usage = """
@@ -422,7 +418,16 @@ abstract class OffsetWindowFunction
       value of `default` is null. If the value of `input` at the `offset`th row is null,
       null is returned. If there is no such an offset row (e.g., when the offset is 1, the last
       row of the window does not have any subsequent row), `default` is returned.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * input - a string expression to evaluate `offset` rows after the current row.
+      * offset - an int expression which is rows to jump ahead in the partition.
+      * default - a string expression which is to use when the offset is larger than the window.
+          The default value is null.
+  """,
+  since = "2.0.0",
+  group = "window_funcs")
 case class Lead(input: Expression, offset: Expression, default: Expression)
     extends OffsetWindowFunction {
 
@@ -440,10 +445,6 @@ case class Lead(input: Expression, offset: Expression, default: Expression)
  * the window. Offsets start at 0, which is the current row. The offset must be constant
  * integer value. The default offset is 1. When the value of `input` is null at the `offset`th row,
  * null is returned. If there is no such offset row, the `default` expression is evaluated.
- *
- * @param input expression to evaluate `offset` rows before the current row.
- * @param offset rows to jump back in the partition.
- * @param default to use when the offset row does not exist.
  */
 @ExpressionDescription(
   usage = """
@@ -452,7 +453,15 @@ case class Lead(input: Expression, offset: Expression, default: Expression)
       value of `default` is null. If the value of `input` at the `offset`th row is null,
       null is returned. If there is no such offset row (e.g., when the offset is 1, the first
       row of the window does not have any previous row), `default` is returned.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * input - a string expression to evaluate `offset` rows before the current row.
+      * offset - an int expression which is rows to jump back in the partition.
+      * default - a string expression which is to use when the offset row does not exist.
+  """,
+  since = "2.0.0",
+  group = "window_funcs")
 case class Lag(input: Expression, offset: Expression, default: Expression)
     extends OffsetWindowFunction {
 
@@ -509,7 +518,9 @@ object SizeBasedWindowFunction {
   usage = """
     _FUNC_() - Assigns a unique, sequential number to each row, starting with one,
       according to the ordering of rows within the window partition.
-  """)
+  """,
+  since = "2.0.0",
+  group = "window_funcs")
 case class RowNumber() extends RowNumberLike {
   override val evaluateExpression = rowNumber
   override def prettyName: String = "row_number"
@@ -526,7 +537,9 @@ case class RowNumber() extends RowNumberLike {
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the position of a value relative to all values in the partition.
-  """)
+  """,
+  since = "2.0.0",
+  group = "window_funcs")
 case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
   override def dataType: DataType = DoubleType
   // The frame for CUME_DIST is Range based instead of Row based, because CUME_DIST must
@@ -554,14 +567,19 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
  * threshold is increased by the bucket size (plus one extra if the current bucket is padded).
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
- *
- * @param buckets number of buckets to divide the rows in. Default value is 1.
  */
 @ExpressionDescription(
   usage = """
     _FUNC_(n) - Divides the rows for each window partition into `n` buckets ranging
       from 1 to at most `n`.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * buckets - an int expression which is number of buckets to divide the rows in.
+          Default value is 1.
+  """,
+  since = "2.0.0",
+  group = "window_funcs")
 case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindowFunction {
   def this() = this(Literal(1))
 
@@ -674,17 +692,21 @@ abstract class RankLike extends AggregateWindowFunction {
  * will produce gaps in the sequence.
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
- *
- * @param children to base the rank on; a change in the value of one the children will trigger a
- *                 change in rank. This is an internal parameter and will be assigned by the
- *                 Analyser.
  */
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the number
       of rows preceding or equal to the current row in the ordering of the partition. The values
       will produce gaps in the sequence.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * children - this is to base the rank on; a change in the value of one the children will
+          trigger a change in rank. This is an internal parameter and will be assigned by the
+          Analyser.
+  """,
+  since = "2.0.0",
+  group = "window_funcs")
 case class Rank(children: Seq[Expression]) extends RankLike {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): Rank = Rank(order)
@@ -696,17 +718,21 @@ case class Rank(children: Seq[Expression]) extends RankLike {
  * ranking sequence.
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
- *
- * @param children to base the rank on; a change in the value of one the children will trigger a
- *                 change in rank. This is an internal parameter and will be assigned by the
- *                 Analyser.
  */
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the
       previously assigned rank value. Unlike the function rank, dense_rank will not produce gaps
       in the ranking sequence.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * children - this is to base the rank on; a change in the value of one the children will
+          trigger a change in rank. This is an internal parameter and will be assigned by the
+          Analyser.
+  """,
+  since = "2.0.0",
+  group = "window_funcs")
 case class DenseRank(children: Seq[Expression]) extends RankLike {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): DenseRank = DenseRank(order)
@@ -726,15 +752,19 @@ case class DenseRank(children: Seq[Expression]) extends RankLike {
  * row counts in the its numerator.
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
- *
- * @param children to base the rank on; a change in the value of one of the children will trigger a
- *                 change in rank. This is an internal parameter and will be assigned by the
- *                 Analyser.
  */
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the percentage ranking of a value in a group of values.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * children - this is to base the rank on; a change in the value of one the children will
+          trigger a change in rank. This is an internal parameter and will be assigned by the
+          Analyser.
+  """,
+  since = "2.0.0",
+  group = "window_funcs")
 case class PercentRank(children: Seq[Expression]) extends RankLike with SizeBasedWindowFunction {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): PercentRank = PercentRank(order)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
index 073b45af51caf..55e06cb9e8471 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.catalyst.expressions.xml
 
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckFailure
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
@@ -160,7 +160,8 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract {
   """)
 // scalastyle:on line.size.limit
 case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract {
-  override def prettyName: String = "xpath_double"
+  override def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("xpath_double")
   override def dataType: DataType = DoubleType
 
   override def nullSafeEval(xml: Any, path: Any): Any = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
index deceec73dda30..c574a20da0b5b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
@@ -49,19 +49,21 @@ sealed trait IdentifierWithDatabase {
 
 /**
  * Encapsulates an identifier that is either a alias name or an identifier that has table
- * name and optionally a database name.
+ * name and a qualifier.
  * The SubqueryAlias node keeps track of the qualifier using the information in this structure
- * @param identifier - Is an alias name or a table name
- * @param database - Is a database name and is optional
+ * @param name - Is an alias name or a table name
+ * @param qualifier - Is a qualifier
  */
-case class AliasIdentifier(identifier: String, database: Option[String])
-  extends IdentifierWithDatabase {
+case class AliasIdentifier(name: String, qualifier: Seq[String]) {
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+  def this(identifier: String) = this(identifier, Seq())
 
-  def this(identifier: String) = this(identifier, None)
+  override def toString: String = (qualifier :+ name).quoted
 }
 
 object AliasIdentifier {
-  def apply(identifier: String): AliasIdentifier = new AliasIdentifier(identifier)
+  def apply(name: String): AliasIdentifier = new AliasIdentifier(name)
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
index cdf4b4689e821..70a673bb42457 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
@@ -27,6 +27,7 @@ import com.fasterxml.jackson.core.json.JsonReadFeature
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 
 /**
  * Options for parsing JSON data into Spark SQL rows.
@@ -88,10 +89,14 @@ private[sql] class JSONOptions(
   val zoneId: ZoneId = DateTimeUtils.getZoneId(
     parameters.getOrElse(DateTimeUtils.TIMEZONE_OPTION, defaultTimeZoneId))
 
-  val dateFormat: String = parameters.getOrElse("dateFormat", "uuuu-MM-dd")
+  val dateFormat: String = parameters.getOrElse("dateFormat", DateFormatter.defaultPattern)
 
-  val timestampFormat: String =
-    parameters.getOrElse("timestampFormat", "uuuu-MM-dd'T'HH:mm:ss.SSSXXX")
+  val timestampFormat: String = parameters.getOrElse("timestampFormat",
+    if (SQLConf.get.legacyTimeParserPolicy == LegacyBehaviorPolicy.LEGACY) {
+      s"${DateFormatter.defaultPattern}'T'HH:mm:ss.SSSXXX"
+    } else {
+      s"${DateFormatter.defaultPattern}'T'HH:mm:ss[.SSS][XXX]"
+    })
 
   val multiLine = parameters.get("multiLine").map(_.toBoolean).getOrElse(false)
 
@@ -128,7 +133,7 @@ private[sql] class JSONOptions(
    * Enables inferring of TimestampType from strings matched to the timestamp pattern
    * defined by the timestampFormat option.
    */
-  val inferTimestamp: Boolean = parameters.get("inferTimestamp").map(_.toBoolean).getOrElse(true)
+  val inferTimestamp: Boolean = parameters.get("inferTimestamp").map(_.toBoolean).getOrElse(false)
 
   /** Build a Jackson [[JsonFactory]] using JSON options. */
   def buildJsonFactory(): JsonFactory = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
index 9c63593ea1752..fb0ca323af1ff 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.core._
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.types._
 
 /**
@@ -80,11 +81,15 @@ private[sql] class JacksonGenerator(
   private val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = false)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = false)
 
   private def makeWriter(dataType: DataType): ValueWriter = dataType match {
     case NullType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index c44025ca8bcfd..70f7f8f468f3a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -21,15 +21,17 @@ import java.io.{ByteArrayOutputStream, CharConversionException}
 import java.nio.charset.MalformedInputException
 
 import scala.collection.mutable.ArrayBuffer
-import scala.util.Try
 import scala.util.control.NonFatal
 
 import com.fasterxml.jackson.core._
 
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 import org.apache.spark.util.Utils
@@ -54,14 +56,18 @@ class JacksonParser(
 
   private val factory = options.buildJsonFactory()
 
-  private val timestampFormatter = TimestampFormatter(
+  private lazy val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
-  private val dateFormatter = DateFormatter(
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = true)
+  private lazy val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = true)
 
   /**
    * Create a converter which converts the JSON documents held by the `JsonParser`
@@ -80,7 +86,7 @@ class JacksonParser(
     val elementConverter = makeConverter(st)
     val fieldConverters = st.map(_.dataType).map(makeConverter).toArray
     (parser: JsonParser) => parseJsonToken[Iterable[InternalRow]](parser, st) {
-      case START_OBJECT => Some(convertObject(parser, st, fieldConverters))
+      case START_OBJECT => Some(convertObject(parser, st, fieldConverters, isRoot = true))
         // SPARK-3308: support reading top level JSON arrays and take every element
         // in such an array as a row
         //
@@ -226,7 +232,15 @@ class JacksonParser(
     case TimestampType =>
       (parser: JsonParser) => parseJsonToken[java.lang.Long](parser, dataType) {
         case VALUE_STRING if parser.getTextLength >= 1 =>
-          timestampFormatter.parse(parser.getText)
+          try {
+            timestampFormatter.parse(parser.getText)
+          } catch {
+            case NonFatal(e) =>
+              // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
+              // compatibility.
+              val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(parser.getText))
+              DateTimeUtils.stringToTimestamp(str, options.zoneId).getOrElse(throw e)
+          }
 
         case VALUE_NUMBER_INT =>
           parser.getLongValue * 1000000L
@@ -235,7 +249,23 @@ class JacksonParser(
     case DateType =>
       (parser: JsonParser) => parseJsonToken[java.lang.Integer](parser, dataType) {
         case VALUE_STRING if parser.getTextLength >= 1 =>
-          dateFormatter.parse(parser.getText)
+          try {
+            dateFormatter.parse(parser.getText)
+          } catch {
+            case NonFatal(e) =>
+              // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
+              // compatibility.
+              val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(parser.getText))
+              DateTimeUtils.stringToDate(str, options.zoneId).getOrElse {
+                // In Spark 1.5.0, we store the data as number of days since epoch in string.
+                // So, we just convert it to Int.
+                try {
+                  RebaseDateTime.rebaseJulianToGregorianDays(parser.getText.toInt)
+                } catch {
+                  case _: NumberFormatException => throw e
+                }
+              }.asInstanceOf[Integer]
+          }
       }
 
     case BinaryType =>
@@ -307,6 +337,8 @@ class JacksonParser(
     }
   }
 
+  private val allowEmptyString = SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_EMPTY_STRING_IN_JSON)
+
   /**
    * This function throws an exception for failed conversion. For empty string on data types
    * except for string and binary types, this also throws an exception.
@@ -315,7 +347,16 @@ class JacksonParser(
       parser: JsonParser,
       dataType: DataType): PartialFunction[JsonToken, R] = {
 
-    // SPARK-25040: Disallow empty strings for data types except for string and binary types.
+    // SPARK-25040: Disallows empty strings for data types except for string and binary types.
+    // But treats empty strings as null for certain types if the legacy config is enabled.
+    case VALUE_STRING if parser.getTextLength < 1 && allowEmptyString =>
+      dataType match {
+        case FloatType | DoubleType | TimestampType | DateType =>
+          throw new RuntimeException(
+            s"Failed to parse an empty string for data type ${dataType.catalogString}")
+        case _ => null
+      }
+
     case VALUE_STRING if parser.getTextLength < 1 =>
       throw new RuntimeException(
         s"Failed to parse an empty string for data type ${dataType.catalogString}")
@@ -334,7 +375,8 @@ class JacksonParser(
   private def convertObject(
       parser: JsonParser,
       schema: StructType,
-      fieldConverters: Array[ValueConverter]): InternalRow = {
+      fieldConverters: Array[ValueConverter],
+      isRoot: Boolean = false): InternalRow = {
     val row = new GenericInternalRow(schema.length)
     var badRecordException: Option[Throwable] = None
 
@@ -344,7 +386,8 @@ class JacksonParser(
           try {
             row.update(index, fieldConverters(index).apply(parser))
           } catch {
-            case NonFatal(e) =>
+            case e: SparkUpgradeException => throw e
+            case NonFatal(e) if isRoot =>
               badRecordException = badRecordException.orElse(Some(e))
               parser.skipChildren()
           }
@@ -415,6 +458,7 @@ class JacksonParser(
         }
       }
     } catch {
+      case e: SparkUpgradeException => throw e
       case e @ (_: RuntimeException | _: JsonProcessingException | _: MalformedInputException) =>
         // JSON parser currently doesn't support partial results for corrupted records.
         // For such records, all fields other than the field configured by
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
index f030955ee6e7f..de396a4c63458 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.analysis.TypeCoercion
 import org.apache.spark.sql.catalyst.expressions.ExprUtils
 import org.apache.spark.sql.catalyst.json.JacksonUtils.nextUntil
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
@@ -40,7 +41,9 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable {
   private val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = true)
 
   /**
    * Infer the type of a collection of json records in three stages:
@@ -90,12 +93,10 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable {
     }
     json.sparkContext.runJob(mergedTypesFromPartitions, foldPartition, mergeResult)
 
-    canonicalizeType(rootType, options) match {
-      case Some(st: StructType) => st
-      case _ =>
-        // canonicalizeType erases all empty structs, including the only one we want to keep
-        StructType(Nil)
-    }
+    canonicalizeType(rootType, options)
+      .find(_.isInstanceOf[StructType])
+      // canonicalizeType erases all empty structs, including the only one we want to keep
+      .getOrElse(StructType(Nil)).asInstanceOf[StructType]
   }
 
   /**
@@ -196,7 +197,8 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable {
    * Recursively canonicalizes inferred types, e.g., removes StructTypes with no fields,
    * drops NullTypes or converts them to StringType based on provided options.
    */
-  private def canonicalizeType(tpe: DataType, options: JSONOptions): Option[DataType] = tpe match {
+  private[catalyst] def canonicalizeType(
+      tpe: DataType, options: JSONOptions): Option[DataType] = tpe match {
     case at: ArrayType =>
       canonicalizeType(at.elementType, options)
         .map(t => at.copy(elementType = t))
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
index 28dc8e9d0d5f3..f79dabf758c14 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
@@ -41,14 +41,14 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] {
         createNamedStruct.valExprs(ordinal)
 
       // Remove redundant array indexing.
-      case GetArrayStructFields(CreateArray(elems), field, ordinal, _, _) =>
+      case GetArrayStructFields(CreateArray(elems, useStringTypeWhenEmpty), field, ordinal, _, _) =>
         // Instead of selecting the field on the entire array, select it from each member
         // of the array. Pushing down the operation this way may open other optimizations
         // opportunities (i.e. struct(...,x,...).x)
-        CreateArray(elems.map(GetStructField(_, ordinal, Some(field.name))))
+        CreateArray(elems.map(GetStructField(_, ordinal, Some(field.name))), useStringTypeWhenEmpty)
 
       // Remove redundant map lookup.
-      case ga @ GetArrayItem(CreateArray(elems), IntegerLiteral(idx)) =>
+      case ga @ GetArrayItem(CreateArray(elems, _), IntegerLiteral(idx)) =>
         // Instead of creating the array and then selecting one row, remove array creation
         // altogether.
         if (idx >= 0 && idx < elems.size) {
@@ -58,7 +58,7 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] {
           // out of bounds, mimic the runtime behavior and return null
           Literal(null, ga.dataType)
         }
-      case GetMapValue(CreateMap(elems), key) => CaseKeyWhen(key, elems)
+      case GetMapValue(CreateMap(elems, _), key) => CaseKeyWhen(key, elems)
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
index f92d8f5b8e534..93c608dc71a24 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
@@ -344,14 +344,11 @@ object JoinReorderDP extends PredicateHelper with Logging {
     }
 
     def betterThan(other: JoinPlan, conf: SQLConf): Boolean = {
-      if (other.planCost.card == 0 || other.planCost.size == 0) {
-        false
-      } else {
-        val relativeRows = BigDecimal(this.planCost.card) / BigDecimal(other.planCost.card)
-        val relativeSize = BigDecimal(this.planCost.size) / BigDecimal(other.planCost.size)
-        relativeRows * conf.joinReorderCardWeight +
-          relativeSize * (1 - conf.joinReorderCardWeight) < 1
-      }
+      val thisCost = BigDecimal(this.planCost.card) * conf.joinReorderCardWeight +
+        BigDecimal(this.planCost.size) * (1 - conf.joinReorderCardWeight)
+      val otherCost = BigDecimal(other.planCost.card) * conf.joinReorderCardWeight +
+        BigDecimal(other.planCost.size) * (1 - conf.joinReorderCardWeight)
+      thisCost < otherCost
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
index ea85014a37bd8..becd7d1dd1738 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
@@ -104,10 +104,13 @@ object NestedColumnAliasing {
   /**
    * Return two maps in order to replace nested fields to aliases.
    *
+   * If `exclusiveAttrs` is given, any nested field accessors of these attributes
+   * won't be considered in nested fields aliasing.
+   *
    * 1. ExtractValue -> Alias: A new alias is created for each nested field.
    * 2. ExprId -> Seq[Alias]: A reference attribute has multiple aliases pointing it.
    */
-  def getAliasSubMap(exprList: Seq[Expression])
+  def getAliasSubMap(exprList: Seq[Expression], exclusiveAttrs: Seq[Attribute] = Seq.empty)
     : Option[(Map[ExtractValue, Alias], Map[ExprId, Seq[Alias]])] = {
     val (nestedFieldReferences, otherRootReferences) =
       exprList.flatMap(collectRootReferenceAndExtractValue).partition {
@@ -115,9 +118,12 @@ object NestedColumnAliasing {
         case _ => false
       }
 
+    // Note that when we group by extractors with their references, we should remove
+    // cosmetic variations.
+    val exclusiveAttrSet = AttributeSet(exclusiveAttrs ++ otherRootReferences)
     val aliasSub = nestedFieldReferences.asInstanceOf[Seq[ExtractValue]]
-      .filter(!_.references.subsetOf(AttributeSet(otherRootReferences)))
-      .groupBy(_.references.head)
+      .filter(!_.references.subsetOf(exclusiveAttrSet))
+      .groupBy(_.references.head.canonicalized.asInstanceOf[Attribute])
       .flatMap { case (attr, nestedFields: Seq[ExtractValue]) =>
         // Each expression can contain multiple nested fields.
         // Note that we keep the original names to deliver to parquet in a case-sensitive way.
@@ -128,8 +134,13 @@ object NestedColumnAliasing {
 
         // If all nested fields of `attr` are used, we don't need to introduce new aliases.
         // By default, ColumnPruning rule uses `attr` already.
+        // Note that we need to remove cosmetic variations first, so we only count a
+        // nested field once.
         if (nestedFieldToAlias.nonEmpty &&
-            nestedFieldToAlias.length < totalFieldNum(attr.dataType)) {
+            nestedFields.map(_.canonicalized)
+              .distinct
+              .map { nestedField => totalFieldNum(nestedField.dataType) }
+              .sum < totalFieldNum(attr.dataType)) {
           Some(attr.exprId -> nestedFieldToAlias)
         } else {
           None
@@ -155,53 +166,6 @@ object NestedColumnAliasing {
     case MapType(keyType, valueType, _) => totalFieldNum(keyType) + totalFieldNum(valueType)
     case _ => 1 // UDT and others
   }
-}
-
-/**
- * This prunes unnessary nested columns from `Generate` and optional `Project` on top
- * of it.
- */
-object GeneratorNestedColumnAliasing {
-  def unapply(plan: LogicalPlan): Option[LogicalPlan] = plan match {
-    // Either `nestedPruningOnExpressions` or `nestedSchemaPruningEnabled` is enabled, we
-    // need to prune nested columns through Project and under Generate. The difference is
-    // when `nestedSchemaPruningEnabled` is on, nested columns will be pruned further at
-    // file format readers if it is supported.
-    case Project(projectList, g: Generate) if (SQLConf.get.nestedPruningOnExpressions ||
-        SQLConf.get.nestedSchemaPruningEnabled) && canPruneGenerator(g.generator) =>
-      // On top on `Generate`, a `Project` that might have nested column accessors.
-      // We try to get alias maps for both project list and generator's children expressions.
-      NestedColumnAliasing.getAliasSubMap(projectList ++ g.generator.children).map {
-        case (nestedFieldToAlias, attrToAliases) =>
-          val newChild = pruneGenerate(g, nestedFieldToAlias, attrToAliases)
-          Project(NestedColumnAliasing.getNewProjectList(projectList, nestedFieldToAlias), newChild)
-      }
-
-    case g: Generate if SQLConf.get.nestedSchemaPruningEnabled &&
-        canPruneGenerator(g.generator) =>
-      NestedColumnAliasing.getAliasSubMap(g.generator.children).map {
-        case (nestedFieldToAlias, attrToAliases) =>
-          pruneGenerate(g, nestedFieldToAlias, attrToAliases)
-      }
-
-    case _ =>
-      None
-  }
-
-  private def pruneGenerate(
-      g: Generate,
-      nestedFieldToAlias: Map[ExtractValue, Alias],
-      attrToAliases: Map[ExprId, Seq[Alias]]): LogicalPlan = {
-    val newGenerator = g.generator.transform {
-      case f: ExtractValue if nestedFieldToAlias.contains(f) =>
-        nestedFieldToAlias(f).toAttribute
-    }.asInstanceOf[Generator]
-
-    // Defer updating `Generate.unrequiredChildIndex` to next round of `ColumnPruning`.
-    val newGenerate = g.copy(generator = newGenerator)
-
-    NestedColumnAliasing.replaceChildrenWithAliases(newGenerate, attrToAliases)
-  }
 
   /**
    * This is a while-list for pruning nested fields at `Generator`.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
index ea01d9e63eef7..592652212b22e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
-import org.apache.spark.sql.catalyst.expressions.{Alias, And, ArrayTransform, CreateArray, CreateMap, CreateNamedStruct, CreateStruct, EqualTo, ExpectsInputTypes, Expression, GetStructField, KnownFloatingPointNormalized, LambdaFunction, NamedLambdaVariable, UnaryExpression}
+import org.apache.spark.sql.catalyst.expressions.{Alias, And, ArrayTransform, CreateArray, CreateMap, CreateNamedStruct, CreateStruct, EqualTo, ExpectsInputTypes, Expression, GetStructField, If, IsNull, KnownFloatingPointNormalized, LambdaFunction, Literal, NamedLambdaVariable, UnaryExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Subquery, Window}
@@ -56,10 +56,6 @@ import org.apache.spark.sql.types._
 object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
 
   def apply(plan: LogicalPlan): LogicalPlan = plan match {
-    // A subquery will be rewritten into join later, and will go through this rule
-    // eventually. Here we skip subquery, as we only need to run this rule once.
-    case _: Subquery => plan
-
     case _ => plan transform {
       case w: Window if w.partitionSpec.exists(p => needNormalize(p)) =>
         // Although the `windowExpressions` may refer to `partitionSpec` expressions, we don't need
@@ -114,20 +110,21 @@ object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
     case CreateNamedStruct(children) =>
       CreateNamedStruct(children.map(normalize))
 
-    case CreateArray(children) =>
-      CreateArray(children.map(normalize))
+    case CreateArray(children, useStringTypeWhenEmpty) =>
+      CreateArray(children.map(normalize), useStringTypeWhenEmpty)
 
-    case CreateMap(children) =>
-      CreateMap(children.map(normalize))
+    case CreateMap(children, useStringTypeWhenEmpty) =>
+      CreateMap(children.map(normalize), useStringTypeWhenEmpty)
 
     case _ if expr.dataType == FloatType || expr.dataType == DoubleType =>
       KnownFloatingPointNormalized(NormalizeNaNAndZero(expr))
 
     case _ if expr.dataType.isInstanceOf[StructType] =>
-      val fields = expr.dataType.asInstanceOf[StructType].fields.indices.map { i =>
-        normalize(GetStructField(expr, i))
+      val fields = expr.dataType.asInstanceOf[StructType].fieldNames.zipWithIndex.map {
+        case (name, i) => Seq(Literal(name), normalize(GetStructField(expr, i)))
       }
-      CreateStruct(fields)
+      val struct = CreateNamedStruct(fields.flatten.toSeq)
+      KnownFloatingPointNormalized(If(IsNull(expr), Literal(null, struct.dataType), struct))
 
     case _ if expr.dataType.isInstanceOf[ArrayType] =>
       val ArrayType(et, containsNull) = expr.dataType
@@ -137,6 +134,28 @@ object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
 
     case _ => throw new IllegalStateException(s"fail to normalize $expr")
   }
+
+  val FLOAT_NORMALIZER: Any => Any = (input: Any) => {
+    val f = input.asInstanceOf[Float]
+    if (f.isNaN) {
+      Float.NaN
+    } else if (f == -0.0f) {
+      0.0f
+    } else {
+      f
+    }
+  }
+
+  val DOUBLE_NORMALIZER: Any => Any = (input: Any) => {
+    val d = input.asInstanceOf[Double]
+    if (d.isNaN) {
+      Double.NaN
+    } else if (d == -0.0d) {
+      0.0d
+    } else {
+      d
+    }
+  }
 }
 
 case class NormalizeNaNAndZero(child: Expression) extends UnaryExpression with ExpectsInputTypes {
@@ -146,27 +165,8 @@ case class NormalizeNaNAndZero(child: Expression) extends UnaryExpression with E
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(FloatType, DoubleType))
 
   private lazy val normalizer: Any => Any = child.dataType match {
-    case FloatType => (input: Any) => {
-      val f = input.asInstanceOf[Float]
-      if (f.isNaN) {
-        Float.NaN
-      } else if (f == -0.0f) {
-        0.0f
-      } else {
-        f
-      }
-    }
-
-    case DoubleType => (input: Any) => {
-      val d = input.asInstanceOf[Double]
-      if (d.isNaN) {
-        Double.NaN
-      } else if (d == -0.0d) {
-        0.0d
-      } else {
-        d
-      }
-    }
+    case FloatType => NormalizeFloatingNumbers.FLOAT_NORMALIZER
+    case DoubleType => NormalizeFloatingNumbers.DOUBLE_NORMALIZER
   }
 
   override def nullSafeEval(input: Any): Any = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 0fdf6b022d885..4629cbbfa7f51 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -53,7 +53,10 @@ abstract class Optimizer(catalogManager: CatalogManager)
       "PartitionPruning",
       "Extract Python UDFs")
 
-  protected def fixedPoint = FixedPoint(SQLConf.get.optimizerMaxIterations)
+  protected def fixedPoint =
+    FixedPoint(
+      SQLConf.get.optimizerMaxIterations,
+      maxIterationsSetting = SQLConf.OPTIMIZER_MAX_ITERATIONS.key)
 
   /**
    * Defines the default rule batches in the Optimizer.
@@ -598,24 +601,32 @@ object ColumnPruning extends Rule[LogicalPlan] {
       s.copy(child = prunedChild(child, s.references))
 
     // prune unrequired references
-    case p @ Project(_, g: Generate) =>
-      val currP = if (p.references != g.outputSet) {
-        val requiredAttrs = p.references -- g.producedAttributes ++ g.generator.references
-        val newChild = prunedChild(g.child, requiredAttrs)
-        val unrequired = g.generator.references -- p.references
-        val unrequiredIndices = newChild.output.zipWithIndex.filter(t => unrequired.contains(t._1))
-          .map(_._2)
-        p.copy(child = g.copy(child = newChild, unrequiredChildIndex = unrequiredIndices))
-      } else {
-        p
-      }
-      // If we can prune nested column on Project + Generate, do it now.
-      // Otherwise by transforming down to Generate, it could be pruned individually,
-      // and causes nested column on top Project unable to resolve.
-      GeneratorNestedColumnAliasing.unapply(currP).getOrElse(currP)
-
-    // prune unrequired nested fields from `Generate`.
-    case GeneratorNestedColumnAliasing(p) => p
+    case p @ Project(_, g: Generate) if p.references != g.outputSet =>
+      val requiredAttrs = p.references -- g.producedAttributes ++ g.generator.references
+      val newChild = prunedChild(g.child, requiredAttrs)
+      val unrequired = g.generator.references -- p.references
+      val unrequiredIndices = newChild.output.zipWithIndex.filter(t => unrequired.contains(t._1))
+        .map(_._2)
+      p.copy(child = g.copy(child = newChild, unrequiredChildIndex = unrequiredIndices))
+
+    // prune unrequired nested fields
+    case p @ Project(projectList, g: Generate) if SQLConf.get.nestedPruningOnExpressions &&
+        NestedColumnAliasing.canPruneGenerator(g.generator) =>
+      val exprsToPrune = projectList ++ g.generator.children
+      NestedColumnAliasing.getAliasSubMap(exprsToPrune, g.qualifiedGeneratorOutput).map {
+        case (nestedFieldToAlias, attrToAliases) =>
+          val newGenerator = g.generator.transform {
+            case f: ExtractValue if nestedFieldToAlias.contains(f) =>
+              nestedFieldToAlias(f).toAttribute
+          }.asInstanceOf[Generator]
+
+          // Defer updating `Generate.unrequiredChildIndex` to next round of `ColumnPruning`.
+          val newGenerate = g.copy(generator = newGenerator)
+
+          val newChild = NestedColumnAliasing.replaceChildrenWithAliases(newGenerate, attrToAliases)
+
+          Project(NestedColumnAliasing.getNewProjectList(projectList, nestedFieldToAlias), newChild)
+      }.getOrElse(p)
 
     // Eliminate unneeded attributes from right side of a Left Existence Join.
     case j @ Join(_, right, LeftExistence(_), _, _) =>
@@ -960,7 +971,7 @@ object CombineFilters extends Rule[LogicalPlan] with PredicateHelper {
 /**
  * Removes Sort operation. This can happen:
  * 1) if the sort order is empty or the sort order does not have any reference
- * 2) if the child is already sorted
+ * 2) if the Sort operator is a local sort and the child is already sorted
  * 3) if there is another Sort operator separated by 0...n Project/Filter operators
  * 4) if the Sort operator is within Join separated by 0...n Project/Filter operators only,
  *    and the Join conditions is deterministic
@@ -968,12 +979,18 @@ object CombineFilters extends Rule[LogicalPlan] with PredicateHelper {
  *    and the aggregate function is order irrelevant
  */
 object EliminateSorts extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan transform applyLocally
+
+  private val applyLocally: PartialFunction[LogicalPlan, LogicalPlan] = {
     case s @ Sort(orders, _, child) if orders.isEmpty || orders.exists(_.child.foldable) =>
       val newOrders = orders.filterNot(_.child.foldable)
-      if (newOrders.isEmpty) child else s.copy(order = newOrders)
-    case Sort(orders, true, child) if SortOrder.orderingSatisfies(child.outputOrdering, orders) =>
-      child
+      if (newOrders.isEmpty) {
+        applyLocally.lift(child).getOrElse(child)
+      } else {
+        s.copy(order = newOrders)
+      }
+    case Sort(orders, false, child) if SortOrder.orderingSatisfies(child.outputOrdering, orders) =>
+      applyLocally.lift(child).getOrElse(child)
     case s @ Sort(_, _, child) => s.copy(child = recursiveRemoveSort(child))
     case j @ Join(originLeft, originRight, _, cond, _) if cond.forall(_.deterministic) =>
       j.copy(left = recursiveRemoveSort(originLeft), right = recursiveRemoveSort(originRight))
@@ -1193,9 +1210,10 @@ object PushPredicateThroughNonJoin extends Rule[LogicalPlan] with PredicateHelpe
 
   def getAliasMap(plan: Aggregate): AttributeMap[Expression] = {
     // Find all the aliased expressions in the aggregate list that don't include any actual
-    // AggregateExpression, and create a map from the alias to the expression
+    // AggregateExpression or PythonUDF, and create a map from the alias to the expression
     val aliasMap = plan.aggregateExpressions.collect {
-      case a: Alias if a.child.find(_.isInstanceOf[AggregateExpression]).isEmpty =>
+      case a: Alias if a.child.find(e => e.isInstanceOf[AggregateExpression] ||
+          PythonUDF.isGroupedAggPandasUDF(e)).isEmpty =>
         (a.toAttribute, a.child)
     }
     AttributeMap(aliasMap)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala
index b19e13870aa65..7535f9c9c32d7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala
@@ -85,8 +85,8 @@ object PropagateEmptyRelation extends Rule[LogicalPlan] with PredicateHelper wit
       case _: Filter => empty(p)
       case _: Sample => empty(p)
       case _: Sort => empty(p)
-      case _: GlobalLimit => empty(p)
-      case _: LocalLimit => empty(p)
+      case _: GlobalLimit if !p.isStreaming => empty(p)
+      case _: LocalLimit if !p.isStreaming => empty(p)
       case _: Repartition => empty(p)
       case _: RepartitionByExpression => empty(p)
       // An aggregate with non-empty group expression will return one output row per group when the
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala
index b8edf985dbda3..33b398e11cde9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala
@@ -64,8 +64,7 @@ object ReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
       case af @ ArrayFilter(_, lf @ LambdaFunction(func, _, _)) =>
         val newLambda = lf.copy(function = replaceNullWithFalse(func))
         af.copy(function = newLambda)
-      case ae @ ArrayExists(_, lf @ LambdaFunction(func, _, _))
-          if !SQLConf.get.getConf(SQLConf.LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC) =>
+      case ae @ ArrayExists(_, lf @ LambdaFunction(func, _, _), false) =>
         val newLambda = lf.copy(function = replaceNullWithFalse(func))
         ae.copy(function = newLambda)
       case mf @ MapFilter(_, lf @ LambdaFunction(func, _, _)) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
index e5571069a7c41..15aa02ff677de 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
@@ -257,7 +257,7 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
 
         // Select the result of the first aggregate in the last aggregate.
         val result = AggregateExpression(
-          aggregate.First(evalWithinGroup(regularGroupId, operator.toAttribute), Literal(true)),
+          aggregate.First(evalWithinGroup(regularGroupId, operator.toAttribute), true),
           mode = Complete,
           isDistinct = false)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index bd400f86ea2c1..7cbeb47c30be9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -525,27 +525,33 @@ object LikeSimplification extends Rule[LogicalPlan] {
   private val equalTo = "([^_%]*)".r
 
   def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
-    case Like(input, Literal(pattern, StringType), escapeChar) =>
+    case l @ Like(input, Literal(pattern, StringType), escapeChar) =>
       if (pattern == null) {
         // If pattern is null, return null value directly, since "col like null" == null.
         Literal(null, BooleanType)
       } else {
-        val escapeStr = String.valueOf(escapeChar)
         pattern.toString match {
-          case startsWith(prefix) if !prefix.endsWith(escapeStr) =>
+          // There are three different situations when pattern containing escapeChar:
+          // 1. pattern contains invalid escape sequence, e.g. 'm\aca'
+          // 2. pattern contains escaped wildcard character, e.g. 'ma\%ca'
+          // 3. pattern contains escaped escape character, e.g. 'ma\\ca'
+          // Although there are patterns can be optimized if we handle the escape first, we just
+          // skip this rule if pattern contains any escapeChar for simplicity.
+          case p if p.contains(escapeChar) => l
+          case startsWith(prefix) =>
             StartsWith(input, Literal(prefix))
           case endsWith(postfix) =>
             EndsWith(input, Literal(postfix))
           // 'a%a' pattern is basically same with 'a%' && '%a'.
           // However, the additional `Length` condition is required to prevent 'a' match 'a%a'.
-          case startsAndEndsWith(prefix, postfix) if !prefix.endsWith(escapeStr) =>
+          case startsAndEndsWith(prefix, postfix) =>
             And(GreaterThanOrEqual(Length(input), Literal(prefix.length + postfix.length)),
               And(StartsWith(input, Literal(prefix)), EndsWith(input, Literal(postfix))))
-          case contains(infix) if !infix.endsWith(escapeStr) =>
+          case contains(infix) =>
             Contains(input, Literal(infix))
           case equalTo(str) =>
             EqualTo(input, Literal(str))
-          case _ => Like(input, Literal.create(pattern, StringType), escapeChar)
+          case _ => l
         }
       }
   }
@@ -620,59 +626,82 @@ object NullPropagation extends Rule[LogicalPlan] {
  */
 object FoldablePropagation extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = {
-    var foldableMap = AttributeMap(plan.flatMap {
-      case Project(projectList, _) => projectList.collect {
-        case a: Alias if a.child.foldable => (a.toAttribute, a)
-      }
-      case _ => Nil
-    })
-    val replaceFoldable: PartialFunction[Expression, Expression] = {
-      case a: AttributeReference if foldableMap.contains(a) => foldableMap(a)
+    CleanupAliases(propagateFoldables(plan)._1)
+  }
+
+  private def propagateFoldables(plan: LogicalPlan): (LogicalPlan, AttributeMap[Alias]) = {
+    plan match {
+      case p: Project =>
+        val (newChild, foldableMap) = propagateFoldables(p.child)
+        val newProject =
+          replaceFoldable(p.withNewChildren(Seq(newChild)).asInstanceOf[Project], foldableMap)
+        val newFoldableMap = AttributeMap(newProject.projectList.collect {
+          case a: Alias if a.child.foldable => (a.toAttribute, a)
+        })
+        (newProject, newFoldableMap)
+
+      // We can not replace the attributes in `Expand.output`. If there are other non-leaf
+      // operators that have the `output` field, we should put them here too.
+      case e: Expand =>
+        val (newChild, foldableMap) = propagateFoldables(e.child)
+        val expandWithNewChildren = e.withNewChildren(Seq(newChild)).asInstanceOf[Expand]
+        val newExpand = if (foldableMap.isEmpty) {
+          expandWithNewChildren
+        } else {
+          val newProjections = expandWithNewChildren.projections.map(_.map(_.transform {
+            case a: AttributeReference if foldableMap.contains(a) => foldableMap(a)
+          }))
+          if (newProjections == expandWithNewChildren.projections) {
+            expandWithNewChildren
+          } else {
+            expandWithNewChildren.copy(projections = newProjections)
+          }
+        }
+        (newExpand, foldableMap)
+
+      case u: UnaryNode if canPropagateFoldables(u) =>
+        val (newChild, foldableMap) = propagateFoldables(u.child)
+        val newU = replaceFoldable(u.withNewChildren(Seq(newChild)), foldableMap)
+        (newU, foldableMap)
+
+      // Join derives the output attributes from its child while they are actually not the
+      // same attributes. For example, the output of outer join is not always picked from its
+      // children, but can also be null. We should exclude these miss-derived attributes when
+      // propagating the foldable expressions.
+      // TODO(cloud-fan): It seems more reasonable to use new attributes as the output attributes
+      // of outer join.
+      case j: Join =>
+        val (newChildren, foldableMaps) = j.children.map(propagateFoldables).unzip
+        val foldableMap = AttributeMap(
+          foldableMaps.foldLeft(Iterable.empty[(Attribute, Alias)])(_ ++ _.baseMap.values).toSeq)
+        val newJoin =
+          replaceFoldable(j.withNewChildren(newChildren).asInstanceOf[Join], foldableMap)
+        val missDerivedAttrsSet: AttributeSet = AttributeSet(newJoin.joinType match {
+          case _: InnerLike | LeftExistence(_) => Nil
+          case LeftOuter => newJoin.right.output
+          case RightOuter => newJoin.left.output
+          case FullOuter => newJoin.left.output ++ newJoin.right.output
+        })
+        val newFoldableMap = AttributeMap(foldableMap.baseMap.values.filterNot {
+          case (attr, _) => missDerivedAttrsSet.contains(attr)
+        }.toSeq)
+        (newJoin, newFoldableMap)
+
+      // For other plans, they are not safe to apply foldable propagation, and they should not
+      // propagate foldable expressions from children.
+      case o =>
+        val newOther = o.mapChildren(propagateFoldables(_)._1)
+        (newOther, AttributeMap.empty)
     }
+  }
 
+  private def replaceFoldable(plan: LogicalPlan, foldableMap: AttributeMap[Alias]): plan.type = {
     if (foldableMap.isEmpty) {
       plan
     } else {
-      CleanupAliases(plan.transformUp {
-        // We can only propagate foldables for a subset of unary nodes.
-        case u: UnaryNode if foldableMap.nonEmpty && canPropagateFoldables(u) =>
-          u.transformExpressions(replaceFoldable)
-
-        // Join derives the output attributes from its child while they are actually not the
-        // same attributes. For example, the output of outer join is not always picked from its
-        // children, but can also be null. We should exclude these miss-derived attributes when
-        // propagating the foldable expressions.
-        // TODO(cloud-fan): It seems more reasonable to use new attributes as the output attributes
-        // of outer join.
-        case j @ Join(left, right, joinType, _, _) if foldableMap.nonEmpty =>
-          val newJoin = j.transformExpressions(replaceFoldable)
-          val missDerivedAttrsSet: AttributeSet = AttributeSet(joinType match {
-            case _: InnerLike | LeftExistence(_) => Nil
-            case LeftOuter => right.output
-            case RightOuter => left.output
-            case FullOuter => left.output ++ right.output
-          })
-          foldableMap = AttributeMap(foldableMap.baseMap.values.filterNot {
-            case (attr, _) => missDerivedAttrsSet.contains(attr)
-          }.toSeq)
-          newJoin
-
-        // We can not replace the attributes in `Expand.output`. If there are other non-leaf
-        // operators that have the `output` field, we should put them here too.
-        case expand: Expand if foldableMap.nonEmpty =>
-          expand.copy(projections = expand.projections.map { projection =>
-            projection.map(_.transform(replaceFoldable))
-          })
-
-        // For other plans, they are not safe to apply foldable propagation, and they should not
-        // propagate foldable expressions from children.
-        case other if foldableMap.nonEmpty =>
-          val childrenOutputSet = AttributeSet(other.children.flatMap(_.output))
-          foldableMap = AttributeMap(foldableMap.baseMap.values.filterNot {
-            case (attr, _) => childrenOutputSet.contains(attr)
-          }.toSeq)
-          other
-      })
+      plan transformExpressions {
+        case a: AttributeReference if foldableMap.contains(a) => foldableMap(a)
+      }
     }
   }
 
@@ -680,7 +709,7 @@ object FoldablePropagation extends Rule[LogicalPlan] {
    * Whitelist of all [[UnaryNode]]s for which allow foldable propagation.
    */
   private def canPropagateFoldables(u: UnaryNode): Boolean = u match {
-    case _: Project => true
+    // Handling `Project` is moved to `propagateFoldables`.
     case _: Filter => true
     case _: SubqueryAlias => true
     case _: Aggregate => true
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
index c79bf3e20b776..ba7e85285db2f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
@@ -79,13 +79,13 @@ object ComputeCurrentTime extends Rule[LogicalPlan] {
     val currentTime = Literal.create(timestamp, timeExpr.dataType)
 
     plan transformAllExpressions {
-      case CurrentDate(Some(timeZoneId)) =>
+      case currentDate @ CurrentDate(Some(timeZoneId)) =>
         currentDates.getOrElseUpdate(timeZoneId, {
           Literal.create(
-            LocalDate.now(DateTimeUtils.getZoneId(timeZoneId)),
+            DateTimeUtils.millisToDays(DateTimeUtils.toMillis(timestamp), currentDate.zoneId),
             DateType)
         })
-      case CurrentTimestamp() => currentTime
+      case CurrentTimestamp() | Now() => currentTime
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
index 0fa29e87e5f39..9643f5827b910 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
@@ -162,10 +162,18 @@ object ObjectSerializerPruning extends Rule[LogicalPlan] {
    * Note: we should do `transformUp` explicitly to change data types.
    */
   private def alignNullTypeInIf(expr: Expression) = expr.transformUp {
-    case i @ If(_: IsNull, Literal(null, dt), ser) if !dt.sameType(ser.dataType) =>
+    case i @ If(IsNullCondition(), Literal(null, dt), ser) if !dt.sameType(ser.dataType) =>
       i.copy(trueValue = Literal(null, ser.dataType))
   }
 
+  object IsNullCondition {
+    def unapply(expr: Expression): Boolean = expr match {
+      case _: IsNull => true
+      case i: Invoke if i.functionName == "isNullAt" => true
+      case _ => false
+    }
+  }
+
   /**
    * This method prunes given serializer expression by given pruned data type. For example,
    * given a serializer creating struct(a int, b int) and pruned data type struct(a int),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index e9ad84472904d..91519c021481c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -104,6 +104,10 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     withOrigin(ctx)(StructType(visitColTypeList(ctx.colTypeList)))
   }
 
+  def parseRawDataType(ctx: SingleDataTypeContext): DataType = withOrigin(ctx) {
+    typedVisit[DataType](ctx.dataType())
+  }
+
   /* ********************************************************************************************
    * Plan parsing
    * ******************************************************************************************** */
@@ -460,6 +464,18 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         }
       }
     }
+    if (matchedActions.isEmpty && notMatchedActions.isEmpty) {
+      throw new ParseException("There must be at least one WHEN clause in a MERGE statement", ctx)
+    }
+    // children being empty means that the condition is not set
+    if (matchedActions.length == 2 && matchedActions.head.condition.isEmpty) {
+      throw new ParseException("When there are 2 MATCHED clauses in a MERGE statement, " +
+        "the first MATCHED clause must have a condition", ctx)
+    }
+    if (matchedActions.groupBy(_.getClass).mapValues(_.size).exists(_._2 > 1)) {
+      throw new ParseException(
+        "UPDATE and DELETE can appear at most once in MATCHED clauses in a MERGE statement", ctx)
+    }
 
     MergeIntoTable(
       aliasedTarget,
@@ -504,6 +520,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   protected def visitStringConstant(ctx: ConstantContext): String = withOrigin(ctx) {
     ctx match {
+      case _: NullLiteralContext => null
       case s: StringLiteralContext => createString(s)
       case o => o.getText
     }
@@ -613,7 +630,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case p: Predicate => p
       case e => Cast(e, BooleanType)
     }
-    Filter(predicate, plan)
+    UnresolvedHaving(predicate, plan)
   }
 
   /**
@@ -707,7 +724,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val withProject = if (aggregationClause == null && havingClause != null) {
       if (conf.getConf(SQLConf.LEGACY_HAVING_WITHOUT_GROUP_BY_AS_WHERE)) {
         // If the legacy conf is set, treat HAVING without GROUP BY as WHERE.
-        withHavingClause(havingClause, createProject())
+        val predicate = expression(havingClause.booleanExpression) match {
+          case p: Predicate => p
+          case e => Cast(e, BooleanType)
+        }
+        Filter(predicate, createProject())
       } else {
         // According to SQL standard, HAVING without GROUP BY means global aggregate.
         withHavingClause(havingClause, Aggregate(Nil, namedExpressions, withFilter))
@@ -1392,9 +1413,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
             throw new ParseException("Invalid escape string." +
               "Escape string must contains only one character.", ctx)
           }
-          str
+          str.charAt(0)
         }.getOrElse('\\')
-        invertIfNotDefined(Like(e, expression(ctx.pattern), Literal(escapeChar)))
+        invertIfNotDefined(Like(e, expression(ctx.pattern), escapeChar))
       case SqlBaseParser.RLIKE =>
         invertIfNotDefined(RLike(e, expression(ctx.pattern)))
       case SqlBaseParser.NULL if ctx.NOT != null =>
@@ -1402,12 +1423,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case SqlBaseParser.NULL =>
         IsNull(e)
       case SqlBaseParser.TRUE => ctx.NOT match {
-        case null => IsTrue(e)
-        case _ => IsNotTrue(e)
+        case null => EqualNullSafe(e, Literal(true))
+        case _ => Not(EqualNullSafe(e, Literal(true)))
       }
       case SqlBaseParser.FALSE => ctx.NOT match {
-        case null => IsFalse(e)
-        case _ => IsNotFalse(e)
+        case null => EqualNullSafe(e, Literal(false))
+        case _ => Not(EqualNullSafe(e, Literal(false)))
       }
       case SqlBaseParser.UNKNOWN => ctx.NOT match {
         case null => IsUnknown(e)
@@ -1503,7 +1524,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Create a [[CreateStruct]] expression.
    */
   override def visitStruct(ctx: StructContext): Expression = withOrigin(ctx) {
-    CreateStruct(ctx.argument.asScala.map(expression))
+    CreateStruct.create(ctx.argument.asScala.map(expression))
   }
 
   /**
@@ -1511,7 +1532,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitFirst(ctx: FirstContext): Expression = withOrigin(ctx) {
     val ignoreNullsExpr = ctx.IGNORE != null
-    First(expression(ctx.expression), Literal(ignoreNullsExpr)).toAggregateExpression()
+    First(expression(ctx.expression), ignoreNullsExpr).toAggregateExpression()
   }
 
   /**
@@ -1519,7 +1540,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitLast(ctx: LastContext): Expression = withOrigin(ctx) {
     val ignoreNullsExpr = ctx.IGNORE != null
-    Last(expression(ctx.expression), Literal(ignoreNullsExpr)).toAggregateExpression()
+    Last(expression(ctx.expression), ignoreNullsExpr).toAggregateExpression()
   }
 
   /**
@@ -1533,12 +1554,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Create a Extract expression.
    */
   override def visitExtract(ctx: ExtractContext): Expression = withOrigin(ctx) {
-    val fieldStr = ctx.field.getText
-    val source = expression(ctx.source)
-    val extractField = DatePart.parseExtractField(fieldStr, source, {
-      throw new ParseException(s"Literals of type '$fieldStr' are currently not supported.", ctx)
-    })
-    new DatePart(Literal(fieldStr), expression(ctx.source), extractField)
+    val arguments = Seq(Literal(ctx.field.getText), expression(ctx.source))
+    UnresolvedFunction("extract", arguments, isDistinct = false)
   }
 
   /**
@@ -2088,14 +2105,23 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitMultiUnitsInterval(ctx: MultiUnitsIntervalContext): CalendarInterval = {
     withOrigin(ctx) {
-      val units = ctx.intervalUnit().asScala
+      val units = ctx.unit.asScala
       val values = ctx.intervalValue().asScala
       try {
         assert(units.length == values.length)
         val kvs = units.indices.map { i =>
           val u = units(i).getText
           val v = if (values(i).STRING() != null) {
-            string(values(i).STRING())
+            val value = string(values(i).STRING())
+            // SPARK-32840: For invalid cases, e.g. INTERVAL '1 day 2' hour,
+            // INTERVAL 'interval 1' day, we need to check ahead before they are concatenated with
+            // units and become valid ones, e.g. '1 day 2 hour'.
+            // Ideally, we only ensure the value parts don't contain any units here.
+            if (value.exists(Character.isLetter)) {
+              throw new ParseException("Can only use numbers in the interval value part for" +
+                s" multiple unit value pairs interval form, but got invalid value: $value", ctx)
+            }
+            value
           } else {
             values(i).getText
           }
@@ -2751,9 +2777,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       operationNotAllowed("CREATE EXTERNAL TABLE ...", ctx)
     }
     val schema = Option(ctx.colTypeList()).map(createSchema)
-    val defaultProvider = conf.defaultDataSourceName
-    val provider =
-      Option(ctx.tableProvider).map(_.multipartIdentifier.getText).getOrElse(defaultProvider)
+    val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText)
     val (partitioning, bucketSpec, properties, options, location, comment) =
       visitCreateTableClauses(ctx.createTableClauses())
 
@@ -2769,7 +2793,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case Some(query) =>
         CreateTableAsSelectStatement(
           table, query, partitioning, bucketSpec, properties, provider, options, location, comment,
-          ifNotExists = ifNotExists)
+          writeOptions = Map.empty, ifNotExists = ifNotExists)
 
       case None if temp =>
         // CREATE TEMPORARY TABLE ... USING ... is not supported by the catalyst parser.
@@ -2813,7 +2837,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val (partitioning, bucketSpec, properties, options, location, comment) =
       visitCreateTableClauses(ctx.createTableClauses())
     val schema = Option(ctx.colTypeList()).map(createSchema)
-    val provider = ctx.tableProvider.multipartIdentifier.getText
+    val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText)
     val orCreate = ctx.replaceTableHeader().CREATE() != null
 
     Option(ctx.query).map(plan) match {
@@ -2824,7 +2848,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
 
       case Some(query) =>
         ReplaceTableAsSelectStatement(table, query, partitioning, bucketSpec, properties,
-          provider, options, location, comment, orCreate = orCreate)
+          provider, options, location, comment, writeOptions = Map.empty, orCreate = orCreate)
 
       case _ =>
         ReplaceTableStatement(table, schema.getOrElse(new StructType), partitioning,
@@ -2887,6 +2911,16 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec))
   }
 
+  /**
+   * Create a [[ShowViews]] command.
+   */
+  override def visitShowViews(ctx: ShowViewsContext): LogicalPlan = withOrigin(ctx) {
+    val multiPart = Option(ctx.multipartIdentifier).map(visitMultipartIdentifier)
+    ShowViews(
+      UnresolvedNamespace(multiPart.getOrElse(Seq.empty[String])),
+      Option(ctx.pattern).map(string))
+  }
+
   override def visitColPosition(ctx: ColPositionContext): ColumnPosition = {
     ctx.position.getType match {
       case SqlBaseParser.FIRST => ColumnPosition.first()
@@ -2940,55 +2974,61 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   }
 
   /**
-   * Parse a [[AlterTableAlterColumnStatement]] command.
+   * Parse a [[AlterTableAlterColumnStatement]] command to alter a column's property.
    *
    * For example:
    * {{{
    *   ALTER TABLE table1 ALTER COLUMN a.b.c TYPE bigint
-   *   ALTER TABLE table1 ALTER COLUMN a.b.c TYPE bigint COMMENT 'new comment'
+   *   ALTER TABLE table1 ALTER COLUMN a.b.c SET NOT NULL
+   *   ALTER TABLE table1 ALTER COLUMN a.b.c DROP NOT NULL
    *   ALTER TABLE table1 ALTER COLUMN a.b.c COMMENT 'new comment'
+   *   ALTER TABLE table1 ALTER COLUMN a.b.c FIRST
+   *   ALTER TABLE table1 ALTER COLUMN a.b.c AFTER x
    * }}}
    */
-  override def visitAlterTableColumn(
-      ctx: AlterTableColumnContext): LogicalPlan = withOrigin(ctx) {
-    val verb = if (ctx.CHANGE != null) "CHANGE" else "ALTER"
-    if (ctx.dataType == null && ctx.commentSpec() == null && ctx.colPosition == null) {
+  override def visitAlterTableAlterColumn(
+      ctx: AlterTableAlterColumnContext): LogicalPlan = withOrigin(ctx) {
+    val action = ctx.alterColumnAction
+    if (action == null) {
+      val verb = if (ctx.CHANGE != null) "CHANGE" else "ALTER"
       operationNotAllowed(
-        s"ALTER TABLE table $verb COLUMN requires a TYPE or a COMMENT or a FIRST/AFTER", ctx)
+        s"ALTER TABLE table $verb COLUMN requires a TYPE, a SET/DROP, a COMMENT, or a FIRST/AFTER",
+        ctx)
     }
 
+    val dataType = if (action.dataType != null) {
+      Some(typedVisit[DataType](action.dataType))
+    } else {
+      None
+    }
+    val nullable = if (action.setOrDrop != null) {
+      action.setOrDrop.getType match {
+        case SqlBaseParser.SET => Some(false)
+        case SqlBaseParser.DROP => Some(true)
+      }
+    } else {
+      None
+    }
+    val comment = if (action.commentSpec != null) {
+      Some(visitCommentSpec(action.commentSpec()))
+    } else {
+      None
+    }
+    val position = if (action.colPosition != null) {
+      Some(typedVisit[ColumnPosition](action.colPosition))
+    } else {
+      None
+    }
+
+    assert(Seq(dataType, nullable, comment, position).count(_.nonEmpty) == 1)
+
     AlterTableAlterColumnStatement(
       visitMultipartIdentifier(ctx.table),
       typedVisit[Seq[String]](ctx.column),
-      dataType = Option(ctx.dataType).map(typedVisit[DataType]),
-      nullable = None,
-      comment = Option(ctx.commentSpec()).map(visitCommentSpec),
-      position = Option(ctx.colPosition).map(typedVisit[ColumnPosition]))
-  }
-
-  /**
-   * Parse a [[AlterTableAlterColumnStatement]] command to change column nullability.
-   *
-   * For example:
-   * {{{
-   *   ALTER TABLE table1 ALTER COLUMN a.b.c SET NOT NULL
-   *   ALTER TABLE table1 ALTER COLUMN a.b.c DROP NOT NULL
-   * }}}
-   */
-  override def visitAlterColumnNullability(ctx: AlterColumnNullabilityContext): LogicalPlan = {
-    withOrigin(ctx) {
-      val nullable = ctx.setOrDrop.getType match {
-        case SqlBaseParser.SET => false
-        case SqlBaseParser.DROP => true
-      }
-      AlterTableAlterColumnStatement(
-        visitMultipartIdentifier(ctx.table),
-        typedVisit[Seq[String]](ctx.column),
-        dataType = None,
-        nullable = Some(nullable),
-        comment = None,
-        position = None)
-    }
+      dataType = dataType,
+      nullable = nullable,
+      comment = comment,
+      position = position)
   }
 
   /**
@@ -3159,7 +3199,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     }
     if (ctx.identifier != null &&
         ctx.identifier.getText.toLowerCase(Locale.ROOT) != "noscan") {
-      throw new ParseException(s"Expected `NOSCAN` instead of `${ctx.identifier.getText}`", ctx)
+      throw new ParseException(s"Expected `NOSCAN` instead of `${ctx.identifier.getText}`",
+        ctx.identifier())
     }
 
     val tableName = visitMultipartIdentifier(ctx.multipartIdentifier())
@@ -3505,7 +3546,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   override def visitShowTblProperties(
       ctx: ShowTblPropertiesContext): LogicalPlan = withOrigin(ctx) {
     ShowTableProperties(
-      UnresolvedTable(visitMultipartIdentifier(ctx.table)),
+      UnresolvedTableOrView(visitMultipartIdentifier(ctx.table)),
       Option(ctx.key).map(visitTablePropertyKey))
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
index c96f2da9a5289..fab282f15f215 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
@@ -40,6 +40,11 @@ abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Log
     astBuilder.visitSingleDataType(parser.singleDataType())
   }
 
+  /** Similar to `parseDataType`, but without CHAR/VARCHAR replacement. */
+  override def parseRawDataType(sqlText: String): DataType = parse(sqlText) { parser =>
+    astBuilder.parseRawDataType(parser.singleDataType())
+  }
+
   /** Creates Expression for a given SQL string. */
   override def parseExpression(sqlText: String): Expression = parse(sqlText) { parser =>
     astBuilder.visitSingleExpression(parser.singleExpression())
@@ -93,7 +98,6 @@ abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Log
     lexer.addErrorListener(ParseErrorListener)
     lexer.legacy_setops_precedence_enbled = conf.setOpsPrecedenceEnforced
     lexer.legacy_exponent_literal_as_decimal_enabled = conf.exponentLiteralAsDecimalEnabled
-    lexer.legacy_create_hive_table_by_default_enabled = conf.createHiveTableByDefaultEnabled
     lexer.SQL_standard_keyword_behavior = conf.ansiEnabled
 
     val tokenStream = new CommonTokenStream(lexer)
@@ -103,7 +107,6 @@ abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Log
     parser.addErrorListener(ParseErrorListener)
     parser.legacy_setops_precedence_enbled = conf.setOpsPrecedenceEnforced
     parser.legacy_exponent_literal_as_decimal_enabled = conf.exponentLiteralAsDecimalEnabled
-    parser.legacy_create_hive_table_by_default_enabled = conf.createHiveTableByDefaultEnabled
     parser.SQL_standard_keyword_behavior = conf.ansiEnabled
 
     try {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala
index 77e357ad073da..d724933bc1029 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala
@@ -70,4 +70,10 @@ trait ParserInterface {
    */
   @throws[ParseException]("Text cannot be parsed to a DataType")
   def parseDataType(sqlText: String): DataType
+
+  /**
+   * Parse a string to a raw [[DataType]] without CHAR/VARCHAR replacement.
+   */
+  @throws[ParseException]("Text cannot be parsed to a DataType")
+  def parseRawDataType(sqlText: String): DataType
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala
index a3779698a5ac3..f2dab941cb8b2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala
@@ -83,6 +83,11 @@ object ParserUtils {
     node.getText.slice(1, node.getText.size - 1)
   }
 
+  /** Collect the entries if any. */
+  def entry(key: String, value: Token): Seq[(String, String)] = {
+    Option(value).toSeq.map(x => key -> string(x))
+  }
+
   /** Get the origin (line and position) of the token. */
   def position(token: Token): Origin = {
     val opt = Option(token)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index c5abb6378ff7c..840f1334d07c1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -17,13 +17,24 @@
 
 package org.apache.spark.sql.catalyst.plans
 
+import scala.collection.mutable
+
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, TreeNode, TreeNodeTag}
-import org.apache.spark.sql.catalyst.util.StringUtils.PlanStringConcat
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DataType, StructType}
 
+/**
+ * An abstraction of the Spark SQL query plan tree, which can be logical or physical. This class
+ * defines some basic properties of a query plan node, as well as some new transform APIs to
+ * transform the expressions of the plan node.
+ *
+ * Note that, the query plan is a mutually recursive structure:
+ *   QueryPlan -> Expression (subquery) -> QueryPlan
+ * The tree traverse APIs like `transform`, `foreach`, `collect`, etc. that are
+ * inherited from `TreeNode`, do not traverse into query plans inside subqueries.
+ */
 abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanType] {
   self: PlanType =>
 
@@ -134,7 +145,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
 
   /**
    * Returns the result of running [[transformExpressions]] on this node
-   * and all its children.
+   * and all its children. Note that this method skips expressions inside subqueries.
    */
   def transformAllExpressions(rule: PartialFunction[Expression, Expression]): this.type = {
     transform {
@@ -159,6 +170,101 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
     }.toSeq
   }
 
+  /**
+   * A variant of `transformUp`, which takes care of the case that the rule replaces a plan node
+   * with a new one that has different output expr IDs, by updating the attribute references in
+   * the parent nodes accordingly.
+   *
+   * @param rule the function to transform plan nodes, and return new nodes with attributes mapping
+   *             from old attributes to new attributes. The attribute mapping will be used to
+   *             rewrite attribute references in the parent nodes.
+   * @param skipCond a boolean condition to indicate if we can skip transforming a plan node to save
+   *                 time.
+   */
+  def transformUpWithNewOutput(
+      rule: PartialFunction[PlanType, (PlanType, Seq[(Attribute, Attribute)])],
+      skipCond: PlanType => Boolean = _ => false): PlanType = {
+    def rewrite(plan: PlanType): (PlanType, Seq[(Attribute, Attribute)]) = {
+      if (skipCond(plan)) {
+        plan -> Nil
+      } else {
+        val attrMapping = new mutable.ArrayBuffer[(Attribute, Attribute)]()
+        var newPlan = plan.mapChildren { child =>
+          val (newChild, childAttrMapping) = rewrite(child)
+          attrMapping ++= childAttrMapping
+          newChild
+        }
+
+        val attrMappingForCurrentPlan = attrMapping.filter {
+          // The `attrMappingForCurrentPlan` is used to replace the attributes of the
+          // current `plan`, so the `oldAttr` must be part of `plan.references`.
+          case (oldAttr, _) => plan.references.contains(oldAttr)
+        }
+
+        if (attrMappingForCurrentPlan.nonEmpty) {
+          assert(!attrMappingForCurrentPlan.groupBy(_._1.exprId)
+            .exists(_._2.map(_._2.exprId).distinct.length > 1),
+            "Found duplicate rewrite attributes")
+
+          val attributeRewrites = AttributeMap(attrMappingForCurrentPlan)
+          // Using attrMapping from the children plans to rewrite their parent node.
+          // Note that we shouldn't rewrite a node using attrMapping from its sibling nodes.
+          newPlan = newPlan.transformExpressions {
+            case a: AttributeReference =>
+              updateAttr(a, attributeRewrites)
+            case pe: PlanExpression[PlanType] =>
+              pe.withNewPlan(updateOuterReferencesInSubquery(pe.plan, attributeRewrites))
+          }
+        }
+
+        val (planAfterRule, newAttrMapping) = CurrentOrigin.withOrigin(origin) {
+          rule.applyOrElse(newPlan, (plan: PlanType) => plan -> Nil)
+        }
+
+        val newValidAttrMapping = newAttrMapping.filter {
+          case (a1, a2) => a1.exprId != a2.exprId
+        }
+
+        // Updates the `attrMapping` entries that are obsoleted by generated entries in `rule`.
+        // For example, `attrMapping` has a mapping entry 'id#1 -> id#2' and `rule`
+        // generates a new entry 'id#2 -> id#3'. In this case, we need to update
+        // the corresponding old entry from 'id#1 -> id#2' to '#id#1 -> #id#3'.
+        val updatedAttrMap = AttributeMap(newValidAttrMapping)
+        val transferAttrMapping = attrMapping.map {
+          case (a1, a2) => (a1, updatedAttrMap.getOrElse(a2, a2))
+        }
+        val newOtherAttrMapping = {
+          val existingAttrMappingSet = transferAttrMapping.map(_._2).toSet
+          newValidAttrMapping.filterNot { case (_, a) => existingAttrMappingSet.contains(a) }
+        }
+        planAfterRule -> (transferAttrMapping ++ newOtherAttrMapping).toSeq
+      }
+    }
+    rewrite(this)._1
+  }
+
+  private def updateAttr(attr: Attribute, attrMap: AttributeMap[Attribute]): Attribute = {
+    val exprId = attrMap.getOrElse(attr, attr).exprId
+    attr.withExprId(exprId)
+  }
+
+  /**
+   * The outer plan may have old references and the function below updates the
+   * outer references to refer to the new attributes.
+   */
+  private def updateOuterReferencesInSubquery(
+      plan: PlanType,
+      attrMap: AttributeMap[Attribute]): PlanType = {
+    plan.transformDown { case currentFragment =>
+      currentFragment.transformExpressions {
+        case OuterReference(a: AttributeReference) =>
+          OuterReference(updateAttr(a, attrMap))
+        case pe: PlanExpression[PlanType] =>
+          pe.withNewPlan(updateOuterReferencesInSubquery(pe.plan, attrMap))
+      }
+    }
+  }
+
   lazy val schema: StructType = StructType.fromAttributes(output)
 
   /** Returns the output schema in the tree format. */
@@ -186,16 +292,29 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
   }
 
   def verboseStringWithOperatorId(): String = {
-    val codegenIdStr =
-      getTagValue(QueryPlan.CODEGEN_ID_TAG).map(id => s"[codegen id : $id]").getOrElse("")
-    val operatorId = getTagValue(QueryPlan.OP_ID_TAG).map(id => s"$id").getOrElse("unknown")
-    s"""
-       |($operatorId) $nodeName $codegenIdStr
-     """.stripMargin
+    val argumentString = argString(SQLConf.get.maxToStringFields)
+
+    if (argumentString.nonEmpty) {
+      s"""
+         |$formattedNodeName
+         |Arguments: $argumentString
+         |""".stripMargin
+    } else {
+      s"""
+         |$formattedNodeName
+         |""".stripMargin
+    }
+  }
+
+  protected def formattedNodeName: String = {
+    val opId = getTagValue(QueryPlan.OP_ID_TAG).map(id => s"$id").getOrElse("unknown")
+    val codegenId =
+      getTagValue(QueryPlan.CODEGEN_ID_TAG).map(id => s" [codegen id : $id]").getOrElse("")
+    s"($opId) $nodeName$codegenId"
   }
 
   /**
-   * All the subqueries of current plan.
+   * All the top-level subqueries of the current plan node. Nested subqueries are not included.
    */
   def subqueries: Seq[PlanType] = {
     expressions.flatMap(_.collect {
@@ -204,21 +323,21 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
   }
 
   /**
-   * Returns a sequence containing the result of applying a partial function to all elements in this
-   * plan, also considering all the plans in its (nested) subqueries
-   */
-  def collectInPlanAndSubqueries[B](f: PartialFunction[PlanType, B]): Seq[B] =
-    (this +: subqueriesAll).flatMap(_.collect(f))
-
-  /**
-   * Returns a sequence containing the subqueries in this plan, also including the (nested)
-   * subquries in its children
+   * All the subqueries of the current plan node and all its children. Nested subqueries are also
+   * included.
    */
   def subqueriesAll: Seq[PlanType] = {
     val subqueries = this.flatMap(_.subqueries)
     subqueries ++ subqueries.flatMap(_.subqueriesAll)
   }
 
+  /**
+   * A variant of `collect`. This method not only apply the given function to all elements in this
+   * plan, also considering all the plans in its (nested) subqueries
+   */
+  def collectWithSubqueries[B](f: PartialFunction[PlanType, B]): Seq[B] =
+    (this +: subqueriesAll).flatMap(_.collect(f))
+
   override def innerChildren: Seq[QueryPlan[_]] = subqueries
 
   /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
index 9404a809b453c..9b6fee64be8f2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.plans.logical
 
 import org.apache.spark.sql.catalyst.analysis.CheckAnalysis
-import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, TreeNode}
 import org.apache.spark.util.Utils
@@ -46,7 +46,7 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
    * Recursively marks all nodes in this plan tree as analyzed.
    * This should only be called by [[CheckAnalysis]].
    */
-  private[catalyst] def setAnalyzed(): Unit = {
+  private[sql] def setAnalyzed(): Unit = {
     if (!_analyzed) {
       _analyzed = true
       children.foreach(_.setAnalyzed())
@@ -120,6 +120,19 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
     }
   }
 
+  /**
+   * A variant of `transformUpWithNewOutput`, which skips touching already analyzed plan.
+   */
+  def resolveOperatorsUpWithNewOutput(
+      rule: PartialFunction[LogicalPlan, (LogicalPlan, Seq[(Attribute, Attribute)])])
+  : LogicalPlan = {
+    if (!analyzed) {
+      transformUpWithNewOutput(rule, skipCond = _.analyzed)
+    } else {
+      self
+    }
+  }
+
   /**
    * Recursively transforms the expressions of a tree, skipping nodes that have already
    * been analyzed.
@@ -167,6 +180,11 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
     super.transformAllExpressions(rule)
   }
 
+  override def clone(): LogicalPlan = {
+    val cloned = super.clone()
+    if (analyzed) cloned.setAnalyzed()
+    cloned
+  }
 }
 
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 40db8b6f49dc4..54e5ff7aeb754 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression,
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, RangePartitioning, RoundRobinPartitioning}
 import org.apache.spark.sql.catalyst.util.truncatedString
+import org.apache.spark.sql.connector.catalog.Identifier
 import org.apache.spark.sql.types._
 import org.apache.spark.util.random.RandomSampler
 
@@ -849,18 +850,18 @@ case class Tail(limitExpr: Expression, child: LogicalPlan) extends OrderPreservi
 /**
  * Aliased subquery.
  *
- * @param name the alias identifier for this subquery.
+ * @param identifier the alias identifier for this subquery.
  * @param child the logical plan of this subquery.
  */
 case class SubqueryAlias(
-    name: AliasIdentifier,
+    identifier: AliasIdentifier,
     child: LogicalPlan)
   extends OrderPreservingUnaryNode {
 
-  def alias: String = name.identifier
+  def alias: String = identifier.name
 
   override def output: Seq[Attribute] = {
-    val qualifierList = name.database.map(Seq(_, alias)).getOrElse(Seq(alias))
+    val qualifierList = identifier.qualifier :+ alias
     child.output.map(_.withQualifier(qualifierList))
   }
   override def doCanonicalize(): LogicalPlan = child.canonicalized
@@ -877,7 +878,13 @@ object SubqueryAlias {
       identifier: String,
       database: String,
       child: LogicalPlan): SubqueryAlias = {
-    SubqueryAlias(AliasIdentifier(identifier, Some(database)), child)
+    SubqueryAlias(AliasIdentifier(identifier, Seq(database)), child)
+  }
+
+  def apply(
+      multipartIdentifier: Seq[String],
+      child: LogicalPlan): SubqueryAlias = {
+    SubqueryAlias(AliasIdentifier(multipartIdentifier.last, multipartIdentifier.init), child)
   }
 }
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala
index f26e5662ee856..a325b61fcc5a9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala
@@ -186,7 +186,8 @@ trait HintErrorHandler {
    * @param parameters the hint parameters
    * @param invalidRelations the set of relation names that cannot be associated
    */
-  def hintRelationsNotFound(name: String, parameters: Seq[Any], invalidRelations: Set[String]): Unit
+  def hintRelationsNotFound(
+    name: String, parameters: Seq[Any], invalidRelations: Set[Seq[String]]): Unit
 
   /**
    * Callback for a join hint specified on a relation that is not part of a join.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
index 1e6b67bf78b70..7d7aae20c65c3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
@@ -64,7 +64,7 @@ case class CreateTableStatement(
     partitioning: Seq[Transform],
     bucketSpec: Option[BucketSpec],
     properties: Map[String, String],
-    provider: String,
+    provider: Option[String],
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
@@ -79,10 +79,11 @@ case class CreateTableAsSelectStatement(
     partitioning: Seq[Transform],
     bucketSpec: Option[BucketSpec],
     properties: Map[String, String],
-    provider: String,
+    provider: Option[String],
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
+    writeOptions: Map[String, String],
     ifNotExists: Boolean) extends ParsedStatement {
 
   override def children: Seq[LogicalPlan] = Seq(asSelect)
@@ -114,7 +115,7 @@ case class ReplaceTableStatement(
     partitioning: Seq[Transform],
     bucketSpec: Option[BucketSpec],
     properties: Map[String, String],
-    provider: String,
+    provider: Option[String],
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
@@ -129,10 +130,11 @@ case class ReplaceTableAsSelectStatement(
     partitioning: Seq[Transform],
     bucketSpec: Option[BucketSpec],
     properties: Map[String, String],
-    provider: String,
+    provider: Option[String],
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
+    writeOptions: Map[String, String],
     orCreate: Boolean) extends ParsedStatement {
 
   override def children: Seq[LogicalPlan] = Seq(asSelect)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala
index da36db7ae1f5f..a586988fd3253 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala
@@ -53,7 +53,8 @@ object SizeInBytesOnlyStatsPlanVisitor extends LogicalPlanVisitor[Statistics] {
    */
   override def default(p: LogicalPlan): Statistics = p match {
     case p: LeafNode => p.computeStats()
-    case _: LogicalPlan => Statistics(sizeInBytes = p.children.map(_.stats.sizeInBytes).product)
+    case _: LogicalPlan =>
+      Statistics(sizeInBytes = p.children.map(_.stats.sizeInBytes).filter(_ > 0L).product)
   }
 
   override def visitAggregate(p: Aggregate): Statistics = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index c04e56355a68f..9077f7ad02b69 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.plans.logical
 
 import org.apache.spark.sql.catalyst.analysis.{NamedRelation, UnresolvedException}
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, Unevaluable}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, Expression, Unevaluable}
 import org.apache.spark.sql.catalyst.plans.DescribeTableSchema
 import org.apache.spark.sql.connector.catalog._
 import org.apache.spark.sql.connector.catalog.TableChange.{AddColumn, ColumnChange}
@@ -35,20 +35,20 @@ trait V2WriteCommand extends Command {
 
   override def children: Seq[LogicalPlan] = Seq(query)
 
-  override lazy val resolved: Boolean = outputResolved
+  override lazy val resolved: Boolean = table.resolved && query.resolved && outputResolved
 
   def outputResolved: Boolean = {
+    assert(table.resolved && query.resolved,
+      "`outputResolved` can only be called when `table` and `query` are both resolved.")
     // If the table doesn't require schema match, we don't need to resolve the output columns.
-    table.skipSchemaResolution || {
-      table.resolved && query.resolved && query.output.size == table.output.size &&
-        query.output.zip(table.output).forall {
-          case (inAttr, outAttr) =>
-            // names and types must match, nullability must be compatible
-            inAttr.name == outAttr.name &&
-              DataType.equalsIgnoreCompatibleNullability(outAttr.dataType, inAttr.dataType) &&
-              (outAttr.nullable || !inAttr.nullable)
-        }
-    }
+    table.skipSchemaResolution || (query.output.size == table.output.size &&
+      query.output.zip(table.output).forall {
+        case (inAttr, outAttr) =>
+          // names and types must match, nullability must be compatible
+          inAttr.name == outAttr.name &&
+            DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outAttr.dataType) &&
+            (outAttr.nullable || !inAttr.nullable)
+      })
   }
 }
 
@@ -86,7 +86,10 @@ case class OverwriteByExpression(
     query: LogicalPlan,
     writeOptions: Map[String, String],
     isByName: Boolean) extends V2WriteCommand {
-  override lazy val resolved: Boolean = outputResolved && deleteExpr.resolved
+  override lazy val resolved: Boolean = {
+    table.resolved && query.resolved && outputResolved && deleteExpr.resolved
+  }
+  override def inputSet: AttributeSet = AttributeSet(table.output)
 }
 
 object OverwriteByExpression {
@@ -346,25 +349,25 @@ case class MergeIntoTable(
   override def children: Seq[LogicalPlan] = Seq(targetTable, sourceTable)
 }
 
-sealed abstract class MergeAction(
-    condition: Option[Expression]) extends Expression with Unevaluable {
+sealed abstract class MergeAction extends Expression with Unevaluable {
+  def condition: Option[Expression]
   override def foldable: Boolean = false
   override def nullable: Boolean = false
   override def dataType: DataType = throw new UnresolvedException(this, "nullable")
   override def children: Seq[Expression] = condition.toSeq
 }
 
-case class DeleteAction(condition: Option[Expression]) extends MergeAction(condition)
+case class DeleteAction(condition: Option[Expression]) extends MergeAction
 
 case class UpdateAction(
     condition: Option[Expression],
-    assignments: Seq[Assignment]) extends MergeAction(condition) {
+    assignments: Seq[Assignment]) extends MergeAction {
   override def children: Seq[Expression] = condition.toSeq ++ assignments
 }
 
 case class InsertAction(
     condition: Option[Expression],
-    assignments: Seq[Assignment]) extends MergeAction(condition) {
+    assignments: Seq[Assignment]) extends MergeAction {
   override def children: Seq[Expression] = condition.toSeq ++ assignments
 }
 
@@ -436,6 +439,22 @@ case class ShowTables(
     AttributeReference("tableName", StringType, nullable = false)())
 }
 
+/**
+ * The logical plan of the SHOW VIEWS command that works for v1 and v2 catalogs.
+ *
+ * Notes: v2 catalogs do not support views API yet, the command will fallback to
+ * v1 ShowViewsCommand during ResolveSessionCatalog.
+ */
+case class ShowViews(
+    namespace: LogicalPlan,
+    pattern: Option[String]) extends Command {
+  override def children: Seq[LogicalPlan] = Seq(namespace)
+
+  override val output: Seq[Attribute] = Seq(
+    AttributeReference("namespace", StringType, nullable = false)(),
+    AttributeReference("viewName", StringType, nullable = false)())
+}
+
 /**
  * The logical plan of the USE/USE NAMESPACE command that works for v2 catalogs.
  */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
index 875c46d2a417a..8efc3593d72f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
@@ -37,6 +37,10 @@ case class QueryExecutionMetering() {
     timeEffectiveRunsMap.clear()
   }
 
+  def getMetrics(): QueryExecutionMetrics = {
+    QueryExecutionMetrics(totalTime, totalNumRuns, totalNumEffectiveRuns, totalEffectiveTime)
+  }
+
   def totalTime: Long = {
     timeMap.sum()
   }
@@ -45,6 +49,14 @@ case class QueryExecutionMetering() {
     numRunsMap.sum()
   }
 
+  def totalNumEffectiveRuns: Long = {
+    numEffectiveRunsMap.sum()
+  }
+
+  def totalEffectiveTime: Long = {
+    timeEffectiveRunsMap.sum()
+  }
+
   def incExecutionTimeBy(ruleName: String, delta: Long): Unit = {
     timeMap.addAndGet(ruleName, delta)
   }
@@ -95,3 +107,18 @@ case class QueryExecutionMetering() {
      """.stripMargin
   }
 }
+
+case class QueryExecutionMetrics(
+    time: Long,
+    numRuns: Long,
+    numEffectiveRuns: Long,
+    timeEffective: Long) {
+
+  def -(metrics: QueryExecutionMetrics): QueryExecutionMetrics = {
+    QueryExecutionMetrics(
+      this.time - metrics.time,
+      this.numRuns - metrics.numRuns,
+      this.numEffectiveRuns - metrics.numEffectiveRuns,
+      this.timeEffective - metrics.timeEffective)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
index 287ae0e8e9f67..bff04d317d4d2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
@@ -21,6 +21,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.QueryPlanningTracker
 import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.trees.TreeNode
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND
 import org.apache.spark.sql.catalyst.util.sideBySide
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.util.Utils
@@ -37,6 +38,10 @@ object RuleExecutor {
   def resetMetrics(): Unit = {
     queryExecutionMeter.resetMetrics()
   }
+
+  def getCurrentMetrics(): QueryExecutionMetrics = {
+    queryExecutionMeter.getMetrics()
+  }
 }
 
 abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
@@ -45,7 +50,17 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
    * An execution strategy for rules that indicates the maximum number of executions. If the
    * execution reaches fix point (i.e. converge) before maxIterations, it will stop.
    */
-  abstract class Strategy { def maxIterations: Int }
+  abstract class Strategy {
+
+    /** The maximum number of executions. */
+    def maxIterations: Int
+
+    /** Whether to throw exception when exceeding the maximum number. */
+    def errorOnExceed: Boolean = false
+
+    /** The key of SQLConf setting to tune maxIterations */
+    def maxIterationsSetting: String = null
+  }
 
   /** A strategy that is run once and idempotent. */
   case object Once extends Strategy { val maxIterations = 1 }
@@ -54,7 +69,10 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
    * A strategy that runs until fix point or maxIterations times, whichever comes first.
    * Especially, a FixedPoint(1) batch is supposed to run only once.
    */
-  case class FixedPoint(maxIterations: Int) extends Strategy
+  case class FixedPoint(
+    override val maxIterations: Int,
+    override val errorOnExceed: Boolean = false,
+    override val maxIterationsSetting: String = null) extends Strategy
 
   /** A batch of rules. */
   protected case class Batch(name: String, strategy: Strategy, rules: Rule[TreeType]*)
@@ -108,6 +126,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
     val queryExecutionMetrics = RuleExecutor.queryExecutionMeter
     val planChangeLogger = new PlanChangeLogger()
     val tracker: Option[QueryPlanningTracker] = QueryPlanningTracker.get
+    val beforeMetrics = RuleExecutor.getCurrentMetrics()
 
     // Run the structural integrity checker against the initial input
     if (!isPlanIntegral(plan)) {
@@ -155,8 +174,14 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
         if (iteration > batch.strategy.maxIterations) {
           // Only log if this is a rule that is supposed to run more than once.
           if (iteration != 2) {
-            val message = s"Max iterations (${iteration - 1}) reached for batch ${batch.name}"
-            if (Utils.isTesting) {
+            val endingMsg = if (batch.strategy.maxIterationsSetting == null) {
+              "."
+            } else {
+              s", please set '${batch.strategy.maxIterationsSetting}' to a larger value."
+            }
+            val message = s"Max iterations (${iteration - 1}) reached for batch ${batch.name}" +
+              s"$endingMsg"
+            if (Utils.isTesting || batch.strategy.errorOnExceed) {
               throw new TreeNodeException(curPlan, message, null)
             } else {
               logWarning(message)
@@ -180,6 +205,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
 
       planChangeLogger.logBatch(batch.name, batchStartPlan, curPlan)
     }
+    planChangeLogger.logMetrics(RuleExecutor.getCurrentMetrics() - beforeMetrics)
 
     curPlan
   }
@@ -212,7 +238,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
             s"""
                |=== Result of Batch ${batchName} ===
                |${sideBySide(oldPlan.treeString, newPlan.treeString).mkString("\n")}
-          """.stripMargin
+            """.stripMargin
           } else {
             s"Batch ${batchName} has no effect."
           }
@@ -222,6 +248,21 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
       }
     }
 
+    def logMetrics(metrics: QueryExecutionMetrics): Unit = {
+      val totalTime = metrics.time / NANOS_PER_SECOND.toDouble
+      val totalTimeEffective = metrics.timeEffective / NANOS_PER_SECOND.toDouble
+      val message =
+        s"""
+           |=== Metrics of Executed Rules ===
+           |Total number of runs: ${metrics.numRuns}
+           |Total time: ${totalTime} seconds
+           |Total number of effective runs: ${metrics.numEffectiveRuns}
+           |Total time of effective runs: ${totalTimeEffective} seconds
+        """.stripMargin
+
+      logBasedOnLevel(message)
+    }
+
     private def logBasedOnLevel(f: => String): Unit = {
       logLevel match {
         case "TRACE" => logTrace(f)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index ba1eeb38e247e..4dc3627cd6a50 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -27,7 +27,7 @@ import org.json4s.JsonAST._
 import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
 
-import org.apache.spark.sql.catalyst.IdentifierWithDatabase
+import org.apache.spark.sql.catalyst.{AliasIdentifier, IdentifierWithDatabase}
 import org.apache.spark.sql.catalyst.ScalaReflection._
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, FunctionResource}
 import org.apache.spark.sql.catalyst.errors._
@@ -40,6 +40,7 @@ import org.apache.spark.sql.catalyst.util.truncatedString
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.storage.StorageLevel
+import org.apache.spark.util.Utils
 
 /** Used by [[TreeNode.getNodeNumbered]] when traversing the tree for a given number */
 private class MutableInt(var i: Int)
@@ -91,7 +92,12 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
   private val tags: mutable.Map[TreeNodeTag[_], Any] = mutable.Map.empty
 
   protected def copyTagsFrom(other: BaseType): Unit = {
-    tags ++= other.tags
+    // SPARK-32753: it only makes sense to copy tags to a new node
+    // but it's too expensive to detect other cases likes node removal
+    // so we make a compromise here to copy tags to node with no tags
+    if (tags.isEmpty) {
+      tags ++= other.tags
+    }
   }
 
   def setTagValue[T](tag: TreeNodeTag[T], value: T): Unit = {
@@ -114,7 +120,30 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
 
   lazy val containsChild: Set[TreeNode[_]] = children.toSet
 
-  private lazy val _hashCode: Int = scala.util.hashing.MurmurHash3.productHash(this)
+  // Copied from Scala 2.13.1
+  // github.com/scala/scala/blob/v2.13.1/src/library/scala/util/hashing/MurmurHash3.scala#L56-L73
+  // to prevent the issue https://github.com/scala/bug/issues/10495
+  // TODO(SPARK-30848): Remove this once we drop Scala 2.12.
+  private final def productHash(x: Product, seed: Int, ignorePrefix: Boolean = false): Int = {
+    val arr = x.productArity
+    // Case objects have the hashCode inlined directly into the
+    // synthetic hashCode method, but this method should still give
+    // a correct result if passed a case object.
+    if (arr == 0) {
+      x.productPrefix.hashCode
+    } else {
+      var h = seed
+      if (!ignorePrefix) h = scala.util.hashing.MurmurHash3.mix(h, x.productPrefix.hashCode)
+      var i = 0
+      while (i < arr) {
+        h = scala.util.hashing.MurmurHash3.mix(h, x.productElement(i).##)
+        i += 1
+      }
+      scala.util.hashing.MurmurHash3.finalizeHash(h, arr)
+    }
+  }
+
+  private lazy val _hashCode: Int = productHash(this, scala.util.hashing.MurmurHash3.productSeed)
   override def hashCode(): Int = _hashCode
 
   /**
@@ -488,11 +517,13 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     mapChildren(_.clone(), forceCopy = true)
   }
 
+  private def simpleClassName: String = Utils.getSimpleName(this.getClass)
+
   /**
    * Returns the name of this type of TreeNode.  Defaults to the class name.
    * Note that we remove the "Exec" suffix for physical operators here.
    */
-  def nodeName: String = getClass.getSimpleName.replaceAll("Exec$", "")
+  def nodeName: String = simpleClassName.replaceAll("Exec$", "")
 
   /**
    * The arguments that should be included in the arg string.  Defaults to the `productIterator`.
@@ -711,7 +742,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
   protected def jsonFields: List[JField] = {
     val fieldNames = getConstructorParameterNames(getClass)
     val fieldValues = productIterator.toSeq ++ otherCopyArgs
-    assert(fieldNames.length == fieldValues.length, s"${getClass.getSimpleName} fields: " +
+    assert(fieldNames.length == fieldValues.length, s"$simpleClassName fields: " +
       fieldNames.mkString(", ") + s", values: " + fieldValues.mkString(", "))
 
     fieldNames.zip(fieldValues).map {
@@ -765,7 +796,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
       try {
         val fieldNames = getConstructorParameterNames(p.getClass)
         val fieldValues = p.productIterator.toSeq
-        assert(fieldNames.length == fieldValues.length, s"${getClass.getSimpleName} fields: " +
+        assert(fieldNames.length == fieldValues.length, s"$simpleClassName fields: " +
           fieldNames.mkString(", ") + s", values: " + fieldValues.mkString(", "))
         ("product-class" -> JString(p.getClass.getName)) :: fieldNames.zip(fieldValues).map {
           case (name, value) => name -> parseToJson(value)
@@ -780,6 +811,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     case exprId: ExprId => true
     case field: StructField => true
     case id: IdentifierWithDatabase => true
+    case alias: AliasIdentifier => true
     case join: JoinType => true
     case spec: BucketSpec => true
     case catalog: CatalogTable => true
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
index 98934368205ec..0185b5743e446 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.util
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.array.ByteArrayMethods
 
@@ -29,12 +30,11 @@ import org.apache.spark.unsafe.array.ByteArrayMethods
  */
 class ArrayBasedMapBuilder(keyType: DataType, valueType: DataType) extends Serializable {
   assert(!keyType.existsRecursively(_.isInstanceOf[MapType]), "key of map cannot be/contain map")
-  assert(keyType != NullType, "map key cannot be null type.")
 
   private lazy val keyToIndex = keyType match {
     // Binary type data is `byte[]`, which can't use `==` to check equality.
-    case _: AtomicType | _: CalendarIntervalType if !keyType.isInstanceOf[BinaryType] =>
-      new java.util.HashMap[Any, Int]()
+    case _: AtomicType | _: CalendarIntervalType | _: NullType
+      if !keyType.isInstanceOf[BinaryType] => new java.util.HashMap[Any, Int]()
     case _ =>
       // for complex types, use interpreted ordering to be able to compare unsafe data with safe
       // data, e.g. UnsafeRow vs GenericInternalRow.
@@ -48,6 +48,8 @@ class ArrayBasedMapBuilder(keyType: DataType, valueType: DataType) extends Seria
   private lazy val keyGetter = InternalRow.getAccessor(keyType)
   private lazy val valueGetter = InternalRow.getAccessor(valueType)
 
+  private val mapKeyDedupPolicy = SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY)
+
   def put(key: Any, value: Any): Unit = {
     if (key == null) {
       throw new RuntimeException("Cannot use null as map key.")
@@ -63,8 +65,17 @@ class ArrayBasedMapBuilder(keyType: DataType, valueType: DataType) extends Seria
       keys.append(key)
       values.append(value)
     } else {
-      // Overwrite the previous value, as the policy is last wins.
-      values(index) = value
+      if (mapKeyDedupPolicy == SQLConf.MapKeyDedupPolicy.EXCEPTION.toString) {
+        throw new RuntimeException(s"Duplicate map key $key was found, please check the input " +
+          "data. If you want to remove the duplicated keys, you can set " +
+          s"${SQLConf.MAP_KEY_DEDUP_POLICY.key} to ${SQLConf.MapKeyDedupPolicy.LAST_WIN} so that " +
+          "the key inserted at last takes precedence.")
+      } else if (mapKeyDedupPolicy == SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+        // Overwrite the previous value, as the policy is last wins.
+        values(index) = value
+      } else {
+        throw new IllegalStateException("Unknown map key dedup policy: " + mapKeyDedupPolicy)
+      }
     }
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala
new file mode 100644
index 0000000000000..e75429c58cc7b
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import com.fasterxml.jackson.core.{JsonGenerator, JsonParser}
+import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer, JsonSerializer, SerializerProvider}
+import org.json4s.JsonAST.JValue
+import org.json4s.jackson.{JValueDeserializer, JValueSerializer}
+
+import org.apache.spark.sql.types.DataType
+
+object DataTypeJsonUtils {
+  /**
+   * Jackson serializer for [[DataType]]. Internally this delegates to json4s based serialization.
+   */
+  class DataTypeJsonSerializer extends JsonSerializer[DataType] {
+    private val delegate = new JValueSerializer
+    override def serialize(
+      value: DataType,
+      gen: JsonGenerator,
+      provider: SerializerProvider): Unit = {
+      delegate.serialize(value.jsonValue, gen, provider)
+    }
+  }
+
+  /**
+   * Jackson deserializer for [[DataType]]. Internally this delegates to json4s based
+   * deserialization.
+   */
+  class DataTypeJsonDeserializer extends JsonDeserializer[DataType] {
+    private val delegate = new JValueDeserializer(classOf[Any])
+
+    override def deserialize(
+      jsonParser: JsonParser,
+      deserializationContext: DeserializationContext): DataType = {
+      val json = delegate.deserialize(jsonParser, deserializationContext)
+      DataType.parseDataType(json.asInstanceOf[JValue])
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index 7f982b019c8d1..07fa09d64400f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -17,48 +17,184 @@
 
 package org.apache.spark.sql.catalyst.util
 
+import java.text.SimpleDateFormat
 import java.time.{LocalDate, ZoneId}
-import java.util.Locale
+import java.util.{Date, Locale}
 
-import DateTimeUtils.{convertSpecialDate, localDateToDays}
+import org.apache.commons.lang3.time.FastDateFormat
+
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 
 sealed trait DateFormatter extends Serializable {
   def parse(s: String): Int // returns days since epoch
+
   def format(days: Int): String
+  def format(date: Date): String
+  def format(localDate: LocalDate): String
+
+  def validatePatternString(): Unit
 }
 
 class Iso8601DateFormatter(
     pattern: String,
     zoneId: ZoneId,
-    locale: Locale) extends DateFormatter with DateTimeFormatterHelper {
+    locale: Locale,
+    legacyFormat: LegacyDateFormats.LegacyDateFormat,
+    isParsing: Boolean)
+  extends DateFormatter with DateTimeFormatterHelper {
 
   @transient
-  private lazy val formatter = getOrCreateFormatter(pattern, locale)
+  private lazy val formatter = getOrCreateFormatter(pattern, locale, isParsing)
+
+  @transient
+  private lazy val legacyFormatter = DateFormatter.getLegacyFormatter(
+    pattern, zoneId, locale, legacyFormat)
 
   override def parse(s: String): Int = {
     val specialDate = convertSpecialDate(s.trim, zoneId)
     specialDate.getOrElse {
-      val localDate = LocalDate.parse(s, formatter)
-      localDateToDays(localDate)
+      try {
+        val localDate = toLocalDate(formatter.parse(s))
+        localDateToDays(localDate)
+      } catch checkParsedDiff(s, legacyFormatter.parse)
     }
   }
 
+  override def format(localDate: LocalDate): String = {
+    try {
+      localDate.format(formatter)
+    } catch checkFormattedDiff(toJavaDate(localDateToDays(localDate)),
+      (d: Date) => format(d))
+  }
+
+  override def format(days: Int): String = {
+    format(LocalDate.ofEpochDay(days))
+  }
+
+  override def format(date: Date): String = {
+    legacyFormatter.format(date)
+  }
+
+  override def validatePatternString(): Unit = {
+    try {
+      formatter
+    } catch checkLegacyFormatter(pattern, legacyFormatter.validatePatternString)
+  }
+}
+
+trait LegacyDateFormatter extends DateFormatter {
+  def parseToDate(s: String): Date
+
+  override def parse(s: String): Int = {
+    fromJavaDate(new java.sql.Date(parseToDate(s).getTime))
+  }
+
   override def format(days: Int): String = {
-    LocalDate.ofEpochDay(days).format(formatter)
+    format(DateTimeUtils.toJavaDate(days))
   }
+
+  override def format(localDate: LocalDate): String = {
+    format(localDateToDays(localDate))
+  }
+}
+
+/**
+ * The legacy formatter is based on Apache Commons FastDateFormat. The formatter uses the default
+ * JVM time zone intentionally for compatibility with Spark 2.4 and earlier versions.
+ *
+ * Note: Using of the default JVM time zone makes the formatter compatible with the legacy
+ *       `DateTimeUtils` methods `toJavaDate` and `fromJavaDate` that are based on the default
+ *       JVM time zone too.
+ *
+ * @param pattern `java.text.SimpleDateFormat` compatible pattern.
+ * @param locale The locale overrides the system locale and is used in parsing/formatting.
+ */
+class LegacyFastDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
+  @transient
+  private lazy val fdf = FastDateFormat.getInstance(pattern, locale)
+  override def parseToDate(s: String): Date = fdf.parse(s)
+  override def format(d: Date): String = fdf.format(d)
+
+  override def validatePatternString(): Unit = fdf
+}
+
+// scalastyle:off line.size.limit
+/**
+ * The legacy formatter is based on `java.text.SimpleDateFormat`. The formatter uses the default
+ * JVM time zone intentionally for compatibility with Spark 2.4 and earlier versions.
+ *
+ * Note: Using of the default JVM time zone makes the formatter compatible with the legacy
+ *       `DateTimeUtils` methods `toJavaDate` and `fromJavaDate` that are based on the default
+ *       JVM time zone too.
+ *
+ * @param pattern The pattern describing the date and time format.
+ *                See <a href="https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html">
+ *                Date and Time Patterns</a>
+ * @param locale  The locale whose date format symbols should be used. It overrides the system
+ *                locale in parsing/formatting.
+ */
+// scalastyle:on line.size.limit
+class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
+  @transient
+  private lazy val sdf = new SimpleDateFormat(pattern, locale)
+  override def parseToDate(s: String): Date = sdf.parse(s)
+  override def format(d: Date): String = sdf.format(d)
+
+  override def validatePatternString(): Unit = sdf
 }
 
 object DateFormatter {
-  val defaultPattern: String = "uuuu-MM-dd"
+  import LegacyDateFormats._
+
   val defaultLocale: Locale = Locale.US
 
-  def apply(format: String, zoneId: ZoneId, locale: Locale): DateFormatter = {
-    new Iso8601DateFormatter(format, zoneId, locale)
+  val defaultPattern: String = "yyyy-MM-dd"
+
+  private def getFormatter(
+      format: Option[String],
+      zoneId: ZoneId,
+      locale: Locale = defaultLocale,
+      legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
+      isParsing: Boolean): DateFormatter = {
+    val pattern = format.getOrElse(defaultPattern)
+    if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
+      getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
+    } else {
+      val df = new Iso8601DateFormatter(pattern, zoneId, locale, legacyFormat, isParsing)
+      df.validatePatternString()
+      df
+    }
   }
 
-  def apply(format: String, zoneId: ZoneId): DateFormatter = {
-    apply(format, zoneId, defaultLocale)
+  def getLegacyFormatter(
+      pattern: String,
+      zoneId: ZoneId,
+      locale: Locale,
+      legacyFormat: LegacyDateFormat): DateFormatter = {
+    legacyFormat match {
+      case FAST_DATE_FORMAT =>
+        new LegacyFastDateFormatter(pattern, locale)
+      case SIMPLE_DATE_FORMAT | LENIENT_SIMPLE_DATE_FORMAT =>
+        new LegacySimpleDateFormatter(pattern, locale)
+    }
   }
 
-  def apply(zoneId: ZoneId): DateFormatter = apply(defaultPattern, zoneId)
+  def apply(
+      format: String,
+      zoneId: ZoneId,
+      locale: Locale,
+      legacyFormat: LegacyDateFormat,
+      isParsing: Boolean): DateFormatter = {
+    getFormatter(Some(format), zoneId, locale, legacyFormat, isParsing)
+  }
+
+  def apply(format: String, zoneId: ZoneId, isParsing: Boolean = false): DateFormatter = {
+    getFormatter(Some(format), zoneId, isParsing = isParsing)
+  }
+
+  def apply(zoneId: ZoneId): DateFormatter = {
+    getFormatter(None, zoneId, isParsing = false)
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index a7b6309baf614..f02b2d08c0935 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -21,24 +21,90 @@ import java.time._
 import java.time.chrono.IsoChronology
 import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverStyle}
 import java.time.temporal.{ChronoField, TemporalAccessor, TemporalQueries}
-import java.util.Locale
+import java.util.{Date, Locale}
 
 import com.google.common.cache.CacheBuilder
 
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 
 trait DateTimeFormatterHelper {
+  private def getOrDefault(accessor: TemporalAccessor, field: ChronoField, default: Int): Int = {
+    if (accessor.isSupported(field)) {
+      accessor.get(field)
+    } else {
+      default
+    }
+  }
+
+  private def verifyLocalDate(
+      accessor: TemporalAccessor, field: ChronoField, candidate: LocalDate): Unit = {
+    if (accessor.isSupported(field)) {
+      val actual = accessor.get(field)
+      val expected = candidate.get(field)
+      if (actual != expected) {
+        throw new DateTimeException(s"Conflict found: Field $field $actual differs from" +
+          s" $field $expected derived from $candidate")
+      }
+    }
+  }
+
+  protected def toLocalDate(accessor: TemporalAccessor): LocalDate = {
+    val localDate = accessor.query(TemporalQueries.localDate())
+    // If all the date fields are specified, return the local date directly.
+    if (localDate != null) return localDate
+
+    // Users may want to parse only a few datetime fields from a string and extract these fields
+    // later, and we should provide default values for missing fields.
+    // To be compatible with Spark 2.4, we pick 1970 as the default value of year.
+    val year = getOrDefault(accessor, ChronoField.YEAR, 1970)
+    if (accessor.isSupported(ChronoField.DAY_OF_YEAR)) {
+      val dayOfYear = accessor.get(ChronoField.DAY_OF_YEAR)
+      val date = LocalDate.ofYearDay(year, dayOfYear)
+      verifyLocalDate(accessor, ChronoField.MONTH_OF_YEAR, date)
+      verifyLocalDate(accessor, ChronoField.DAY_OF_MONTH, date)
+      date
+    } else {
+      val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
+      val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
+      LocalDate.of(year, month, day)
+    }
+  }
+
+  private def toLocalTime(accessor: TemporalAccessor): LocalTime = {
+    val localTime = accessor.query(TemporalQueries.localTime())
+    // If all the time fields are specified, return the local time directly.
+    if (localTime != null) return localTime
+
+    val hour = if (accessor.isSupported(ChronoField.HOUR_OF_DAY)) {
+      accessor.get(ChronoField.HOUR_OF_DAY)
+    } else if (accessor.isSupported(ChronoField.HOUR_OF_AMPM)) {
+      // When we reach here, it means am/pm is not specified. Here we assume it's am.
+      // All of CLOCK_HOUR_OF_AMPM(h)/HOUR_OF_DAY(H)/CLOCK_HOUR_OF_DAY(k)/HOUR_OF_AMPM(K) will
+      // be resolved to HOUR_OF_AMPM here, we do not need to handle them separately
+      accessor.get(ChronoField.HOUR_OF_AMPM)
+    } else if (accessor.isSupported(ChronoField.AMPM_OF_DAY) &&
+      accessor.get(ChronoField.AMPM_OF_DAY) == 1) {
+      // When reach here, the `hour` part is missing, and PM is specified.
+      // None of CLOCK_HOUR_OF_AMPM(h)/HOUR_OF_DAY(H)/CLOCK_HOUR_OF_DAY(k)/HOUR_OF_AMPM(K) is
+      // specified
+      12
+    } else {
+      0
+    }
+    val minute = getOrDefault(accessor, ChronoField.MINUTE_OF_HOUR, 0)
+    val second = getOrDefault(accessor, ChronoField.SECOND_OF_MINUTE, 0)
+    val nanoSecond = getOrDefault(accessor, ChronoField.NANO_OF_SECOND, 0)
+    LocalTime.of(hour, minute, second, nanoSecond)
+  }
+
   // Converts the parsed temporal object to ZonedDateTime. It sets time components to zeros
   // if they does not exist in the parsed object.
-  protected def toZonedDateTime(
-      temporalAccessor: TemporalAccessor,
-      zoneId: ZoneId): ZonedDateTime = {
-    // Parsed input might not have time related part. In that case, time component is set to zeros.
-    val parsedLocalTime = temporalAccessor.query(TemporalQueries.localTime)
-    val localTime = if (parsedLocalTime == null) LocalTime.MIDNIGHT else parsedLocalTime
-    // Parsed input must have date component. At least, year must present in temporalAccessor.
-    val localDate = temporalAccessor.query(TemporalQueries.localDate)
-
+  protected def toZonedDateTime(accessor: TemporalAccessor, zoneId: ZoneId): ZonedDateTime = {
+    val localDate = toLocalDate(accessor)
+    val localTime = toLocalTime(accessor)
     ZonedDateTime.of(localDate, localTime, zoneId)
   }
 
@@ -48,21 +114,95 @@ trait DateTimeFormatterHelper {
   // In this way, synchronised is intentionally omitted in this method to make parallel calls
   // less synchronised.
   // The Cache.get method is not used here to avoid creation of additional instances of Callable.
-  protected def getOrCreateFormatter(pattern: String, locale: Locale): DateTimeFormatter = {
-    val key = (pattern, locale)
+  protected def getOrCreateFormatter(
+      pattern: String,
+      locale: Locale,
+      isParsing: Boolean): DateTimeFormatter = {
+    val newPattern = convertIncompatiblePattern(pattern, isParsing)
+    val useVarLen = isParsing && newPattern.contains('S')
+    val key = (newPattern, locale, useVarLen)
     var formatter = cache.getIfPresent(key)
     if (formatter == null) {
-      formatter = buildFormatter(pattern, locale)
+      formatter = buildFormatter(newPattern, locale, useVarLen)
       cache.put(key, formatter)
     }
     formatter
   }
+
+  private def needConvertToSparkUpgradeException(e: Throwable): Boolean = e match {
+    case _: DateTimeException if SQLConf.get.legacyTimeParserPolicy == EXCEPTION => true
+    case _ => false
+  }
+  // When legacy time parser policy set to EXCEPTION, check whether we will get different results
+  // between legacy parser and new parser. If new parser fails but legacy parser works, throw a
+  // SparkUpgradeException. On the contrary, if the legacy policy set to CORRECTED,
+  // DateTimeParseException will address by the caller side.
+  protected def checkParsedDiff[T](
+      s: String, legacyParseFunc: String => T): PartialFunction[Throwable, T] = {
+    case e if needConvertToSparkUpgradeException(e) =>
+      try {
+        legacyParseFunc(s)
+      } catch {
+        case _: Throwable => throw e
+      }
+      throw new SparkUpgradeException("3.0", s"Fail to parse '$s' in the new parser. You can " +
+        s"set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY to restore the behavior " +
+        s"before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.", e)
+  }
+
+  // When legacy time parser policy set to EXCEPTION, check whether we will get different results
+  // between legacy formatter and new formatter. If new formatter fails but legacy formatter works,
+  // throw a SparkUpgradeException. On the contrary, if the legacy policy set to CORRECTED,
+  // DateTimeParseException will address by the caller side.
+  protected def checkFormattedDiff[T <: Date](
+      d: T,
+      legacyFormatFunc: T => String): PartialFunction[Throwable, String] = {
+    case e if needConvertToSparkUpgradeException(e) =>
+      val resultCandidate = try {
+        legacyFormatFunc(d)
+      } catch {
+        case _: Throwable => throw e
+      }
+      throw new SparkUpgradeException("3.0", s"Fail to format it to '$resultCandidate' in the new" +
+        s" formatter. You can set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY to restore" +
+        " the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid" +
+        " datetime string.", e)
+  }
+
+  /**
+   * When the new DateTimeFormatter failed to initialize because of invalid datetime pattern, it
+   * will throw IllegalArgumentException. If the pattern can be recognized by the legacy formatter
+   * it will raise SparkUpgradeException to tell users to restore the previous behavior via LEGACY
+   * policy or follow our guide to correct their pattern. Otherwise, the original
+   * IllegalArgumentException will be thrown.
+   *
+   * @param pattern the date time pattern
+   * @param tryLegacyFormatter a func to capture exception, identically which forces a legacy
+   *                           datetime formatter to be initialized
+   */
+  protected def checkLegacyFormatter(
+      pattern: String,
+      tryLegacyFormatter: => Unit): PartialFunction[Throwable, DateTimeFormatter] = {
+    case e: IllegalArgumentException =>
+      try {
+        tryLegacyFormatter
+      } catch {
+        case _: Throwable => throw e
+      }
+      throw new SparkUpgradeException("3.0", s"Fail to recognize '$pattern' pattern in the" +
+        s" DateTimeFormatter. 1) You can set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY" +
+        s" to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern" +
+        s" with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html",
+        e)
+  }
 }
 
 private object DateTimeFormatterHelper {
   val cache = CacheBuilder.newBuilder()
     .maximumSize(128)
-    .build[(String, Locale), DateTimeFormatter]()
+    .build[(String, Locale, Boolean), DateTimeFormatter]()
+
+  final val extractor = "^([^S]*)(S*)(.*)$".r
 
   def createBuilder(): DateTimeFormatterBuilder = {
     new DateTimeFormatterBuilder().parseCaseInsensitive()
@@ -70,18 +210,49 @@ private object DateTimeFormatterHelper {
 
   def toFormatter(builder: DateTimeFormatterBuilder, locale: Locale): DateTimeFormatter = {
     builder
-      .parseDefaulting(ChronoField.ERA, 1)
-      .parseDefaulting(ChronoField.MONTH_OF_YEAR, 1)
-      .parseDefaulting(ChronoField.DAY_OF_MONTH, 1)
-      .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
-      .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
       .toFormatter(locale)
       .withChronology(IsoChronology.INSTANCE)
       .withResolverStyle(ResolverStyle.STRICT)
   }
 
-  def buildFormatter(pattern: String, locale: Locale): DateTimeFormatter = {
-    val builder = createBuilder().appendPattern(pattern)
+  /**
+   * Building a formatter for parsing seconds fraction with variable length
+   */
+  def createBuilderWithVarLengthSecondFraction(
+      pattern: String): DateTimeFormatterBuilder = {
+    val builder = createBuilder()
+    pattern.split("'").zipWithIndex.foreach {
+      // Split string starting with the regex itself which is `'` here will produce an extra empty
+      // string at res(0). So when the first element here is empty string we do not need append `'`
+      // literal to the DateTimeFormatterBuilder.
+      case ("", idx) if idx != 0 => builder.appendLiteral("'")
+      case (pattenPart, idx) if idx % 2 == 0 =>
+        var rest = pattenPart
+        while (rest.nonEmpty) {
+          rest match {
+            case extractor(prefix, secondFraction, suffix) =>
+              builder.appendPattern(prefix)
+              if (secondFraction.nonEmpty) {
+                builder.appendFraction(ChronoField.NANO_OF_SECOND, 1, secondFraction.length, false)
+              }
+              rest = suffix
+            case _ => throw new IllegalArgumentException(s"Unrecognized datetime pattern: $pattern")
+          }
+        }
+      case (patternPart, _) => builder.appendLiteral(patternPart)
+    }
+    builder
+  }
+
+  def buildFormatter(
+      pattern: String,
+      locale: Locale,
+      varLenEnabled: Boolean): DateTimeFormatter = {
+    val builder = if (varLenEnabled) {
+      createBuilderWithVarLengthSecondFraction(pattern)
+    } else {
+      createBuilder().appendPattern(pattern)
+    }
     toFormatter(builder, locale)
   }
 
@@ -95,4 +266,86 @@ private object DateTimeFormatterHelper {
       .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
     toFormatter(builder, TimestampFormatter.defaultLocale)
   }
+
+  private final val bugInStandAloneForm = {
+    // Java 8 has a bug for stand-alone form. See https://bugs.openjdk.java.net/browse/JDK-8114833
+    // Note: we only check the US locale so that it's a static check. It can produce false-negative
+    // as some locales are not affected by the bug. Since `L`/`q` is rarely used, we choose to not
+    // complicate the check here.
+    // TODO: remove it when we drop Java 8 support.
+    val formatter = DateTimeFormatter.ofPattern("LLL qqq", Locale.US)
+    formatter.format(LocalDate.of(2000, 1, 1)) == "1 1"
+  }
+  // SPARK-31892: The week-based date fields are rarely used and really confusing for parsing values
+  // to datetime, especially when they are mixed with other non-week-based ones;
+  // SPARK-31879: It's also difficult for us to restore the behavior of week-based date fields
+  // formatting, in DateTimeFormatter the first day of week for week-based date fields become
+  // localized, for the default Locale.US, it uses Sunday as the first day of week, while in Spark
+  // 2.4, the SimpleDateFormat uses Monday as the first day of week.
+  final val weekBasedLetters = Set('Y', 'W', 'w', 'u', 'e', 'c')
+  final val unsupportedLetters = Set('A', 'n', 'N', 'p')
+  // The quarter fields will also be parsed strangely, e.g. when the pattern contains `yMd` and can
+  // be directly resolved then the `q` do check for whether the month is valid, but if the date
+  // fields is incomplete, e.g. `yM`, the checking will be bypassed.
+  final val unsupportedLettersForParsing = Set('E', 'F', 'q', 'Q')
+  final val unsupportedPatternLengths = {
+    // SPARK-31771: Disable Narrow-form TextStyle to avoid silent data change, as it is Full-form in
+    // 2.4
+    Seq("G", "M", "L", "E", "Q", "q").map(_ * 5) ++
+      // SPARK-31867: Disable year pattern longer than 10 which will cause Java time library throw
+      // unchecked `ArrayIndexOutOfBoundsException` by the `NumberPrinterParser` for formatting. It
+      // makes the call side difficult to handle exceptions and easily leads to silent data change
+      // because of the exceptions being suppressed.
+      // SPARK-32424: The max year that we can actually handle is 6 digits, otherwise, it will
+      // overflow
+      Seq("y").map(_ * 7)
+  }.toSet
+
+  /**
+   * In Spark 3.0, we switch to the Proleptic Gregorian calendar and use DateTimeFormatter for
+   * parsing/formatting datetime values. The pattern string is incompatible with the one defined
+   * by SimpleDateFormat in Spark 2.4 and earlier. This function converts all incompatible pattern
+   * for the new parser in Spark 3.0. See more details in SPARK-31030.
+   * @param pattern The input pattern.
+   * @return The pattern for new parser
+   */
+  def convertIncompatiblePattern(pattern: String, isParsing: Boolean): String = {
+    val eraDesignatorContained = pattern.split("'").zipWithIndex.exists {
+      case (patternPart, index) =>
+        // Text can be quoted using single quotes, we only check the non-quote parts.
+        index % 2 == 0 && patternPart.contains("G")
+    }
+    (pattern + " ").split("'").zipWithIndex.map {
+      case (patternPart, index) =>
+        if (index % 2 == 0) {
+          for (c <- patternPart if weekBasedLetters.contains(c)) {
+            throw new IllegalArgumentException(s"All week-based patterns are unsupported since" +
+              s" Spark 3.0, detected: $c, Please use the SQL function EXTRACT instead")
+          }
+          for (c <- patternPart if unsupportedLetters.contains(c) ||
+            (isParsing && unsupportedLettersForParsing.contains(c))) {
+            throw new IllegalArgumentException(s"Illegal pattern character: $c")
+          }
+          for (style <- unsupportedPatternLengths if patternPart.contains(style)) {
+            throw new IllegalArgumentException(s"Too many pattern letters: ${style.head}")
+          }
+          if (bugInStandAloneForm && (patternPart.contains("LLL") || patternPart.contains("qqq"))) {
+            throw new IllegalArgumentException("Java 8 has a bug to support stand-alone " +
+              "form (3 or more 'L' or 'q' in the pattern string). Please use 'M' or 'Q' instead, " +
+              "or upgrade your Java version. For more details, please read " +
+              "https://bugs.openjdk.java.net/browse/JDK-8114833")
+          }
+          // In DateTimeFormatter, 'u' supports negative years. We substitute 'y' to 'u' here for
+          // keeping the support in Spark 3.0. If parse failed in Spark 3.0, fall back to 'y'.
+          // We only do this substitution when there is no era designator found in the pattern.
+          if (!eraDesignatorContained) {
+            patternPart.replace("y", "u")
+          } else {
+            patternPart
+          }
+        } else {
+          patternPart
+        }
+    }.mkString("'").stripSuffix(" ")
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 6800abb2ae109..344a6ae428881 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -26,7 +26,10 @@ import java.util.concurrent.TimeUnit._
 
 import scala.util.control.NonFatal
 
+import sun.util.calendar.ZoneInfo
+
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.RebaseDateTime._
 import org.apache.spark.sql.types.Decimal
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
@@ -47,12 +50,8 @@ object DateTimeUtils {
   // it's 2440587.5, rounding up to compatible with Hive
   final val JULIAN_DAY_OF_EPOCH = 2440588
 
-  // number of days between 1.1.1970 and 1.1.2001
-  final val to2001 = -11323
+  final val julianCommonEraStart = Timestamp.valueOf("0001-01-01 00:00:00")
 
-  // this is year -17999, calculation: 50 * daysIn400Year
-  final val YearZero = -17999
-  final val toYearZero = to2001 + 7304850
   final val TimeZoneGMT = TimeZone.getTimeZone("GMT")
   final val TimeZoneUTC = TimeZone.getTimeZone("UTC")
 
@@ -67,24 +66,22 @@ object DateTimeUtils {
 
   // we should use the exact day as Int, for example, (year, month, day) -> day
   def millisToDays(millisUtc: Long): SQLDate = {
-    millisToDays(millisUtc, defaultTimeZone())
+    millisToDays(millisUtc, defaultTimeZone().toZoneId)
   }
 
-  def millisToDays(millisUtc: Long, timeZone: TimeZone): SQLDate = {
-    // SPARK-6785: use Math.floorDiv so negative number of days (dates before 1970)
-    // will correctly work as input for function toJavaDate(Int)
-    val millisLocal = millisUtc + timeZone.getOffset(millisUtc)
-    Math.floorDiv(millisLocal, MILLIS_PER_DAY).toInt
+  def millisToDays(millisUtc: Long, zoneId: ZoneId): SQLDate = {
+    val instant = microsToInstant(fromMillis(millisUtc))
+    localDateToDays(LocalDateTime.ofInstant(instant, zoneId).toLocalDate)
   }
 
   // reverse of millisToDays
   def daysToMillis(days: SQLDate): Long = {
-    daysToMillis(days, defaultTimeZone())
+    daysToMillis(days, defaultTimeZone().toZoneId)
   }
 
-  def daysToMillis(days: SQLDate, timeZone: TimeZone): Long = {
-    val millisLocal = days.toLong * MILLIS_PER_DAY
-    millisLocal - getOffsetFromLocalMillis(millisLocal, timeZone)
+  def daysToMillis(days: SQLDate, zoneId: ZoneId): Long = {
+    val instant = daysToLocalDate(days).atStartOfDay(zoneId).toInstant
+    toMillis(instantToMicros(instant))
   }
 
   // Converts Timestamp to string according to Hive TimestampWritable convention.
@@ -93,31 +90,99 @@ object DateTimeUtils {
   }
 
   /**
-   * Returns the number of days since epoch from java.sql.Date.
+   * Converts a local date at the default JVM time zone to the number of days since 1970-01-01
+   * in the hybrid calendar (Julian + Gregorian) by discarding the time part. The resulted days are
+   * rebased from the hybrid to Proleptic Gregorian calendar. The days rebasing is performed via
+   * UTC time zone for simplicity because the difference between two calendars is the same in
+   * any given time zone and UTC time zone.
+   *
+   * Note: The date is shifted by the offset of the default JVM time zone for backward compatibility
+   *       with Spark 2.4 and earlier versions. The goal of the shift is to get a local date derived
+   *       from the number of days that has the same date fields (year, month, day) as the original
+   *       `date` at the default JVM time zone.
+   *
+   * @param date It represents a specific instant in time based on the hybrid calendar which
+   *             combines Julian and Gregorian calendars.
+   * @return The number of days since the epoch in Proleptic Gregorian calendar.
    */
   def fromJavaDate(date: Date): SQLDate = {
-    millisToDays(date.getTime)
+    val millisUtc = date.getTime
+    val millisLocal = millisUtc + TimeZone.getDefault.getOffset(millisUtc)
+    val julianDays = Math.toIntExact(Math.floorDiv(millisLocal, MILLIS_PER_DAY))
+    rebaseJulianToGregorianDays(julianDays)
   }
 
   /**
-   * Returns a java.sql.Date from number of days since epoch.
+   * Converts days since the epoch 1970-01-01 in Proleptic Gregorian calendar to a local date
+   * at the default JVM time zone in the hybrid calendar (Julian + Gregorian). It rebases the given
+   * days from Proleptic Gregorian to the hybrid calendar at UTC time zone for simplicity because
+   * the difference between two calendars doesn't depend on any time zone. The result is shifted
+   * by the time zone offset in wall clock to have the same date fields (year, month, day)
+   * at the default JVM time zone as the input `daysSinceEpoch` in Proleptic Gregorian calendar.
+   *
+   * Note: The date is shifted by the offset of the default JVM time zone for backward compatibility
+   *       with Spark 2.4 and earlier versions.
+   *
+   * @param daysSinceEpoch The number of days since 1970-01-01 in Proleptic Gregorian calendar.
+   * @return A local date in the hybrid calendar as `java.sql.Date` from number of days since epoch.
    */
   def toJavaDate(daysSinceEpoch: SQLDate): Date = {
-    new Date(daysToMillis(daysSinceEpoch))
+    val rebasedDays = rebaseGregorianToJulianDays(daysSinceEpoch)
+    val localMillis = Math.multiplyExact(rebasedDays, MILLIS_PER_DAY)
+    val timeZoneOffset = TimeZone.getDefault match {
+      case zoneInfo: ZoneInfo => zoneInfo.getOffsetsByWall(localMillis, null)
+      case timeZone: TimeZone => timeZone.getOffset(localMillis - timeZone.getRawOffset)
+    }
+    new Date(localMillis - timeZoneOffset)
   }
 
   /**
-   * Returns a java.sql.Timestamp from number of micros since epoch.
+   * Converts microseconds since the epoch to an instance of `java.sql.Timestamp`
+   * via creating a local timestamp at the system time zone in Proleptic Gregorian
+   * calendar, extracting date and time fields like `year` and `hours`, and forming
+   * new timestamp in the hybrid calendar from the extracted fields.
+   *
+   * The conversion is based on the JVM system time zone because the `java.sql.Timestamp`
+   * uses the time zone internally.
+   *
+   * The method performs the conversion via local timestamp fields to have the same date-time
+   * representation as `year`, `month`, `day`, ..., `seconds` in the original calendar
+   * and in the target calendar.
+   *
+   * @param us The number of microseconds since 1970-01-01T00:00:00.000000Z.
+   * @return A `java.sql.Timestamp` from number of micros since epoch.
    */
   def toJavaTimestamp(us: SQLTimestamp): Timestamp = {
-    Timestamp.from(microsToInstant(us))
+    val rebasedMicros = rebaseGregorianToJulianMicros(us)
+    val seconds = Math.floorDiv(rebasedMicros, MICROS_PER_SECOND)
+    val ts = new Timestamp(seconds * MILLIS_PER_SECOND)
+    val nanos = (rebasedMicros - seconds * MICROS_PER_SECOND) * NANOS_PER_MICROS
+    ts.setNanos(nanos.toInt)
+    ts
   }
 
   /**
-   * Returns the number of micros since epoch from java.sql.Timestamp.
+   * Converts an instance of `java.sql.Timestamp` to the number of microseconds since
+   * 1970-01-01T00:00:00.000000Z. It extracts date-time fields from the input, builds
+   * a local timestamp in Proleptic Gregorian calendar from the fields, and binds
+   * the timestamp to the system time zone. The resulted instant is converted to
+   * microseconds since the epoch.
+   *
+   * The conversion is performed via the system time zone because it is used internally
+   * in `java.sql.Timestamp` while extracting date-time fields.
+   *
+   * The goal of the function is to have the same local date-time in the original calendar
+   * - the hybrid calendar (Julian + Gregorian) and in the target calendar which is
+   * Proleptic Gregorian calendar, see SPARK-26651.
+   *
+   * @param t It represents a specific instant in time based on
+   *          the hybrid calendar which combines Julian and
+   *          Gregorian calendars.
+   * @return The number of micros since epoch from `java.sql.Timestamp`.
    */
   def fromJavaTimestamp(t: Timestamp): SQLTimestamp = {
-    instantToMicros(t.toInstant)
+    val micros = fromMillis(t.getTime) + (t.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS
+    rebaseJulianToGregorianMicros(micros)
   }
 
   /**
@@ -127,7 +192,9 @@ object DateTimeUtils {
   def fromJulianDay(day: Int, nanoseconds: Long): SQLTimestamp = {
     // use Long to avoid rounding errors
     val seconds = (day - JULIAN_DAY_OF_EPOCH).toLong * SECONDS_PER_DAY
-    SECONDS.toMicros(seconds) + NANOSECONDS.toMicros(nanoseconds)
+    val micros = SECONDS.toMicros(seconds) + NANOSECONDS.toMicros(nanoseconds)
+    val rebased = rebaseJulianToGregorianMicros(micros)
+    rebased
   }
 
   /**
@@ -136,7 +203,7 @@ object DateTimeUtils {
    * Note: support timestamp since 4717 BC (without negative nanoseconds, compatible with Hive).
    */
   def toJulianDay(us: SQLTimestamp): (Int, Long) = {
-    val julian_us = us + JULIAN_DAY_OF_EPOCH * MICROS_PER_DAY
+    val julian_us = rebaseGregorianToJulianMicros(us) + JULIAN_DAY_OF_EPOCH * MICROS_PER_DAY
     val day = julian_us / MICROS_PER_DAY
     val micros = julian_us % MICROS_PER_DAY
     (day.toInt, MICROSECONDS.toNanos(micros))
@@ -157,7 +224,7 @@ object DateTimeUtils {
    * Converts milliseconds since epoch to SQLTimestamp.
    */
   def fromMillis(millis: Long): SQLTimestamp = {
-    MILLISECONDS.toMicros(millis)
+    Math.multiplyExact(millis, MICROS_PER_MILLIS)
   }
 
   def microsToEpochDays(epochMicros: SQLTimestamp, zoneId: ZoneId): SQLDate = {
@@ -171,6 +238,21 @@ object DateTimeUtils {
     instantToMicros(localDateTime.atZone(zoneId).toInstant)
   }
 
+  // A method called by JSON/CSV parser to clean up the legacy timestamp string by removing the
+  // "GMT" string.
+  def cleanLegacyTimestampStr(s: String): String = {
+    val indexOfGMT = s.indexOf("GMT")
+    if (indexOfGMT != -1) {
+      // ISO8601 with a weird time zone specifier (2000-01-01T00:00GMT+01:00)
+      val s0 = s.substring(0, indexOfGMT)
+      val s1 = s.substring(indexOfGMT + 3)
+      // Mapped to 2000-01-01T00:00+01:00
+      s0 + s1
+    } else {
+      s
+    }
+  }
+
   /**
    * Trim and parse a given UTF8 date string to the corresponding a corresponding [[Long]] value.
    * The return type is [[Option]] in order to distinguish between 0L and null. The following
@@ -180,28 +262,28 @@ object DateTimeUtils {
    * `yyyy-[m]m`
    * `yyyy-[m]m-[d]d`
    * `yyyy-[m]m-[d]d `
-   * `yyyy-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]`
-   * `yyyy-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]Z`
-   * `yyyy-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]-[h]h:[m]m`
-   * `yyyy-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]+[h]h:[m]m`
-   * `yyyy-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]`
-   * `yyyy-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]Z`
-   * `yyyy-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]-[h]h:[m]m`
-   * `yyyy-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]+[h]h:[m]m`
-   * `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]`
-   * `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]Z`
-   * `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]-[h]h:[m]m`
-   * `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]+[h]h:[m]m`
-   * `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]`
-   * `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]Z`
-   * `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]-[h]h:[m]m`
-   * `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]+[h]h:[m]m`
+   * `yyyy-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
+   * `yyyy-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
+   * `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
+   * `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
+   *
+   * where `zone_id` should have one of the forms:
+   *   - Z - Zulu time zone UTC+0
+   *   - +|-[h]h:[m]m
+   *   - A short id, see https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html#SHORT_IDS
+   *   - An id with one of the prefixes UTC+, UTC-, GMT+, GMT-, UT+ or UT-,
+   *     and a suffix in the formats:
+   *     - +|-h[h]
+   *     - +|-hh[:]mm
+   *     - +|-hh:mm:ss
+   *     - +|-hhmmss
+   *  - Region-based zone IDs in the form `area/city`, such as `Europe/Paris`
    */
   def stringToTimestamp(s: UTF8String, timeZoneId: ZoneId): Option[SQLTimestamp] = {
     if (s == null) {
       return None
     }
-    var tz: Option[Byte] = None
+    var tz: Option[String] = None
     val segments: Array[Int] = Array[Int](1, 1, 1, 0, 0, 0, 0, 0, 0)
     var i = 0
     var currentSegmentValue = 0
@@ -252,22 +334,21 @@ object DateTimeUtils {
             return None
           }
         } else if (i == 5 || i == 6) {
-          if (b == 'Z') {
+          if (b == '-' || b == '+') {
             segments(i) = currentSegmentValue
             currentSegmentValue = 0
             i += 1
-            tz = Some(43)
-          } else if (b == '-' || b == '+') {
+            tz = Some(new String(bytes, j, 1))
+          } else if (b == '.' && i == 5) {
             segments(i) = currentSegmentValue
             currentSegmentValue = 0
             i += 1
-            tz = Some(b)
-          } else if (b == '.' && i == 5) {
+          } else {
             segments(i) = currentSegmentValue
             currentSegmentValue = 0
             i += 1
-          } else {
-            return None
+            tz = Some(new String(bytes, j, bytes.length - j))
+            j = bytes.length - 1
           }
           if (i == 6  && b != '.') {
             i += 1
@@ -307,11 +388,11 @@ object DateTimeUtils {
       digitsMilli -= 1
     }
     try {
-      val zoneId = if (tz.isEmpty) {
-        timeZoneId
-      } else {
-        val sign = if (tz.get.toChar == '-') -1 else 1
-        ZoneOffset.ofHoursMinutes(sign * segments(7), sign * segments(8))
+      val zoneId = tz match {
+        case None => timeZoneId
+        case Some("+") => ZoneOffset.ofHoursMinutes(segments(7), segments(8))
+        case Some("-") => ZoneOffset.ofHoursMinutes(-segments(7), -segments(8))
+        case Some(zoneName: String) => getZoneId(zoneName.trim)
       }
       val nanoseconds = MICROSECONDS.toNanos(segments(6))
       val localTime = LocalTime.of(segments(3), segments(4), segments(5), nanoseconds.toInt)
@@ -413,63 +494,47 @@ object DateTimeUtils {
     }
   }
 
-  /**
-   * Returns the microseconds since year zero (-17999) from microseconds since epoch.
-   */
-  private def absoluteMicroSecond(microsec: SQLTimestamp): SQLTimestamp = {
-    microsec + toYearZero * MICROS_PER_DAY
-  }
-
-  private def localTimestamp(microsec: SQLTimestamp, timeZone: TimeZone): SQLTimestamp = {
-    val zoneOffsetUs = MILLISECONDS.toMicros(timeZone.getOffset(MICROSECONDS.toMillis(microsec)))
-    absoluteMicroSecond(microsec) + zoneOffsetUs
+  private def localTimestamp(microsec: SQLTimestamp, zoneId: ZoneId): LocalDateTime = {
+    microsToInstant(microsec).atZone(zoneId).toLocalDateTime
   }
 
   /**
    * Returns the hour value of a given timestamp value. The timestamp is expressed in microseconds.
    */
-  def getHours(microsec: SQLTimestamp, timeZone: TimeZone): Int = {
-    (MICROSECONDS.toHours(localTimestamp(microsec, timeZone)) % 24).toInt
+  def getHours(microsec: SQLTimestamp, zoneId: ZoneId): Int = {
+    localTimestamp(microsec, zoneId).getHour
   }
 
   /**
    * Returns the minute value of a given timestamp value. The timestamp is expressed in
    * microseconds.
    */
-  def getMinutes(microsec: SQLTimestamp, timeZone: TimeZone): Int = {
-    (MICROSECONDS.toMinutes(localTimestamp(microsec, timeZone)) % 60).toInt
+  def getMinutes(microsec: SQLTimestamp, zoneId: ZoneId): Int = {
+    localTimestamp(microsec, zoneId).getMinute
   }
 
   /**
    * Returns the second value of a given timestamp value. The timestamp is expressed in
    * microseconds.
    */
-  def getSeconds(microsec: SQLTimestamp, timeZone: TimeZone): Int = {
-    (MICROSECONDS.toSeconds(localTimestamp(microsec, timeZone)) % 60).toInt
+  def getSeconds(microsec: SQLTimestamp, zoneId: ZoneId): Int = {
+    localTimestamp(microsec, zoneId).getSecond
   }
 
   /**
    * Returns the seconds part and its fractional part with microseconds.
    */
-  def getSecondsWithFraction(microsec: SQLTimestamp, timeZone: TimeZone): Decimal = {
-    val secFrac = localTimestamp(microsec, timeZone) % (MILLIS_PER_MINUTE * MICROS_PER_MILLIS)
-    Decimal(secFrac, 8, 6)
-  }
-
-  /**
-   * Returns seconds, including fractional parts, multiplied by 1000. The timestamp
-   * is expressed in microseconds since the epoch.
-   */
-  def getMilliseconds(timestamp: SQLTimestamp, timeZone: TimeZone): Decimal = {
-    Decimal(getMicroseconds(timestamp, timeZone), 8, 3)
+  def getSecondsWithFraction(microsec: SQLTimestamp, zoneId: ZoneId): Decimal = {
+    Decimal(getMicroseconds(microsec, zoneId), 8, 6)
   }
 
   /**
    * Returns seconds, including fractional parts, multiplied by 1000000. The timestamp
    * is expressed in microseconds since the epoch.
    */
-  def getMicroseconds(timestamp: SQLTimestamp, timeZone: TimeZone): Int = {
-    Math.floorMod(localTimestamp(timestamp, timeZone), MICROS_PER_SECOND * 60).toInt
+  def getMicroseconds(timestamp: SQLTimestamp, zoneId: ZoneId): Int = {
+    val lt = localTimestamp(timestamp, zoneId)
+    (lt.getLong(ChronoField.MICRO_OF_SECOND) + lt.getSecond * MICROS_PER_SECOND).toInt
   }
 
   /**
@@ -480,24 +545,6 @@ object DateTimeUtils {
     LocalDate.ofEpochDay(date).getDayOfYear
   }
 
-  private def extractFromYear(date: SQLDate, divider: Int): Int = {
-    val localDate = daysToLocalDate(date)
-    val yearOfEra = localDate.get(ChronoField.YEAR_OF_ERA)
-    var result = yearOfEra / divider
-    if ((yearOfEra % divider) != 0 || yearOfEra <= 1) result += 1
-    if (localDate.get(ChronoField.ERA) == 0) result = -result
-    result
-  }
-
-  /** Returns the millennium for the given date. The date is expressed in days since 1.1.1970. */
-  def getMillennium(date: SQLDate): Int = extractFromYear(date, 1000)
-
-  /** Returns the century for the given date. The date is expressed in days since 1.1.1970. */
-  def getCentury(date: SQLDate): Int = extractFromYear(date, 100)
-
-  /** Returns the decade for the given date. The date is expressed in days since 1.1.1970. */
-  def getDecade(date: SQLDate): Int = Math.floorDiv(getYear(date), 10)
-
   /**
    * Returns the year value for the given date. The date is expressed in days
    * since 1.1.1970.
@@ -510,7 +557,7 @@ object DateTimeUtils {
    * Returns the year which conforms to ISO 8601. Each ISO 8601 week-numbering
    * year begins with the Monday of the week containing the 4th of January.
    */
-  def getIsoYear(date: SQLDate): Int = {
+  def getWeekBasedYear(date: SQLDate): Int = {
     daysToLocalDate(date).get(IsoFields.WEEK_BASED_YEAR)
   }
 
@@ -573,6 +620,22 @@ object DateTimeUtils {
     instantToMicros(resultTimestamp.toInstant)
   }
 
+  /**
+   * Add the date and the interval's months and days.
+   * Returns a date value, expressed in days since 1.1.1970.
+   *
+   * @throws DateTimeException if the result exceeds the supported date range
+   * @throws IllegalArgumentException if the interval has `microseconds` part
+   */
+  def dateAddInterval(
+     start: SQLDate,
+     interval: CalendarInterval): SQLDate = {
+    require(interval.microseconds == 0,
+      "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date")
+    val ld = LocalDate.ofEpochDay(start).plusMonths(interval.months).plusDays(interval.days)
+    localDateToDays(ld)
+  }
+
   /**
    * Returns number of months between time1 and time2. time1 and time2 are expressed in
    * microseconds since 1.1.1970. If time1 is later than time2, the result is positive.
@@ -587,11 +650,11 @@ object DateTimeUtils {
       time1: SQLTimestamp,
       time2: SQLTimestamp,
       roundOff: Boolean,
-      timeZone: TimeZone): Double = {
-    val millis1 = MICROSECONDS.toMillis(time1)
-    val millis2 = MICROSECONDS.toMillis(time2)
-    val date1 = millisToDays(millis1, timeZone)
-    val date2 = millisToDays(millis2, timeZone)
+      zoneId: ZoneId): Double = {
+    val millis1 = toMillis(time1)
+    val millis2 = toMillis(time2)
+    val date1 = millisToDays(millis1, zoneId)
+    val date2 = millisToDays(millis2, zoneId)
     val (year1, monthInYear1, dayInMonth1, daysToMonthEnd1) = splitDate(date1)
     val (year2, monthInYear2, dayInMonth2, daysToMonthEnd2) = splitDate(date2)
 
@@ -605,8 +668,8 @@ object DateTimeUtils {
     }
     // using milliseconds can cause precision loss with more than 8 digits
     // we follow Hive's implementation which uses seconds
-    val secondsInDay1 = MILLISECONDS.toSeconds(millis1 - daysToMillis(date1, timeZone))
-    val secondsInDay2 = MILLISECONDS.toSeconds(millis2 - daysToMillis(date2, timeZone))
+    val secondsInDay1 = MILLISECONDS.toSeconds(millis1 - daysToMillis(date1, zoneId))
+    val secondsInDay2 = MILLISECONDS.toSeconds(millis2 - daysToMillis(date2, zoneId))
     val secondsDiff = (dayInMonth1 - dayInMonth2) * SECONDS_PER_DAY + secondsInDay1 - secondsInDay2
     val secondsInMonth = DAYS.toSeconds(31)
     val diff = monthDiff + secondsDiff / secondsInMonth.toDouble
@@ -673,76 +736,65 @@ object DateTimeUtils {
   private[sql] val TRUNC_TO_MINUTE = 3
   private[sql] val TRUNC_TO_HOUR = 4
   private[sql] val TRUNC_TO_DAY = 5
-  // The levels from TRUNC_TO_WEEK to TRUNC_TO_MILLENNIUM are used in truncations
+  // The levels from TRUNC_TO_WEEK to TRUNC_TO_YEAR are used in truncations
   // of DATE and TIMESTAMP values.
   private[sql] val TRUNC_TO_WEEK = 6
   private[sql] val MIN_LEVEL_OF_DATE_TRUNC = TRUNC_TO_WEEK
   private[sql] val TRUNC_TO_MONTH = 7
   private[sql] val TRUNC_TO_QUARTER = 8
   private[sql] val TRUNC_TO_YEAR = 9
-  private[sql] val TRUNC_TO_DECADE = 10
-  private[sql] val TRUNC_TO_CENTURY = 11
-  private[sql] val TRUNC_TO_MILLENNIUM = 12
 
   /**
    * Returns the trunc date from original date and trunc level.
-   * Trunc level should be generated using `parseTruncLevel()`, should be between 0 and 6.
+   * Trunc level should be generated using `parseTruncLevel()`, should be between 6 and 9.
    */
   def truncDate(d: SQLDate, level: Int): SQLDate = {
-    def truncToYearLevel(divider: Int, adjust: Int): SQLDate = {
-      val oldYear = getYear(d)
-      var newYear = Math.floorDiv(oldYear, divider)
-      if (adjust > 0 && Math.floorMod(oldYear, divider) == 0) {
-        newYear -= 1
-      }
-      newYear = newYear * divider + adjust
-      localDateToDays(LocalDate.of(newYear, 1, 1))
-    }
     level match {
       case TRUNC_TO_WEEK => getNextDateForDayOfWeek(d - 7, MONDAY)
       case TRUNC_TO_MONTH => d - DateTimeUtils.getDayOfMonth(d) + 1
       case TRUNC_TO_QUARTER =>
         localDateToDays(daysToLocalDate(d).`with`(IsoFields.DAY_OF_QUARTER, 1L))
       case TRUNC_TO_YEAR => d - DateTimeUtils.getDayInYear(d) + 1
-      case TRUNC_TO_DECADE => truncToYearLevel(10, 0)
-      case TRUNC_TO_CENTURY => truncToYearLevel(100, 1)
-      case TRUNC_TO_MILLENNIUM => truncToYearLevel(1000, 1)
       case _ =>
         // caller make sure that this should never be reached
         sys.error(s"Invalid trunc level: $level")
     }
   }
 
+  private def truncToUnit(t: SQLTimestamp, zoneId: ZoneId, unit: ChronoUnit): SQLTimestamp = {
+    val truncated = microsToInstant(t).atZone(zoneId).truncatedTo(unit)
+    instantToMicros(truncated.toInstant)
+  }
+
   /**
    * Returns the trunc date time from original date time and trunc level.
-   * Trunc level should be generated using `parseTruncLevel()`, should be between 0 and 12.
-   */
-  def truncTimestamp(t: SQLTimestamp, level: Int, timeZone: TimeZone): SQLTimestamp = {
-    if (level == TRUNC_TO_MICROSECOND) return t
-    var millis = MICROSECONDS.toMillis(t)
-    val truncated = level match {
-      case TRUNC_TO_MILLISECOND => millis
-      case TRUNC_TO_SECOND =>
-        millis - millis % MILLIS_PER_SECOND
-      case TRUNC_TO_MINUTE =>
-        millis - millis % MILLIS_PER_MINUTE
-      case TRUNC_TO_HOUR =>
-        val offset = timeZone.getOffset(millis)
-        millis += offset
-        millis - millis % MILLIS_PER_HOUR - offset
-      case TRUNC_TO_DAY =>
-        val offset = timeZone.getOffset(millis)
-        millis += offset
-        millis - millis % MILLIS_PER_DAY - offset
-      case _ => // Try to truncate date levels
-        val dDays = millisToDays(millis, timeZone)
-        daysToMillis(truncDate(dDays, level), timeZone)
+   * Trunc level should be generated using `parseTruncLevel()`, should be between 0 and 9.
+   */
+  def truncTimestamp(t: SQLTimestamp, level: Int, zoneId: ZoneId): SQLTimestamp = {
+    // Time zone offsets have a maximum precision of seconds (see `java.time.ZoneOffset`). Hence
+    // truncation to microsecond, millisecond, and second can be done
+    // without using time zone information. This results in a performance improvement.
+    level match {
+      case TRUNC_TO_MICROSECOND => t
+      case TRUNC_TO_MINUTE => truncToUnit(t, zoneId, ChronoUnit.MINUTES)
+      case TRUNC_TO_HOUR => truncToUnit(t, zoneId, ChronoUnit.HOURS)
+      case TRUNC_TO_DAY => truncToUnit(t, zoneId, ChronoUnit.DAYS)
+      case _ =>
+        val millis = toMillis(t)
+        val truncated = level match {
+          case TRUNC_TO_MILLISECOND => millis
+          case TRUNC_TO_SECOND =>
+            millis - Math.floorMod(millis, MILLIS_PER_SECOND)
+          case _ => // Try to truncate date levels
+            val dDays = millisToDays(millis, zoneId)
+            daysToMillis(truncDate(dDays, level), zoneId)
+        }
+        fromMillis(truncated)
     }
-    truncated * MICROS_PER_MILLIS
   }
 
   /**
-   * Returns the truncate level, could be from TRUNC_TO_MICROSECOND to TRUNC_TO_MILLENNIUM,
+   * Returns the truncate level, could be from TRUNC_TO_MICROSECOND to TRUNC_TO_YEAR,
    * or TRUNC_INVALID, TRUNC_INVALID means unsupported truncate level.
    */
   def parseTruncLevel(format: UTF8String): Int = {
@@ -760,40 +812,11 @@ object DateTimeUtils {
         case "MON" | "MONTH" | "MM" => TRUNC_TO_MONTH
         case "QUARTER" => TRUNC_TO_QUARTER
         case "YEAR" | "YYYY" | "YY" => TRUNC_TO_YEAR
-        case "DECADE" => TRUNC_TO_DECADE
-        case "CENTURY" => TRUNC_TO_CENTURY
-        case "MILLENNIUM" => TRUNC_TO_MILLENNIUM
         case _ => TRUNC_INVALID
       }
     }
   }
 
-  /**
-   * Lookup the offset for given millis seconds since 1970-01-01 00:00:00 in given timezone.
-   * TODO: Improve handling of normalization differences.
-   * TODO: Replace with JSR-310 or similar system - see SPARK-16788
-   */
-  private[sql] def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone): Long = {
-    var guess = tz.getRawOffset
-    // the actual offset should be calculated based on milliseconds in UTC
-    val offset = tz.getOffset(millisLocal - guess)
-    if (offset != guess) {
-      guess = tz.getOffset(millisLocal - offset)
-      if (guess != offset) {
-        // fallback to do the reverse lookup using java.time.LocalDateTime
-        // this should only happen near the start or end of DST
-        val localDate = LocalDate.ofEpochDay(MILLISECONDS.toDays(millisLocal))
-        val localTime = LocalTime.ofNanoOfDay(MILLISECONDS.toNanos(
-          Math.floorMod(millisLocal, MILLIS_PER_DAY)))
-        val localDateTime = LocalDateTime.of(localDate, localTime)
-        val millisEpoch = localDateTime.atZone(tz.toZoneId).toInstant.toEpochMilli
-
-        guess = (millisLocal - millisEpoch).toInt
-      }
-    }
-    guess
-  }
-
   /**
    * Convert the timestamp `ts` from one timezone to another.
    *
@@ -801,27 +824,9 @@ object DateTimeUtils {
    * mapping, the conversion here may return wrong result, we should make the timestamp
    * timezone-aware.
    */
-  def convertTz(ts: SQLTimestamp, fromZone: TimeZone, toZone: TimeZone): SQLTimestamp = {
-    // We always use local timezone to parse or format a timestamp
-    val localZone = defaultTimeZone()
-    val utcTs = if (fromZone.getID == localZone.getID) {
-      ts
-    } else {
-      // get the human time using local time zone, that actually is in fromZone.
-      val localZoneOffsetMs = localZone.getOffset(MICROSECONDS.toMillis(ts))
-      val localTsUs = ts + MILLISECONDS.toMicros(localZoneOffsetMs)  // in fromZone
-      val offsetFromLocalMs = getOffsetFromLocalMillis(MICROSECONDS.toMillis(localTsUs), fromZone)
-      localTsUs - MILLISECONDS.toMicros(offsetFromLocalMs)
-    }
-    if (toZone.getID == localZone.getID) {
-      utcTs
-    } else {
-      val toZoneOffsetMs = toZone.getOffset(MICROSECONDS.toMillis(utcTs))
-      val localTsUs = utcTs + MILLISECONDS.toMicros(toZoneOffsetMs)  // in toZone
-      // treat it as local timezone, convert to UTC (we could get the expected human time back)
-      val offsetFromLocalMs = getOffsetFromLocalMillis(MICROSECONDS.toMillis(localTsUs), localZone)
-      localTsUs - MILLISECONDS.toMicros(offsetFromLocalMs)
-    }
+  def convertTz(ts: SQLTimestamp, fromZone: ZoneId, toZone: ZoneId): SQLTimestamp = {
+    val rebasedDateTime = microsToInstant(ts).atZone(toZone).toLocalDateTime.atZone(fromZone)
+    instantToMicros(rebasedDateTime.toInstant)
   }
 
   /**
@@ -829,7 +834,7 @@ object DateTimeUtils {
    * representation in their timezone.
    */
   def fromUTCTime(time: SQLTimestamp, timeZone: String): SQLTimestamp = {
-    convertTz(time, TimeZoneGMT, getTimeZone(timeZone))
+    convertTz(time, ZoneOffset.UTC, getZoneId(timeZone))
   }
 
   /**
@@ -837,18 +842,7 @@ object DateTimeUtils {
    * string representation in their timezone.
    */
   def toUTCTime(time: SQLTimestamp, timeZone: String): SQLTimestamp = {
-    convertTz(time, getTimeZone(timeZone), TimeZoneGMT)
-  }
-
-  /**
-   * Returns the number of seconds with fractional part in microsecond precision
-   * since 1970-01-01 00:00:00 local time.
-   */
-  def getEpoch(timestamp: SQLTimestamp, zoneId: ZoneId): Decimal = {
-    val offset = SECONDS.toMicros(
-      zoneId.getRules.getOffset(microsToInstant(timestamp)).getTotalSeconds)
-    val sinceEpoch = timestamp + offset
-    Decimal(sinceEpoch, 20, 6)
+    convertTz(time, getZoneId(timeZone), ZoneOffset.UTC)
   }
 
   def currentTimestamp(): SQLTimestamp = instantToMicros(Instant.now())
@@ -863,9 +857,9 @@ object DateTimeUtils {
 
   /**
    * Extracts special values from an input string ignoring case.
-   * @param input - a trimmed string
-   * @param zoneId - zone identifier used to get the current date.
-   * @return some special value in lower case or None.
+   * @param input A trimmed string
+   * @param zoneId Zone identifier used to get the current date.
+   * @return Some special value in lower case or None.
    */
   private def extractSpecialValue(input: String, zoneId: ZoneId): Option[String] = {
     def isValid(value: String, timeZoneId: String): Boolean = {
@@ -892,9 +886,9 @@ object DateTimeUtils {
 
   /**
    * Converts notational shorthands that are converted to ordinary timestamps.
-   * @param input - a trimmed string
-   * @param zoneId - zone identifier used to get the current date.
-   * @return some of microseconds since the epoch if the conversion completed
+   * @param input A trimmed string
+   * @param zoneId Zone identifier used to get the current date.
+   * @return Some of microseconds since the epoch if the conversion completed
    *         successfully otherwise None.
    */
   def convertSpecialTimestamp(input: String, zoneId: ZoneId): Option[SQLTimestamp] = {
@@ -918,9 +912,9 @@ object DateTimeUtils {
 
   /**
    * Converts notational shorthands that are converted to ordinary dates.
-   * @param input - a trimmed string
-   * @param zoneId - zone identifier used to get the current date.
-   * @return some of days since the epoch if the conversion completed successfully otherwise None.
+   * @param input A trimmed string
+   * @param zoneId Zone identifier used to get the current date.
+   * @return Some of days since the epoch if the conversion completed successfully otherwise None.
    */
   def convertSpecialDate(input: String, zoneId: ZoneId): Option[SQLDate] = {
     extractSpecialValue(input, zoneId).flatMap {
@@ -942,9 +936,9 @@ object DateTimeUtils {
 
   /**
    * Subtracts two dates.
-   * @param endDate - the end date, exclusive
-   * @param startDate - the start date, inclusive
-   * @return an interval between two dates. The interval can be negative
+   * @param endDate The end date, exclusive
+   * @param startDate The start date, inclusive
+   * @return An interval between two dates. The interval can be negative
    *         if the end date is before the start date.
    */
   def subtractDates(endDate: SQLDate, startDate: SQLDate): CalendarInterval = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala
index ea619c6a7666c..6471a746f2edf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala
@@ -22,6 +22,7 @@ import java.util
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.XxHash64Function
+import org.apache.spark.sql.catalyst.optimizer.NormalizeFloatingNumbers.{DOUBLE_NORMALIZER, FLOAT_NORMALIZER}
 import org.apache.spark.sql.types._
 
 // A helper class for HyperLogLogPlusPlus.
@@ -88,7 +89,12 @@ class HyperLogLogPlusPlusHelper(relativeSD: Double) extends Serializable {
    *
    * Variable names in the HLL++ paper match variable names in the code.
    */
-  def update(buffer: InternalRow, bufferOffset: Int, value: Any, dataType: DataType): Unit = {
+  def update(buffer: InternalRow, bufferOffset: Int, _value: Any, dataType: DataType): Unit = {
+    val value = dataType match {
+      case FloatType => FLOAT_NORMALIZER.apply(_value)
+      case DoubleType => DOUBLE_NORMALIZER.apply(_value)
+      case _ => _value
+    }
     // Create the hashed value 'x'.
     val x = XxHash64Function.hash(value, dataType, 42L)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 7692299a46ef5..af5061b005cb2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -17,12 +17,12 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.math.BigDecimal
 import java.util.concurrent.TimeUnit
 
 import scala.util.control.NonFatal
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.fromMillis
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.Decimal
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -49,57 +49,25 @@ object IntervalUtils {
     interval.months / MONTHS_PER_YEAR
   }
 
-  def getMillenniums(interval: CalendarInterval): Int = {
-    getYears(interval) / YEARS_PER_MILLENNIUM
-  }
-
-  def getCenturies(interval: CalendarInterval): Int = {
-    getYears(interval) / YEARS_PER_CENTURY
-  }
-
-  def getDecades(interval: CalendarInterval): Int = {
-    getYears(interval) / YEARS_PER_DECADE
-  }
-
   def getMonths(interval: CalendarInterval): Byte = {
     (interval.months % MONTHS_PER_YEAR).toByte
   }
 
-  def getQuarters(interval: CalendarInterval): Byte = {
-    (getMonths(interval) / MONTHS_PER_QUARTER + 1).toByte
-  }
-
   def getDays(interval: CalendarInterval): Int = {
-    interval.days
+    val daysInMicroseconds = (interval.microseconds / MICROS_PER_DAY).toInt
+    Math.addExact(interval.days, daysInMicroseconds)
   }
 
   def getHours(interval: CalendarInterval): Long = {
-    interval.microseconds / MICROS_PER_HOUR
+    (interval.microseconds % MICROS_PER_DAY) / MICROS_PER_HOUR
   }
 
   def getMinutes(interval: CalendarInterval): Byte = {
     ((interval.microseconds % MICROS_PER_HOUR) / MICROS_PER_MINUTE).toByte
   }
 
-  def getMicroseconds(interval: CalendarInterval): Long = {
-    interval.microseconds % MICROS_PER_MINUTE
-  }
-
   def getSeconds(interval: CalendarInterval): Decimal = {
-    Decimal(getMicroseconds(interval), 8, 6)
-  }
-
-  def getMilliseconds(interval: CalendarInterval): Decimal = {
-    Decimal(getMicroseconds(interval), 8, 3)
-  }
-
-  // Returns total number of seconds with microseconds fractional part in the given interval.
-  def getEpoch(interval: CalendarInterval): Decimal = {
-    var result = interval.microseconds
-    result += MICROS_PER_DAY * interval.days
-    result += MICROS_PER_YEAR * (interval.months / MONTHS_PER_YEAR)
-    result += MICROS_PER_MONTH * (interval.months % MONTHS_PER_YEAR)
-    Decimal(result, 18, 6)
+    Decimal(interval.microseconds % MICROS_PER_MINUTE, 8, 6)
   }
 
   private def toLongWithRange(
@@ -135,8 +103,7 @@ object IntervalUtils {
             s"Error parsing interval year-month string: ${e.getMessage}", e)
       }
     }
-    assert(input.length == input.trim.length)
-    input match {
+    input.trim match {
       case yearMonthPattern("-", yearStr, monthStr) =>
         negateExact(toInterval(yearStr, monthStr))
       case yearMonthPattern(_, yearStr, monthStr) =>
@@ -176,6 +143,9 @@ object IntervalUtils {
   private val dayTimePatternLegacy =
     "^([+|-])?((\\d+) )?((\\d+):)?(\\d+):(\\d+)(\\.(\\d+))?$".r
 
+  private val fallbackNotice = s"set ${SQLConf.LEGACY_FROM_DAYTIME_STRING.key} to true " +
+    "to restore the behavior before Spark 3.0."
+
   /**
    * Legacy method of parsing a string in a day-time format. It ignores the `from` bound,
    * and takes into account only the `to` bound by truncating the result. For example,
@@ -195,7 +165,8 @@ object IntervalUtils {
     require(input != null, "Interval day-time string must be not null")
     assert(input.length == input.trim.length)
     val m = dayTimePatternLegacy.pattern.matcher(input)
-    require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input")
+    require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input, " +
+      s"$fallbackNotice")
 
     try {
       val sign = if (m.group(1) != null && m.group(1) == "-") -1 else 1
@@ -295,8 +266,9 @@ object IntervalUtils {
     val regexp = dayTimePattern.get(from -> to)
     require(regexp.isDefined, s"Cannot support (interval '$input' $from to $to) expression")
     val pattern = regexp.get.pattern
-    val m = pattern.matcher(input)
-    require(m.matches, s"Interval string must match day-time format of '$pattern': $input")
+    val m = pattern.matcher(input.trim)
+    require(m.matches, s"Interval string must match day-time format of '$pattern': $input, " +
+      s"$fallbackNotice")
     var micros: Long = 0L
     var days: Int = 0
     unitsRange(to, from).foreach {
@@ -399,22 +371,38 @@ object IntervalUtils {
   }
 
   /**
-   * Makes an interval from months, days and micros with the fractional part by
-   * adding the month fraction to days and the days fraction to micros.
-   *
-   * @throws ArithmeticException if the result overflows any field value
+   * Makes an interval from months, days and micros with the fractional part.
+   * The overflow style here follows the way of ansi sql standard and the natural rules for
+   * intervals as defined in the Gregorian calendar. Thus, the days fraction will be added
+   * to microseconds but the months fraction will not be added to days, and it will throw
+   * exception if any part overflows.
    */
   private def fromDoubles(
       monthsWithFraction: Double,
       daysWithFraction: Double,
       microsWithFraction: Double): CalendarInterval = {
     val truncatedMonths = Math.toIntExact(monthsWithFraction.toLong)
-    val days = daysWithFraction + DAYS_PER_MONTH * (monthsWithFraction - truncatedMonths)
-    val truncatedDays = Math.toIntExact(days.toLong)
-    val micros = microsWithFraction + MICROS_PER_DAY * (days - truncatedDays)
+    val truncatedDays = Math.toIntExact(daysWithFraction.toLong)
+    val micros = microsWithFraction + MICROS_PER_DAY * (daysWithFraction - truncatedDays)
     new CalendarInterval(truncatedMonths, truncatedDays, micros.round)
   }
 
+  /**
+   * Makes an interval from months, days and micros with the fractional part.
+   * The overflow style here follows the way of casting [[java.lang.Double]] to integrals and the
+   * natural rules for intervals as defined in the Gregorian calendar. Thus, the days fraction
+   * will be added to microseconds but the months fraction will not be added to days, and there may
+   * be rounding or truncation in months(or day and microseconds) part.
+   */
+  private def safeFromDoubles(
+      monthsWithFraction: Double,
+      daysWithFraction: Double,
+      microsWithFraction: Double): CalendarInterval = {
+    val truncatedDays = daysWithFraction.toInt
+    val micros = microsWithFraction + MICROS_PER_DAY * (daysWithFraction - truncatedDays)
+    new CalendarInterval(monthsWithFraction.toInt, truncatedDays, micros.round)
+  }
+
   /**
    * Unary minus, return the negated the calendar interval value.
    *
@@ -478,6 +466,13 @@ object IntervalUtils {
     new CalendarInterval(months, days, microseconds)
   }
 
+  /**
+   * Return a new calendar interval instance of the left interval times a multiplier.
+   */
+  def multiply(interval: CalendarInterval, num: Double): CalendarInterval = {
+    safeFromDoubles(num * interval.months, num * interval.days, num * interval.microseconds)
+  }
+
   /**
    * Return a new calendar interval instance of the left interval times a multiplier.
    *
@@ -487,6 +482,14 @@ object IntervalUtils {
     fromDoubles(num * interval.months, num * interval.days, num * interval.microseconds)
   }
 
+  /**
+   * Return a new calendar interval instance of the left interval divides by a dividend.
+   */
+  def divide(interval: CalendarInterval, num: Double): CalendarInterval = {
+    if (num == 0) return null
+    safeFromDoubles(interval.months / num, interval.days / num, interval.microseconds / num)
+  }
+
   /**
    * Return a new calendar interval instance of the left interval divides by a dividend.
    *
@@ -699,9 +702,7 @@ object IntervalUtils {
                   microseconds = Math.addExact(microseconds, minutesUs)
                   i += minuteStr.numBytes()
                 } else if (s.matchAt(millisStr, i)) {
-                  val millisUs = Math.multiplyExact(
-                    currentValue,
-                    MICROS_PER_MILLIS)
+                  val millisUs = fromMillis(currentValue)
                   microseconds = Math.addExact(microseconds, millisUs)
                   i += millisStr.numBytes()
                 } else if (s.matchAt(microsStr, i)) {
@@ -734,7 +735,10 @@ object IntervalUtils {
     val result = state match {
       case UNIT_SUFFIX | UNIT_END | TRIM_BEFORE_SIGN =>
         new CalendarInterval(months, days, microseconds)
-      case _ => null
+      case TRIM_BEFORE_VALUE => throwIAE(s"expect a number after '$currentWord' but hit EOL")
+      case VALUE | VALUE_FRACTIONAL_PART =>
+        throwIAE(s"expect a unit name after '$currentWord' but hit EOL")
+      case _ => throwIAE(s"unknown error when parsing '$currentWord'")
     }
 
     result
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala
index 3a0490d07733d..e07aa0ffa42fc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala
@@ -254,7 +254,7 @@ class QuantileSummaries(
 
     // Target rank
     val rank = math.ceil(quantile * count).toLong
-    val targetError = relativeError * count
+    val targetError = sampled.map(s => s.delta + s.g).max / 2
     // Minimum rank at current sample
     var minRank = 0L
     var i = 0
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
new file mode 100644
index 0000000000000..f93878df38cb2
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -0,0 +1,488 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId}
+import java.time.temporal.ChronoField
+import java.util.{Calendar, TimeZone}
+import java.util.Calendar.{DAY_OF_MONTH, DST_OFFSET, ERA, HOUR_OF_DAY, MINUTE, MONTH, SECOND, YEAR, ZONE_OFFSET}
+
+import scala.collection.mutable.AnyRefMap
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper}
+
+import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+
+/**
+ * The collection of functions for rebasing days and microseconds from/to the hybrid calendar
+ * (Julian + Gregorian since 1582-10-15) which is used by Spark 2.4 and earlier versions
+ * to/from Proleptic Gregorian calendar which is used by Spark since version 3.0. See SPARK-26651.
+ */
+object RebaseDateTime {
+  /**
+   * Rebases days since the epoch from an original to an target calendar, for instance,
+   * from a hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
+   *
+   * It finds the latest switch day which is less than the given `days`, and adds the difference
+   * in days associated with the switch days to the given `days`.
+   * The function is based on linear search which starts from the most recent switch days.
+   * This allows to perform less comparisons for modern dates.
+   *
+   * @param switches The days when difference in days between original and target calendar
+   *                 was changed.
+   * @param diffs The differences in days between calendars.
+   * @param days The number of days since the epoch 1970-01-01 to be rebased
+   *             to the target calendar.
+   * @return The rebased days.
+   */
+  private def rebaseDays(switches: Array[Int], diffs: Array[Int], days: Int): Int = {
+    var i = switches.length
+    do { i -= 1 } while (i > 0 && days < switches(i))
+    days + diffs(i)
+  }
+
+  // The differences in days between Julian and Proleptic Gregorian dates.
+  // The diff at the index `i` is applicable for all days in the date interval:
+  // [julianGregDiffSwitchDay(i), julianGregDiffSwitchDay(i+1))
+  private val julianGregDiffs = Array(2, 1, 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, 0)
+  // The sorted days in Julian calendar when difference in days between Julian and
+  // Proleptic Gregorian calendars was changed.
+  // The starting point is the `0001-01-01` (-719164 days since the epoch in
+  // Julian calendar). This array is not applicable for dates before the staring point.
+  // Rebasing switch days and diffs `julianGregDiffSwitchDay` and `julianGregDiffs`
+  // was generated by the `localRebaseJulianToGregorianDays` function.
+  private val julianGregDiffSwitchDay = Array(
+    -719164, -682945, -646420, -609895, -536845, -500320, -463795,
+    -390745, -354220, -317695, -244645, -208120, -171595, -141427)
+
+  final val lastSwitchJulianDay: Int = julianGregDiffSwitchDay.last
+
+  // The first days of Common Era (CE) which is mapped to the '0001-01-01' date in Julian calendar.
+  private final val julianCommonEraStartDay = julianGregDiffSwitchDay(0)
+
+  /**
+   * Converts the given number of days since the epoch day 1970-01-01 to a local date in Julian
+   * calendar, interprets the result as a local date in Proleptic Gregorian calendar, and takes the
+   * number of days since the epoch from the Gregorian local date.
+   *
+   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
+   * calendar. See SPARK-26651.
+   *
+   * For example:
+   *   Julian calendar: 1582-01-01 -> -141704
+   *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
+   * The code below converts -141704 to -141714.
+   *
+   * @param days The number of days since the epoch in Julian calendar. It can be negative.
+   * @return The rebased number of days in Gregorian calendar.
+   */
+  private[sql] def localRebaseJulianToGregorianDays(days: Int): Int = {
+    val utcCal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setTimeZone(TimeZoneUTC)
+      .setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
+      .build()
+    val localDate = LocalDate.of(
+      utcCal.get(YEAR),
+      utcCal.get(MONTH) + 1,
+      // The number of days will be added later to handle non-existing
+      // Julian dates in Proleptic Gregorian calendar.
+      // For example, 1000-02-29 exists in Julian calendar because 1000
+      // is a leap year but it is not a leap year in Gregorian calendar.
+      1)
+      .`with`(ChronoField.ERA, utcCal.get(ERA))
+      .plusDays(utcCal.get(DAY_OF_MONTH) - 1)
+    Math.toIntExact(localDate.toEpochDay)
+  }
+
+  /**
+   * An optimized version of [[localRebaseJulianToGregorianDays(Int)]]. This method leverages the
+   * pre-calculated rebasing array to save calculation. For dates of Before Common Era, the
+   * function falls back to the regular unoptimized version.
+   *
+   * @param days The number of days since the epoch in Julian calendar. It can be negative.
+   * @return The rebased number of days in Gregorian calendar.
+   */
+  def rebaseJulianToGregorianDays(days: Int): Int = {
+    if (days < julianCommonEraStartDay) {
+      localRebaseJulianToGregorianDays(days)
+    } else {
+      rebaseDays(julianGregDiffSwitchDay, julianGregDiffs, days)
+    }
+  }
+
+  // The differences in days between Proleptic Gregorian and Julian dates.
+  // The diff at the index `i` is applicable for all days in the date interval:
+  // [gregJulianDiffSwitchDay(i), gregJulianDiffSwitchDay(i+1))
+  private val gregJulianDiffs = Array(
+    -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+  // The sorted days in Proleptic Gregorian calendar when difference in days between
+  // Proleptic Gregorian and Julian was changed.
+  // The starting point is the `0001-01-01` (-719162 days since the epoch in
+  // Proleptic Gregorian calendar). This array is not applicable for dates before the staring point.
+  // Rebasing switch days and diffs `gregJulianDiffSwitchDay` and `gregJulianDiffs`
+  // was generated by the `localRebaseGregorianToJulianDays` function.
+  private val gregJulianDiffSwitchDay = Array(
+    -719162, -682944, -646420, -609896, -536847, -500323, -463799, -390750,
+    -354226, -317702, -244653, -208129, -171605, -141436, -141435, -141434,
+    -141433, -141432, -141431, -141430, -141429, -141428, -141427)
+
+  final val lastSwitchGregorianDay: Int = gregJulianDiffSwitchDay.last
+
+  // The first days of Common Era (CE) which is mapped to the '0001-01-01' date
+  // in Proleptic Gregorian calendar.
+  private final val gregorianCommonEraStartDay = gregJulianDiffSwitchDay(0)
+
+  private final val gregorianStartDate = LocalDate.of(1582, 10, 15)
+  private final val julianEndDate = LocalDate.of(1582, 10, 4)
+
+  /**
+   * Converts the given number of days since the epoch day 1970-01-01 to a local date in Proleptic
+   * Gregorian calendar, interprets the result as a local date in Julian calendar, and takes the
+   * number of days since the epoch from the Julian local date.
+   *
+   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
+   * calendar. See SPARK-26651.
+   *
+   * For example:
+   *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
+   *   Julian calendar: 1582-01-01 -> -141704
+   * The code below converts -141714 to -141704.
+   *
+   * @param days The number of days since the epoch in Proleptic Gregorian calendar.
+   *             It can be negative.
+   * @return The rebased number of days in Julian calendar.
+   */
+  private[sql] def localRebaseGregorianToJulianDays(days: Int): Int = {
+    var localDate = LocalDate.ofEpochDay(days)
+    if (localDate.isAfter(julianEndDate) && localDate.isBefore(gregorianStartDate)) {
+      localDate = gregorianStartDate
+    }
+    val utcCal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setTimeZone(TimeZoneUTC)
+      .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
+      .build()
+    Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY))
+  }
+
+  /**
+   * An optimized version of [[localRebaseGregorianToJulianDays(Int)]]. This method leverages the
+   * pre-calculated rebasing array to save calculation. For dates of Before Common Era, the
+   * function falls back to the regular unoptimized version.
+   *
+   * @param days The number of days since the epoch in Gregorian calendar. It can be negative.
+   * @return The rebased number of days since the epoch in Julian calendar.
+   */
+  def rebaseGregorianToJulianDays(days: Int): Int = {
+    if (days < gregorianCommonEraStartDay) {
+      localRebaseGregorianToJulianDays(days)
+    } else {
+      rebaseDays(gregJulianDiffSwitchDay, gregJulianDiffs, days)
+    }
+  }
+
+
+  /**
+   * The class describes JSON records with microseconds rebasing info.
+   * Here is an example of JSON file:
+   * {{{
+   *   [
+   *     {
+   *       "tz": "Europe/Paris",
+   *       "switches": [-123, 0],
+   *       "diffs": [422000000, 0]
+   *     }
+   *   ]
+   * }}}
+   *
+   * @param tz One of time zone ID which is expected to be acceptable by `ZoneId.of`.
+   * @param switches An ordered array of seconds since the epoch when the diff between
+   *                 two calendars are changed.
+   * @param diffs Differences in seconds associated with elements of `switches`.
+   */
+  private case class JsonRebaseRecord(tz: String, switches: Array[Long], diffs: Array[Long])
+
+  /**
+   * Rebasing info used to convert microseconds from an original to a target calendar.
+   *
+   * @param switches An ordered array of microseconds since the epoch when the diff between
+   *                 two calendars are changed.
+   * @param diffs Differences in microseconds associated with elements of `switches`.
+   */
+  private[sql] case class RebaseInfo(switches: Array[Long], diffs: Array[Long])
+
+  /**
+   * Rebases micros since the epoch from an original to an target calendar, for instance,
+   * from a hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
+   *
+   * It finds the latest switch micros which is less than the given `micros`, and adds the
+   * difference in micros associated with the switch micros to the given `micros`.
+   * The function is based on linear search which starts from the most recent switch micros.
+   * This allows to perform less comparisons for modern timestamps.
+   *
+   * @param rebaseInfo The rebasing info contains an ordered micros when difference in micros
+   *                   between original and target calendar was changed,
+   *                   and differences in micros between calendars
+   * @param micros The number of micros since the epoch 1970-01-01T00:00:00Z to be rebased
+   *               to the target calendar. It can be negative.
+   * @return The rebased micros.
+   */
+  private def rebaseMicros(rebaseInfo: RebaseInfo, micros: Long): Long = {
+    val switches = rebaseInfo.switches
+    var i = switches.length
+    do { i -= 1 } while (i > 0 && micros < switches(i))
+    micros + rebaseInfo.diffs(i)
+  }
+
+  // Loads rebasing info from an JSON file. JSON records in the files should conform to
+  // `JsonRebaseRecord`. AnyRefMap is used here instead of Scala's immutable map because
+  // it is 2 times faster in DateTimeRebaseBenchmark.
+  private[sql] def loadRebaseRecords(fileName: String): AnyRefMap[String, RebaseInfo] = {
+    val file = Thread.currentThread().getContextClassLoader.getResource(fileName)
+    val mapper = new ObjectMapper() with ScalaObjectMapper
+    mapper.registerModule(DefaultScalaModule)
+    val jsonRebaseRecords = mapper.readValue[Seq[JsonRebaseRecord]](file)
+    val anyRefMap = new AnyRefMap[String, RebaseInfo]((3 * jsonRebaseRecords.size) / 2)
+    jsonRebaseRecords.foreach { jsonRecord =>
+      val rebaseInfo = RebaseInfo(jsonRecord.switches, jsonRecord.diffs)
+      var i = 0
+      while (i < rebaseInfo.switches.length) {
+        rebaseInfo.switches(i) = rebaseInfo.switches(i) * MICROS_PER_SECOND
+        rebaseInfo.diffs(i) = rebaseInfo.diffs(i) * MICROS_PER_SECOND
+        i += 1
+      }
+      anyRefMap.update(jsonRecord.tz, rebaseInfo)
+    }
+    anyRefMap
+  }
+
+  /**
+   * A map of time zone IDs to its ordered time points (instants in microseconds since the epoch)
+   * when the difference between 2 instances associated with the same local timestamp in
+   * Proleptic Gregorian and the hybrid calendar was changed, and to the diff at the index `i` is
+   * applicable for all microseconds in the time interval:
+   *   [gregJulianDiffSwitchMicros(i), gregJulianDiffSwitchMicros(i+1))
+   */
+  private val gregJulianRebaseMap = loadRebaseRecords("gregorian-julian-rebase-micros.json")
+
+  private def getLastSwitchTs(rebaseMap: AnyRefMap[String, RebaseInfo]): Long = {
+    val latestTs = rebaseMap.values.map(_.switches.last).max
+    require(rebaseMap.values.forall(_.diffs.last == 0),
+      s"Differences between Julian and Gregorian calendar after ${microsToInstant(latestTs)} " +
+      "are expected to be zero for all available time zones.")
+    latestTs
+  }
+  // The switch time point after which all diffs between Gregorian and Julian calendars
+  // across all time zones are zero
+  final val lastSwitchGregorianTs: Long = getLastSwitchTs(gregJulianRebaseMap)
+
+  private final val gregorianStartTs = LocalDateTime.of(gregorianStartDate, LocalTime.MIDNIGHT)
+  private final val julianEndTs = LocalDateTime.of(
+    julianEndDate,
+    LocalTime.of(23, 59, 59, 999999999))
+
+  /**
+   * Converts the given number of microseconds since the epoch '1970-01-01T00:00:00Z', to a local
+   * date-time in Proleptic Gregorian calendar with timezone identified by `zoneId`, interprets the
+   * result as a local date-time in Julian calendar with the same timezone, and takes microseconds
+   * since the epoch from the Julian local date-time.
+   *
+   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
+   * calendar. See SPARK-26651.
+   *
+   * For example:
+   *   Proleptic Gregorian calendar: 1582-01-01 00:00:00.123456 -> -12244061221876544
+   *   Julian calendar: 1582-01-01 00:00:00.123456 -> -12243196799876544
+   * The code below converts -12244061221876544 to -12243196799876544.
+   *
+   * @param zoneId The time zone ID at which the rebasing should be performed.
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
+   *               in Proleptic Gregorian calendar. It can be negative.
+   * @return The rebased microseconds since the epoch in Julian calendar.
+   */
+  private[sql] def rebaseGregorianToJulianMicros(zoneId: ZoneId, micros: Long): Long = {
+    val instant = microsToInstant(micros)
+    val zonedDateTime = instant.atZone(zoneId)
+    var ldt = zonedDateTime.toLocalDateTime
+    if (ldt.isAfter(julianEndTs) && ldt.isBefore(gregorianStartTs)) {
+      ldt = LocalDateTime.of(gregorianStartDate, ldt.toLocalTime)
+    }
+    val cal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setDate(ldt.getYear, ldt.getMonthValue - 1, ldt.getDayOfMonth)
+      .setTimeOfDay(ldt.getHour, ldt.getMinute, ldt.getSecond)
+      .build()
+    // A local timestamp can have 2 instants in the cases of switching from:
+    //  1. Summer to winter time.
+    //  2. One standard time zone to another one. For example, Asia/Hong_Kong switched from JST
+    //     to HKT on 18 November, 1945 01:59:59 AM.
+    // Below we check that the original `instant` is earlier or later instant. If it is an earlier
+    // instant, we take the standard and DST offsets of the previous day otherwise of the next one.
+    val trans = zoneId.getRules.getTransition(ldt)
+    if (trans != null && trans.isOverlap) {
+      val cloned = cal.clone().asInstanceOf[Calendar]
+      // Does the current offset belong to the offset before the transition.
+      // If so, we will take zone offsets from the previous day otherwise from the next day.
+      // This assumes that transitions cannot happen often than once per 2 days.
+      val shift = if (trans.getOffsetBefore == zonedDateTime.getOffset) -1 else 1
+      cloned.add(DAY_OF_MONTH, shift)
+      cal.set(ZONE_OFFSET, cloned.get(ZONE_OFFSET))
+      cal.set(DST_OFFSET, cloned.get(DST_OFFSET))
+    }
+    fromMillis(cal.getTimeInMillis) + ldt.get(ChronoField.MICRO_OF_SECOND)
+  }
+
+  /**
+   * An optimized version of [[rebaseGregorianToJulianMicros(ZoneId, Long)]]. This method leverages
+   * the pre-calculated rebasing maps to save calculation. If the rebasing map doesn't contain
+   * information about the current JVM system time zone or `micros` is related to Before Common Era,
+   * the function falls back to the regular unoptimized version.
+   *
+   * Note: The function assumes that the input micros was derived from a local timestamp
+   *       at the default system JVM time zone in Proleptic Gregorian calendar.
+   *
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
+   *               in Proleptic Gregorian calendar. It can be negative.
+   * @return The rebased microseconds since the epoch in Julian calendar.
+   */
+  def rebaseGregorianToJulianMicros(micros: Long): Long = {
+    if (micros >= lastSwitchGregorianTs) {
+      micros
+    } else {
+      val timeZone = TimeZone.getDefault
+      val tzId = timeZone.getID
+      val rebaseRecord = gregJulianRebaseMap.getOrNull(tzId)
+      if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
+        rebaseGregorianToJulianMicros(timeZone.toZoneId, micros)
+      } else {
+        rebaseMicros(rebaseRecord, micros)
+      }
+    }
+  }
+
+  /**
+   * Converts the given number of microseconds since the epoch '1970-01-01T00:00:00Z', to a local
+   * date-time in Julian calendar with timezone identified by `zoneId`, interprets the result as a
+   * local date-time in Proleptic Gregorian calendar with the same timezone, and takes microseconds
+   * since the epoch from the Gregorian local date-time.
+   *
+   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
+   * calendar. See SPARK-26651.
+   *
+   * For example:
+   *   Julian calendar: 1582-01-01 00:00:00.123456 -> -12243196799876544
+   *   Proleptic Gregorian calendar: 1582-01-01 00:00:00.123456 -> -12244061221876544
+   * The code below converts -12243196799876544 to -12244061221876544.
+   *
+   * @param zoneId The time zone ID at which the rebasing should be performed.
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
+   *               in the Julian calendar. It can be negative.
+   * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
+   */
+  private[sql] def rebaseJulianToGregorianMicros(zoneId: ZoneId, micros: Long): Long = {
+    val cal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setInstant(toMillis(micros))
+      .build()
+    val localDateTime = LocalDateTime.of(
+      cal.get(YEAR),
+      cal.get(MONTH) + 1,
+      // The number of days will be added later to handle non-existing
+      // Julian dates in Proleptic Gregorian calendar.
+      // For example, 1000-02-29 exists in Julian calendar because 1000
+      // is a leap year but it is not a leap year in Gregorian calendar.
+      1,
+      cal.get(HOUR_OF_DAY),
+      cal.get(MINUTE),
+      cal.get(SECOND),
+      (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
+      .`with`(ChronoField.ERA, cal.get(ERA))
+      .plusDays(cal.get(DAY_OF_MONTH) - 1)
+    val zonedDateTime = localDateTime.atZone(zoneId)
+    // In the case of local timestamp overlapping, we need to choose the correct time instant
+    // which is related to the original local timestamp. We look ahead of 1 day, and if the next
+    // date has the same standard zone and DST offsets, the current local timestamp belongs to
+    // the period after the transition. In that case, we take the later zoned date time otherwise
+    // earlier one. Here, we assume that transitions happen not often than once per day.
+    val trans = zoneId.getRules.getTransition(localDateTime)
+    val adjustedZdt = if (trans != null && trans.isOverlap) {
+      val dstOffset = cal.get(DST_OFFSET)
+      val zoneOffset = cal.get(ZONE_OFFSET)
+      cal.add(DAY_OF_MONTH, 1)
+      if (zoneOffset == cal.get(ZONE_OFFSET) && dstOffset == cal.get(DST_OFFSET)) {
+        zonedDateTime.withLaterOffsetAtOverlap()
+      } else {
+        zonedDateTime.withEarlierOffsetAtOverlap()
+      }
+    } else zonedDateTime
+    instantToMicros(adjustedZdt.toInstant)
+  }
+
+  // The rebasing maps to convert microseconds from the hybrid calendar (Julian + Gregorian)
+  // to Proleptic Gregorian calendar. It maps time zone IDs to ordered timestamps (ascending order)
+  // where at every timestamps the difference between 2 calendars was changed, and to ordered
+  // differences between 2 calendars. The diff at the index `i` is applicable for all timestamps
+  // in the interval: [julianGregDiffSwitchMicros(i), julianGregDiffSwitchMicros(i+1))
+  private val julianGregRebaseMap = loadRebaseRecords("julian-gregorian-rebase-micros.json")
+
+  // The switch time point after which all diffs between Julian and Gregorian calendars
+  // across all time zones are zero
+  final val lastSwitchJulianTs: Long = getLastSwitchTs(julianGregRebaseMap)
+
+  /**
+   * An optimized version of [[rebaseJulianToGregorianMicros(ZoneId, Long)]]. This method leverages
+   * the pre-calculated rebasing maps to save calculation. If the rebasing map doesn't contain
+   * information about the current JVM system time zone or `micros` is related to Before Common Era,
+   * the function falls back to the regular unoptimized version.
+   *
+   * Note: The function assumes that the input micros was derived from a local timestamp
+   *       at the default system JVM time zone in the Julian calendar.
+   *
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
+   *               in the Julian calendar. It can be negative.
+   * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
+   */
+  def rebaseJulianToGregorianMicros(micros: Long): Long = {
+    if (micros >= lastSwitchJulianTs) {
+      micros
+    } else {
+      val timeZone = TimeZone.getDefault
+      val tzId = timeZone.getID
+      val rebaseRecord = julianGregRebaseMap.getOrNull(tzId)
+      if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
+        rebaseJulianToGregorianMicros(timeZone.toZoneId, micros)
+      } else {
+        rebaseMicros(rebaseRecord, micros)
+      }
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala
new file mode 100644
index 0000000000000..3b7f748c2817b
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+object SQLOrderingUtil {
+
+  /**
+   * A special version of double comparison that follows SQL semantic:
+   *  1. NaN == NaN
+   *  2. NaN is greater than any non-NaN double
+   *  3. -0.0 == 0.0
+   */
+  def compareDoubles(x: Double, y: Double): Int = {
+    if (x == y) 0 else java.lang.Double.compare(x, y)
+  }
+
+  /**
+   * A special version of float comparison that follows SQL semantic:
+   *  1. NaN == NaN
+   *  2. NaN is greater than any non-NaN float
+   *  3. -0.0 == 0.0
+   */
+  def compareFloats(x: Float, y: Float): Int = {
+    if (x == y) 0 else java.lang.Float.compare(x, y)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
index b42ae4e45366e..2a416d6c97584 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
@@ -123,7 +123,11 @@ object StringUtils extends Logging {
           val stringToAppend = if (available >= sLen) s else s.substring(0, available)
           strings.append(stringToAppend)
         }
-        length += sLen
+
+        // Keeps the total length of appended strings. Note that we need to cap the length at
+        // `ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH`; otherwise, we will overflow
+        // length causing StringIndexOutOfBoundsException in the substring call above.
+        length = Math.min(length.toLong + sLen, ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH).toInt
       }
     }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 5be4807083fa3..093ebb27e0dc5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -17,15 +17,23 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.text.ParseException
+import java.sql.Timestamp
+import java.text.{ParseException, ParsePosition, SimpleDateFormat}
 import java.time._
-import java.time.format.DateTimeParseException
+import java.time.format.{DateTimeFormatter, DateTimeParseException}
 import java.time.temporal.ChronoField.MICRO_OF_SECOND
 import java.time.temporal.TemporalQueries
-import java.util.Locale
-import java.util.concurrent.TimeUnit.SECONDS
+import java.util.{Calendar, GregorianCalendar, Locale, TimeZone}
 
-import DateTimeUtils.convertSpecialTimestamp
+import org.apache.commons.lang3.time.FastDateFormat
+
+import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.{LegacyDateFormat, LENIENT_SIMPLE_DATE_FORMAT}
+import org.apache.spark.sql.catalyst.util.RebaseDateTime._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
+import org.apache.spark.sql.types.Decimal
 
 sealed trait TimestampFormatter extends Serializable {
   /**
@@ -41,38 +49,69 @@ sealed trait TimestampFormatter extends Serializable {
   @throws(classOf[DateTimeParseException])
   @throws(classOf[DateTimeException])
   def parse(s: String): Long
+
   def format(us: Long): String
+  def format(ts: Timestamp): String
+  def format(instant: Instant): String
+  def validatePatternString(): Unit
 }
 
 class Iso8601TimestampFormatter(
     pattern: String,
     zoneId: ZoneId,
-    locale: Locale) extends TimestampFormatter with DateTimeFormatterHelper {
+    locale: Locale,
+    legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
+    isParsing: Boolean)
+  extends TimestampFormatter with DateTimeFormatterHelper {
   @transient
-  protected lazy val formatter = getOrCreateFormatter(pattern, locale)
+  protected lazy val formatter: DateTimeFormatter =
+    getOrCreateFormatter(pattern, locale, isParsing)
+
+  @transient
+  protected lazy val legacyFormatter = TimestampFormatter.getLegacyFormatter(
+    pattern, zoneId, locale, legacyFormat)
 
   override def parse(s: String): Long = {
     val specialDate = convertSpecialTimestamp(s.trim, zoneId)
     specialDate.getOrElse {
-      val parsed = formatter.parse(s)
-      val parsedZoneId = parsed.query(TemporalQueries.zone())
-      val timeZoneId = if (parsedZoneId == null) zoneId else parsedZoneId
-      val zonedDateTime = toZonedDateTime(parsed, timeZoneId)
-      val epochSeconds = zonedDateTime.toEpochSecond
-      val microsOfSecond = zonedDateTime.get(MICRO_OF_SECOND)
-
-      Math.addExact(SECONDS.toMicros(epochSeconds), microsOfSecond)
+      try {
+        val parsed = formatter.parse(s)
+        val parsedZoneId = parsed.query(TemporalQueries.zone())
+        val timeZoneId = if (parsedZoneId == null) zoneId else parsedZoneId
+        val zonedDateTime = toZonedDateTime(parsed, timeZoneId)
+        val epochSeconds = zonedDateTime.toEpochSecond
+        val microsOfSecond = zonedDateTime.get(MICRO_OF_SECOND)
+
+        Math.addExact(Math.multiplyExact(epochSeconds, MICROS_PER_SECOND), microsOfSecond)
+      } catch checkParsedDiff(s, legacyFormatter.parse)
     }
   }
 
+  override def format(instant: Instant): String = {
+    try {
+      formatter.withZone(zoneId).format(instant)
+    } catch checkFormattedDiff(toJavaTimestamp(instantToMicros(instant)),
+      (t: Timestamp) => format(t))
+  }
+
   override def format(us: Long): String = {
     val instant = DateTimeUtils.microsToInstant(us)
-    formatter.withZone(zoneId).format(instant)
+    format(instant)
+  }
+
+  override def format(ts: Timestamp): String = {
+    legacyFormatter.format(ts)
+  }
+
+  override def validatePatternString(): Unit = {
+    try {
+      formatter
+    } catch checkLegacyFormatter(pattern, legacyFormatter.validatePatternString)
   }
 }
 
 /**
- * The formatter parses/formats timestamps according to the pattern `uuuu-MM-dd HH:mm:ss.[..fff..]`
+ * The formatter parses/formats timestamps according to the pattern `yyyy-MM-dd HH:mm:ss.[..fff..]`
  * where `[..fff..]` is a fraction of second up to microsecond resolution. The formatter does not
  * output trailing zeros in the fraction. For example, the timestamp `2019-03-05 15:00:01.123400` is
  * formatted as the string `2019-03-05 15:00:01.1234`.
@@ -80,26 +119,208 @@ class Iso8601TimestampFormatter(
  * @param zoneId the time zone identifier in which the formatter parses or format timestamps
  */
 class FractionTimestampFormatter(zoneId: ZoneId)
-  extends Iso8601TimestampFormatter("", zoneId, TimestampFormatter.defaultLocale) {
+  extends Iso8601TimestampFormatter(
+    TimestampFormatter.defaultPattern,
+    zoneId,
+    TimestampFormatter.defaultLocale,
+    LegacyDateFormats.FAST_DATE_FORMAT,
+    isParsing = false) {
 
   @transient
   override protected lazy val formatter = DateTimeFormatterHelper.fractionFormatter
+
+  // The new formatter will omit the trailing 0 in the timestamp string, but the legacy formatter
+  // can't. Here we borrow the code from Spark 2.4 DateTimeUtils.timestampToString to omit the
+  // trailing 0 for the legacy formatter as well.
+  override def format(ts: Timestamp): String = {
+    val timestampString = ts.toString
+    val formatted = legacyFormatter.format(ts)
+
+    if (timestampString.length > 19 && timestampString.substring(19) != ".0") {
+      formatted + timestampString.substring(19)
+    } else {
+      formatted
+    }
+  }
+}
+
+/**
+ * The custom sub-class of `GregorianCalendar` is needed to get access to
+ * protected `fields` immediately after parsing. We cannot use
+ * the `get()` method because it performs normalization of the fraction
+ * part. Accordingly, the `MILLISECOND` field doesn't contain original value.
+ *
+ * Also this class allows to set raw value to the `MILLISECOND` field
+ * directly before formatting.
+ */
+class MicrosCalendar(tz: TimeZone, digitsInFraction: Int)
+  extends GregorianCalendar(tz, Locale.US) {
+  // Converts parsed `MILLISECOND` field to seconds fraction in microsecond precision.
+  // For example if the fraction pattern is `SSSS` then `digitsInFraction` = 4, and
+  // if the `MILLISECOND` field was parsed to `1234`.
+  def getMicros(): SQLTimestamp = {
+    // Append 6 zeros to the field: 1234 -> 1234000000
+    val d = fields(Calendar.MILLISECOND) * MICROS_PER_SECOND
+    // Take the first 6 digits from `d`: 1234000000 -> 123400
+    // The rest contains exactly `digitsInFraction`: `0000` = 10 ^ digitsInFraction
+    // So, the result is `(1234 * 1000000) / (10 ^ digitsInFraction)
+    d / Decimal.POW_10(digitsInFraction)
+  }
+
+  // Converts the seconds fraction in microsecond precision to a value
+  // that can be correctly formatted according to the specified fraction pattern.
+  // The method performs operations opposite to `getMicros()`.
+  def setMicros(micros: Long): Unit = {
+    val d = micros * Decimal.POW_10(digitsInFraction)
+    fields(Calendar.MILLISECOND) = (d / MICROS_PER_SECOND).toInt
+  }
+}
+
+class LegacyFastTimestampFormatter(
+    pattern: String,
+    zoneId: ZoneId,
+    locale: Locale) extends TimestampFormatter {
+
+  @transient private lazy val fastDateFormat =
+    FastDateFormat.getInstance(pattern, TimeZone.getTimeZone(zoneId), locale)
+  @transient private lazy val cal = new MicrosCalendar(
+    fastDateFormat.getTimeZone,
+    fastDateFormat.getPattern.count(_ == 'S'))
+
+  override def parse(s: String): SQLTimestamp = {
+    cal.clear() // Clear the calendar because it can be re-used many times
+    if (!fastDateFormat.parse(s, new ParsePosition(0), cal)) {
+      throw new IllegalArgumentException(s"'$s' is an invalid timestamp")
+    }
+    val micros = cal.getMicros()
+    cal.set(Calendar.MILLISECOND, 0)
+    val julianMicros = Math.addExact(fromMillis(cal.getTimeInMillis), micros)
+    rebaseJulianToGregorianMicros(julianMicros)
+  }
+
+  override def format(timestamp: SQLTimestamp): String = {
+    val julianMicros = rebaseGregorianToJulianMicros(timestamp)
+    cal.setTimeInMillis(Math.floorDiv(julianMicros, MICROS_PER_SECOND) * MILLIS_PER_SECOND)
+    cal.setMicros(Math.floorMod(julianMicros, MICROS_PER_SECOND))
+    fastDateFormat.format(cal)
+  }
+
+  override def format(ts: Timestamp): String = {
+    if (ts.getNanos == 0) {
+      fastDateFormat.format(ts)
+    } else {
+      format(fromJavaTimestamp(ts))
+    }
+  }
+
+  override def format(instant: Instant): String = {
+    format(instantToMicros(instant))
+  }
+
+  override def validatePatternString(): Unit = fastDateFormat
+}
+
+class LegacySimpleTimestampFormatter(
+    pattern: String,
+    zoneId: ZoneId,
+    locale: Locale,
+    lenient: Boolean = true) extends TimestampFormatter {
+  @transient private lazy val sdf = {
+    val formatter = new SimpleDateFormat(pattern, locale)
+    formatter.setTimeZone(TimeZone.getTimeZone(zoneId))
+    formatter.setLenient(lenient)
+    formatter
+  }
+
+  override def parse(s: String): Long = {
+    fromJavaTimestamp(new Timestamp(sdf.parse(s).getTime))
+  }
+
+  override def format(us: Long): String = {
+    sdf.format(toJavaTimestamp(us))
+  }
+
+  override def format(ts: Timestamp): String = {
+    sdf.format(ts)
+  }
+
+  override def format(instant: Instant): String = {
+    format(instantToMicros(instant))
+  }
+
+  override def validatePatternString(): Unit = sdf
+}
+
+object LegacyDateFormats extends Enumeration {
+  type LegacyDateFormat = Value
+  val FAST_DATE_FORMAT, SIMPLE_DATE_FORMAT, LENIENT_SIMPLE_DATE_FORMAT = Value
 }
 
 object TimestampFormatter {
-  val defaultPattern: String = "uuuu-MM-dd HH:mm:ss"
+  import LegacyDateFormats._
+
   val defaultLocale: Locale = Locale.US
 
-  def apply(format: String, zoneId: ZoneId, locale: Locale): TimestampFormatter = {
-    new Iso8601TimestampFormatter(format, zoneId, locale)
+  def defaultPattern(): String = s"${DateFormatter.defaultPattern} HH:mm:ss"
+
+  private def getFormatter(
+      format: Option[String],
+      zoneId: ZoneId,
+      locale: Locale = defaultLocale,
+      legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
+      isParsing: Boolean): TimestampFormatter = {
+    val pattern = format.getOrElse(defaultPattern)
+    if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
+      getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
+    } else {
+      val tf = new Iso8601TimestampFormatter(
+        pattern, zoneId, locale, legacyFormat, isParsing)
+      tf.validatePatternString()
+      tf
+    }
+  }
+
+  def getLegacyFormatter(
+      pattern: String,
+      zoneId: ZoneId,
+      locale: Locale,
+      legacyFormat: LegacyDateFormat): TimestampFormatter = {
+    legacyFormat match {
+      case FAST_DATE_FORMAT =>
+        new LegacyFastTimestampFormatter(pattern, zoneId, locale)
+      case SIMPLE_DATE_FORMAT =>
+        new LegacySimpleTimestampFormatter(pattern, zoneId, locale, lenient = false)
+      case LENIENT_SIMPLE_DATE_FORMAT =>
+        new LegacySimpleTimestampFormatter(pattern, zoneId, locale, lenient = true)
+    }
+  }
+
+  def apply(
+      format: String,
+      zoneId: ZoneId,
+      locale: Locale,
+      legacyFormat: LegacyDateFormat,
+      isParsing: Boolean): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, locale, legacyFormat, isParsing)
+  }
+
+  def apply(
+      format: String,
+      zoneId: ZoneId,
+      legacyFormat: LegacyDateFormat,
+      isParsing: Boolean): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, defaultLocale, legacyFormat, isParsing)
   }
 
-  def apply(format: String, zoneId: ZoneId): TimestampFormatter = {
-    apply(format, zoneId, defaultLocale)
+  def apply(
+      format: String,
+      zoneId: ZoneId,
+      isParsing: Boolean): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, isParsing = isParsing)
   }
 
   def apply(zoneId: ZoneId): TimestampFormatter = {
-    apply(defaultPattern, zoneId, defaultLocale)
+    getFormatter(None, zoneId, isParsing = false)
   }
 
   def getFractionFormatter(zoneId: ZoneId): TimestampFormatter = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
index 3478af8783af6..d90804f4b6ff6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
@@ -20,7 +20,9 @@ package org.apache.spark.sql.connector.catalog
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.BucketSpec
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.connector.expressions.{BucketTransform, IdentityTransform, LogicalExpressions, Transform}
+import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Conversion helpers for working with v2 [[CatalogPlugin]].
@@ -84,15 +86,15 @@ private[sql] object CatalogV2Implicits {
   }
 
   implicit class NamespaceHelper(namespace: Array[String]) {
-    def quoted: String = namespace.map(quote).mkString(".")
+    def quoted: String = namespace.map(quoteIfNeeded).mkString(".")
   }
 
   implicit class IdentifierHelper(ident: Identifier) {
     def quoted: String = {
       if (ident.namespace.nonEmpty) {
-        ident.namespace.map(quote).mkString(".") + "." + quote(ident.name)
+        ident.namespace.map(quoteIfNeeded).mkString(".") + "." + quoteIfNeeded(ident.name)
       } else {
-        quote(ident.name)
+        quoteIfNeeded(ident.name)
       }
     }
 
@@ -122,14 +124,20 @@ private[sql] object CatalogV2Implicits {
           s"$quoted is not a valid TableIdentifier as it has more than 2 name parts.")
     }
 
-    def quoted: String = parts.map(quote).mkString(".")
+    def quoted: String = parts.map(quoteIfNeeded).mkString(".")
   }
 
-  def quote(part: String): String = {
+  def quoteIfNeeded(part: String): String = {
     if (part.contains(".") || part.contains("`")) {
       s"`${part.replace("`", "``")}`"
     } else {
       part
     }
   }
+
+  private lazy val catalystSqlParser = new CatalystSqlParser(SQLConf.get)
+
+  def parseColumnPath(name: String): Seq[String] = {
+    catalystSqlParser.parseMultipartIdentifier(name)
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
index 0fabe4df6c9a4..e1f329352592f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
@@ -22,11 +22,13 @@ import java.util.Collections
 
 import scala.collection.JavaConverters._
 
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.{NamedRelation, NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException, UnresolvedV2Relation}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.catalyst.plans.logical.AlterTable
 import org.apache.spark.sql.connector.catalog.TableChange._
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.types.{ArrayType, MapType, StructField, StructType}
+import org.apache.spark.sql.types.{ArrayType, DataType, HIVE_TYPE_STRING, HiveStringType, MapType, StructField, StructType}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.Utils
 
@@ -298,10 +300,9 @@ private[sql] object CatalogV2Util {
       options: Map[String, String],
       location: Option[String],
       comment: Option[String],
-      provider: String): Map[String, String] = {
-    properties ++
-      options ++
-      Map(TableCatalog.PROP_PROVIDER -> provider) ++
+      provider: Option[String]): Map[String, String] = {
+    properties ++ options ++
+      provider.map(TableCatalog.PROP_PROVIDER -> _) ++
       comment.map(TableCatalog.PROP_COMMENT -> _) ++
       location.map(TableCatalog.PROP_LOCATION -> _)
   }
@@ -330,4 +331,19 @@ private[sql] object CatalogV2Util {
       .getOrElse(catalogManager.v2SessionCatalog)
       .asTableCatalog
   }
+
+  def failCharType(dt: DataType): Unit = {
+    if (HiveStringType.containsCharType(dt)) {
+      throw new AnalysisException(
+        "Cannot use CHAR type in non-Hive-Serde tables, please use STRING type instead.")
+    }
+  }
+
+  def assertNoCharTypeInSchema(schema: StructType): Unit = {
+    schema.foreach { f =>
+      if (f.metadata.contains(HIVE_TYPE_STRING)) {
+        failCharType(CatalystSqlParser.parseRawDataType(f.metadata.getString(HIVE_TYPE_STRING)))
+      }
+    }
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala
new file mode 100644
index 0000000000000..cc4395e7bd62d
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog
+
+import java.lang.reflect.InvocationTargetException
+import java.util
+import java.util.NoSuchElementException
+import java.util.regex.Pattern
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.util.Utils
+
+private[sql] object Catalogs {
+  /**
+   * Load and configure a catalog by name.
+   * <p>
+   * This loads, instantiates, and initializes the catalog plugin for each call; it does not cache
+   * or reuse instances.
+   *
+   * @param name a String catalog name
+   * @param conf a SQLConf
+   * @return an initialized CatalogPlugin
+   * @throws CatalogNotFoundException if the plugin class cannot be found
+   * @throws org.apache.spark.SparkException           if the plugin class cannot be instantiated
+   */
+  @throws[CatalogNotFoundException]
+  @throws[SparkException]
+  def load(name: String, conf: SQLConf): CatalogPlugin = {
+    val pluginClassName = try {
+      conf.getConfString("spark.sql.catalog." + name)
+    } catch {
+      case _: NoSuchElementException =>
+        throw new CatalogNotFoundException(
+          s"Catalog '$name' plugin class not found: spark.sql.catalog.$name is not defined")
+    }
+    val loader = Utils.getContextOrSparkClassLoader
+    try {
+      val pluginClass = loader.loadClass(pluginClassName)
+      if (!classOf[CatalogPlugin].isAssignableFrom(pluginClass)) {
+        throw new SparkException(
+          s"Plugin class for catalog '$name' does not implement CatalogPlugin: $pluginClassName")
+      }
+      val plugin = pluginClass.getDeclaredConstructor().newInstance().asInstanceOf[CatalogPlugin]
+      plugin.initialize(name, catalogOptions(name, conf))
+      plugin
+    } catch {
+      case _: ClassNotFoundException =>
+        throw new SparkException(
+          s"Cannot find catalog plugin class for catalog '$name': $pluginClassName")
+      case e: NoSuchMethodException =>
+        throw new SparkException(
+          s"Failed to find public no-arg constructor for catalog '$name': $pluginClassName)", e)
+      case e: IllegalAccessException =>
+        throw new SparkException(
+          s"Failed to call public no-arg constructor for catalog '$name': $pluginClassName)", e)
+      case e: InstantiationException =>
+        throw new SparkException("Cannot instantiate abstract catalog plugin class for " +
+          s"catalog '$name': $pluginClassName", e.getCause)
+      case e: InvocationTargetException =>
+        throw new SparkException("Failed during instantiating constructor for catalog " +
+          s"'$name': $pluginClassName", e.getCause)
+    }
+  }
+
+  /**
+   * Extracts a named catalog's configuration from a SQLConf.
+   *
+   * @param name a catalog name
+   * @param conf a SQLConf
+   * @return a case insensitive string map of options starting with spark.sql.catalog.(name).
+   */
+  private def catalogOptions(name: String, conf: SQLConf) = {
+    val prefix = Pattern.compile("^spark\\.sql\\.catalog\\." + name + "\\.(.+)")
+    val options = new util.HashMap[String, String]
+    conf.getAllConfs.foreach {
+      case (key, value) =>
+        val matcher = prefix.matcher(key)
+        if (matcher.matches && matcher.groupCount > 0) options.put(matcher.group(1), value)
+    }
+    new CaseInsensitiveStringMap(options)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
index 080ddf1d027e9..10c15747ec4ce 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.connector.catalog
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 
@@ -56,8 +57,14 @@ private[sql] trait LookupCatalog extends Logging {
    * Extract session catalog and identifier from a multi-part identifier.
    */
   object SessionCatalogAndIdentifier {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+
     def unapply(parts: Seq[String]): Option[(CatalogPlugin, Identifier)] = parts match {
       case CatalogAndIdentifier(catalog, ident) if CatalogV2Util.isSessionCatalog(catalog) =>
+        if (ident.namespace.length != 1) {
+          throw new AnalysisException(
+            s"The namespace in session catalog must have exactly one name part: ${parts.quoted}")
+        }
         Some(catalog, ident)
       case _ => None
     }
@@ -94,6 +101,10 @@ private[sql] trait LookupCatalog extends Logging {
    * Extract catalog and identifier from a multi-part name with the current catalog if needed.
    * Catalog name takes precedence over identifier, but for a single-part name, identifier takes
    * precedence over catalog name.
+   *
+   * Note that, this pattern is used to look up permanent catalog objects like table, view,
+   * function, etc. If you need to look up temp objects like temp view, please do it separately
+   * before calling this pattern, as temp objects don't belong to any catalog.
    */
   object CatalogAndIdentifier {
     import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
@@ -103,16 +114,7 @@ private[sql] trait LookupCatalog extends Logging {
     def unapply(nameParts: Seq[String]): Option[(CatalogPlugin, Identifier)] = {
       assert(nameParts.nonEmpty)
       if (nameParts.length == 1) {
-        // If the current catalog is session catalog, the current namespace is not used because
-        // the single-part name could be referencing a temp view, which doesn't belong to any
-        // namespaces. An empty namespace will be resolved inside the session catalog
-        // implementation when a relation is looked up.
-        val ns = if (CatalogV2Util.isSessionCatalog(currentCatalog)) {
-          Array.empty[String]
-        } else {
-          catalogManager.currentNamespace
-        }
-        Some((currentCatalog, Identifier.of(ns, nameParts.head)))
+        Some((currentCatalog, Identifier.of(catalogManager.currentNamespace, nameParts.head)))
       } else if (nameParts.head.equalsIgnoreCase(globalTempDB)) {
         // Conceptually global temp views are in a special reserved catalog. However, the v2 catalog
         // API does not support view yet, and we have to use v1 commands to deal with global temp
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala
index 91e0c58a1c6d0..70fc9689e6087 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala
@@ -24,6 +24,7 @@ import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.CatalogTable
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.connector.expressions.{LogicalExpressions, Transform}
 import org.apache.spark.sql.types.StructType
 
@@ -35,20 +36,12 @@ private[sql] case class V1Table(v1Table: CatalogTable) extends Table {
     def quoted: String = {
       identifier.database match {
         case Some(db) =>
-          Seq(db, identifier.table).map(quote).mkString(".")
+          Seq(db, identifier.table).map(quoteIfNeeded).mkString(".")
         case _ =>
-          quote(identifier.table)
+          quoteIfNeeded(identifier.table)
 
       }
     }
-
-    private def quote(part: String): String = {
-      if (part.contains(".") || part.contains("`")) {
-        s"`${part.replace("`", "``")}`"
-      } else {
-        part
-      }
-    }
   }
 
   def catalogTable: CatalogTable = v1Table
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
index 84adddf2671f5..ea26847ac3cef 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
@@ -64,7 +64,7 @@ private[sql] object LogicalExpressions {
 /**
  * Allows Spark to rewrite the given references of the transform during analysis.
  */
-sealed trait RewritableTransform extends Transform {
+private[sql] sealed trait RewritableTransform extends Transform {
   /** Creates a copy of this transform with the new analyzed references. */
   def withReferences(newReferences: Seq[NamedReference]): Transform
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 04572c38be8dd..0cf980cfc0477 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -24,6 +24,7 @@ import java.util.zip.Deflater
 
 import scala.collection.JavaConverters._
 import scala.collection.immutable
+import scala.util.Try
 import scala.util.matching.Regex
 
 import org.apache.hadoop.fs.Path
@@ -33,10 +34,12 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.{IGNORE_MISSING_FILES => SPARK_IGNORE_MISSING_FILES}
 import org.apache.spark.network.util.ByteUnit
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.{HintErrorLogger, Resolver}
 import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
 import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator
 import org.apache.spark.sql.catalyst.plans.logical.HintErrorHandler
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
 import org.apache.spark.unsafe.array.ByteArrayMethods
 import org.apache.spark.util.Utils
@@ -174,6 +177,7 @@ object SQLConf {
   val ANALYZER_MAX_ITERATIONS = buildConf("spark.sql.analyzer.maxIterations")
     .internal()
     .doc("The max number of iterations the analyzer runs.")
+    .version("3.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -182,12 +186,14 @@ object SQLConf {
       "specified by their rule names and separated by comma. It is not guaranteed that all the " +
       "rules in this configuration will eventually be excluded, as some rules are necessary " +
       "for correctness. The optimizer will log the rules that have indeed been excluded.")
+    .version("2.4.0")
     .stringConf
     .createOptional
 
   val OPTIMIZER_MAX_ITERATIONS = buildConf("spark.sql.optimizer.maxIterations")
     .internal()
     .doc("The max number of iterations the optimizer runs.")
+    .version("2.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -195,6 +201,7 @@ object SQLConf {
     buildConf("spark.sql.optimizer.inSetConversionThreshold")
       .internal()
       .doc("The threshold of set size for InSet conversion.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -203,6 +210,7 @@ object SQLConf {
       .internal()
       .doc("Configures the max set size in InSet for which Spark will generate code with " +
         "switch statements. This is applicable only to bytes, shorts, ints, dates.")
+      .version("3.0.0")
       .intConf
       .checkValue(threshold => threshold >= 0 && threshold <= 600, "The max set size " +
         "for using switch statements in InSet must be non-negative and less than or equal to 600")
@@ -213,6 +221,7 @@ object SQLConf {
     .doc("Configures the log level for logging the change from the original plan to the new " +
       "plan after a rule or batch is applied. The value can be 'trace', 'debug', 'info', " +
       "'warn', or 'error'. The default log level is 'trace'.")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValue(logLevel => Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR").contains(logLevel),
@@ -224,6 +233,7 @@ object SQLConf {
     .internal()
     .doc("Configures a list of rules to be logged in the optimizer, in which the rules are " +
       "specified by their rule names and separated by comma.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
@@ -231,12 +241,14 @@ object SQLConf {
     .internal()
     .doc("Configures a list of batches to be logged in the optimizer, in which the batches " +
       "are specified by their batch names and separated by comma.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
   val DYNAMIC_PARTITION_PRUNING_ENABLED =
     buildConf("spark.sql.optimizer.dynamicPartitionPruning.enabled")
       .doc("When true, we will generate predicate for partition column when it's used as join key")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -246,6 +258,7 @@ object SQLConf {
       .doc("When true, distinct count statistics will be used for computing the data size of the " +
         "partitioned table after dynamic partition pruning, in order to evaluate if it is worth " +
         "adding an extra subquery as the pruning filter if broadcast reuse is not applicable.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -256,26 +269,30 @@ object SQLConf {
       "used as the fallback filter ratio for computing the data size of the partitioned table " +
       "after dynamic partition pruning, in order to evaluate if it is worth adding an extra " +
       "subquery as the pruning filter if broadcast reuse is not applicable.")
+    .version("3.0.0")
     .doubleConf
     .createWithDefault(0.5)
 
-  val DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST =
-    buildConf("spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcast")
+  val DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY =
+    buildConf("spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly")
       .internal()
-      .doc("When true, dynamic partition pruning will seek to reuse the broadcast results from " +
-        "a broadcast hash join operation.")
+      .doc("When true, dynamic partition pruning will only apply when the broadcast exchange of " +
+        "a broadcast hash join operation can be reused as the dynamic pruning filter.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val COMPRESS_CACHED = buildConf("spark.sql.inMemoryColumnarStorage.compressed")
     .doc("When set to true Spark SQL will automatically select a compression codec for each " +
       "column based on statistics of the data.")
+    .version("1.0.1")
     .booleanConf
     .createWithDefault(true)
 
   val COLUMN_BATCH_SIZE = buildConf("spark.sql.inMemoryColumnarStorage.batchSize")
     .doc("Controls the size of batches for columnar caching.  Larger batch sizes can improve " +
       "memory utilization and compression, but risk OOMs when caching data.")
+    .version("1.1.1")
     .intConf
     .createWithDefault(10000)
 
@@ -283,6 +300,7 @@ object SQLConf {
     buildConf("spark.sql.inMemoryColumnarStorage.partitionPruning")
       .internal()
       .doc("When true, enable partition pruning for in-memory columnar tables.")
+      .version("1.2.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -290,12 +308,14 @@ object SQLConf {
     buildConf("spark.sql.inMemoryTableScanStatistics.enable")
       .internal()
       .doc("When true, enable in-memory table scan accumulators.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val CACHE_VECTORIZED_READER_ENABLED =
     buildConf("spark.sql.inMemoryColumnarStorage.enableVectorizedReader")
       .doc("Enables vectorized reader for columnar caching.")
+      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -303,12 +323,14 @@ object SQLConf {
     buildConf("spark.sql.columnVector.offheap.enabled")
       .internal()
       .doc("When true, use OffHeapColumnVector in ColumnarBatch.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   val PREFER_SORTMERGEJOIN = buildConf("spark.sql.join.preferSortMergeJoin")
     .internal()
     .doc("When true, prefer sort merge join over shuffle hash join.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -317,6 +339,7 @@ object SQLConf {
     .doc("When true, enable use of radix sort when possible. Radix sort is much faster but " +
       "requires additional memory to be reserved up-front. The memory overhead may be " +
       "significant when sorting very small rows (up to 50% more in this case).")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -324,17 +347,19 @@ object SQLConf {
     .doc("Configures the maximum size in bytes for a table that will be broadcast to all worker " +
       "nodes when performing a join.  By setting this value to -1 broadcasting can be disabled. " +
       "Note that currently statistics are only supported for Hive Metastore tables where the " +
-      "command <code>ANALYZE TABLE &lt;tableName&gt; COMPUTE STATISTICS noscan</code> has been " +
+      "command `ANALYZE TABLE <tableName> COMPUTE STATISTICS noscan` has been " +
       "run, and file-based data source tables where the statistics are computed directly on " +
       "the files of data.")
+    .version("1.1.0")
     .bytesConf(ByteUnit.BYTE)
-    .createWithDefault(10L * 1024 * 1024)
+    .createWithDefaultString("10MB")
 
   val LIMIT_SCALE_UP_FACTOR = buildConf("spark.sql.limit.scaleUpFactor")
     .internal()
     .doc("Minimal increase rate in number of partitions between attempts when executing a take " +
       "on a query. Higher values lead to more partitions read. Lower values might lead to " +
       "longer execution times as more jobs will be run")
+    .version("2.1.1")
     .intConf
     .createWithDefault(4)
 
@@ -342,6 +367,7 @@ object SQLConf {
     buildConf("spark.sql.hive.advancedPartitionPredicatePushdown.enabled")
       .internal()
       .doc("When true, advanced partition predicate pushdown into Hive metastore is enabled.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -349,108 +375,145 @@ object SQLConf {
     .doc("The default number of partitions to use when shuffling data for joins or aggregations. " +
       "Note: For structured streaming, this configuration cannot be changed between query " +
       "restarts from the same checkpoint location.")
+    .version("1.1.0")
     .intConf
     .checkValue(_ > 0, "The value of spark.sql.shuffle.partitions must be positive")
     .createWithDefault(200)
 
+  val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE =
+    buildConf("spark.sql.adaptive.shuffle.targetPostShuffleInputSize")
+      .internal()
+      .doc("(Deprecated since Spark 3.0)")
+      .version("1.6.0")
+      .bytesConf(ByteUnit.BYTE)
+      .createWithDefaultString("64MB")
+
   val ADAPTIVE_EXECUTION_ENABLED = buildConf("spark.sql.adaptive.enabled")
-    .doc("When true, enable adaptive query execution.")
+    .doc("When true, enable adaptive query execution, which re-optimizes the query plan in the " +
+      "middle of query execution, based on accurate runtime statistics.")
+    .version("1.6.0")
     .booleanConf
     .createWithDefault(false)
 
-  val REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED =
-    buildConf("spark.sql.adaptive.shuffle.reducePostShufflePartitions.enabled")
-      .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled, this enables reducing " +
-        "the number of post-shuffle partitions based on map output statistics.")
-      .booleanConf
-      .createWithDefault(true)
+  val ADAPTIVE_EXECUTION_FORCE_APPLY = buildConf("spark.sql.adaptive.forceApply")
+    .internal()
+    .doc("Adaptive query execution is skipped when the query does not have exchanges or " +
+      "sub-queries. By setting this config to true (together with " +
+      s"'${ADAPTIVE_EXECUTION_ENABLED.key}' set to true), Spark will force apply adaptive query " +
+      "execution for all supported queries.")
+    .version("3.0.0")
+    .booleanConf
+    .createWithDefault(false)
 
-  val FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED =
-    buildConf("spark.sql.adaptive.shuffle.fetchShuffleBlocksInBatch.enabled")
-      .doc("Whether to fetch the continuous shuffle blocks in batch. Instead of fetching blocks " +
-        "one by one, fetching continuous shuffle blocks for the same map task in batch can " +
-        "reduce IO and improve performance. Note, multiple continuous blocks exist in single " +
-        s"fetch request only happen when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
-        s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled, this feature also depends " +
-        "on a relocatable serializer, the concatenation support codec in use and the new version " +
-        "shuffle fetch protocol.")
+  val ADAPTIVE_EXECUTION_LOG_LEVEL = buildConf("spark.sql.adaptive.logLevel")
+    .internal()
+    .doc("Configures the log level for adaptive execution logging of plan changes. The value " +
+      "can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'debug'.")
+    .version("3.0.0")
+    .stringConf
+    .transform(_.toUpperCase(Locale.ROOT))
+    .checkValues(Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR"))
+    .createWithDefault("debug")
+
+  val ADVISORY_PARTITION_SIZE_IN_BYTES =
+    buildConf("spark.sql.adaptive.advisoryPartitionSizeInBytes")
+      .doc("The advisory size in bytes of the shuffle partition during adaptive optimization " +
+        s"(when ${ADAPTIVE_EXECUTION_ENABLED.key} is true). It takes effect when Spark " +
+        "coalesces small shuffle partitions or splits skewed shuffle partition.")
+      .version("3.0.0")
+      .fallbackConf(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
+
+  val COALESCE_PARTITIONS_ENABLED =
+    buildConf("spark.sql.adaptive.coalescePartitions.enabled")
+      .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark will coalesce " +
+        "contiguous shuffle partitions according to the target size (specified by " +
+        s"'${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'), to avoid too many small tasks.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
-  val SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS =
-    buildConf("spark.sql.adaptive.shuffle.minNumPostShufflePartitions")
-      .doc("The advisory minimum number of post-shuffle partitions used when " +
-        s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
-        s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled.")
+  val COALESCE_PARTITIONS_MIN_PARTITION_NUM =
+    buildConf("spark.sql.adaptive.coalescePartitions.minPartitionNum")
+      .doc("The minimum number of shuffle partitions after coalescing. If not set, the default " +
+        "value is the default parallelism of the Spark cluster. This configuration only " +
+        s"has an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
+        s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true.")
+      .version("3.0.0")
       .intConf
-      .checkValue(_ > 0, "The minimum shuffle partition number " +
-        "must be a positive integer.")
-      .createWithDefault(1)
+      .checkValue(_ > 0, "The minimum number of partitions must be positive.")
+      .createOptional
 
-  val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE =
-    buildConf("spark.sql.adaptive.shuffle.targetPostShuffleInputSize")
-      .doc("The target post-shuffle input size in bytes of a task. This configuration only has " +
-        s"an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
-        s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled.")
-      .bytesConf(ByteUnit.BYTE)
-      .createWithDefault(64 * 1024 * 1024)
-
-  val SHUFFLE_MAX_NUM_POSTSHUFFLE_PARTITIONS =
-    buildConf("spark.sql.adaptive.shuffle.maxNumPostShufflePartitions")
-      .doc("The advisory maximum number of post-shuffle partitions used in adaptive execution. " +
-        "This is used as the initial number of pre-shuffle partitions. By default it equals to " +
-        "spark.sql.shuffle.partitions. This configuration only has an effect when " +
-        s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
-        s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled.")
+  val COALESCE_PARTITIONS_INITIAL_PARTITION_NUM =
+    buildConf("spark.sql.adaptive.coalescePartitions.initialPartitionNum")
+      .doc("The initial number of shuffle partitions before coalescing. By default it equals to " +
+        s"${SHUFFLE_PARTITIONS.key}. This configuration only has an effect when " +
+        s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and '${COALESCE_PARTITIONS_ENABLED.key}' " +
+        "are both true.")
+      .version("3.0.0")
       .intConf
-      .checkValue(_ > 0, "The maximum shuffle partition number " +
-        "must be a positive integer.")
+      .checkValue(_ > 0, "The initial number of partitions must be positive.")
       .createOptional
 
-  val LOCAL_SHUFFLE_READER_ENABLED =
-    buildConf("spark.sql.adaptive.shuffle.localShuffleReader.enabled")
-    .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled, this enables the " +
-      "optimization of converting the shuffle reader to local shuffle reader for the shuffle " +
-      "exchange of the broadcast hash join in probe side.")
-    .booleanConf
-    .createWithDefault(true)
-
-  val ADAPTIVE_EXECUTION_SKEWED_JOIN_ENABLED =
-    buildConf("spark.sql.adaptive.optimizeSkewedJoin.enabled")
-    .doc("When true and adaptive execution is enabled, a skewed join is automatically handled at " +
-      "runtime.")
-    .booleanConf
-    .createWithDefault(true)
+  val FETCH_SHUFFLE_BLOCKS_IN_BATCH =
+    buildConf("spark.sql.adaptive.fetchShuffleBlocksInBatch")
+      .internal()
+      .doc("Whether to fetch the contiguous shuffle blocks in batch. Instead of fetching blocks " +
+        "one by one, fetching contiguous shuffle blocks for the same map task in batch can " +
+        "reduce IO and improve performance. Note, multiple contiguous blocks exist in single " +
+        s"fetch request only happen when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
+        s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true. This feature also depends " +
+        "on a relocatable serializer, the concatenation support codec in use and the new version " +
+        "shuffle fetch protocol.")
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(true)
 
-  val ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD =
-    buildConf("spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionSizeThreshold")
-      .doc("Configures the minimum size in bytes for a partition that is considered as a skewed " +
-        "partition in adaptive skewed join.")
-      .bytesConf(ByteUnit.BYTE)
-      .createWithDefault(64 * 1024 * 1024)
+  val LOCAL_SHUFFLE_READER_ENABLED =
+    buildConf("spark.sql.adaptive.localShuffleReader.enabled")
+      .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark tries to use local " +
+        "shuffle reader to read the shuffle data when the shuffle partitioning is not needed, " +
+        "for example, after converting sort-merge join to broadcast-hash join.")
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(true)
 
-  val ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR =
-    buildConf("spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionFactor")
-      .doc("A partition is considered as a skewed partition if its size is larger than" +
-        " this factor multiple the median partition size and also larger than " +
-        s" ${ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key}")
-      .intConf
-      .createWithDefault(10)
+  val SKEW_JOIN_ENABLED =
+    buildConf("spark.sql.adaptive.skewJoin.enabled")
+      .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark dynamically " +
+        "handles skew in sort-merge join by splitting (and replicating if needed) skewed " +
+        "partitions.")
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(true)
 
-  val ADAPTIVE_EXECUTION_SKEWED_PARTITION_MAX_SPLITS =
-    buildConf("spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionMaxSplits")
-      .doc("Configures the maximum number of task to handle a skewed partition in adaptive skewed" +
-        "join.")
+  val SKEW_JOIN_SKEWED_PARTITION_FACTOR =
+    buildConf("spark.sql.adaptive.skewJoin.skewedPartitionFactor")
+      .doc("A partition is considered as skewed if its size is larger than this factor " +
+        "multiplying the median partition size and also larger than " +
+        "'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes'")
+      .version("3.0.0")
       .intConf
-      .checkValue( _ >= 1, "The split size at least be 1")
+      .checkValue(_ > 0, "The skew factor must be positive.")
       .createWithDefault(5)
 
+  val SKEW_JOIN_SKEWED_PARTITION_THRESHOLD =
+    buildConf("spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes")
+      .doc("A partition is considered as skewed if its size in bytes is larger than this " +
+        s"threshold and also larger than '${SKEW_JOIN_SKEWED_PARTITION_FACTOR.key}' " +
+        "multiplying the median partition size. Ideally this config should be set larger " +
+        s"than '${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'.")
+      .version("3.0.0")
+      .bytesConf(ByteUnit.BYTE)
+      .createWithDefaultString("256MB")
+
   val NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN =
     buildConf("spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin")
+      .internal()
       .doc("The relation with a non-empty partition ratio lower than this config will not be " +
         "considered as the build side of a broadcast-hash join in adaptive execution regardless " +
         "of its size.This configuration only has an effect when " +
-        s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled.")
+        s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is true.")
+      .version("3.0.0")
       .doubleConf
       .checkValue(_ >= 0, "The non-empty partition ratio must be positive number.")
       .createWithDefault(0.2)
@@ -459,6 +522,7 @@ object SQLConf {
     buildConf("spark.sql.subexpressionElimination.enabled")
       .internal()
       .doc("When true, common subexpressions will be eliminated.")
+      .version("1.6.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -466,6 +530,7 @@ object SQLConf {
     .internal()
     .doc("Whether the query analyzer should be case sensitive or not. " +
       "Default to case insensitive. It is highly discouraged to turn on case sensitive mode.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -475,6 +540,7 @@ object SQLConf {
       "plan to optimize them. Constraint propagation can sometimes be computationally expensive " +
       "for certain kinds of query plans (such as those with a large number of predicates and " +
       "aliases) which might negatively impact overall runtime.")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -483,6 +549,7 @@ object SQLConf {
     .doc("When true, string literals (including regex patterns) remain escaped in our SQL " +
       "parser. The default is false since Spark 2.0. Setting it to true can restore the behavior " +
       "prior to Spark 2.0.")
+    .version("2.2.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -491,6 +558,7 @@ object SQLConf {
     .doc("When estimating the output data size of a table scan, multiply the file size with this " +
       "factor as the estimated data size, in case the data is compressed in the file and lead to" +
       " a heavily underestimated result.")
+    .version("2.3.1")
     .doubleConf
     .checkValue(_ > 0, "the value of fileDataSizeFactor must be greater than 0")
     .createWithDefault(1.0)
@@ -499,6 +567,7 @@ object SQLConf {
     .doc("When true, the Parquet data source merges schemas collected from all data files, " +
          "otherwise the schema is picked from the summary file or a random data file " +
          "if no summary file is available.")
+    .version("1.5.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -507,6 +576,7 @@ object SQLConf {
          "summary files and we will ignore them when merging schema. Otherwise, if this is " +
          "false, which is the default, we will merge all part-files. This should be considered " +
          "as expert-only option, and shouldn't be enabled before knowing what it means exactly.")
+    .version("1.5.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -515,6 +585,7 @@ object SQLConf {
       "Spark SQL, do not differentiate between binary data and strings when writing out the " +
       "Parquet schema. This flag tells Spark SQL to interpret binary data as a string to provide " +
       "compatibility with these systems.")
+    .version("1.1.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -523,6 +594,7 @@ object SQLConf {
       "Spark would also store Timestamp as INT96 because we need to avoid precision lost of the " +
       "nanoseconds field. This flag tells Spark SQL to interpret INT96 data as a timestamp to " +
       "provide compatibility with these systems.")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -530,6 +602,7 @@ object SQLConf {
     .doc("This controls whether timestamp adjustments should be applied to INT96 data when " +
       "converting to timestamps, for data written by Impala.  This is necessary because Impala " +
       "stores INT96 data with a different timezone offset than Hive & Spark.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -543,10 +616,11 @@ object SQLConf {
       "is a standard timestamp type in Parquet, which stores number of microseconds from the " +
       "Unix epoch. TIMESTAMP_MILLIS is also standard, but with millisecond precision, which " +
       "means Spark has to truncate the microsecond portion of its timestamp value.")
+    .version("2.3.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(ParquetOutputTimestampType.values.map(_.toString))
-    .createWithDefault(ParquetOutputTimestampType.TIMESTAMP_MICROS.toString)
+    .createWithDefault(ParquetOutputTimestampType.INT96.toString)
 
   val PARQUET_COMPRESSION = buildConf("spark.sql.parquet.compression.codec")
     .doc("Sets the compression codec used when writing Parquet files. If either `compression` or " +
@@ -554,6 +628,7 @@ object SQLConf {
       "precedence would be `compression`, `parquet.compression`, " +
       "`spark.sql.parquet.compression.codec`. Acceptable values include: none, uncompressed, " +
       "snappy, gzip, lzo, brotli, lz4, zstd.")
+    .version("1.1.1")
     .stringConf
     .transform(_.toLowerCase(Locale.ROOT))
     .checkValues(Set("none", "uncompressed", "snappy", "gzip", "lzo", "lz4", "brotli", "zstd"))
@@ -561,6 +636,7 @@ object SQLConf {
 
   val PARQUET_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.parquet.filterPushdown")
     .doc("Enables Parquet filter push-down optimization when set to true.")
+    .version("1.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -568,6 +644,7 @@ object SQLConf {
     .doc("If true, enables Parquet filter push-down optimization for Date. " +
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
       "enabled.")
+    .version("2.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -577,15 +654,17 @@ object SQLConf {
       .doc("If true, enables Parquet filter push-down optimization for Timestamp. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled and Timestamp stored as TIMESTAMP_MICROS or TIMESTAMP_MILLIS type.")
-    .internal()
-    .booleanConf
-    .createWithDefault(true)
+      .version("2.4.0")
+      .internal()
+      .booleanConf
+      .createWithDefault(true)
 
   val PARQUET_FILTER_PUSHDOWN_DECIMAL_ENABLED =
     buildConf("spark.sql.parquet.filterPushdown.decimal")
       .doc("If true, enables Parquet filter push-down optimization for Decimal. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled.")
+      .version("2.4.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -595,6 +674,7 @@ object SQLConf {
     .doc("If true, enables Parquet filter push-down optimization for string startsWith function. " +
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
       "enabled.")
+    .version("2.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -607,6 +687,7 @@ object SQLConf {
         "By setting this value to 0 this feature can be disabled. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled.")
+      .version("2.4.0")
       .internal()
       .intConf
       .checkValue(threshold => threshold >= 0, "The threshold must not be negative.")
@@ -618,6 +699,7 @@ object SQLConf {
       "systems such as Apache Hive and Apache Impala use. If false, the newer format in Parquet " +
       "will be used. For example, decimals will be written in int-based format. If Parquet " +
       "output is intended for use with systems that do not support this newer format, set to true.")
+    .version("1.6.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -627,6 +709,7 @@ object SQLConf {
       "of org.apache.parquet.hadoop.ParquetOutputCommitter. If it is not, then metadata " +
       "summaries will never be created, irrespective of the value of " +
       "parquet.summary.metadata.level")
+    .version("1.5.0")
     .internal()
     .stringConf
     .createWithDefault("org.apache.parquet.hadoop.ParquetOutputCommitter")
@@ -634,6 +717,7 @@ object SQLConf {
   val PARQUET_VECTORIZED_READER_ENABLED =
     buildConf("spark.sql.parquet.enableVectorizedReader")
       .doc("Enables vectorized parquet decoding.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -643,12 +727,14 @@ object SQLConf {
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' " +
       "is enabled and the vectorized reader is not used. You can ensure the vectorized reader " +
       s"is not used by setting '${PARQUET_VECTORIZED_READER_ENABLED.key}' to false.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val PARQUET_VECTORIZED_READER_BATCH_SIZE = buildConf("spark.sql.parquet.columnarReaderBatchSize")
     .doc("The number of rows to include in a parquet vectorized reader batch. The number should " +
       "be carefully chosen to minimize overhead and avoid OOMs in reading data.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(4096)
 
@@ -657,6 +743,7 @@ object SQLConf {
       "`orc.compress` is specified in the table-specific options/properties, the precedence " +
       "would be `compression`, `orc.compress`, `spark.sql.orc.compression.codec`." +
       "Acceptable values include: none, uncompressed, snappy, zlib, lzo.")
+    .version("2.3.0")
     .stringConf
     .transform(_.toLowerCase(Locale.ROOT))
     .checkValues(Set("none", "uncompressed", "snappy", "zlib", "lzo"))
@@ -665,6 +752,7 @@ object SQLConf {
   val ORC_IMPLEMENTATION = buildConf("spark.sql.orc.impl")
     .doc("When native, use the native version of ORC support instead of the ORC library in Hive. " +
       "It is 'hive' by default prior to Spark 2.4.")
+    .version("2.3.0")
     .internal()
     .stringConf
     .checkValues(Set("hive", "native"))
@@ -672,23 +760,27 @@ object SQLConf {
 
   val ORC_VECTORIZED_READER_ENABLED = buildConf("spark.sql.orc.enableVectorizedReader")
     .doc("Enables vectorized orc decoding.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORC_VECTORIZED_READER_BATCH_SIZE = buildConf("spark.sql.orc.columnarReaderBatchSize")
     .doc("The number of rows to include in a orc vectorized reader batch. The number should " +
       "be carefully chosen to minimize overhead and avoid OOMs in reading data.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(4096)
 
   val ORC_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.orc.filterPushdown")
     .doc("When true, enable filter pushdown for ORC files.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORC_SCHEMA_MERGING_ENABLED = buildConf("spark.sql.orc.mergeSchema")
     .doc("When true, the Orc data source merges schemas collected from all data files, " +
       "otherwise the schema is picked from a random data file.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -696,6 +788,7 @@ object SQLConf {
     .doc("When true, check all the partition paths under the table\'s root directory " +
          "when reading data stored in HDFS. This configuration will be deprecated in the future " +
          s"releases and replaced by ${SPARK_IGNORE_MISSING_FILES.key}.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -705,6 +798,7 @@ object SQLConf {
            "unmatching partitions can be eliminated earlier. This only affects Hive tables " +
            "not converted to filesource relations (see HiveUtils.CONVERT_METASTORE_PARQUET and " +
            "HiveUtils.CONVERT_METASTORE_ORC for more information).")
+      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -714,6 +808,7 @@ object SQLConf {
            "This includes both datasource and converted Hive tables. When partition management " +
            "is enabled, datasource tables store partition in the Hive metastore, and use the " +
            "metastore to prune partitions during query planning.")
+      .version("2.1.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -722,6 +817,7 @@ object SQLConf {
       .doc("When nonzero, enable caching of partition file metadata in memory. All tables share " +
            "a cache that can use up to specified num bytes for file metadata. This conf only " +
            "has an effect when hive filesource partition management is enabled.")
+      .version("2.1.1")
       .longConf
       .createWithDefault(250 * 1024 * 1024)
 
@@ -737,6 +833,7 @@ object SQLConf {
       "and write it back to the table properties), INFER_ONLY (infer the schema but don't " +
       "attempt to write it to the table properties) and NEVER_INFER (the default mode-- fallback " +
       "to using the case-insensitive metastore schema instead of inferring).")
+    .version("2.1.1")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(HiveCaseSensitiveInferenceMode.values.map(_.toString))
@@ -747,25 +844,31 @@ object SQLConf {
     .doc("When true, enable the metadata-only query optimization that use the table's metadata " +
       "to produce the partition columns instead of table scans. It applies when all the columns " +
       "scanned are partition columns and the query has an aggregate operator that satisfies " +
-      "distinct semantics. By default the optimization is disabled, since it may return " +
-      "incorrect results when the files are empty.")
+      "distinct semantics. By default the optimization is disabled, and deprecated as of Spark " +
+      "3.0 since it may return incorrect results when the files are empty, see also SPARK-26709." +
+      "It will be removed in the future releases. If you must use, use 'SparkSessionExtensions' " +
+      "instead to inject it as a custom rule.")
+    .version("2.1.1")
     .booleanConf
     .createWithDefault(false)
 
   val COLUMN_NAME_OF_CORRUPT_RECORD = buildConf("spark.sql.columnNameOfCorruptRecord")
     .doc("The name of internal column for storing raw/un-parsed JSON and CSV records that fail " +
       "to parse.")
+    .version("1.2.0")
     .stringConf
     .createWithDefault("_corrupt_record")
 
   val BROADCAST_TIMEOUT = buildConf("spark.sql.broadcastTimeout")
     .doc("Timeout in seconds for the broadcast wait time in broadcast joins.")
+    .version("1.3.0")
     .timeConf(TimeUnit.SECONDS)
-    .createWithDefault(5 * 60)
+    .createWithDefaultString(s"${5 * 60}")
 
   // This is only used for the thriftserver
   val THRIFTSERVER_POOL = buildConf("spark.sql.thriftserver.scheduler.pool")
     .doc("Set a Fair Scheduler pool for a JDBC client session.")
+    .version("1.1.1")
     .stringConf
     .createOptional
 
@@ -773,23 +876,27 @@ object SQLConf {
     buildConf("spark.sql.thriftServer.incrementalCollect")
       .internal()
       .doc("When true, enable incremental collection for execution in Thrift Server.")
+      .version("2.0.3")
       .booleanConf
       .createWithDefault(false)
 
   val THRIFTSERVER_UI_STATEMENT_LIMIT =
     buildConf("spark.sql.thriftserver.ui.retainedStatements")
       .doc("The number of SQL statements kept in the JDBC/ODBC web UI history.")
+      .version("1.4.0")
       .intConf
       .createWithDefault(200)
 
   val THRIFTSERVER_UI_SESSION_LIMIT = buildConf("spark.sql.thriftserver.ui.retainedSessions")
     .doc("The number of SQL client sessions kept in the JDBC/ODBC web UI history.")
+    .version("1.4.0")
     .intConf
     .createWithDefault(200)
 
   // This is used to set the default data source
   val DEFAULT_DATA_SOURCE_NAME = buildConf("spark.sql.sources.default")
     .doc("The default data source to use in input/output.")
+    .version("1.3.0")
     .stringConf
     .createWithDefault("parquet")
 
@@ -798,6 +905,7 @@ object SQLConf {
     .doc("When true, a table created by a Hive CTAS statement (no USING clause) " +
       "without specifying any storage property will be converted to a data source table, " +
       s"using the data source set by ${DEFAULT_DATA_SOURCE_NAME.key}.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -806,22 +914,26 @@ object SQLConf {
       .doc("When true, fast stats (number of files and total size of all files) will be gathered" +
         " in parallel while repairing table partitions to avoid the sequential listing in Hive" +
         " metastore.")
+      .version("2.0.1")
       .booleanConf
       .createWithDefault(true)
 
   val PARTITION_COLUMN_TYPE_INFERENCE =
     buildConf("spark.sql.sources.partitionColumnTypeInference.enabled")
       .doc("When true, automatically infer the data types for partitioned columns.")
+      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
   val BUCKETING_ENABLED = buildConf("spark.sql.sources.bucketing.enabled")
     .doc("When false, we will treat bucketed table as normal table")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val BUCKETING_MAX_BUCKETS = buildConf("spark.sql.sources.bucketing.maxBuckets")
-    .doc("The maximum number of buckets allowed. Defaults to 100000")
+    .doc("The maximum number of buckets allowed.")
+    .version("2.4.0")
     .intConf
     .checkValue(_ > 0, "the value of spark.sql.sources.bucketing.maxBuckets must be greater than 0")
     .createWithDefault(100000)
@@ -830,36 +942,42 @@ object SQLConf {
     .internal()
     .doc("When false, we will throw an error if a query contains a cartesian product without " +
         "explicit CROSS JOIN syntax.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORDER_BY_ORDINAL = buildConf("spark.sql.orderByOrdinal")
     .doc("When true, the ordinal numbers are treated as the position in the select list. " +
          "When false, the ordinal numbers in order/sort by clause are ignored.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val GROUP_BY_ORDINAL = buildConf("spark.sql.groupByOrdinal")
     .doc("When true, the ordinal numbers in group by clauses are treated as the position " +
       "in the select list. When false, the ordinal numbers are ignored.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val GROUP_BY_ALIASES = buildConf("spark.sql.groupByAliases")
     .doc("When true, aliases in a select list can be used in group by clauses. When false, " +
       "an analysis exception is thrown in the case.")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
   // The output committer class used by data sources. The specified class needs to be a
   // subclass of org.apache.hadoop.mapreduce.OutputCommitter.
   val OUTPUT_COMMITTER_CLASS = buildConf("spark.sql.sources.outputCommitterClass")
+    .version("1.4.0")
     .internal()
     .stringConf
     .createOptional
 
   val FILE_COMMIT_PROTOCOL_CLASS =
     buildConf("spark.sql.sources.commitProtocolClass")
+      .version("2.1.1")
       .internal()
       .stringConf
       .createWithDefault(
@@ -869,8 +987,9 @@ object SQLConf {
     buildConf("spark.sql.sources.parallelPartitionDiscovery.threshold")
       .doc("The maximum number of paths allowed for listing files at driver side. If the number " +
         "of detected paths exceeds this value during partition discovery, it tries to list the " +
-        "files with another Spark distributed job. This applies to Parquet, ORC, CSV, JSON and " +
-        "LibSVM data sources.")
+        "files with another Spark distributed job. This configuration is effective only when " +
+        "using file-based sources such as Parquet, JSON and ORC.")
+      .version("1.5.0")
       .intConf
       .checkValue(parallel => parallel >= 0, "The maximum number of paths allowed for listing " +
         "files at driver side must not be negative")
@@ -880,17 +999,19 @@ object SQLConf {
     buildConf("spark.sql.sources.parallelPartitionDiscovery.parallelism")
       .doc("The number of parallelism to list a collection of path recursively, Set the " +
         "number to prevent file listing from generating too many tasks.")
+      .version("2.1.1")
       .internal()
       .intConf
       .createWithDefault(10000)
 
   val IGNORE_DATA_LOCALITY =
-    buildConf("spark.sql.sources.ignoreDataLocality.enabled")
+    buildConf("spark.sql.sources.ignoreDataLocality")
       .doc("If true, Spark will not fetch the block locations for each file on " +
         "listing files. This speeds up file listing, but the scheduler cannot " +
         "schedule tasks to take advantage of data locality. It can be particularly " +
         "useful if data is read from a remote cluster so the scheduler could never " +
         "take advantage of locality anyway.")
+      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(false)
@@ -899,19 +1020,22 @@ object SQLConf {
   // See SPARK-6231.
   val DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY =
     buildConf("spark.sql.selfJoinAutoResolveAmbiguity")
+      .version("1.4.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
 
   val FAIL_AMBIGUOUS_SELF_JOIN_ENABLED =
-    buildConf("spark.sql.analyzer.failAmbiguousSelfJoin.enabled")
+    buildConf("spark.sql.analyzer.failAmbiguousSelfJoin")
       .doc("When true, fail the Dataset query if it contains ambiguous self-join.")
+      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
 
   // Whether to retain group by columns or not in GroupedData.agg.
   val DATAFRAME_RETAIN_GROUP_COLUMNS = buildConf("spark.sql.retainGroupColumns")
+    .version("1.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -919,12 +1043,14 @@ object SQLConf {
   val DATAFRAME_PIVOT_MAX_VALUES = buildConf("spark.sql.pivotMaxValues")
     .doc("When doing a pivot without specifying values for the pivot column this is the maximum " +
       "number of (distinct) values that will be collected without error.")
+    .version("1.6.0")
     .intConf
     .createWithDefault(10000)
 
   val RUN_SQL_ON_FILES = buildConf("spark.sql.runSQLOnFiles")
     .internal()
     .doc("When true, we could use `datasource`.`path` as table in SQL query.")
+    .version("1.6.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -932,6 +1058,7 @@ object SQLConf {
     .internal()
     .doc("When true, the whole stage (of multiple operators) will be compiled into single java" +
       " method.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -940,6 +1067,7 @@ object SQLConf {
     .internal()
     .doc("When true, embed the (whole-stage) codegen stage ID into " +
       "the class name of the generated class as a suffix")
+    .version("2.3.1")
     .booleanConf
     .createWithDefault(true)
 
@@ -947,6 +1075,7 @@ object SQLConf {
     .internal()
     .doc("The maximum number of fields (including nested fields) that will be supported before" +
       " deactivating whole-stage codegen.")
+    .version("2.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -956,6 +1085,7 @@ object SQLConf {
       "interpreted if any compile error happens. Disabling fallback if `CODEGEN_ONLY`. " +
       "`NO_CODEGEN` skips codegen and goes interpreted path always. Note that " +
       "this config works only for tests.")
+    .version("2.4.0")
     .internal()
     .stringConf
     .checkValues(CodegenObjectFactoryMode.values.map(_.toString))
@@ -965,12 +1095,14 @@ object SQLConf {
     .internal()
     .doc("When true, (whole stage) codegen could be temporary disabled for the part of query that" +
       " fail to compile generated code")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val CODEGEN_LOGGING_MAX_LINES = buildConf("spark.sql.codegen.logging.maxLines")
     .internal()
     .doc("The maximum number of codegen lines to log when errors occur. Use -1 for unlimited.")
+    .version("2.3.0")
     .intConf
     .checkValue(maxLines => maxLines >= -1, "The maximum must be a positive integer, 0 to " +
       "disable logging or -1 to apply no limit.")
@@ -984,6 +1116,7 @@ object SQLConf {
       "is the largest bytecode size possible for a valid Java method. When running on HotSpot, " +
       s"it may be preferable to set the value to ${CodeGenerator.DEFAULT_JVM_HUGE_METHOD_LIMIT} " +
       "to match HotSpot's implementation.")
+    .version("2.3.0")
     .intConf
     .createWithDefault(65535)
 
@@ -995,6 +1128,7 @@ object SQLConf {
       "generated, so use the code length as metric. When running on HotSpot, a function's " +
       "bytecode should not go beyond 8KB, otherwise it will not be JITted; it also should not " +
       "be too small, otherwise there will be many function calls.")
+    .version("3.0.0")
     .intConf
     .checkValue(threshold => threshold > 0, "The threshold must be a positive integer.")
     .createWithDefault(1024)
@@ -1005,6 +1139,7 @@ object SQLConf {
       .doc("When true, whole stage codegen would put the logic of consuming rows of each " +
         "physical operator into individual methods, instead of a single big method. This can be " +
         "used to avoid oversized function that can miss the opportunity of JIT optimization.")
+      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1012,8 +1147,9 @@ object SQLConf {
     .doc("The maximum number of bytes to pack into a single partition when reading files. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
+    .version("2.0.0")
     .bytesConf(ByteUnit.BYTE)
-    .createWithDefault(128 * 1024 * 1024) // parquet.block.size
+    .createWithDefaultString("128MB") // parquet.block.size
 
   val FILES_OPEN_COST_IN_BYTES = buildConf("spark.sql.files.openCostInBytes")
     .internal()
@@ -1022,6 +1158,7 @@ object SQLConf {
       " over estimated, then the partitions with small files will be faster than partitions with" +
       " bigger files (which is scheduled first). This configuration is effective only when using" +
       " file-based sources such as Parquet, JSON and ORC.")
+    .version("2.0.0")
     .longConf
     .createWithDefault(4 * 1024 * 1024)
 
@@ -1030,6 +1167,7 @@ object SQLConf {
       "encountering corrupted files and the contents that have been read will still be returned. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
+    .version("2.1.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -1038,24 +1176,35 @@ object SQLConf {
       "encountering missing files and the contents that have been read will still be returned. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val MAX_RECORDS_PER_FILE = buildConf("spark.sql.files.maxRecordsPerFile")
     .doc("Maximum number of records to write out to a single file. " +
       "If this value is zero or negative, there is no limit.")
+    .version("2.2.0")
     .longConf
     .createWithDefault(0)
 
   val EXCHANGE_REUSE_ENABLED = buildConf("spark.sql.exchange.reuse")
     .internal()
     .doc("When true, the planner will try to find out duplicated exchanges and re-use them.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
-  val SUBQUERY_REUSE_ENABLED = buildConf("spark.sql.execution.subquery.reuse.enabled")
+  val SUBQUERY_REUSE_ENABLED = buildConf("spark.sql.execution.reuseSubquery")
     .internal()
     .doc("When true, the planner will try to find out duplicated subqueries and re-use them.")
+    .version("3.0.0")
+    .booleanConf
+    .createWithDefault(true)
+
+  val REMOVE_REDUNDANT_SORTS_ENABLED = buildConf("spark.sql.execution.removeRedundantSorts")
+    .internal()
+    .doc("Whether to remove redundant physical sort node")
+    .version("2.4.8")
     .booleanConf
     .createWithDefault(true)
 
@@ -1067,6 +1216,7 @@ object SQLConf {
           "be a subclass of StateStoreProvider, and must have a zero-arg constructor. " +
           "Note: For structured streaming, this configuration cannot be changed between query " +
           "restarts from the same checkpoint location.")
+      .version("2.3.0")
       .stringConf
       .createWithDefault(
         "org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider")
@@ -1076,6 +1226,7 @@ object SQLConf {
       .internal()
       .doc("Minimum number of state store delta files that needs to be generated before they " +
         "consolidated into snapshots.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -1083,24 +1234,28 @@ object SQLConf {
     buildConf("spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion")
       .internal()
       .doc("State format version used by flatMapGroupsWithState operation in a streaming query")
+      .version("2.4.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
 
   val CHECKPOINT_LOCATION = buildConf("spark.sql.streaming.checkpointLocation")
     .doc("The default location for storing checkpoint data for streaming queries.")
+    .version("2.0.0")
     .stringConf
     .createOptional
 
   val FORCE_DELETE_TEMP_CHECKPOINT_LOCATION =
-    buildConf("spark.sql.streaming.forceDeleteTempCheckpointLocation.enabled")
+    buildConf("spark.sql.streaming.forceDeleteTempCheckpointLocation")
       .doc("When true, enable temporary checkpoint locations force delete.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val MIN_BATCHES_TO_RETAIN = buildConf("spark.sql.streaming.minBatchesToRetain")
     .internal()
     .doc("The minimum number of batches that must be retained and made recoverable.")
+    .version("2.1.1")
     .intConf
     .createWithDefault(100)
 
@@ -1110,6 +1265,7 @@ object SQLConf {
       "loading from files. The value adjusts a trade-off between memory usage vs cache miss: " +
       "'2' covers both success and direct failure cases, '1' covers only success case, " +
       "and '0' covers extreme case - disable cache to maximize memory size of executors.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(2)
 
@@ -1119,6 +1275,7 @@ object SQLConf {
       .doc("State format version used by streaming aggregation operations in a streaming query. " +
         "State between versions are tend to be incompatible, so state format version shouldn't " +
         "be modified after running.")
+      .version("2.4.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
@@ -1129,6 +1286,7 @@ object SQLConf {
       "If we find a concurrent active run for a streaming query (in the same or different " +
       "SparkSessions on the same cluster) and this flag is true, we will stop the old streaming " +
       "query run to start the new one.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1138,6 +1296,7 @@ object SQLConf {
       .doc("State format version used by streaming join operations in a streaming query. " +
         "State between versions are tend to be incompatible, so state format version shouldn't " +
         "be modified after running.")
+      .version("3.0.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
@@ -1147,24 +1306,15 @@ object SQLConf {
       .internal()
       .doc("When true, the logical plan for streaming query will be checked for unsupported" +
         " operations.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
-  val STREAMING_UI_ENABLED =
-    buildConf("spark.sql.streaming.ui.enabled")
-      .doc("Whether to run the structured streaming UI for the Spark application.")
-      .booleanConf
-      .createWithDefault(true)
-
-  val STREAMING_UI_INACTIVE_QUERY_RETENTION =
-    buildConf("spark.sql.streaming.ui.numInactiveQueries")
-      .doc("The number of inactive queries to retain for structured streaming ui.")
-      .intConf
-      .createWithDefault(100)
-
   val VARIABLE_SUBSTITUTE_ENABLED =
     buildConf("spark.sql.variable.substitute")
-      .doc("This enables substitution using syntax like ${var} ${system:var} and ${env:var}.")
+      .doc("This enables substitution using syntax like `${var}`, `${system:var}`, " +
+        "and `${env:var}`.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1174,7 +1324,8 @@ object SQLConf {
       .doc("Enable two-level aggregate hash map. When enabled, records will first be " +
         "inserted/looked-up at a 1st-level, small, fast map, and then fallback to a " +
         "2nd-level, larger, slower map when 1st level is full or keys cannot be found. " +
-        "When disabled, records go directly to the 2nd level. Defaults to true.")
+        "When disabled, records go directly to the 2nd level.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1182,6 +1333,7 @@ object SQLConf {
     buildConf("spark.sql.codegen.aggregate.map.vectorized.enable")
       .internal()
       .doc("Enable vectorized aggregate hash map. This is for testing/benchmarking only.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1191,6 +1343,7 @@ object SQLConf {
       .doc("When true, the code generator would split aggregate code into individual methods " +
         "instead of a single big method. This can be used to avoid oversized function that " +
         "can miss the opportunity of JIT optimization.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1202,6 +1355,7 @@ object SQLConf {
         "However the DAG depth may become too large and cause unexpected behavior. This " +
         "configuration puts a limit on this: when the depth of a view exceeds this value during " +
         "analysis, we terminate the resolution to avoid potential errors.")
+      .version("2.2.0")
       .intConf
       .checkValue(depth => depth > 0, "The maximum depth of a view reference in a nested view " +
         "must be positive.")
@@ -1209,6 +1363,7 @@ object SQLConf {
 
   val STREAMING_FILE_COMMIT_PROTOCOL_CLASS =
     buildConf("spark.sql.streaming.commitProtocolClass")
+      .version("2.1.0")
       .internal()
       .stringConf
       .createWithDefault("org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol")
@@ -1221,6 +1376,7 @@ object SQLConf {
         "'max' which chooses the maximum across multiple operators. " +
         "Note: This configuration cannot be changed between query restarts from the same " +
         "checkpoint location.")
+      .version("2.4.0")
       .stringConf
       .transform(_.toLowerCase(Locale.ROOT))
       .checkValue(
@@ -1235,6 +1391,7 @@ object SQLConf {
       .doc("In the case of ObjectHashAggregateExec, when the size of the in-memory hash map " +
         "grows too large, we will fall back to sort-based aggregation. This option sets a row " +
         "count threshold for the size of the hash map.")
+      .version("2.2.0")
       .intConf
       // We are trying to be conservative and use a relatively small default count threshold here
       // since the state object of some TypedImperativeAggregate function can be quite large (e.g.
@@ -1244,6 +1401,7 @@ object SQLConf {
   val USE_OBJECT_HASH_AGG = buildConf("spark.sql.execution.useObjectHashAggregateExec")
     .internal()
     .doc("Decides if we use ObjectHashAggregateExec")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1252,12 +1410,14 @@ object SQLConf {
       .doc("Whether to ignore null fields when generating JSON objects in JSON data source and " +
         "JSON functions such as to_json. " +
         "If false, it generates null for null fields in JSON objects.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val FILE_SINK_LOG_DELETION = buildConf("spark.sql.streaming.fileSink.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream sink.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1266,6 +1426,7 @@ object SQLConf {
       .internal()
       .doc("Number of log files after which all the previous files " +
         "are compacted into the next log file.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -1273,12 +1434,14 @@ object SQLConf {
     buildConf("spark.sql.streaming.fileSink.log.cleanupDelay")
       .internal()
       .doc("How long that a file is guaranteed to be visible for all readers.")
+      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes
 
   val FILE_SOURCE_LOG_DELETION = buildConf("spark.sql.streaming.fileSource.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream source.")
+    .version("2.0.1")
     .booleanConf
     .createWithDefault(true)
 
@@ -1287,6 +1450,7 @@ object SQLConf {
       .internal()
       .doc("Number of log files after which all the previous files " +
         "are compacted into the next log file.")
+      .version("2.0.1")
       .intConf
       .createWithDefault(10)
 
@@ -1294,6 +1458,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.fileSource.log.cleanupDelay")
       .internal()
       .doc("How long in milliseconds a file is guaranteed to be visible for all readers.")
+      .version("2.0.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes
 
@@ -1303,12 +1468,14 @@ object SQLConf {
       .doc("When true, force the schema of streaming file source to be nullable (including all " +
         "the fields). Otherwise, the schema might not be compatible with actual data, which " +
         "leads to corruptions.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val FILE_SOURCE_CLEANER_NUM_THREADS =
     buildConf("spark.sql.streaming.fileSource.cleaner.numThreads")
       .doc("Number of threads used in the file source completed file cleaner.")
+      .version("3.0.0")
       .intConf
       .createWithDefault(1)
 
@@ -1316,6 +1483,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.schemaInference")
       .internal()
       .doc("Whether file-based streaming sources will infer its own schema")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1323,20 +1491,23 @@ object SQLConf {
     buildConf("spark.sql.streaming.pollingDelay")
       .internal()
       .doc("How long to delay polling new data when no data is available")
+      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(10L)
 
   val STREAMING_STOP_TIMEOUT =
     buildConf("spark.sql.streaming.stopTimeout")
-      .doc("How long to wait for the streaming execution thread to stop when calling the " +
-        "streaming query's stop() method in milliseconds. 0 or negative values wait indefinitely.")
+      .doc("How long to wait in milliseconds for the streaming execution thread to stop when " +
+        "calling the streaming query's stop() method. 0 or negative values wait indefinitely.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
-      .createWithDefault(0L)
+      .createWithDefaultString("0")
 
   val STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL =
     buildConf("spark.sql.streaming.noDataProgressEventInterval")
       .internal()
       .doc("How long to wait between two progress events when there is no data")
+      .version("2.1.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(10000L)
 
@@ -1345,18 +1516,21 @@ object SQLConf {
       .doc(
         "Whether streaming micro-batch engine will execute batches without data " +
           "for eager state management for stateful streaming queries.")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
   val STREAMING_METRICS_ENABLED =
     buildConf("spark.sql.streaming.metricsEnabled")
       .doc("Whether Dropwizard/Codahale metrics will be reported for active streaming queries.")
+      .version("2.0.2")
       .booleanConf
       .createWithDefault(false)
 
   val STREAMING_PROGRESS_RETENTION =
     buildConf("spark.sql.streaming.numRecentProgressUpdates")
       .doc("The number of progress updates to retain for a streaming query")
+      .version("2.1.1")
       .intConf
       .createWithDefault(100)
 
@@ -1364,6 +1538,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.checkpointFileManagerClass")
       .doc("The class used to write checkpoint files atomically. This class must be a subclass " +
         "of the interface CheckpointFileManager.")
+      .version("2.4.0")
       .internal()
       .stringConf
 
@@ -1371,6 +1546,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.checkpoint.escapedPathCheck.enabled")
       .doc("Whether to detect a streaming query may pick up an incorrect checkpoint path due " +
         "to SPARK-26824.")
+      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -1381,6 +1557,7 @@ object SQLConf {
       .doc("When true, SQL commands use parallel file listing, " +
         "as opposed to single thread listing. " +
         "This usually speeds up commands that need to list many directories.")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1390,6 +1567,7 @@ object SQLConf {
       s"which is larger than `${AUTO_BROADCASTJOIN_THRESHOLD.key}` to be more conservative. " +
       "That is to say by default the optimizer will not choose to broadcast a table unless it " +
       "knows for sure its size is small enough.")
+    .version("1.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(Long.MaxValue)
 
@@ -1400,14 +1578,16 @@ object SQLConf {
       "For non-partitioned data source tables, it will be automatically recalculated if table " +
       "statistics are not available. For partitioned data source and partitioned Hive tables, " +
       s"It is '${DEFAULT_SIZE_IN_BYTES.key}' if table statistics are not available.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(false)
 
   val NDV_MAX_ERROR =
     buildConf("spark.sql.statistics.ndv.maxError")
       .internal()
-      .doc("The maximum estimation error allowed in HyperLogLog++ algorithm when generating " +
-        "column level statistics.")
+      .doc("The maximum relative standard deviation allowed in HyperLogLog++ algorithm " +
+        "when generating column level statistics.")
+      .version("2.1.1")
       .doubleConf
       .createWithDefault(0.05)
 
@@ -1418,6 +1598,7 @@ object SQLConf {
         "histogram. Note that collecting histograms takes extra cost. For example, collecting " +
         "column statistics usually takes only one table scan, but generating equi-height " +
         "histogram will cause an extra table scan.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1425,6 +1606,7 @@ object SQLConf {
     buildConf("spark.sql.statistics.histogram.numBins")
       .internal()
       .doc("The number of bins when generating histograms.")
+      .version("2.3.0")
       .intConf
       .checkValue(num => num > 1, "The number of bins must be greater than 1.")
       .createWithDefault(254)
@@ -1435,6 +1617,7 @@ object SQLConf {
       .doc("Accuracy of percentile approximation when generating equi-height histograms. " +
         "Larger value means better accuracy. The relative error can be deduced by " +
         "1.0 / PERCENTILE_ACCURACY.")
+      .version("2.3.0")
       .intConf
       .createWithDefault(10000)
 
@@ -1443,30 +1626,35 @@ object SQLConf {
       .doc("Enables automatic update for table size once table's data is changed. Note that if " +
         "the total number of files of the table is very large, this can be expensive and slow " +
         "down data change commands.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   val CBO_ENABLED =
     buildConf("spark.sql.cbo.enabled")
       .doc("Enables CBO for estimation of plan statistics when set true.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val PLAN_STATS_ENABLED =
     buildConf("spark.sql.cbo.planStats.enabled")
       .doc("When true, the logical plan will fetch row counts and column statistics from catalog.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val JOIN_REORDER_ENABLED =
     buildConf("spark.sql.cbo.joinReorder.enabled")
       .doc("Enables join reorder in CBO.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val JOIN_REORDER_DP_THRESHOLD =
     buildConf("spark.sql.cbo.joinReorder.dp.threshold")
       .doc("The maximum number of joined nodes allowed in the dynamic programming algorithm.")
+      .version("2.2.0")
       .intConf
       .checkValue(number => number > 0, "The maximum number must be a positive integer.")
       .createWithDefault(12)
@@ -1476,6 +1664,7 @@ object SQLConf {
       .internal()
       .doc("The weight of cardinality (number of rows) for plan cost comparison in join reorder: " +
         "rows * weight + size * (1 - weight).")
+      .version("2.2.0")
       .doubleConf
       .checkValue(weight => weight >= 0 && weight <= 1, "The weight value must be in [0, 1].")
       .createWithDefault(0.7)
@@ -1483,11 +1672,13 @@ object SQLConf {
   val JOIN_REORDER_DP_STAR_FILTER =
     buildConf("spark.sql.cbo.joinReorder.dp.star.filter")
       .doc("Applies star-join filter heuristics to cost based join enumeration.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val STARSCHEMA_DETECTION = buildConf("spark.sql.cbo.starSchemaDetection")
     .doc("When true, it enables join reordering based on star schema detection. ")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1495,19 +1686,31 @@ object SQLConf {
     .internal()
     .doc("Specifies the upper limit of the ratio between the largest fact tables" +
       " for a star join to be considered. ")
+    .version("2.2.0")
     .doubleConf
     .createWithDefault(0.9)
 
-  val SESSION_LOCAL_TIMEZONE =
-    buildConf("spark.sql.session.timeZone")
-      .doc("""The ID of session local timezone, e.g. "GMT", "America/Los_Angeles", etc.""")
-      .stringConf
-      .createWithDefaultFunction(() => TimeZone.getDefault.getID)
+  private def isValidTimezone(zone: String): Boolean = {
+    Try { DateTimeUtils.getZoneId(zone) }.isSuccess
+  }
+
+  val SESSION_LOCAL_TIMEZONE = buildConf("spark.sql.session.timeZone")
+    .doc("The ID of session local timezone in the format of either region-based zone IDs or " +
+      "zone offsets. Region IDs must have the form 'area/city', such as 'America/Los_Angeles'. " +
+      "Zone offsets must be in the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. " +
+      "Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not " +
+      "recommended to use because they can be ambiguous.")
+    .version("2.2.0")
+    .stringConf
+    .checkValue(isValidTimezone, s"Cannot resolve the given timezone with" +
+      " ZoneId.of(_, ZoneId.SHORT_IDS)")
+    .createWithDefaultFunction(() => TimeZone.getDefault.getID)
 
   val WINDOW_EXEC_BUFFER_IN_MEMORY_THRESHOLD =
     buildConf("spark.sql.windowExec.buffer.in.memory.threshold")
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the window operator")
+      .version("2.2.1")
       .intConf
       .createWithDefault(4096)
 
@@ -1515,6 +1718,7 @@ object SQLConf {
     buildConf("spark.sql.windowExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by window operator")
+      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
@@ -1523,6 +1727,7 @@ object SQLConf {
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the sort merge " +
         "join operator")
+      .version("2.2.1")
       .intConf
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1530,6 +1735,7 @@ object SQLConf {
     buildConf("spark.sql.sortMergeJoinExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by sort merge join operator")
+      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
@@ -1538,6 +1744,7 @@ object SQLConf {
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the cartesian " +
         "product operator")
+      .version("2.2.1")
       .intConf
       .createWithDefault(4096)
 
@@ -1545,12 +1752,14 @@ object SQLConf {
     buildConf("spark.sql.cartesianProductExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by cartesian product operator")
+      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
   val SUPPORT_QUOTED_REGEX_COLUMN_NAME = buildConf("spark.sql.parser.quotedRegexColumnNames")
     .doc("When true, quoted Identifiers (using backticks) in SELECT statement are interpreted" +
       " as regular expressions.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1559,12 +1768,14 @@ object SQLConf {
       .internal()
       .doc("Number of points to sample per partition in order to determine the range boundaries" +
           " for range partitioning, typically used in global sorting (without limit).")
+      .version("2.3.0")
       .intConf
       .createWithDefault(100)
 
   val ARROW_EXECUTION_ENABLED =
     buildConf("spark.sql.execution.arrow.enabled")
       .doc("(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.enabled'.)")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1576,8 +1787,18 @@ object SQLConf {
         "2. pyspark.sql.SparkSession.createDataFrame when its input is a Pandas DataFrame " +
         "The following data types are unsupported: " +
         "BinaryType, MapType, ArrayType of TimestampType, and nested StructType.")
+      .version("3.0.0")
       .fallbackConf(ARROW_EXECUTION_ENABLED)
 
+  val PYSPARK_JVM_STACKTRACE_ENABLED =
+    buildConf("spark.sql.pyspark.jvmStacktrace.enabled")
+      .doc("When true, it shows the JVM stacktrace in the user-facing PySpark exception " +
+        "together with Python stacktrace. By default, it is disabled and hides JVM stacktrace " +
+        "and shows a Python-friendly exception only.")
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val ARROW_SPARKR_EXECUTION_ENABLED =
     buildConf("spark.sql.execution.arrow.sparkr.enabled")
       .doc("When true, make use of Apache Arrow for columnar data transfers in SparkR. " +
@@ -1588,6 +1809,7 @@ object SQLConf {
         "4. gapply " +
         "The following data types are unsupported: " +
         "FloatType, BinaryType, ArrayType, StructType and MapType.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1595,6 +1817,7 @@ object SQLConf {
     buildConf("spark.sql.execution.arrow.fallback.enabled")
       .doc("(Deprecated since Spark 3.0, please set " +
         "'spark.sql.execution.arrow.pyspark.fallback.enabled'.)")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1602,22 +1825,25 @@ object SQLConf {
     buildConf("spark.sql.execution.arrow.pyspark.fallback.enabled")
       .doc(s"When true, optimizations enabled by '${ARROW_PYSPARK_EXECUTION_ENABLED.key}' will " +
         "fallback automatically to non-optimized implementations if an error occurs.")
+      .version("3.0.0")
       .fallbackConf(ARROW_FALLBACK_ENABLED)
 
   val ARROW_EXECUTION_MAX_RECORDS_PER_BATCH =
     buildConf("spark.sql.execution.arrow.maxRecordsPerBatch")
       .doc("When using Apache Arrow, limit the maximum number of records that can be written " +
         "to a single ArrowRecordBatch in memory. If set to zero or negative there is no limit.")
+      .version("2.3.0")
       .intConf
       .createWithDefault(10000)
 
   val PANDAS_UDF_BUFFER_SIZE =
-    buildConf("spark.sql.pandas.udf.buffer.size")
+    buildConf("spark.sql.execution.pandas.udf.buffer.size")
       .doc(
-        s"Same as ${BUFFER_SIZE} but only applies to Pandas UDF executions. If it is not set, " +
-        s"the fallback is ${BUFFER_SIZE}. Note that Pandas execution requires more than 4 bytes. " +
-        "Lowering this value could make small Pandas UDF batch iterated and pipelined; however, " +
-        "it might degrade performance. See SPARK-27870.")
+        s"Same as `${BUFFER_SIZE.key}` but only applies to Pandas UDF executions. If it is not " +
+        s"set, the fallback is `${BUFFER_SIZE.key}`. Note that Pandas execution requires more " +
+        "than 4 bytes. Lowering this value could make small Pandas UDF batch iterated and " +
+        "pipelined; however, it might degrade performance. See SPARK-27870.")
+      .version("3.0.0")
       .fallbackConf(BUFFER_SIZE)
 
   val PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME =
@@ -1627,16 +1853,18 @@ object SQLConf {
         "to use position if not. When false, a grouped map Pandas UDF will assign columns from " +
         "the returned Pandas DataFrame based on position, regardless of column label type. " +
         "This configuration will be deprecated in future releases.")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
   val PANDAS_ARROW_SAFE_TYPE_CONVERSION =
-    buildConf("spark.sql.execution.pandas.arrowSafeTypeConversion")
+    buildConf("spark.sql.execution.pandas.convertToArrowArraySafely")
       .internal()
       .doc("When true, Arrow will perform safe type conversion when converting " +
         "Pandas.Series to Arrow array during serialization. Arrow will raise errors " +
         "when detecting unsafe type conversion like overflow. When false, disabling Arrow's type " +
         "check and do type conversions anyway. This config only works for Arrow 0.11.0+.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1650,6 +1878,7 @@ object SQLConf {
       " column names of the left node are distinct. If all the conditions are met, the" +
       " rule will replace the except operation with a Filter by flipping the filter" +
       " condition(s) of the right node.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1660,6 +1889,7 @@ object SQLConf {
         "happens according to Hive behavior and SQL ANSI 2011 specification, ie. rounding the " +
         "decimal part of the result if an exact representation is not possible. Otherwise, NULL " +
         "is returned in those cases, as previously.")
+      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1669,6 +1899,7 @@ object SQLConf {
       .doc("When integral literal is used in decimal operations, pick a minimum precision " +
         "required by the literal if this config is true, to make the resulting precision and/or " +
         "scale smaller. This can reduce the possibility of precision lose and/or overflow.")
+      .version("2.3.3")
       .booleanConf
       .createWithDefault(true)
 
@@ -1678,6 +1909,7 @@ object SQLConf {
         "information. The values of options whose names that match this regex will be redacted " +
         "in the explain output. This redaction is applied on top of the global redaction " +
         s"configuration defined by ${SECRET_REDACTION_PATTERN.key}.")
+    .version("2.2.2")
     .regexConf
     .createWithDefault("(?i)url".r)
 
@@ -1687,17 +1919,20 @@ object SQLConf {
         "information. When this regex matches a string part, that string part is replaced by a " +
         "dummy value. This is currently used to redact the output of SQL explain commands. " +
         "When this conf is not set, the value from `spark.redaction.string.regex` is used.")
+      .version("2.3.0")
       .fallbackConf(org.apache.spark.internal.config.STRING_REDACTION_PATTERN)
 
   val CONCAT_BINARY_AS_STRING = buildConf("spark.sql.function.concatBinaryAsString")
     .doc("When this option is set to false and all inputs are binary, `functions.concat` returns " +
       "an output as binary. Otherwise, it returns as a string.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val ELT_OUTPUT_AS_STRING = buildConf("spark.sql.function.eltOutputAsString")
     .doc("When this option is set to false and all inputs are binary, `elt` returns " +
       "an output as binary. Otherwise, it returns as a string.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1708,6 +1943,7 @@ object SQLConf {
         "user-specified schema. If the validation fails, a runtime exception is thrown. " +
         "When this option is set to false, the partition column value will be converted to null " +
         "if it can not be casted to corresponding user-specified schema.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1715,6 +1951,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.continuous.epochBacklogQueueSize")
       .doc("The max number of entries to be stored in queue to wait for late epochs. " +
         "If this parameter is exceeded by the size of the queue, stream will stop with an error.")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
@@ -1723,6 +1960,7 @@ object SQLConf {
     .internal()
     .doc("The size (measured in number of rows) of the queue used in continuous execution to" +
       " buffer the results of a ContinuousDataReader.")
+    .version("2.3.0")
     .intConf
     .createWithDefault(1024)
 
@@ -1731,6 +1969,7 @@ object SQLConf {
       .internal()
       .doc("The interval at which continuous execution readers will poll to check whether" +
         " the epoch has advanced on the driver.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(100)
 
@@ -1739,12 +1978,14 @@ object SQLConf {
     .doc("A comma-separated list of data source short names or fully qualified data source " +
       "implementation class names for which Data Source V2 code path is disabled. These data " +
       "sources will fallback to Data Source V1 code path.")
+    .version("3.0.0")
     .stringConf
     .createWithDefault("avro,csv,json,kafka,orc,parquet,text")
 
   val DISABLED_V2_STREAMING_WRITERS = buildConf("spark.sql.streaming.disabledV2Writers")
     .doc("A comma-separated list of fully qualified data source register class names for which" +
       " StreamWriteSupport is disabled. Writes to these sources will fall back to the V1 Sinks.")
+    .version("2.3.1")
     .stringConf
     .createWithDefault("")
 
@@ -1755,9 +1996,24 @@ object SQLConf {
         "A comma-separated list of fully qualified data source register class names for which " +
           "MicroBatchReadSupport is disabled. Reads from these sources will fall back to the " +
           "V1 Sources.")
+      .version("2.4.0")
       .stringConf
       .createWithDefault("")
 
+  val FASTFAIL_ON_FILEFORMAT_OUTPUT =
+    buildConf("spark.sql.execution.fastFailOnFileFormatOutput")
+      .internal()
+      .doc("Whether to fast fail task execution when writing output to FileFormat datasource. " +
+        "If this is enabled, in `FileFormatWriter` we will catch `FileAlreadyExistsException` " +
+        "and fast fail output task without further task retry. Only enabling this if you know " +
+        "the `FileAlreadyExistsException` of the output task is unrecoverable, i.e., further " +
+        "task attempts won't be able to success. If the `FileAlreadyExistsException` might be " +
+        "recoverable, you should keep this as disabled and let Spark to retry output tasks. " +
+        "This is disabled by default.")
+      .version("3.0.2")
+      .booleanConf
+      .createWithDefault(false)
+
   object PartitionOverwriteMode extends Enumeration {
     val STATIC, DYNAMIC = Value
   }
@@ -1775,6 +2031,7 @@ object SQLConf {
         "(which takes precedence over this setting), e.g. " +
         "dataframe.write.option(\"partitionOverwriteMode\", \"dynamic\").save(path)."
       )
+      .version("2.3.0")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(PartitionOverwriteMode.values.map(_.toString))
@@ -1799,6 +2056,7 @@ object SQLConf {
         "in type coercion, e.g. converting `double` to `int` or `decimal` to `double` is " +
         "not allowed."
       )
+      .version("3.0.0")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(StoreAssignmentPolicy.values.map(_.toString))
@@ -1809,6 +2067,7 @@ object SQLConf {
       "throw a runtime exception if an overflow occurs in any operation on integral/decimal " +
       "field. 2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in " +
       "the SQL parser.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1821,8 +2080,9 @@ object SQLConf {
         "issues. Turn on this config to insert a local sort before actually doing repartition " +
         "to generate consistent repartition results. The performance of repartition() may go " +
         "down since we insert extra local sort before it.")
-        .booleanConf
-        .createWithDefault(true)
+      .version("2.1.4")
+      .booleanConf
+      .createWithDefault(true)
 
   val NESTED_SCHEMA_PRUNING_ENABLED =
     buildConf("spark.sql.optimizer.nestedSchemaPruning.enabled")
@@ -1831,15 +2091,30 @@ object SQLConf {
         "satisfying a query. This optimization allows columnar file format readers to avoid " +
         "reading unnecessary nested column data. Currently Parquet and ORC are the " +
         "data sources that implement this optimization.")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
+  val NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST =
+    buildConf("spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources")
+      .internal()
+      .doc("A comma-separated list of data source short names or fully qualified data source " +
+        "implementation class names for which Spark tries to push down predicates for nested " +
+        "columns and/or names containing `dots` to data sources. This configuration is only " +
+        "effective with file-based data source in DSv1. Currently, Parquet implements " +
+        "both optimizations while ORC only supports predicates for names containing `dots`. The " +
+        "other data sources don't support this feature yet. So the default value is 'parquet,orc'.")
+      .version("3.0.0")
+      .stringConf
+      .createWithDefault("parquet,orc")
+
   val SERIALIZER_NESTED_SCHEMA_PRUNING_ENABLED =
     buildConf("spark.sql.optimizer.serializer.nestedSchemaPruning.enabled")
       .internal()
       .doc("Prune nested fields from object serialization operator which are unnecessary in " +
         "satisfying a query. This optimization allows object serializers to avoid " +
         "executing unnecessary nested expressions.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1850,6 +2125,7 @@ object SQLConf {
         "satisfying a query. Note that this optimization doesn't prune nested fields from " +
         "physical data source scanning. For pruning nested fields from scanning, please use " +
         "`spark.sql.optimizer.nestedSchemaPruning.enabled` config.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1859,6 +2135,7 @@ object SQLConf {
       .doc("In SQL queries with a SORT followed by a LIMIT like " +
           "'SELECT x FROM t ORDER BY y LIMIT m', if m is under this threshold, do a top-K sort" +
           " in memory, otherwise do a global sort which spills to disk if necessary.")
+      .version("2.4.0")
       .intConf
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1874,6 +2151,7 @@ object SQLConf {
     .internal()
     .doc("If it is set to true, column names of the requested schema are passed to CSV parser. " +
       "Other column values can be ignored during parsing even if they are malformed.")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1884,6 +2162,7 @@ object SQLConf {
       "Jupyter, the HTML table (generated by _repr_html_) will be returned. For plain Python " +
       "REPL, the returned outputs are formatted like dataframe.show(). In SparkR, the returned " +
       "outputs are showed similar to R data.frame would.")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1892,12 +2171,14 @@ object SQLConf {
       s"effect when ${REPL_EAGER_EVAL_ENABLED.key} is set to true. The valid range of this " +
       "config is from 0 to (Int.MaxValue - 1), so the invalid config like negative and " +
       "greater than (Int.MaxValue - 1) will be normalized to 0 and (Int.MaxValue - 1).")
+    .version("2.4.0")
     .intConf
     .createWithDefault(20)
 
   val REPL_EAGER_EVAL_TRUNCATE = buildConf("spark.sql.repl.eagerEval.truncate")
     .doc("The max number of characters for each cell that is returned by eager evaluation. " +
       s"This only takes effect when ${REPL_EAGER_EVAL_ENABLED.key} is set to true.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(20)
 
@@ -1908,6 +2189,7 @@ object SQLConf {
         "by the fast hash aggregate product operator. The bit is not for actual value, " +
         "but the actual numBuckets is determined by loadFactor " +
         "(e.g: default bit value 16 , the actual numBuckets is ((1 << 16) / 0.5).")
+      .version("2.4.0")
       .intConf
       .checkValue(bit => bit >= 10 && bit <= 30, "The bit value must be in [10, 30].")
       .createWithDefault(16)
@@ -1915,6 +2197,7 @@ object SQLConf {
   val AVRO_COMPRESSION_CODEC = buildConf("spark.sql.avro.compression.codec")
     .doc("Compression codec used in writing of AVRO files. Supported codecs: " +
       "uncompressed, deflate, snappy, bzip2 and xz. Default codec is snappy.")
+    .version("2.4.0")
     .stringConf
     .checkValues(Set("uncompressed", "deflate", "snappy", "bzip2", "xz"))
     .createWithDefault("snappy")
@@ -1923,20 +2206,25 @@ object SQLConf {
     .doc("Compression level for the deflate codec used in writing of AVRO files. " +
       "Valid value must be in the range of from 1 to 9 inclusive or -1. " +
       "The default value is -1 which corresponds to 6 level in the current implementation.")
+    .version("2.4.0")
     .intConf
     .checkValues((1 to 9).toSet + Deflater.DEFAULT_COMPRESSION)
     .createWithDefault(Deflater.DEFAULT_COMPRESSION)
 
   val LEGACY_SIZE_OF_NULL = buildConf("spark.sql.legacy.sizeOfNull")
-    .doc("If it is set to true, size of null returns -1. This behavior was inherited from Hive. " +
-      "The size function returns null for null input if the flag is disabled.")
+    .internal()
+    .doc(s"If it is set to false, or ${ANSI_ENABLED.key} is true, then size of null returns " +
+      "null. Otherwise, it returns -1, which was inherited from Hive.")
+    .version("2.4.0")
     .booleanConf
-    .createWithDefault(false)
+    .createWithDefault(true)
 
   val LEGACY_REPLACE_DATABRICKS_SPARK_AVRO_ENABLED =
     buildConf("spark.sql.legacy.replaceDatabricksSparkAvro.enabled")
+      .internal()
       .doc("If it is set to true, the data source provider com.databricks.spark.avro is mapped " +
         "to the built-in but external Avro data source module for backward compatibility.")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1947,6 +2235,7 @@ object SQLConf {
         "set operations are performed from left to right as they appear in the query. When set " +
         "to false and order of evaluation is not specified by parentheses, INTERSECT operations " +
         "are performed before any UNION, EXCEPT and MINUS operations.")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1955,47 +2244,63 @@ object SQLConf {
       .internal()
       .doc("When set to true, a literal with an exponent (e.g. 1E-30) would be parsed " +
         "as Decimal rather than Double.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val LEGACY_ALLOW_NEGATIVE_SCALE_OF_DECIMAL_ENABLED =
-    buildConf("spark.sql.legacy.allowNegativeScaleOfDecimal.enabled")
+    buildConf("spark.sql.legacy.allowNegativeScaleOfDecimal")
       .internal()
       .doc("When set to true, negative scale of Decimal type is allowed. For example, " +
         "the type of number 1E10BD under legacy mode is DecimalType(2, -9), but is " +
         "Decimal(11, 0) in non legacy mode.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
-  val LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED =
-    buildConf("spark.sql.legacy.createHiveTableByDefault.enabled")
-      .internal()
-      .doc("When set to true, CREATE TABLE syntax without a provider will use hive " +
-        s"instead of the value of ${DEFAULT_DATA_SOURCE_NAME.key}.")
-      .booleanConf
-      .createWithDefault(false)
-
-
-  val LEGACY_INTEGRALDIVIDE_RETURN_LONG = buildConf("spark.sql.legacy.integralDivide.returnBigint")
-    .doc("If it is set to true, the div operator returns always a bigint. This behavior was " +
-      "inherited from Hive. Otherwise, the return type is the data type of the operands.")
-    .internal()
-    .booleanConf
-    .createWithDefault(false)
-
   val LEGACY_BUCKETED_TABLE_SCAN_OUTPUT_ORDERING =
     buildConf("spark.sql.legacy.bucketedTableScan.outputOrdering")
       .internal()
       .doc("When true, the bucketed table scan will list files during planning to figure out the " +
         "output ordering, which is expensive and may make the planning quite slow.")
-    .booleanConf
-    .createWithDefault(false)
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(false)
 
   val LEGACY_HAVING_WITHOUT_GROUP_BY_AS_WHERE =
     buildConf("spark.sql.legacy.parser.havingWithoutGroupByAsWhere")
       .internal()
       .doc("If it is set to true, the parser will treat HAVING without GROUP BY as a normal " +
         "WHERE, which does not follow SQL standard.")
+      .version("2.4.1")
+      .booleanConf
+      .createWithDefault(false)
+
+  val LEGACY_ALLOW_EMPTY_STRING_IN_JSON =
+    buildConf("spark.sql.legacy.json.allowEmptyString.enabled")
+      .internal()
+      .doc("When set to true, the parser of JSON data source treats empty strings as null for " +
+        "some data types such as `IntegerType`.")
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE =
+    buildConf("spark.sql.legacy.createEmptyCollectionUsingStringType")
+      .internal()
+      .doc("When set to true, Spark returns an empty collection with `StringType` as element " +
+        "type if the `array`/`map` function is called without any parameters. Otherwise, Spark " +
+        "returns an empty collection with `NullType` as element type.")
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  val LEGACY_ALLOW_UNTYPED_SCALA_UDF =
+    buildConf("spark.sql.legacy.allowUntypedScalaUDF")
+      .internal()
+      .doc("When set to true, user is allowed to use org.apache.spark.sql.functions." +
+        "udf(f: AnyRef, dataType: DataType). Otherwise, an exception will be thrown at runtime.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2004,6 +2309,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, TRUNCATE TABLE command will not try to set back original " +
         "permission and ACLs when re-creating the table/partition paths.")
+      .version("2.4.6")
       .booleanConf
       .createWithDefault(false)
 
@@ -2013,6 +2319,7 @@ object SQLConf {
       .doc("When set to true, the key attribute resulted from running `Dataset.groupByKey` " +
         "for non-struct key type, will be named as `value`, following the behavior of Spark " +
         "version 2.4 and earlier.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2020,6 +2327,7 @@ object SQLConf {
     .doc("Maximum number of fields of sequence-like entries can be converted to strings " +
       "in debug output. Any elements beyond the limit will be dropped and replaced by a" +
       """ "... N more fields" placeholder.""")
+    .version("3.0.0")
     .intConf
     .createWithDefault(25)
 
@@ -2028,17 +2336,19 @@ object SQLConf {
       "longer, further output will be truncated.  The default setting always generates a full " +
       "plan.  Set this to a lower value such as 8k if plan strings are taking up too much " +
       "memory or are causing OutOfMemory errors in the driver or UI processes.")
+    .version("3.0.0")
     .bytesConf(ByteUnit.BYTE)
     .checkValue(i => i >= 0 && i <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH, "Invalid " +
       "value for 'spark.sql.maxPlanStringLength'.  Length must be a valid string length " +
       "(nonnegative and shorter than the maximum size).")
-    .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
+    .createWithDefaultString(s"${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}")
 
   val SET_COMMAND_REJECTS_SPARK_CORE_CONFS =
     buildConf("spark.sql.legacy.setCommandRejectsSparkCoreConfs")
       .internal()
       .doc("If it is set to true, SET command will fail when the key is registered as " +
         "a SparkConf entry.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2047,6 +2357,7 @@ object SQLConf {
       "java.time.LocalDate classes of Java 8 API are used as external types for " +
       "Catalyst's TimestampType and DateType. If it is set to false, java.sql.Timestamp " +
       "and java.sql.Date are used for the same purpose.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2054,20 +2365,24 @@ object SQLConf {
     .doc("The max length of a file that can be read by the binary file data source. " +
       "Spark will fail fast and not attempt to read the file if its length exceeds this value. " +
       "The theoretical max is Int.MaxValue, though VMs might implement a smaller max.")
+    .version("3.0.0")
     .internal()
     .intConf
     .createWithDefault(Int.MaxValue)
 
   val LEGACY_CAST_DATETIME_TO_STRING =
     buildConf("spark.sql.legacy.typeCoercion.datetimeToString.enabled")
+      .internal()
       .doc("If it is set to true, date/timestamp will cast to string in binary comparisons " +
         "with String")
-    .booleanConf
-    .createWithDefault(false)
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(false)
 
   val DEFAULT_CATALOG = buildConf("spark.sql.defaultCatalog")
     .doc("Name of the default catalog. This will be the current catalog if users have not " +
       "explicitly set the current catalog yet.")
+    .version("3.0.0")
     .stringConf
     .createWithDefault(SESSION_CATALOG_NAME)
 
@@ -2079,23 +2394,66 @@ object SQLConf {
         s"be loaded by the $SESSION_CATALOG_NAME, this catalog must also return the table " +
         s"metadata. To delegate operations to the $SESSION_CATALOG_NAME, implementations can " +
         "extend 'CatalogExtension'.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
-  val LEGACY_LOOSE_UPCAST = buildConf("spark.sql.legacy.looseUpcast")
+  object MapKeyDedupPolicy extends Enumeration {
+    val EXCEPTION, LAST_WIN = Value
+  }
+
+  val MAP_KEY_DEDUP_POLICY = buildConf("spark.sql.mapKeyDedupPolicy")
+    .doc("The policy to deduplicate map keys in builtin function: CreateMap, MapFromArrays, " +
+      "MapFromEntries, StringToMap, MapConcat and TransformKeys. When EXCEPTION, the query " +
+      "fails if duplicated map keys are detected. When LAST_WIN, the map key that is inserted " +
+      "at last takes precedence.")
+    .version("3.0.0")
+    .stringConf
+    .transform(_.toUpperCase(Locale.ROOT))
+    .checkValues(MapKeyDedupPolicy.values.map(_.toString))
+    .createWithDefault(MapKeyDedupPolicy.EXCEPTION.toString)
+
+  val LEGACY_LOOSE_UPCAST = buildConf("spark.sql.legacy.doLooseUpcast")
+    .internal()
     .doc("When true, the upcast will be loose and allows string to atomic types.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
-  val LEGACY_CTE_PRECEDENCE_ENABLED = buildConf("spark.sql.legacy.ctePrecedence.enabled")
+  object LegacyBehaviorPolicy extends Enumeration {
+    val EXCEPTION, LEGACY, CORRECTED = Value
+  }
+
+  val LEGACY_CTE_PRECEDENCE_POLICY = buildConf("spark.sql.legacy.ctePrecedencePolicy")
     .internal()
-    .doc("When true, outer CTE definitions takes precedence over inner definitions.")
-    .booleanConf
-    .createWithDefault(false)
+    .doc("When LEGACY, outer CTE definitions takes precedence over inner definitions. If set to " +
+      "CORRECTED, inner CTE definitions take precedence. The default value is EXCEPTION, " +
+      "AnalysisException is thrown while name conflict is detected in nested CTE. This config " +
+      "will be removed in future versions and CORRECTED will be the only behavior.")
+    .version("3.0.0")
+    .stringConf
+    .transform(_.toUpperCase(Locale.ROOT))
+    .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+    .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
+
+  val LEGACY_TIME_PARSER_POLICY = buildConf("spark.sql.legacy.timeParserPolicy")
+    .internal()
+    .doc("When LEGACY, java.text.SimpleDateFormat is used for formatting and parsing " +
+      "dates/timestamps in a locale-sensitive manner, which is the approach before Spark 3.0. " +
+      "When set to CORRECTED, classes from java.time.* packages are used for the same purpose. " +
+      "The default value is EXCEPTION, RuntimeException is thrown when we will get different " +
+      "results.")
+    .version("3.0.0")
+    .stringConf
+    .transform(_.toUpperCase(Locale.ROOT))
+    .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+    .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
 
   val LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC =
-    buildConf("spark.sql.legacy.arrayExistsFollowsThreeValuedLogic")
+    buildConf("spark.sql.legacy.followThreeValuedLogicInArrayExists")
+      .internal()
       .doc("When true, the ArrayExists will follow the three-valued boolean logic.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2104,9 +2462,10 @@ object SQLConf {
       .doc("A comma-delimited string config of the optional additional remote Maven mirror " +
         "repositories. This is only used for downloading Hive jars in IsolatedClientLoader " +
         "if the default Maven Central repo is unreachable.")
+      .version("3.0.0")
       .stringConf
       .createWithDefault(
-        "https://maven-central.storage-download.googleapis.com/repos/central/data/")
+        "https://maven-central.storage-download.googleapis.com/maven2/")
 
   val LEGACY_FROM_DAYTIME_STRING =
     buildConf("spark.sql.legacy.fromDayTimeString.enabled")
@@ -2116,35 +2475,40 @@ object SQLConf {
         "all interval units out of the specified range. If it is set to `false`, " +
         "`ParseException` is thrown if the input does not match to the pattern " +
         "defined by `from` and `to`.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val LEGACY_PROPERTY_NON_RESERVED =
-    buildConf("spark.sql.legacy.property.nonReserved")
+    buildConf("spark.sql.legacy.notReserveProperties")
       .internal()
       .doc("When true, all database and table properties are not reserved and available for " +
         "create/alter syntaxes. But please be aware that the reserved properties will be " +
         "silently removed.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
-  val LEGACY_ADD_DIRECTORY_USING_RECURSIVE =
-    buildConf("spark.sql.legacy.addDirectory.recursive.enabled")
+  val LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE =
+    buildConf("spark.sql.legacy.addSingleFileInAddFile")
       .internal()
-      .doc("When true, users can add directory by passing path of a directory to ADD FILE " +
-        "command of SQL. If false, then only a single file can be added.")
+      .doc("When true, only a single file can be added using ADD FILE. If false, then users " +
+        "can add directory by passing directory path to ADD FILE.")
+      .version("3.0.0")
       .booleanConf
-      .createWithDefault(true)
+      .createWithDefault(false)
 
   val LEGACY_MSSQLSERVER_NUMERIC_MAPPING_ENABLED =
     buildConf("spark.sql.legacy.mssqlserver.numericMapping.enabled")
       .internal()
       .doc("When true, use legacy MySqlServer SMALLINT and REAL type mapping.")
+      .version("2.4.5")
       .booleanConf
       .createWithDefault(false)
 
   val CSV_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.csv.filterPushdown.enabled")
     .doc("When true, enable filter pushdown to CSV datasource.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -2154,10 +2518,89 @@ object SQLConf {
       .doc("The number of partitions to be handled in one turn when use " +
         "`AlterTableAddPartitionCommand` to add partitions into table. The smaller " +
         "batch size is, the less memory is required for the real handler, e.g. Hive Metastore.")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The value of spark.sql.addPartitionInBatch.size must be positive")
       .createWithDefault(100)
 
+  val LEGACY_ALLOW_HASH_ON_MAPTYPE = buildConf("spark.sql.legacy.allowHashOnMapType")
+    .doc("When set to true, hash expressions can be applied on elements of MapType. Otherwise, " +
+      "an analysis exception will be thrown.")
+    .version("3.0.0")
+    .booleanConf
+    .createWithDefault(false)
+
+  val LEGACY_PARQUET_REBASE_MODE_IN_WRITE =
+    buildConf("spark.sql.legacy.parquet.datetimeRebaseModeInWrite")
+      .internal()
+      .doc("When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar " +
+        "to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. " +
+        "When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. " +
+        "When EXCEPTION, which is the default, Spark will fail the writing if it sees " +
+        "ancient dates/timestamps that are ambiguous between the two calendars. " +
+        "This config influences on writes of the following parquet logical types: DATE, " +
+        "TIMESTAMP_MILLIS, TIMESTAMP_MICROS. Spark always performs rebasing for the INT96 type.")
+      .version("3.0.0")
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+      .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
+
+  val LEGACY_PARQUET_REBASE_MODE_IN_READ =
+    buildConf("spark.sql.legacy.parquet.datetimeRebaseModeInRead")
+      .internal()
+      .doc("When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + " +
+        "Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files. " +
+        "When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. " +
+        "When EXCEPTION, which is the default, Spark will fail the reading if it sees " +
+        "ancient dates/timestamps that are ambiguous between the two calendars. This config is " +
+        "only effective if the writer info (like Spark, Hive) of the Parquet files is unknown. " +
+        "This config influences on reads of the following parquet logical types: DATE, " +
+        "TIMESTAMP_MILLIS, TIMESTAMP_MICROS. Spark always performs rebasing for the INT96 type.")
+      .version("3.0.0")
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+      .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
+
+  val LEGACY_AVRO_REBASE_MODE_IN_WRITE =
+    buildConf("spark.sql.legacy.avro.datetimeRebaseModeInWrite")
+      .internal()
+      .doc("When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar " +
+        "to the legacy hybrid (Julian + Gregorian) calendar when writing Avro files. " +
+        "When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. " +
+        "When EXCEPTION, which is the default, Spark will fail the writing if it sees " +
+        "ancient dates/timestamps that are ambiguous between the two calendars.")
+      .version("3.0.0")
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+      .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
+
+  val LEGACY_AVRO_REBASE_MODE_IN_READ =
+    buildConf("spark.sql.legacy.avro.datetimeRebaseModeInRead")
+      .internal()
+      .doc("When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + " +
+        "Gregorian) calendar to Proleptic Gregorian calendar when reading Avro files. " +
+        "When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. " +
+        "When EXCEPTION, which is the default, Spark will fail the reading if it sees " +
+        "ancient dates/timestamps that are ambiguous between the two calendars. This config is " +
+        "only effective if the writer info (like Spark, Hive) of the Avro files is unknown.")
+      .version("3.0.0")
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+      .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
+
+  val SCRIPT_TRANSFORMATION_EXIT_TIMEOUT =
+    buildConf("spark.sql.scriptTransformation.exitTimeoutInSeconds")
+      .internal()
+      .doc("Timeout for executor to wait for the termination of transformation script when EOF.")
+      .version("3.0.0")
+      .timeConf(TimeUnit.SECONDS)
+      .checkValue(_ > 0, "The timeout value must be positive")
+      .createWithDefault(10L)
+
   /**
    * Holds information about keys that have been deprecated.
    *
@@ -2185,7 +2628,12 @@ object SQLConf {
       DeprecatedConfig(ARROW_EXECUTION_ENABLED.key, "3.0",
         s"Use '${ARROW_PYSPARK_EXECUTION_ENABLED.key}' instead of it."),
       DeprecatedConfig(ARROW_FALLBACK_ENABLED.key, "3.0",
-        s"Use '${ARROW_PYSPARK_FALLBACK_ENABLED.key}' instead of it.")
+        s"Use '${ARROW_PYSPARK_FALLBACK_ENABLED.key}' instead of it."),
+      DeprecatedConfig(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "3.0",
+        s"Use '${ADVISORY_PARTITION_SIZE_IN_BYTES.key}' instead of it."),
+      DeprecatedConfig(OPTIMIZER_METADATA_ONLY.key, "3.0",
+        "Avoid to depend on this optimization to prevent a potential correctness issue. " +
+          "If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule.")
     )
 
     Map(configs.map { cfg => cfg.key -> cfg } : _*)
@@ -2206,6 +2654,10 @@ object SQLConf {
    * The map contains info about removed SQL configs. Keys are SQL config names,
    * map values contain extra information like the version in which the config was removed,
    * config's default value and a comment.
+   *
+   * Please, add a removed SQL configuration property here only when it affects behaviours.
+   * For example, `spark.sql.variable.substitute.depth` was not added as it virtually
+   * became no-op later. By this, it makes migrations to new Spark versions painless.
    */
   val removedSQLConfigs: Map[String, RemovedConfig] = {
     val configs = Seq(
@@ -2216,8 +2668,6 @@ object SQLConf {
         "It was removed to prevent loosing of users data for non-default value."),
       RemovedConfig("spark.sql.legacy.compareDateTimestampInTimestamp", "3.0.0", "true",
         "It was removed to prevent errors like SPARK-23549 for non-default value."),
-      RemovedConfig("spark.sql.variable.substitute.depth", "3.0.0", "40",
-        "It was deprecated since Spark 2.1, and not used in Spark 2.4."),
       RemovedConfig("spark.sql.parquet.int64AsTimestampMillis", "3.0.0", "false",
         "The config was deprecated since Spark 2.3." +
         s"Use '${PARQUET_OUTPUT_TIMESTAMP_TYPE.key}' instead of it."),
@@ -2273,8 +2723,8 @@ class SQLConf extends Serializable with Logging {
   def dynamicPartitionPruningFallbackFilterRatio: Double =
     getConf(DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO)
 
-  def dynamicPartitionPruningReuseBroadcast: Boolean =
-    getConf(DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST)
+  def dynamicPartitionPruningReuseBroadcastOnly: Boolean =
+    getConf(DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY)
 
   def stateStoreProviderClass: String = getConf(STATE_STORE_PROVIDER_CLASS)
 
@@ -2284,10 +2734,6 @@ class SQLConf extends Serializable with Logging {
 
   def isUnsupportedOperationCheckEnabled: Boolean = getConf(UNSUPPORTED_OPERATION_CHECK_ENABLED)
 
-  def isStreamingUIEnabled: Boolean = getConf(STREAMING_UI_ENABLED)
-
-  def streamingUIInactiveQueryRetention: Int = getConf(STREAMING_UI_INACTIVE_QUERY_RETENTION)
-
   def streamingFileCommitProtocolClass: String = getConf(STREAMING_FILE_COMMIT_PROTOCOL_CLASS)
 
   def fileSinkLogDeletion: Boolean = getConf(FILE_SINK_LOG_DELETION)
@@ -2344,23 +2790,26 @@ class SQLConf extends Serializable with Logging {
 
   def cacheVectorizedReaderEnabled: Boolean = getConf(CACHE_VECTORIZED_READER_ENABLED)
 
-  def numShufflePartitions: Int = getConf(SHUFFLE_PARTITIONS)
+  def defaultNumShufflePartitions: Int = getConf(SHUFFLE_PARTITIONS)
+
+  def numShufflePartitions: Int = {
+    if (adaptiveExecutionEnabled && coalesceShufflePartitionsEnabled) {
+      getConf(COALESCE_PARTITIONS_INITIAL_PARTITION_NUM).getOrElse(defaultNumShufflePartitions)
+    } else {
+      defaultNumShufflePartitions
+    }
+  }
 
   def adaptiveExecutionEnabled: Boolean = getConf(ADAPTIVE_EXECUTION_ENABLED)
 
-  def targetPostShuffleInputSize: Long = getConf(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
+  def adaptiveExecutionLogLevel: String = getConf(ADAPTIVE_EXECUTION_LOG_LEVEL)
 
-  def fetchShuffleBlocksInBatchEnabled: Boolean = getConf(FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED)
+  def fetchShuffleBlocksInBatch: Boolean = getConf(FETCH_SHUFFLE_BLOCKS_IN_BATCH)
 
   def nonEmptyPartitionRatioForBroadcastJoin: Double =
     getConf(NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN)
 
-  def reducePostShufflePartitionsEnabled: Boolean = getConf(REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED)
-
-  def minNumPostShufflePartitions: Int = getConf(SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS)
-
-  def maxNumPostShufflePartitions: Int =
-    getConf(SHUFFLE_MAX_NUM_POSTSHUFFLE_PARTITIONS).getOrElse(numShufflePartitions)
+  def coalesceShufflePartitionsEnabled: Boolean = getConf(COALESCE_PARTITIONS_ENABLED)
 
   def minBatchesToRetain: Int = getConf(MIN_BATCHES_TO_RETAIN)
 
@@ -2445,11 +2894,15 @@ class SQLConf extends Serializable with Logging {
 
   def datetimeJava8ApiEnabled: Boolean = getConf(DATETIME_JAVA8API_ENABLED)
 
-  def addDirectoryRecursiveEnabled: Boolean = getConf(LEGACY_ADD_DIRECTORY_USING_RECURSIVE)
+  def addSingleFileInAddFile: Boolean = getConf(LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE)
 
   def legacyMsSqlServerNumericMappingEnabled: Boolean =
     getConf(LEGACY_MSSQLSERVER_NUMERIC_MAPPING_ENABLED)
 
+  def legacyTimeParserPolicy: LegacyBehaviorPolicy.Value = {
+    LegacyBehaviorPolicy.withName(getConf(SQLConf.LEGACY_TIME_PARSER_POLICY))
+  }
+
   /**
    * Returns the [[Resolver]] for the current configuration, which can be used to determine if two
    * identifiers are equal.
@@ -2628,6 +3081,8 @@ class SQLConf extends Serializable with Logging {
 
   def arrowPySparkEnabled: Boolean = getConf(ARROW_PYSPARK_EXECUTION_ENABLED)
 
+  def pysparkJVMStacktraceEnabled: Boolean = getConf(PYSPARK_JVM_STACKTRACE_ENABLED)
+
   def arrowSparkREnabled: Boolean = getConf(ARROW_SPARKR_EXECUTION_ENABLED)
 
   def arrowPySparkFallbackEnabled: Boolean = getConf(ARROW_PYSPARK_FALLBACK_ENABLED)
@@ -2660,6 +3115,8 @@ class SQLConf extends Serializable with Logging {
   def disabledV2StreamingMicroBatchReaders: String =
     getConf(DISABLED_V2_STREAMING_MICROBATCH_READERS)
 
+  def fastFailFileFormatOutput: Boolean = getConf(FASTFAIL_ON_FILEFORMAT_OUTPUT)
+
   def concatBinaryAsString: Boolean = getConf(CONCAT_BINARY_AS_STRING)
 
   def eltOutputAsString: Boolean = getConf(ELT_OUTPUT_AS_STRING)
@@ -2683,7 +3140,10 @@ class SQLConf extends Serializable with Logging {
 
   def csvColumnPruning: Boolean = getConf(SQLConf.CSV_PARSER_COLUMN_PRUNING)
 
-  def legacySizeOfNull: Boolean = getConf(SQLConf.LEGACY_SIZE_OF_NULL)
+  def legacySizeOfNull: Boolean = {
+    // size(null) should return null under ansi mode.
+    getConf(SQLConf.LEGACY_SIZE_OF_NULL) && !getConf(ANSI_ENABLED)
+  }
 
   def isReplEagerEvalEnabled: Boolean = getConf(SQLConf.REPL_EAGER_EVAL_ENABLED)
 
@@ -2706,11 +3166,6 @@ class SQLConf extends Serializable with Logging {
   def allowNegativeScaleOfDecimalEnabled: Boolean =
     getConf(SQLConf.LEGACY_ALLOW_NEGATIVE_SCALE_OF_DECIMAL_ENABLED)
 
-  def createHiveTableByDefaultEnabled: Boolean =
-    getConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED)
-
-  def integralDivideReturnLong: Boolean = getConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG)
-
   def truncateTableIgnorePermissionAcl: Boolean =
     getConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL)
 
@@ -2829,10 +3284,10 @@ class SQLConf extends Serializable with Logging {
    * Return all the configuration definitions that have been defined in [[SQLConf]]. Each
    * definition contains key, defaultValue and doc.
    */
-  def getAllDefinedConfs: Seq[(String, String, String)] = sqlConfEntries.synchronized {
+  def getAllDefinedConfs: Seq[(String, String, String, String)] = sqlConfEntries.synchronized {
     sqlConfEntries.values.asScala.filter(_.isPublic).map { entry =>
       val displayValue = Option(getConfString(entry.key, null)).getOrElse(entry.defaultValueString)
-      (entry.key, displayValue, entry.doc)
+      (entry.key, displayValue, entry.doc, entry.version)
     }.toSeq
   }
 
@@ -2854,16 +3309,41 @@ class SQLConf extends Serializable with Logging {
     settings.containsKey(key)
   }
 
+  /**
+   * Logs a warning message if the given config key is deprecated.
+   */
+  private def logDeprecationWarning(key: String): Unit = {
+    SQLConf.deprecatedSQLConfigs.get(key).foreach {
+      case DeprecatedConfig(configName, version, comment) =>
+        logWarning(
+          s"The SQL config '$configName' has been deprecated in Spark v$version " +
+          s"and may be removed in the future. $comment")
+    }
+  }
+
+  private def requireDefaultValueOfRemovedConf(key: String, value: String): Unit = {
+    SQLConf.removedSQLConfigs.get(key).foreach {
+      case RemovedConfig(configName, version, defaultValue, comment) =>
+        if (value != defaultValue) {
+          throw new AnalysisException(
+            s"The SQL config '$configName' was removed in the version $version. $comment")
+        }
+    }
+  }
+
   protected def setConfWithCheck(key: String, value: String): Unit = {
+    logDeprecationWarning(key)
+    requireDefaultValueOfRemovedConf(key, value)
     settings.put(key, value)
   }
 
   def unsetConf(key: String): Unit = {
+    logDeprecationWarning(key)
     settings.remove(key)
   }
 
   def unsetConf(entry: ConfigEntry[_]): Unit = {
-    settings.remove(entry.key)
+    unsetConf(entry.key)
   }
 
   def clear(): Unit = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
index 66ac9ddb21aaa..9618ff6062635 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
@@ -32,18 +32,24 @@ object StaticSQLConf {
 
   val WAREHOUSE_PATH = buildStaticConf("spark.sql.warehouse.dir")
     .doc("The default location for managed databases and tables.")
+    .version("2.0.0")
     .stringConf
     .createWithDefault(Utils.resolveURI("spark-warehouse").toString)
 
   val CATALOG_IMPLEMENTATION = buildStaticConf("spark.sql.catalogImplementation")
     .internal()
+    .version("2.0.0")
     .stringConf
     .checkValues(Set("hive", "in-memory"))
     .createWithDefault("in-memory")
 
   val GLOBAL_TEMP_DATABASE = buildStaticConf("spark.sql.globalTempDatabase")
     .internal()
+    .version("2.1.0")
     .stringConf
+    // System preserved database should not exists in metastore. However it's hard to guarantee it
+    // for every session, because case-sensitivity differs. Here we always lowercase it to make our
+    // life easier.
     .transform(_.toLowerCase(Locale.ROOT))
     .createWithDefault("global_temp")
 
@@ -55,9 +61,10 @@ object StaticSQLConf {
   // that's why this conf has to be a static SQL conf.
   val SCHEMA_STRING_LENGTH_THRESHOLD =
     buildStaticConf("spark.sql.sources.schemaStringLengthThreshold")
+      .internal()
       .doc("The maximum length allowed in a single cell when " +
         "storing additional schema information in Hive's metastore.")
-      .internal()
+      .version("1.3.1")
       .intConf
       .createWithDefault(4000)
 
@@ -65,6 +72,7 @@ object StaticSQLConf {
     buildStaticConf("spark.sql.filesourceTableRelationCacheSize")
       .internal()
       .doc("The maximum size of the cache that maps qualified table names to table relation plans.")
+      .version("2.2.0")
       .intConf
       .checkValue(cacheSize => cacheSize >= 0, "The maximum size of the cache must not be negative")
       .createWithDefault(1000)
@@ -73,6 +81,7 @@ object StaticSQLConf {
       .internal()
       .doc("When nonzero, enable caching of generated classes for operators and expressions. " +
         "All jobs share the cache that can use up to the specified number for generated classes.")
+      .version("2.4.0")
       .intConf
       .checkValue(maxEntries => maxEntries >= 0, "The maximum must not be negative")
       .createWithDefault(100)
@@ -82,6 +91,7 @@ object StaticSQLConf {
     .doc("When true, put comment in the generated code. Since computing huge comments " +
       "can be extremely expensive in certain cases, such as deeply-nested expressions which " +
       "operate over inputs with wide schemas, default is false.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -90,6 +100,7 @@ object StaticSQLConf {
   val DEBUG_MODE = buildStaticConf("spark.sql.debug")
     .internal()
     .doc("Only used for internal debugging. Not all functions are supported when it is enabled.")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -98,6 +109,7 @@ object StaticSQLConf {
       .doc("When set to true, Hive Thrift server is running in a single session mode. " +
         "All the JDBC/ODBC connections share the temporary views, function registries, " +
         "SQL configuration and the current database.")
+      .version("1.6.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -109,6 +121,7 @@ object StaticSQLConf {
       "applied in the specified order. For the case of parsers, the last parser is used and each " +
       "parser can delegate to its predecessor. For the case of function name conflicts, the last " +
       "registered function name is used.")
+    .version("2.2.0")
     .stringConf
     .toSequence
     .createOptional
@@ -117,6 +130,7 @@ object StaticSQLConf {
     .doc("List of class names implementing QueryExecutionListener that will be automatically " +
       "added to newly created sessions. The classes should have either a no-arg constructor, " +
       "or a constructor that expects a SparkConf argument.")
+    .version("2.3.0")
     .stringConf
     .toSequence
     .createOptional
@@ -125,6 +139,7 @@ object StaticSQLConf {
     .doc("List of class names implementing StreamingQueryListener that will be automatically " +
       "added to newly created sessions. The classes should have either a no-arg constructor, " +
       "or a constructor that expects a SparkConf argument.")
+    .version("2.4.0")
     .stringConf
     .toSequence
     .createOptional
@@ -132,6 +147,7 @@ object StaticSQLConf {
   val UI_RETAINED_EXECUTIONS =
     buildStaticConf("spark.sql.ui.retainedExecutions")
       .doc("Number of executions to retain in the Spark UI.")
+      .version("1.5.0")
       .intConf
       .createWithDefault(1000)
 
@@ -144,6 +160,7 @@ object StaticSQLConf {
         "Notice the number should be carefully chosen since decreasing parallelism might " +
         "cause longer waiting for other broadcasting. Also, increasing parallelism may " +
         "cause memory problem.")
+      .version("3.0.0")
       .intConf
       .checkValue(thres => thres > 0 && thres <= 128, "The threshold must be in (0,128].")
       .createWithDefault(128)
@@ -152,6 +169,7 @@ object StaticSQLConf {
     buildStaticConf("spark.sql.subquery.maxThreadThreshold")
       .internal()
       .doc("The maximum degree of parallelism to execute the subquery.")
+      .version("2.4.6")
       .intConf
       .checkValue(thres => thres > 0 && thres <= 128, "The threshold must be in (0,128].")
       .createWithDefault(16)
@@ -159,6 +177,7 @@ object StaticSQLConf {
   val SQL_EVENT_TRUNCATE_LENGTH = buildStaticConf("spark.sql.event.truncate.length")
     .doc("Threshold of SQL length beyond which it will be truncated before adding to " +
       "event. Defaults to no truncation. If set to 0, callsite will be logged instead.")
+    .version("3.0.0")
     .intConf
     .checkValue(_ >= 0, "Must be set greater or equal to zero")
     .createWithDefault(Int.MaxValue)
@@ -167,13 +186,44 @@ object StaticSQLConf {
     buildStaticConf("spark.sql.legacy.sessionInitWithConfigDefaults")
       .doc("Flag to revert to legacy behavior where a cloned SparkSession receives SparkConf " +
         "defaults, dropping any overrides in its parent SparkSession.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val DEFAULT_URL_STREAM_HANDLER_FACTORY_ENABLED =
     buildStaticConf("spark.sql.defaultUrlStreamHandlerFactory.enabled")
-      .doc("When true, set FsUrlStreamHandlerFactory to support ADD JAR against HDFS locations")
       .internal()
+      .doc(
+        "When true, register Hadoop's FsUrlStreamHandlerFactory to support " +
+        "ADD JAR against HDFS locations. " +
+        "It should be disabled when a different stream protocol handler should be registered " +
+        "to support a particular protocol type, or if Hadoop's FsUrlStreamHandlerFactory " +
+        "conflicts with other protocol types such as `http` or `https`. See also SPARK-25694 " +
+        "and HADOOP-14598.")
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(true)
+
+  val STREAMING_UI_ENABLED =
+    buildStaticConf("spark.sql.streaming.ui.enabled")
+      .doc("Whether to run the Structured Streaming Web UI for the Spark application when the " +
+        "Spark Web UI is enabled.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
+
+  val STREAMING_UI_RETAINED_PROGRESS_UPDATES =
+    buildStaticConf("spark.sql.streaming.ui.retainedProgressUpdates")
+      .doc("The number of progress updates to retain for a streaming query for Structured " +
+        "Streaming UI.")
+      .version("3.0.0")
+      .intConf
+      .createWithDefault(100)
+
+  val STREAMING_UI_RETAINED_QUERIES =
+    buildStaticConf("spark.sql.streaming.ui.retainedQueries")
+      .doc("The number of inactive queries to retain for Structured Streaming UI.")
+      .version("3.0.0")
+      .intConf
+      .createWithDefault(100)
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala
similarity index 60%
rename from sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationUtils.scala
rename to sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala
index f4c4b04bada2a..32be74a345c5a 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala
@@ -15,20 +15,12 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.hive.thriftserver
+package org.apache.spark.sql.internal.connector
 
-import org.apache.spark.sql.catalyst.catalog.CatalogTableType
-import org.apache.spark.sql.catalyst.catalog.CatalogTableType.{EXTERNAL, MANAGED, VIEW}
+import org.apache.spark.sql.connector.write.WriteBuilder
 
-/**
- * Utils for metadata operations.
- */
-private[hive] trait SparkMetadataOperationUtils {
-
-  def tableTypeString(tableType: CatalogTableType): String = tableType match {
-    case EXTERNAL | MANAGED => "TABLE"
-    case VIEW => "VIEW"
-    case t =>
-      throw new IllegalArgumentException(s"Unknown table type is found: $t")
-  }
+// An internal `WriteBuilder` mixin to support UPDATE streaming output mode.
+// TODO: design an official API for streaming output mode UPDATE.
+trait SupportsStreamingUpdate extends WriteBuilder {
+  def update(): WriteBuilder
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
index fc7d33e823a99..2b44a3a861ed9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
@@ -18,13 +18,15 @@
 package org.apache.spark.sql.sources
 
 import org.apache.spark.annotation.{Evolving, Stable}
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.parseColumnPath
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 // This file defines all the filters that we can push down to the data sources.
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 /**
- * A filter predicate for data sources.
+ * A filter predicate for data sources. Mapping between Spark SQL types and filter value
+ * types follow the convention for return type of [[org.apache.spark.sql.Row#get(int)]].
  *
  * @since 1.3.0
  */
@@ -32,6 +34,10 @@ import org.apache.spark.annotation.{Evolving, Stable}
 sealed abstract class Filter {
   /**
    * List of columns that are referenced by this filter.
+   *
+   * Note that, each element in `references` represents a column; `dots` are used as separators
+   * for nested columns. If any part of the names contains `dots`, it is quoted to avoid confusion.
+   *
    * @since 2.1.0
    */
   def references: Array[String]
@@ -40,12 +46,32 @@ sealed abstract class Filter {
     case f: Filter => f.references
     case _ => Array.empty
   }
+
+  /**
+   * List of columns that are referenced by this filter.
+   *
+   * @return each element is a column name as an array of string multi-identifier
+   * @since 3.0.0
+   */
+  def v2references: Array[Array[String]] = {
+    this.references.map(parseColumnPath(_).toArray)
+  }
+
+  /**
+   * If any of the references of this filter contains nested column
+   */
+  private[sql] def containsNestedColumn: Boolean = {
+    this.v2references.exists(_.length > 1)
+  }
 }
 
 /**
- * A filter that evaluates to `true` iff the attribute evaluates to a value
+ * A filter that evaluates to `true` iff the column evaluates to a value
  * equal to `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -58,6 +84,9 @@ case class EqualTo(attribute: String, value: Any) extends Filter {
  * in that it returns `true` (rather than NULL) if both inputs are NULL, and `false`
  * (rather than NULL) if one of the input is NULL and the other is not NULL.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.5.0
  */
 @Stable
@@ -69,6 +98,9 @@ case class EqualNullSafe(attribute: String, value: Any) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to a value
  * greater than `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -80,6 +112,9 @@ case class GreaterThan(attribute: String, value: Any) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to a value
  * greater than or equal to `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -91,6 +126,9 @@ case class GreaterThanOrEqual(attribute: String, value: Any) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to a value
  * less than `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -102,6 +140,9 @@ case class LessThan(attribute: String, value: Any) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to a value
  * less than or equal to `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -112,6 +153,9 @@ case class LessThanOrEqual(attribute: String, value: Any) extends Filter {
 /**
  * A filter that evaluates to `true` iff the attribute evaluates to one of the values in the array.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -120,7 +164,7 @@ case class In(attribute: String, values: Array[Any]) extends Filter {
     var h = attribute.hashCode
     values.foreach { v =>
       h *= 41
-      h += v.hashCode()
+      h += (if (v != null) v.hashCode() else 0)
     }
     h
   }
@@ -139,6 +183,9 @@ case class In(attribute: String, values: Array[Any]) extends Filter {
 /**
  * A filter that evaluates to `true` iff the attribute evaluates to null.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -149,6 +196,9 @@ case class IsNull(attribute: String) extends Filter {
 /**
  * A filter that evaluates to `true` iff the attribute evaluates to a non-null value.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -190,6 +240,9 @@ case class Not(child: Filter) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to
  * a string that starts with `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.1
  */
 @Stable
@@ -201,6 +254,9 @@ case class StringStartsWith(attribute: String, value: String) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to
  * a string that ends with `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.1
  */
 @Stable
@@ -212,6 +268,9 @@ case class StringEndsWith(attribute: String, value: String) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to
  * a string that contains the string `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.1
  */
 @Stable
@@ -221,6 +280,8 @@ case class StringContains(attribute: String, value: String) extends Filter {
 
 /**
  * A filter that always evaluates to `true`.
+ *
+ * @since 3.0.0
  */
 @Evolving
 case class AlwaysTrue() extends Filter {
@@ -233,6 +294,8 @@ object AlwaysTrue extends AlwaysTrue {
 
 /**
  * A filter that always evaluates to `false`.
+ *
+ * @since 3.0.0
  */
 @Evolving
 case class AlwaysFalse() extends Filter {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
index 8a8cea194bf2c..3f70b7647f195 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -21,20 +21,17 @@ import java.util.Locale
 
 import scala.util.control.NonFatal
 
-import com.fasterxml.jackson.core.{JsonGenerator, JsonParser}
-import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer, JsonSerializer, SerializerProvider}
-import com.fasterxml.jackson.databind.`type`.TypeFactory
 import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize}
 import org.json4s._
 import org.json4s.JsonAST.JValue
 import org.json4s.JsonDSL._
-import org.json4s.jackson.{JValueDeserializer, JValueSerializer}
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.annotation.Stable
 import org.apache.spark.sql.catalyst.analysis.Resolver
 import org.apache.spark.sql.catalyst.expressions.{Cast, Expression}
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.catalyst.util.DataTypeJsonUtils.{DataTypeJsonDeserializer, DataTypeJsonSerializer}
 import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
@@ -331,19 +328,19 @@ object DataType {
       ignoreNullability: Boolean = false): Boolean = {
     (from, to) match {
       case (left: ArrayType, right: ArrayType) =>
-        equalsStructurally(left.elementType, right.elementType) &&
+        equalsStructurally(left.elementType, right.elementType, ignoreNullability) &&
           (ignoreNullability || left.containsNull == right.containsNull)
 
       case (left: MapType, right: MapType) =>
-        equalsStructurally(left.keyType, right.keyType) &&
-          equalsStructurally(left.valueType, right.valueType) &&
+        equalsStructurally(left.keyType, right.keyType, ignoreNullability) &&
+          equalsStructurally(left.valueType, right.valueType, ignoreNullability) &&
           (ignoreNullability || left.valueContainsNull == right.valueContainsNull)
 
       case (StructType(fromFields), StructType(toFields)) =>
         fromFields.length == toFields.length &&
           fromFields.zip(toFields)
             .forall { case (l, r) =>
-              equalsStructurally(l.dataType, r.dataType) &&
+              equalsStructurally(l.dataType, r.dataType, ignoreNullability) &&
                 (ignoreNullability || l.nullable == r.nullable)
             }
 
@@ -460,7 +457,7 @@ object DataType {
 
       case (w: AtomicType, r: AtomicType) if storeAssignmentPolicy == STRICT =>
         if (!Cast.canUpCast(w, r)) {
-          addError(s"Cannot safely cast '$context': $w to $r")
+          addError(s"Cannot safely cast '$context': ${w.catalogString} to ${r.catalogString}")
           false
         } else {
           true
@@ -470,7 +467,7 @@ object DataType {
 
       case (w: AtomicType, r: AtomicType) if storeAssignmentPolicy == ANSI =>
         if (!Cast.canANSIStoreAssign(w, r)) {
-          addError(s"Cannot safely cast '$context': $w to $r")
+          addError(s"Cannot safely cast '$context': ${w.catalogString} to ${r.catalogString}")
           false
         } else {
           true
@@ -480,35 +477,9 @@ object DataType {
         true
 
       case (w, r) =>
-        addError(s"Cannot write '$context': $w is incompatible with $r")
+        addError(s"Cannot write '$context': " +
+          s"${w.catalogString} is incompatible with ${r.catalogString}")
         false
     }
   }
 }
-
-/**
- * Jackson serializer for [[DataType]]. Internally this delegates to json4s based serialization.
- */
-class DataTypeJsonSerializer extends JsonSerializer[DataType] {
-  private val delegate = new JValueSerializer
-  override def serialize(
-      value: DataType,
-      gen: JsonGenerator,
-      provider: SerializerProvider): Unit = {
-    delegate.serialize(value.jsonValue, gen, provider)
-  }
-}
-
-/**
- * Jackson deserializer for [[DataType]]. Internally this delegates to json4s based deserialization.
- */
-class DataTypeJsonDeserializer extends JsonDeserializer[DataType] {
-  private val delegate = new JValueDeserializer(classOf[Any])
-
-  override def deserialize(
-      jsonParser: JsonParser,
-      deserializationContext: DeserializationContext): DataType = {
-    val json = delegate.deserialize(jsonParser, deserializationContext)
-    DataType.parseDataType(json.asInstanceOf[JValue])
-  }
-}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 9ce64b09f7870..f32e48e1cc128 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -541,7 +541,7 @@ object Decimal {
   /** Maximum number of decimal digits a Long can represent */
   val MAX_LONG_DIGITS = 18
 
-  private val POW_10 = Array.tabulate[Long](MAX_LONG_DIGITS + 1)(i => math.pow(10, i).toLong)
+  val POW_10 = Array.tabulate[Long](MAX_LONG_DIGITS + 1)(i => math.pow(10, i).toLong)
 
   private val BIG_DEC_ZERO = BigDecimal(0)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
index 05069e24e2329..360a14060c15d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
@@ -159,7 +159,7 @@ object DecimalType extends AbstractDataType {
   private[sql] def checkNegativeScale(scale: Int): Unit = {
     if (scale < 0 && !SQLConf.get.allowNegativeScaleOfDecimalEnabled) {
       throw new AnalysisException(s"Negative scale is not allowed: $scale. " +
-        s"You can use spark.sql.legacy.allowNegativeScaleOfDecimal.enabled=true " +
+        s"You can use spark.sql.legacy.allowNegativeScaleOfDecimal=true " +
         s"to enable legacy mode to allow it.")
     }
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
index 01268a9ff1667..ea4f39d4b19d2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
@@ -22,6 +22,7 @@ import scala.reflect.runtime.universe.typeTag
 import scala.util.Try
 
 import org.apache.spark.annotation.Stable
+import org.apache.spark.sql.catalyst.util.SQLOrderingUtil
 
 /**
  * The data type representing `Double` values. Please use the singleton `DataTypes.DoubleType`.
@@ -38,7 +39,7 @@ class DoubleType private() extends FractionalType {
   private[sql] val numeric = implicitly[Numeric[Double]]
   private[sql] val fractional = implicitly[Fractional[Double]]
   private[sql] val ordering =
-    (x: Double, y: Double) => java.lang.Double.compare(x, y)
+    (x: Double, y: Double) => SQLOrderingUtil.compareDoubles(x, y)
   private[sql] val asIntegral = DoubleType.DoubleAsIfIntegral
 
   override private[sql] def exactNumeric = DoubleExactNumeric
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
index 1491f5904baef..f00046facf693 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
@@ -22,6 +22,7 @@ import scala.reflect.runtime.universe.typeTag
 import scala.util.Try
 
 import org.apache.spark.annotation.Stable
+import org.apache.spark.sql.catalyst.util.SQLOrderingUtil
 
 /**
  * The data type representing `Float` values. Please use the singleton `DataTypes.FloatType`.
@@ -38,7 +39,7 @@ class FloatType private() extends FractionalType {
   private[sql] val numeric = implicitly[Numeric[Float]]
   private[sql] val fractional = implicitly[Fractional[Float]]
   private[sql] val ordering =
-    (x: Float, y: Float) => java.lang.Float.compare(x, y)
+    (x: Float, y: Float) => SQLOrderingUtil.compareFloats(x, y)
   private[sql] val asIntegral = FloatType.FloatAsIfIntegral
 
   override private[sql] def exactNumeric = FloatExactNumeric
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
index 4eb3226c5786e..a29f49ad14a77 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
@@ -53,6 +53,13 @@ object HiveStringType {
     case _: HiveStringType => StringType
     case _ => dt
   }
+
+  def containsCharType(dt: DataType): Boolean = dt match {
+    case ArrayType(et, _) => containsCharType(et)
+    case MapType(kt, vt, _) => containsCharType(kt) || containsCharType(vt)
+    case StructType(fields) => fields.exists(f => containsCharType(f.dataType))
+    case _ => dt.isInstanceOf[CharType]
+  }
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
index 6af16e2dba105..689c30f6b7fa9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
@@ -78,8 +78,8 @@ abstract class UserDefinedType[UserType >: Null] extends DataType with Serializa
    */
   override private[spark] def asNullable: UserDefinedType[UserType] = this
 
-  override private[sql] def acceptsType(dataType: DataType) = dataType match {
-    case other: UserDefinedType[_] =>
+  override private[sql] def acceptsType(dataType: DataType): Boolean = dataType match {
+    case other: UserDefinedType[_] if this.userClass != null && other.userClass != null =>
       this.getClass == other.getClass ||
         this.userClass.isAssignableFrom(other.userClass)
     case _ => false
@@ -90,7 +90,7 @@ abstract class UserDefinedType[UserType >: Null] extends DataType with Serializa
   override def hashCode(): Int = getClass.hashCode()
 
   override def equals(other: Any): Boolean = other match {
-    case that: UserDefinedType[_] => this.acceptsType(that)
+    case that: UserDefinedType[_] => this.getClass == that.getClass
     case _ => false
   }
 
@@ -131,6 +131,11 @@ private[sql] class PythonUserDefinedType(
       ("sqlType" -> sqlType.jsonValue)
   }
 
+  override private[sql] def acceptsType(dataType: DataType): Boolean = dataType match {
+    case other: PythonUserDefinedType => pyUDT == other.pyUDT
+    case _ => false
+  }
+
   override def equals(other: Any): Boolean = other match {
     case that: PythonUserDefinedType => pyUDT == that.pyUDT
     case _ => false
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
index 1ac85360f944f..7026ff7de2e0d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
@@ -20,9 +20,10 @@ package org.apache.spark.sql.types
 import scala.math.Numeric._
 import scala.math.Ordering
 
+import org.apache.spark.sql.catalyst.util.SQLOrderingUtil
 import org.apache.spark.sql.types.Decimal.DecimalIsConflicted
 
-object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOrdering {
+private[sql] object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOrdering {
   private def checkOverflow(res: Int, x: Byte, y: Byte, op: String): Unit = {
     if (res > Byte.MaxValue || res < Byte.MinValue) {
       throw new ArithmeticException(s"$x $op $y caused overflow.")
@@ -56,7 +57,7 @@ object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOrdering {
 }
 
 
-object ShortExactNumeric extends ShortIsIntegral with Ordering.ShortOrdering {
+private[sql] object ShortExactNumeric extends ShortIsIntegral with Ordering.ShortOrdering {
   private def checkOverflow(res: Int, x: Short, y: Short, op: String): Unit = {
     if (res > Short.MaxValue || res < Short.MinValue) {
       throw new ArithmeticException(s"$x $op $y caused overflow.")
@@ -90,7 +91,7 @@ object ShortExactNumeric extends ShortIsIntegral with Ordering.ShortOrdering {
 }
 
 
-object IntegerExactNumeric extends IntIsIntegral with Ordering.IntOrdering {
+private[sql] object IntegerExactNumeric extends IntIsIntegral with Ordering.IntOrdering {
   override def plus(x: Int, y: Int): Int = Math.addExact(x, y)
 
   override def minus(x: Int, y: Int): Int = Math.subtractExact(x, y)
@@ -100,7 +101,7 @@ object IntegerExactNumeric extends IntIsIntegral with Ordering.IntOrdering {
   override def negate(x: Int): Int = Math.negateExact(x)
 }
 
-object LongExactNumeric extends LongIsIntegral with Ordering.LongOrdering {
+private[sql] object LongExactNumeric extends LongIsIntegral with Ordering.LongOrdering {
   override def plus(x: Long, y: Long): Long = Math.addExact(x, y)
 
   override def minus(x: Long, y: Long): Long = Math.subtractExact(x, y)
@@ -117,14 +118,14 @@ object LongExactNumeric extends LongIsIntegral with Ordering.LongOrdering {
     }
 }
 
-object FloatExactNumeric extends FloatIsFractional {
+private[sql] object FloatExactNumeric extends FloatIsFractional {
   private def overflowException(x: Float, dataType: String) =
     throw new ArithmeticException(s"Casting $x to $dataType causes overflow")
 
-  private val intUpperBound = Int.MaxValue.toFloat
-  private val intLowerBound = Int.MinValue.toFloat
-  private val longUpperBound = Long.MaxValue.toFloat
-  private val longLowerBound = Long.MinValue.toFloat
+  private val intUpperBound = Int.MaxValue
+  private val intLowerBound = Int.MinValue
+  private val longUpperBound = Long.MaxValue
+  private val longLowerBound = Long.MinValue
 
   override def toInt(x: Float): Int = {
     // When casting floating values to integral types, Spark uses the method `Numeric.toInt`
@@ -148,17 +149,17 @@ object FloatExactNumeric extends FloatIsFractional {
     }
   }
 
-  override def compare(x: Float, y: Float): Int = java.lang.Float.compare(x, y)
+  override def compare(x: Float, y: Float): Int = SQLOrderingUtil.compareFloats(x, y)
 }
 
-object DoubleExactNumeric extends DoubleIsFractional {
+private[sql] object DoubleExactNumeric extends DoubleIsFractional {
   private def overflowException(x: Double, dataType: String) =
     throw new ArithmeticException(s"Casting $x to $dataType causes overflow")
 
-  private val intUpperBound = Int.MaxValue.toDouble
-  private val intLowerBound = Int.MinValue.toDouble
-  private val longUpperBound = Long.MaxValue.toDouble
-  private val longLowerBound = Long.MinValue.toDouble
+  private val intUpperBound = Int.MaxValue
+  private val intLowerBound = Int.MinValue
+  private val longUpperBound = Long.MaxValue
+  private val longLowerBound = Long.MinValue
 
   override def toInt(x: Double): Int = {
     if (Math.floor(x) <= intUpperBound && Math.ceil(x) >= intLowerBound) {
@@ -176,10 +177,10 @@ object DoubleExactNumeric extends DoubleIsFractional {
     }
   }
 
-  override def compare(x: Double, y: Double): Int = java.lang.Double.compare(x, y)
+  override def compare(x: Double, y: Double): Int = SQLOrderingUtil.compareDoubles(x, y)
 }
 
-object DecimalExactNumeric extends DecimalIsConflicted {
+private[sql] object DecimalExactNumeric extends DecimalIsConflicted {
   override def toInt(x: Decimal): Int = x.roundToInt()
 
   override def toLong(x: Decimal): Long = x.roundToLong()
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
index 2da0d1a51cb29..003ce850c926e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
@@ -27,7 +27,7 @@ import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
-object ArrowUtils {
+private[sql] object ArrowUtils {
 
   val rootAllocator = new RootAllocator(Long.MaxValue)
 
diff --git a/sql/catalyst/src/test/resources/ansi-sql-2016-reserved-keywords.txt b/sql/catalyst/src/test/resources/ansi-sql-2016-reserved-keywords.txt
new file mode 100644
index 0000000000000..921491a4a4761
--- /dev/null
+++ b/sql/catalyst/src/test/resources/ansi-sql-2016-reserved-keywords.txt
@@ -0,0 +1,401 @@
+-- This file comes from: https://github.com/postgres/postgres/tree/master/doc/src/sgml/keywords
+ABS
+ACOS
+ALL
+ALLOCATE
+ALTER
+AND
+ANY
+ARE
+ARRAY
+ARRAY_AGG
+ARRAY_MAX_CARDINALITY
+AS
+ASENSITIVE
+ASIN
+ASYMMETRIC
+AT
+ATAN
+ATOMIC
+AUTHORIZATION
+AVG
+BEGIN
+BEGIN_FRAME
+BEGIN_PARTITION
+BETWEEN
+BIGINT
+BINARY
+BLOB
+BOOLEAN
+BOTH
+BY
+CALL
+CALLED
+CARDINALITY
+CASCADED
+CASE
+CAST
+CEIL
+CEILING
+CHAR
+CHAR_LENGTH
+CHARACTER
+CHARACTER_LENGTH
+CHECK
+CLASSIFIER
+CLOB
+CLOSE
+COALESCE
+COLLATE
+COLLECT
+COLUMN
+COMMIT
+CONDITION
+CONNECT
+CONSTRAINT
+CONTAINS
+CONVERT
+COPY
+CORR
+CORRESPONDING
+COS
+COSH
+COUNT
+COVAR_POP
+COVAR_SAMP
+CREATE
+CROSS
+CUBE
+CUME_DIST
+CURRENT
+CURRENT_CATALOG
+CURRENT_DATE
+CURRENT_DEFAULT_TRANSFORM_GROUP
+CURRENT_PATH
+CURRENT_ROLE
+CURRENT_ROW
+CURRENT_SCHEMA
+CURRENT_TIME
+CURRENT_TIMESTAMP
+CURRENT_TRANSFORM_GROUP_FOR_TYPE
+CURRENT_USER
+CURSOR
+CYCLE
+DATE
+DAY
+DEALLOCATE
+DEC
+DECIMAL
+DECFLOAT
+DECLARE
+DEFAULT
+DEFINE
+DELETE
+DENSE_RANK
+DEREF
+DESCRIBE
+DETERMINISTIC
+DISCONNECT
+DISTINCT
+DOUBLE
+DROP
+DYNAMIC
+EACH
+ELEMENT
+ELSE
+EMPTY
+END
+END_FRAME
+END_PARTITION
+END-EXEC
+EQUALS
+ESCAPE
+EVERY
+EXCEPT
+EXEC
+EXECUTE
+EXISTS
+EXP
+EXTERNAL
+EXTRACT
+FALSE
+FETCH
+FILTER
+FIRST_VALUE
+FLOAT
+FLOOR
+FOR
+FOREIGN
+FRAME_ROW
+FREE
+FROM
+FULL
+FUNCTION
+FUSION
+GET
+GLOBAL
+GRANT
+GROUP
+GROUPING
+GROUPS
+HAVING
+HOLD
+HOUR
+IDENTITY
+IN
+INDICATOR
+INITIAL
+INNER
+INOUT
+INSENSITIVE
+INSERT
+INT
+INTEGER
+INTERSECT
+INTERSECTION
+INTERVAL
+INTO
+IS
+JOIN
+JSON_ARRAY
+JSON_ARRAYAGG
+JSON_EXISTS
+JSON_OBJECT
+JSON_OBJECTAGG
+JSON_QUERY
+JSON_TABLE
+JSON_TABLE_PRIMITIVE
+JSON_VALUE
+LAG
+LANGUAGE
+LARGE
+LAST_VALUE
+LATERAL
+LEAD
+LEADING
+LEFT
+LIKE
+LIKE_REGEX
+LISTAGG
+LN
+LOCAL
+LOCALTIME
+LOCALTIMESTAMP
+LOG
+LOG10
+LOWER
+MATCH
+MATCH_NUMBER
+MATCH_RECOGNIZE
+MATCHES
+MAX
+MEASURES
+MEMBER
+MERGE
+METHOD
+MIN
+MINUTE
+MOD
+MODIFIES
+MODULE
+MONTH
+MULTISET
+NATIONAL
+NATURAL
+NCHAR
+NCLOB
+NEW
+NO
+NONE
+NORMALIZE
+NOT
+NTH_VALUE
+NTILE
+NULL
+NULLIF
+NUMERIC
+OCTET_LENGTH
+OCCURRENCES_REGEX
+OF
+OFFSET
+OLD
+OMIT
+ON
+ONE
+ONLY
+OPEN
+OR
+ORDER
+OUT
+OUTER
+OVER
+OVERLAPS
+OVERLAY
+PARAMETER
+PARTITION
+PATTERN
+PER
+PERCENT
+PERCENT_RANK
+PERCENTILE_CONT
+PERCENTILE_DISC
+PERIOD
+PERMUTE
+PORTION
+POSITION
+POSITION_REGEX
+POWER
+PRECEDES
+PRECISION
+PREPARE
+PRIMARY
+PROCEDURE
+PTF
+RANGE
+RANK
+READS
+REAL
+RECURSIVE
+REF
+REFERENCES
+REFERENCING
+REGR_AVGX
+REGR_AVGY
+REGR_COUNT
+REGR_INTERCEPT
+REGR_R2
+REGR_SLOPE
+REGR_SXX
+REGR_SXY
+REGR_SYY
+RELEASE
+RESULT
+RETURN
+RETURNS
+REVOKE
+RIGHT
+ROLLBACK
+ROLLUP
+ROW
+ROW_NUMBER
+ROWS
+RUNNING
+SAVEPOINT
+SCOPE
+SCROLL
+SEARCH
+SECOND
+SEEK
+SELECT
+SENSITIVE
+SESSION_USER
+SET
+SHOW
+SIMILAR
+SIN
+SINH
+SKIP
+SMALLINT
+SOME
+SPECIFIC
+SPECIFICTYPE
+SQL
+SQLEXCEPTION
+SQLSTATE
+SQLWARNING
+SQRT
+START
+STATIC
+STDDEV_POP
+STDDEV_SAMP
+SUBMULTISET
+SUBSET
+SUBSTRING
+SUBSTRING_REGEX
+SUCCEEDS
+SUM
+SYMMETRIC
+SYSTEM
+SYSTEM_TIME
+SYSTEM_USER
+TABLE
+TABLESAMPLE
+TAN
+TANH
+THEN
+TIME
+TIMESTAMP
+TIMEZONE_HOUR
+TIMEZONE_MINUTE
+TO
+TRAILING
+TRANSLATE
+TRANSLATE_REGEX
+TRANSLATION
+TREAT
+TRIGGER
+TRIM
+TRIM_ARRAY
+TRUE
+TRUNCATE
+UESCAPE
+UNION
+UNIQUE
+UNKNOWN
+UNMATCHED
+UNNEST
+UPDATE
+UPPER
+USER
+USING
+VALUE
+VALUES
+VALUE_OF
+VAR_POP
+VAR_SAMP
+VARBINARY
+VARCHAR
+VARYING
+VERSIONING
+WHEN
+WHENEVER
+WHERE
+WIDTH_BUCKET
+WINDOW
+WITH
+WITHIN
+WITHOUT
+YEAR
+DATALINK
+DLNEWCOPY
+DLPREVIOUSCOPY
+DLURLCOMPLETE
+DLURLCOMPLETEWRITE
+DLURLCOMPLETEONLY
+DLURLPATH
+DLURLPATHWRITE
+DLURLPATHONLY
+DLURLSCHEME
+DLURLSERVER
+DLVALUE
+IMPORT
+XML
+XMLAGG
+XMLATTRIBUTES
+XMLBINARY
+XMLCAST
+XMLCOMMENT
+XMLCONCAT
+XMLDOCUMENT
+XMLELEMENT
+XMLEXISTS
+XMLFOREST
+XMLITERATE
+XMLNAMESPACES
+XMLPARSE
+XMLPI
+XMLQUERY
+XMLSERIALIZE
+XMLTABLE
+XMLTEXT
+XMLVALIDATE
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/HashBenchmark.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/HashBenchmark.scala
index 3b4b80daf0843..3f0121bcf4a63 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/HashBenchmark.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/HashBenchmark.scala
@@ -41,13 +41,13 @@ object HashBenchmark extends BenchmarkBase {
   def test(name: String, schema: StructType, numRows: Int, iters: Int): Unit = {
     runBenchmark(name) {
       val generator = RandomDataGenerator.forType(schema, nullable = false).get
-      val encoder = RowEncoder(schema)
+      val toRow = RowEncoder(schema).createSerializer()
       val attrs = schema.toAttributes
       val safeProjection = GenerateSafeProjection.generate(attrs, attrs)
 
       val rows = (1 to numRows).map(_ =>
         // The output of encoder is UnsafeRow, use safeProjection to turn in into safe format.
-        safeProjection(encoder.toRow(generator().asInstanceOf[Row])).copy()
+        safeProjection(toRow(generator().asInstanceOf[Row])).copy()
       ).toArray
 
       val benchmark = new Benchmark("Hash For " + name, iters * numRows.toLong, output = output)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
index a7c20c34d78bc..3e2dc3f3af5f1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
@@ -18,13 +18,16 @@
 package org.apache.spark.sql
 
 import java.math.MathContext
+import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
 
 import scala.collection.mutable
-import scala.util.Random
+import scala.util.{Random, Try}
 
 import org.apache.spark.sql.catalyst.CatalystTypeConverters
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.{MICROS_PER_MILLIS, MILLIS_PER_DAY}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
 /**
@@ -161,36 +164,89 @@ object RandomDataGenerator {
       })
       case BooleanType => Some(() => rand.nextBoolean())
       case DateType =>
-        val generator =
-          () => {
-            var milliseconds = rand.nextLong() % 253402329599999L
-            // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
-            // for "0001-01-01 00:00:00.000000". We need to find a
-            // number that is greater or equals to this number as a valid timestamp value.
-            while (milliseconds < -62135740800000L) {
-              // 253402329599999L is the number of milliseconds since
-              // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
-              milliseconds = rand.nextLong() % 253402329599999L
-            }
-            DateTimeUtils.toJavaDate((milliseconds / MILLIS_PER_DAY).toInt)
+        def uniformDaysRand(rand: Random): Int = {
+          var milliseconds = rand.nextLong() % 253402329599999L
+          // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
+          // for "0001-01-01 00:00:00.000000". We need to find a
+          // number that is greater or equals to this number as a valid timestamp value.
+          while (milliseconds < -62135740800000L) {
+            // 253402329599999L is the number of milliseconds since
+            // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
+            milliseconds = rand.nextLong() % 253402329599999L
           }
-        Some(generator)
+          (milliseconds / MILLIS_PER_DAY).toInt
+        }
+        val specialDates = Seq(
+          "0001-01-01", // the fist day of Common Era
+          "1582-10-15", // the cutover date from Julian to Gregorian calendar
+          "1970-01-01", // the epoch date
+          "9999-12-31" // the last supported date according to SQL standard
+        )
+        if (SQLConf.get.getConf(SQLConf.DATETIME_JAVA8API_ENABLED)) {
+          randomNumeric[LocalDate](
+            rand,
+            (rand: Random) => LocalDate.ofEpochDay(uniformDaysRand(rand)),
+            specialDates.map(LocalDate.parse))
+        } else {
+          randomNumeric[java.sql.Date](
+            rand,
+            (rand: Random) => {
+              val date = DateTimeUtils.toJavaDate(uniformDaysRand(rand))
+              // The generated `date` is based on the hybrid calendar Julian + Gregorian since
+              // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
+              // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `date` to
+              // a local date in Proleptic Gregorian calendar to satisfy this requirement. Some
+              // years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
+              // As the consequence of that, 29 February of such years might not exist in Proleptic
+              // Gregorian calendar. When this happens, we shift the date by one day.
+              Try { date.toLocalDate; date }.getOrElse(new Date(date.getTime + MILLIS_PER_DAY))
+            },
+            specialDates.map(java.sql.Date.valueOf))
+        }
       case TimestampType =>
-        val generator =
-          () => {
-            var milliseconds = rand.nextLong() % 253402329599999L
-            // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
-            // for "0001-01-01 00:00:00.000000". We need to find a
-            // number that is greater or equals to this number as a valid timestamp value.
-            while (milliseconds < -62135740800000L) {
-              // 253402329599999L is the number of milliseconds since
-              // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
-              milliseconds = rand.nextLong() % 253402329599999L
-            }
-            // DateTimeUtils.toJavaTimestamp takes microsecond.
-            DateTimeUtils.toJavaTimestamp(milliseconds * 1000)
+        def uniformMicorsRand(rand: Random): Long = {
+          var milliseconds = rand.nextLong() % 253402329599999L
+          // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
+          // for "0001-01-01 00:00:00.000000". We need to find a
+          // number that is greater or equals to this number as a valid timestamp value.
+          while (milliseconds < -62135740800000L) {
+            // 253402329599999L is the number of milliseconds since
+            // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
+            milliseconds = rand.nextLong() % 253402329599999L
           }
-        Some(generator)
+          milliseconds * MICROS_PER_MILLIS
+        }
+        val specialTs = Seq(
+          "0001-01-01 00:00:00", // the fist timestamp of Common Era
+          "1582-10-15 23:59:59", // the cutover date from Julian to Gregorian calendar
+          "1970-01-01 00:00:00", // the epoch timestamp
+          "9999-12-31 23:59:59"  // the last supported timestamp according to SQL standard
+        )
+        if (SQLConf.get.getConf(SQLConf.DATETIME_JAVA8API_ENABLED)) {
+          randomNumeric[Instant](
+            rand,
+            (rand: Random) => DateTimeUtils.microsToInstant(uniformMicorsRand(rand)),
+            specialTs.map { s =>
+              val ldt = LocalDateTime.parse(s.replace(" ", "T"))
+              ldt.atZone(ZoneId.systemDefault()).toInstant
+            })
+        } else {
+          randomNumeric[java.sql.Timestamp](
+            rand,
+            (rand: Random) => {
+              // DateTimeUtils.toJavaTimestamp takes microsecond.
+              val ts = DateTimeUtils.toJavaTimestamp(uniformMicorsRand(rand))
+              // The generated `ts` is based on the hybrid calendar Julian + Gregorian since
+              // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
+              // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `ts` to
+              // a local timestamp in Proleptic Gregorian calendar to satisfy this requirement. Some
+              // years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
+              // As the consequence of that, 29 February of such years might not exist in Proleptic
+              // Gregorian calendar. When this happens, we shift the timestamp `ts` by one day.
+              Try { ts.toLocalDateTime; ts }.getOrElse(new Timestamp(ts.getTime + MILLIS_PER_DAY))
+            },
+            specialTs.map(java.sql.Timestamp.valueOf))
+        }
       case CalendarIntervalType => Some(() => {
         val months = rand.nextInt(1000)
         val days = rand.nextInt(10000)
@@ -204,10 +260,10 @@ object RandomDataGenerator {
           new MathContext(precision)).bigDecimal)
       case DoubleType => randomNumeric[Double](
         rand, r => longBitsToDouble(r.nextLong()), Seq(Double.MinValue, Double.MinPositiveValue,
-          Double.MaxValue, Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN, 0.0))
+          Double.MaxValue, Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN, 0.0, -0.0))
       case FloatType => randomNumeric[Float](
         rand, r => intBitsToFloat(r.nextInt()), Seq(Float.MinValue, Float.MinPositiveValue,
-          Float.MaxValue, Float.PositiveInfinity, Float.NegativeInfinity, Float.NaN, 0.0f))
+          Float.MaxValue, Float.PositiveInfinity, Float.NegativeInfinity, Float.NaN, 0.0f, -0.0f))
       case ByteType => randomNumeric[Byte](
         rand, _.nextInt().toByte, Seq(Byte.MinValue, Byte.MaxValue, 0.toByte))
       case IntegerType => randomNumeric[Int](
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
index 3e62ca069e9ea..cb335e5f435a3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
@@ -24,30 +24,36 @@ import scala.util.Random
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.CatalystTypeConverters
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
  * Tests of [[RandomDataGenerator]].
  */
-class RandomDataGeneratorSuite extends SparkFunSuite {
+class RandomDataGeneratorSuite extends SparkFunSuite with SQLHelper {
 
   /**
    * Tests random data generation for the given type by using it to generate random values then
    * converting those values into their Catalyst equivalents using CatalystTypeConverters.
    */
   def testRandomDataGeneration(dataType: DataType, nullable: Boolean = true): Unit = {
-    val toCatalyst = CatalystTypeConverters.createToCatalystConverter(dataType)
-    val generator = RandomDataGenerator.forType(dataType, nullable, new Random(33)).getOrElse {
-      fail(s"Random data generator was not defined for $dataType")
-    }
-    if (nullable) {
-      assert(Iterator.fill(100)(generator()).contains(null))
-    } else {
-      assert(!Iterator.fill(100)(generator()).contains(null))
-    }
-    for (_ <- 1 to 10) {
-      val generatedValue = generator()
-      toCatalyst(generatedValue)
+    Seq(false, true).foreach { java8Api =>
+      withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+        val toCatalyst = CatalystTypeConverters.createToCatalystConverter(dataType)
+        val generator = RandomDataGenerator.forType(dataType, nullable, new Random(33)).getOrElse {
+          fail(s"Random data generator was not defined for $dataType")
+        }
+        if (nullable) {
+          assert(Iterator.fill(100)(generator()).contains(null))
+        } else {
+          assert(!Iterator.fill(100)(generator()).contains(null))
+        }
+        for (_ <- 1 to 10) {
+          val generatedValue = generator()
+          toCatalyst(generatedValue)
+        }
+      }
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
index 42a4cfc91f826..950e313fb727a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
@@ -37,8 +37,8 @@ object UnsafeProjectionBenchmark extends BenchmarkBase {
 
   def generateRows(schema: StructType, numRows: Int): Array[InternalRow] = {
     val generator = RandomDataGenerator.forType(schema, nullable = false).get
-    val encoder = RowEncoder(schema)
-    (1 to numRows).map(_ => encoder.toRow(generator().asInstanceOf[Row]).copy()).toArray
+    val toRow = RowEncoder(schema).createSerializer()
+    (1 to numRows).map(_ => toRow(generator().asInstanceOf[Row]).copy()).toArray
   }
 
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala
new file mode 100644
index 0000000000000..082b01173e7b8
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst
+
+import java.io.File
+import java.nio.file.Files
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.fileToString
+
+trait SQLKeywordUtils extends SQLHelper {
+
+  val sqlSyntaxDefs = {
+    val sqlBasePath = {
+      java.nio.file.Paths.get(sparkHome, "sql", "catalyst", "src", "main", "antlr4", "org",
+        "apache", "spark", "sql", "catalyst", "parser", "SqlBase.g4").toFile
+    }
+    fileToString(sqlBasePath).split("\n")
+  }
+
+  // each element is an array of 4 string: the keyword name, reserve or not in Spark ANSI mode,
+  // Spark default mode, and the SQL standard.
+  val keywordsInDoc: Array[Array[String]] = {
+    val docPath = {
+      java.nio.file.Paths.get(sparkHome, "docs", "sql-ref-ansi-compliance.md").toFile
+    }
+    fileToString(docPath).split("\n")
+      .dropWhile(!_.startsWith("|Keyword|")).drop(2).takeWhile(_.startsWith("|"))
+      .map(_.stripPrefix("|").split("\\|").map(_.trim))
+  }
+
+  private def parseAntlrGrammars[T](startTag: String, endTag: String)
+      (f: PartialFunction[String, Seq[T]]): Set[T] = {
+    val keywords = new mutable.ArrayBuffer[T]
+    val default = (_: String) => Nil
+    var startTagFound = false
+    var parseFinished = false
+    val lineIter = sqlSyntaxDefs.toIterator
+    while (!parseFinished && lineIter.hasNext) {
+      val line = lineIter.next()
+      if (line.trim.startsWith(startTag)) {
+        startTagFound = true
+      } else if (line.trim.startsWith(endTag)) {
+        parseFinished = true
+      } else if (startTagFound) {
+        f.applyOrElse(line, default).foreach { symbol =>
+          keywords += symbol
+        }
+      }
+    }
+    assert(keywords.nonEmpty && startTagFound && parseFinished, "cannot extract keywords from " +
+      s"the `SqlBase.g4` file, so please check if the start/end tags (`$startTag` and `$endTag`) " +
+      "are placed correctly in the file.")
+    keywords.toSet
+  }
+
+  // If a symbol does not have the same string with its literal (e.g., `SETMINUS: 'MINUS';`),
+  // we need to map a symbol to actual literal strings.
+  val symbolsToExpandIntoDifferentLiterals = {
+    val kwDef = """([A-Z_]+):(.+);""".r
+    val keywords = parseAntlrGrammars(
+        "//--SPARK-KEYWORD-LIST-START", "//--SPARK-KEYWORD-LIST-END") {
+      case kwDef(symbol, literalDef) =>
+        val splitDefs = literalDef.split("""\|""")
+        val hasMultipleLiterals = splitDefs.length > 1
+        // The case where a symbol has multiple literal definitions,
+        // e.g., `DATABASES: 'DATABASES' | 'SCHEMAS';`.
+        if (hasMultipleLiterals) {
+          // Filters out inappropriate entries, e.g., `!` in `NOT: 'NOT' | '!';`
+          val litDef = """([A-Z_]+)""".r
+          val literals = splitDefs.map(_.replaceAll("'", "").trim).toSeq.flatMap {
+            case litDef(lit) => Some(lit)
+            case _ => None
+          }
+          (symbol, literals) :: Nil
+        } else {
+          val literal = literalDef.replaceAll("'", "").trim
+          // The case where a symbol string and its literal string are different,
+          // e.g., `SETMINUS: 'MINUS';`.
+          if (symbol != literal) {
+            (symbol, literal :: Nil) :: Nil
+          } else {
+            Nil
+          }
+        }
+    }
+    keywords.toMap
+  }
+
+  // All the SQL keywords defined in `SqlBase.g4`
+  val allCandidateKeywords: Set[String] = {
+    val kwDef = """([A-Z_]+):.+;""".r
+    parseAntlrGrammars(
+        "//--SPARK-KEYWORD-LIST-START", "//--SPARK-KEYWORD-LIST-END") {
+      // Parses a pattern, e.g., `AFTER: 'AFTER';`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+  }
+
+  val nonReservedKeywordsInAnsiMode: Set[String] = {
+    val kwDef = """\s*[\|:]\s*([A-Z_]+)\s*""".r
+    parseAntlrGrammars("//--ANSI-NON-RESERVED-START", "//--ANSI-NON-RESERVED-END") {
+      // Parses a pattern, e.g., `    | AFTER`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+  }
+
+  val reservedKeywordsInAnsiMode = allCandidateKeywords -- nonReservedKeywordsInAnsiMode
+
+  val nonReservedKeywordsInDefaultMode: Set[String] = {
+    val kwDef = """\s*[\|:]\s*([A-Z_]+)\s*""".r
+    parseAntlrGrammars("//--DEFAULT-NON-RESERVED-START", "//--DEFAULT-NON-RESERVED-END") {
+      // Parses a pattern, e.g., `    | AFTER`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+  }
+}
+
+class SQLKeywordSuite extends SparkFunSuite with SQLKeywordUtils {
+  test("all keywords are documented") {
+    val documentedKeywords = keywordsInDoc.map(_.head).toSet
+    if (allCandidateKeywords != documentedKeywords) {
+      val undocumented = (allCandidateKeywords -- documentedKeywords).toSeq.sorted
+      fail("Some keywords are not documented: " + undocumented.mkString(", "))
+    }
+  }
+
+  test("Spark keywords are documented correctly under ANSI mode") {
+    // keywords under ANSI mode should either be reserved or non-reserved.
+    keywordsInDoc.map(_.apply(1)).foreach { desc =>
+      assert(desc == "reserved" || desc == "non-reserved")
+    }
+
+    val nonReservedInDoc = keywordsInDoc.filter(_.apply(1) == "non-reserved").map(_.head).toSet
+    if (nonReservedKeywordsInAnsiMode != nonReservedInDoc) {
+      val misImplemented = ((nonReservedInDoc -- nonReservedKeywordsInAnsiMode) ++
+        (nonReservedKeywordsInAnsiMode -- nonReservedInDoc)).toSeq.sorted
+      fail("Some keywords are documented and implemented inconsistently: " +
+        misImplemented.mkString(", "))
+    }
+  }
+
+  test("Spark keywords are documented correctly under default mode") {
+    // keywords under default mode should either be strict-non-reserved or non-reserved.
+    keywordsInDoc.map(_.apply(2)).foreach { desc =>
+      assert(desc == "strict-non-reserved" || desc == "non-reserved")
+    }
+
+    val nonReservedInDoc = keywordsInDoc.filter(_.apply(2) == "non-reserved").map(_.head).toSet
+    if (nonReservedKeywordsInDefaultMode != nonReservedInDoc) {
+      val misImplemented = ((nonReservedInDoc -- nonReservedKeywordsInDefaultMode) ++
+        (nonReservedKeywordsInDefaultMode -- nonReservedInDoc)).toSeq.sorted
+      fail("Some keywords are documented and implemented inconsistently: " +
+        misImplemented.mkString(", "))
+    }
+  }
+
+  test("SQL 2016 keywords are documented correctly") {
+    withTempDir { dir =>
+      val tmpFile = new File(dir, "tmp")
+      val is = Thread.currentThread().getContextClassLoader
+        .getResourceAsStream("ansi-sql-2016-reserved-keywords.txt")
+      Files.copy(is, tmpFile.toPath)
+      val reservedKeywordsInSql2016 = Files.readAllLines(tmpFile.toPath)
+        .asScala.filterNot(_.startsWith("--")).map(_.trim).toSet
+      val documented = keywordsInDoc.filter(_.last == "reserved").map(_.head).toSet
+      assert((documented -- reservedKeywordsInSql2016).isEmpty)
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index f9cd9c3c398f6..b981a50499bf5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedExtractValue
 import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, If, SpecificInternalRow, UpCast}
 import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, NewInstance}
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
 
 case class PrimitiveData(
     intField: Int,
@@ -48,7 +49,8 @@ case class NullableData(
     decimalField: java.math.BigDecimal,
     dateField: Date,
     timestampField: Timestamp,
-    binaryField: Array[Byte])
+    binaryField: Array[Byte],
+    intervalField: CalendarInterval)
 
 case class OptionalData(
     intField: Option[Int],
@@ -58,7 +60,8 @@ case class OptionalData(
     shortField: Option[Short],
     byteField: Option[Byte],
     booleanField: Option[Boolean],
-    structField: Option[PrimitiveData])
+    structField: Option[PrimitiveData],
+    intervalField: Option[CalendarInterval])
 
 case class ComplexData(
     arrayField: Seq[Int],
@@ -200,7 +203,8 @@ class ScalaReflectionSuite extends SparkFunSuite {
         StructField("decimalField", DecimalType.SYSTEM_DEFAULT, nullable = true),
         StructField("dateField", DateType, nullable = true),
         StructField("timestampField", TimestampType, nullable = true),
-        StructField("binaryField", BinaryType, nullable = true))),
+        StructField("binaryField", BinaryType, nullable = true),
+        StructField("intervalField", CalendarIntervalType, nullable = true))),
       nullable = true))
   }
 
@@ -215,7 +219,8 @@ class ScalaReflectionSuite extends SparkFunSuite {
         StructField("shortField", ShortType, nullable = true),
         StructField("byteField", ByteType, nullable = true),
         StructField("booleanField", BooleanType, nullable = true),
-        StructField("structField", schemaFor[PrimitiveData].dataType, nullable = true))),
+        StructField("structField", schemaFor[PrimitiveData].dataType, nullable = true),
+        StructField("intervalField", CalendarIntervalType, nullable = true))),
       nullable = true))
   }
 
@@ -295,10 +300,10 @@ class ScalaReflectionSuite extends SparkFunSuite {
   test("convert Option[Product] to catalyst") {
     val primitiveData = PrimitiveData(1, 1, 1, 1, 1, 1, true)
     val data = OptionalData(Some(2), Some(2), Some(2), Some(2), Some(2), Some(2), Some(true),
-      Some(primitiveData))
+      Some(primitiveData), Some(new CalendarInterval(1, 2, 3)))
     val dataType = schemaFor[OptionalData].dataType
     val convertedData = InternalRow(2, 2.toLong, 2.toDouble, 2.toFloat, 2.toShort, 2.toByte, true,
-      InternalRow(1, 1, 1, 1, 1, 1, true))
+      InternalRow(1, 1, 1, 1, 1, 1, true), new CalendarInterval(1, 2, 3))
     assert(CatalystTypeConverters.createToCatalystConverter(dataType)(data) === convertedData)
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 7023dbe2a3672..09e0d9c4882fb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.analysis
 import org.scalatest.Assertions._
 
 import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
@@ -164,6 +165,22 @@ class AnalysisErrorSuite extends AnalysisTest {
           UnspecifiedFrame)).as("window")),
     "Distinct window functions are not supported" :: Nil)
 
+  errorTest(
+    "window aggregate function with filter predicate",
+    testRelation2.select(
+      WindowExpression(
+        AggregateExpression(
+          Count(UnresolvedAttribute("b")),
+          Complete,
+          isDistinct = false,
+          filter = Some(UnresolvedAttribute("b") > 1)),
+        WindowSpecDefinition(
+          UnresolvedAttribute("a") :: Nil,
+          SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
+          UnspecifiedFrame)).as("window")),
+    "window aggregate function with filter predicate is not supported" :: Nil
+  )
+
   errorTest(
     "distinct function",
     CatalystSqlParser.parsePlan("SELECT hex(DISTINCT a) FROM TaBlE"),
@@ -191,12 +208,12 @@ class AnalysisErrorSuite extends AnalysisTest {
     "FILTER predicate specified, but aggregate is not an aggregate function" :: Nil)
 
   errorTest(
-    "DISTINCT and FILTER cannot be used in aggregate functions at the same time",
+    "DISTINCT aggregate function with filter predicate",
     CatalystSqlParser.parsePlan("SELECT count(DISTINCT a) FILTER (WHERE c > 1) FROM TaBlE2"),
     "DISTINCT and FILTER cannot be used in aggregate functions at the same time" :: Nil)
 
   errorTest(
-    "FILTER expression is non-deterministic, it cannot be used in aggregate functions",
+    "non-deterministic filter predicate in aggregate functions",
     CatalystSqlParser.parsePlan("SELECT count(a) FILTER (WHERE rand(int(c)) > 1) FROM TaBlE2"),
     "FILTER expression is non-deterministic, it cannot be used in aggregate functions" :: Nil)
 
@@ -416,6 +433,32 @@ class AnalysisErrorSuite extends AnalysisTest {
       :: Nil
   )
 
+  errorTest(
+    "SPARK-30998: unsupported nested inner generators",
+    {
+      val nestedListRelation = LocalRelation(
+        AttributeReference("nestedList", ArrayType(ArrayType(IntegerType)))())
+      nestedListRelation.select(Explode(Explode($"nestedList")))
+    },
+    "Generators are not supported when it's nested in expressions, but got: " +
+      "explode(explode(nestedList))" :: Nil
+  )
+
+  errorTest(
+    "SPARK-30998: unsupported nested inner generators for aggregates",
+    testRelation.select(Explode(Explode(
+      CreateArray(CreateArray(min($"a") :: max($"a") :: Nil) :: Nil)))),
+    "Generators are not supported when it's nested in expressions, but got: " +
+      "explode(explode(array(array(min(a), max(a)))))" :: Nil
+  )
+
+  errorTest(
+    "generator nested in expressions for aggregates",
+    testRelation.select(Explode(CreateArray(min($"a") :: max($"a") :: Nil)) + 1),
+    "Generators are not supported when it's nested in expressions, but got: " +
+      "(explode(array(min(a), max(a))) + 1)" :: Nil
+  )
+
   errorTest(
     "generator appears in operator which is not Project",
     listRelation.sortBy(Explode($"list").asc),
@@ -440,6 +483,14 @@ class AnalysisErrorSuite extends AnalysisTest {
     "Only one generator allowed per select clause but found 2: explode(list), explode(list)" :: Nil
   )
 
+  errorTest(
+    "more than one generators for aggregates in SELECT",
+    testRelation.select(Explode(CreateArray(min($"a") :: Nil)),
+      Explode(CreateArray(max($"a") :: Nil))),
+    "Only one generator allowed per select clause but found 2: " +
+      "explode(array(min(a))), explode(array(max(a)))" :: Nil
+  )
+
   test("SPARK-6452 regression test") {
     // CheckAnalysis should throw AnalysisException when Aggregate contains missing attribute(s)
     // Since we manually construct the logical plan at here and Sum only accept
@@ -636,4 +687,15 @@ class AnalysisErrorSuite extends AnalysisTest {
     assertAnalysisError(plan,
       "Aggregate/Window/Generate expressions are not valid in where clause of the query" :: Nil)
   }
+
+  test("SPARK-30811: CTE should not cause stack overflow when " +
+    "it refers to non-existent table with same name") {
+    val plan = With(
+      UnresolvedRelation(TableIdentifier("t")),
+      Seq("t" -> SubqueryAlias("t",
+        Project(
+          Alias(Literal(1), "x")() :: Nil,
+          UnresolvedRelation(TableIdentifier("t", Option("nonexist")))))))
+    assertAnalysisError(plan, "Table or view not found:" :: Nil)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 5405009c9e208..63c277918a429 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -21,15 +21,19 @@ import java.util.{Locale, TimeZone}
 
 import scala.reflect.ClassTag
 
+import org.apache.log4j.Level
 import org.scalatest.Matchers
 
 import org.apache.spark.api.python.PythonEvalType
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.{AliasIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate.{Count, Sum}
+import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count, Sum}
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
 import org.apache.spark.sql.catalyst.plans.{Cross, Inner}
 import org.apache.spark.sql.catalyst.plans.logical._
@@ -43,6 +47,13 @@ import org.apache.spark.sql.types._
 class AnalysisSuite extends AnalysisTest with Matchers {
   import org.apache.spark.sql.catalyst.analysis.TestRelations._
 
+  test("fail for unresolved plan") {
+    intercept[AnalysisException] {
+      // `testRelation` does not have column `b`.
+      testRelation.select('b).analyze
+    }
+  }
+
   test("union project *") {
     val plan = (1 to 120)
       .map(_ => testRelation)
@@ -324,20 +335,21 @@ class AnalysisSuite extends AnalysisTest with Matchers {
     }
 
     // non-primitive parameters do not need special null handling
-    val udf1 = ScalaUDF((s: String) => "x", StringType, string :: Nil, false :: Nil)
+    val udf1 = ScalaUDF((s: String) => "x", StringType, string :: Nil,
+      Option(ExpressionEncoder[String]()) :: Nil)
     val expected1 = udf1
     checkUDF(udf1, expected1)
 
     // only primitive parameter needs special null handling
     val udf2 = ScalaUDF((s: String, d: Double) => "x", StringType, string :: double :: Nil,
-      false :: true :: Nil)
+      Option(ExpressionEncoder[String]()) :: Option(ExpressionEncoder[Double]()) :: Nil)
     val expected2 =
       If(IsNull(double), nullResult, udf2.copy(children = string :: KnownNotNull(double) :: Nil))
     checkUDF(udf2, expected2)
 
     // special null handling should apply to all primitive parameters
     val udf3 = ScalaUDF((s: Short, d: Double) => "x", StringType, short :: double :: Nil,
-      true :: true :: Nil)
+      Option(ExpressionEncoder[Short]()) :: Option(ExpressionEncoder[Double]()) :: Nil)
     val expected3 = If(
       IsNull(short) || IsNull(double),
       nullResult,
@@ -349,7 +361,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       (s: Short, d: Double) => "x",
       StringType,
       short :: nonNullableDouble :: Nil,
-      true :: true :: Nil)
+      Option(ExpressionEncoder[Short]()) :: Option(ExpressionEncoder[Double]()) :: Nil)
     val expected4 = If(
       IsNull(short),
       nullResult,
@@ -360,8 +372,12 @@ class AnalysisSuite extends AnalysisTest with Matchers {
   test("SPARK-24891 Fix HandleNullInputsForUDF rule") {
     val a = testRelation.output(0)
     val func = (x: Int, y: Int) => x + y
-    val udf1 = ScalaUDF(func, IntegerType, a :: a :: Nil, false :: false :: Nil)
-    val udf2 = ScalaUDF(func, IntegerType, a :: udf1 :: Nil, false :: false :: Nil)
+    val udf1 = ScalaUDF(func, IntegerType, a :: a :: Nil,
+      Option(ExpressionEncoder[java.lang.Integer]()) ::
+        Option(ExpressionEncoder[java.lang.Integer]()) :: Nil)
+    val udf2 = ScalaUDF(func, IntegerType, a :: udf1 :: Nil,
+      Option(ExpressionEncoder[java.lang.Integer]()) ::
+        Option(ExpressionEncoder[java.lang.Integer]()) :: Nil)
     val plan = Project(Alias(udf2, "")() :: Nil, testRelation)
     comparePlans(plan.analyze, plan.analyze.analyze)
   }
@@ -736,5 +752,225 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       b :: ScalarSubquery(subquery, Nil).as("sum") :: Nil,
       CollectMetrics("evt1", count :: Nil, tblB))
     assertAnalysisError(query, "Multiple definitions of observed metrics" :: "evt1" :: Nil)
+
+    // Aggregate with filter predicate - fail
+    val sumWithFilter = sum.transform {
+      case a: AggregateExpression => a.copy(filter = Some(true))
+    }.asInstanceOf[NamedExpression]
+    assertAnalysisError(
+      CollectMetrics("evt1", sumWithFilter :: Nil, testRelation),
+      "aggregates with filter predicate are not allowed" :: Nil)
+  }
+
+  test("Analysis exceed max iterations") {
+    // RuleExecutor only throw exception or log warning when the rule is supposed to run
+    // more than once.
+    val maxIterations = 2
+    val conf = new SQLConf().copy(SQLConf.ANALYZER_MAX_ITERATIONS -> maxIterations)
+    val testAnalyzer = new Analyzer(
+      new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+
+    val plan = testRelation2.select(
+      $"a" / Literal(2) as "div1",
+      $"a" / $"b" as "div2",
+      $"a" / $"c" as "div3",
+      $"a" / $"d" as "div4",
+      $"e" / $"e" as "div5")
+
+    val message = intercept[TreeNodeException[LogicalPlan]] {
+      testAnalyzer.execute(plan)
+    }.getMessage
+    assert(message.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
+      s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
+  }
+
+  test("SPARK-30886 Deprecate two-parameter TRIM/LTRIM/RTRIM") {
+    Seq("trim", "ltrim", "rtrim").foreach { f =>
+      val logAppender = new LogAppender("deprecated two-parameter TRIM/LTRIM/RTRIM functions")
+      def check(count: Int): Unit = {
+        val message = "Two-parameter TRIM/LTRIM/RTRIM function signatures are deprecated."
+        assert(logAppender.loggingEvents.size == count)
+        assert(logAppender.loggingEvents.exists(
+          e => e.getLevel == Level.WARN &&
+            e.getRenderedMessage.contains(message)))
+      }
+
+      withLogAppender(logAppender) {
+        val testAnalyzer1 = new Analyzer(
+          new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+
+        val plan1 = testRelation2.select(
+          UnresolvedFunction(f, $"a" :: Nil, isDistinct = false))
+        testAnalyzer1.execute(plan1)
+        // One-parameter is not deprecated.
+        assert(logAppender.loggingEvents.isEmpty)
+
+        val plan2 = testRelation2.select(
+          UnresolvedFunction(f, $"a" :: $"b" :: Nil, isDistinct = false))
+        testAnalyzer1.execute(plan2)
+        // Deprecation warning is printed out once.
+        check(1)
+
+        val plan3 = testRelation2.select(
+          UnresolvedFunction(f, $"b" :: $"a" :: Nil, isDistinct = false))
+        testAnalyzer1.execute(plan3)
+        // There is no change in the log.
+        check(1)
+
+        // New analyzer from new SessionState
+        val testAnalyzer2 = new Analyzer(
+          new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+        val plan4 = testRelation2.select(
+          UnresolvedFunction(f, $"c" :: $"d" :: Nil, isDistinct = false))
+        testAnalyzer2.execute(plan4)
+        // Additional deprecation warning from new analyzer
+        check(2)
+
+        val plan5 = testRelation2.select(
+          UnresolvedFunction(f, $"c" :: $"d" :: Nil, isDistinct = false))
+        testAnalyzer2.execute(plan5)
+        // There is no change in the log.
+        check(2)
+      }
+    }
+  }
+
+  test("SPARK-32131: Fix wrong column index when we have more than two columns" +
+    " during union and set operations" ) {
+    val firstTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", DoubleType)(),
+      AttributeReference("c", IntegerType)(),
+      AttributeReference("d", FloatType)())
+
+    val secondTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", TimestampType)(),
+      AttributeReference("c", IntegerType)(),
+      AttributeReference("d", FloatType)())
+
+    val thirdTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", DoubleType)(),
+      AttributeReference("c", TimestampType)(),
+      AttributeReference("d", FloatType)())
+
+    val fourthTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", DoubleType)(),
+      AttributeReference("c", IntegerType)(),
+      AttributeReference("d", TimestampType)())
+
+    val r1 = Union(firstTable, secondTable)
+    val r2 = Union(firstTable, thirdTable)
+    val r3 = Union(firstTable, fourthTable)
+    val r4 = Except(firstTable, secondTable, isAll = false)
+    val r5 = Intersect(firstTable, secondTable, isAll = false)
+
+    assertAnalysisError(r1,
+      Seq("Union can only be performed on tables with the compatible column types. " +
+        "timestamp <> double at the second column of the second table"))
+
+    assertAnalysisError(r2,
+      Seq("Union can only be performed on tables with the compatible column types. " +
+        "timestamp <> int at the third column of the second table"))
+
+    assertAnalysisError(r3,
+      Seq("Union can only be performed on tables with the compatible column types. " +
+        "timestamp <> float at the 4th column of the second table"))
+
+    assertAnalysisError(r4,
+      Seq("Except can only be performed on tables with the compatible column types. " +
+        "timestamp <> double at the second column of the second table"))
+
+    assertAnalysisError(r5,
+      Seq("Intersect can only be performed on tables with the compatible column types. " +
+        "timestamp <> double at the second column of the second table"))
+  }
+
+  test("SPARK-32237: Hint in CTE") {
+    val plan = With(
+      Project(
+        Seq(UnresolvedAttribute("cte.a")),
+        UnresolvedRelation(TableIdentifier("cte"))
+      ),
+      Seq(
+        (
+          "cte",
+          SubqueryAlias(
+            AliasIdentifier("cte"),
+            UnresolvedHint(
+              "REPARTITION",
+              Seq(Literal(3)),
+              Project(testRelation.output, testRelation)
+            )
+          )
+        )
+      )
+    )
+    assertAnalysisSuccess(plan)
+  }
+
+  test("SPARK-33197: Make sure changes to ANALYZER_MAX_ITERATIONS take effect at runtime") {
+    // RuleExecutor only throw exception or log warning when the rule is supposed to run
+    // more than once.
+    val maxIterations = 2
+    val maxIterationsEnough = 5
+    withSQLConf(SQLConf.ANALYZER_MAX_ITERATIONS.key -> maxIterations.toString) {
+      val conf = SQLConf.get
+      val testAnalyzer = new Analyzer(
+        new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+
+      val plan = testRelation2.select(
+        $"a" / Literal(2) as "div1",
+        $"a" / $"b" as "div2",
+        $"a" / $"c" as "div3",
+        $"a" / $"d" as "div4",
+        $"e" / $"e" as "div5")
+
+      val message1 = intercept[TreeNodeException[LogicalPlan]] {
+        testAnalyzer.execute(plan)
+      }.getMessage
+      assert(message1.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
+        s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
+
+      withSQLConf(SQLConf.ANALYZER_MAX_ITERATIONS.key -> maxIterationsEnough.toString) {
+        try {
+          testAnalyzer.execute(plan)
+        } catch {
+          case ex: TreeNodeException[_]
+            if ex.getMessage.contains(SQLConf.ANALYZER_MAX_ITERATIONS.key) =>
+              fail("analyzer.execute should not reach max iterations.")
+        }
+      }
+
+      val message2 = intercept[TreeNodeException[LogicalPlan]] {
+        testAnalyzer.execute(plan)
+      }.getMessage
+      assert(message2.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
+        s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
+    }
+  }
+
+  test("SPARK-33733: PullOutNondeterministic should check and collect deterministic field") {
+    val reflect =
+      CallMethodViaReflection(Seq("java.lang.Math", "abs", testRelation.output.head))
+    val udf = ScalaUDF(
+      (s: String) => s,
+      StringType,
+      Literal.create(null, StringType) :: Nil,
+      Option(ExpressionEncoder[String]().resolveAndBind()) :: Nil,
+      udfDeterministic = false)
+
+    Seq(reflect, udf).foreach { e: Expression =>
+      val plan = Sort(Seq(e.asc), false, testRelation)
+      val projected = Alias(e, "_nondeterministic")()
+      val expect =
+        Project(testRelation.output,
+          Sort(Seq(projected.toAttribute.asc), false,
+            Project(testRelation.output :+ projected,
+              testRelation)))
+      checkAnalysis(plan, expect)
+    }
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
index 3f8d409992381..4473c20b2cca6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
@@ -45,6 +45,8 @@ trait AnalysisTest extends PlanTest {
     catalog.createTempView("TaBlE", TestRelations.testRelation, overrideIfExists = true)
     catalog.createTempView("TaBlE2", TestRelations.testRelation2, overrideIfExists = true)
     catalog.createTempView("TaBlE3", TestRelations.testRelation3, overrideIfExists = true)
+    catalog.createGlobalTempView("TaBlE4", TestRelations.testRelation4, overrideIfExists = true)
+    catalog.createGlobalTempView("TaBlE5", TestRelations.testRelation5, overrideIfExists = true)
     new Analyzer(catalog, conf) {
       override val extendedResolutionRules = EliminateSubqueryAliases +: extendedAnalysisRules
     }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
index c01dea96fe2de..3810434cffb53 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
@@ -21,7 +21,7 @@ import java.net.URI
 import java.util.Locale
 
 import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, InMemoryCatalog, SessionCatalog}
-import org.apache.spark.sql.catalyst.expressions.{Alias, AnsiCast, AttributeReference, Cast, Expression, LessThanOrEqual, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Alias, AnsiCast, AttributeReference, Cast, LessThanOrEqual, Literal}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
@@ -143,7 +143,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write", "'table-name'",
-      "Cannot safely cast", "'x'", "'y'", "DoubleType to FloatType"))
+      "Cannot safely cast", "'x'", "'y'", "double to float"))
   }
 
   test("byName: multiple field errors are reported") {
@@ -160,7 +160,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write incompatible data to table", "'table-name'",
-      "Cannot safely cast", "'x'", "DoubleType to FloatType",
+      "Cannot safely cast", "'x'", "double to float",
       "Cannot write nullable values to non-null column", "'x'",
       "Cannot find data for output column", "'y'"))
   }
@@ -176,7 +176,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write", "'table-name'",
-      "Cannot safely cast", "'x'", "'y'", "DoubleType to FloatType"))
+      "Cannot safely cast", "'x'", "'y'", "double to float"))
   }
 
   test("byPosition: multiple field errors are reported") {
@@ -194,7 +194,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write incompatible data to table", "'table-name'",
       "Cannot write nullable values to non-null column", "'x'",
-      "Cannot safely cast", "'x'", "DoubleType to FloatType"))
+      "Cannot safely cast", "'x'", "double to float"))
   }
 }
 
@@ -230,6 +230,15 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
 
   def byPosition(table: NamedRelation, query: LogicalPlan): LogicalPlan
 
+  test("skipSchemaResolution should still require query to be resolved") {
+    val table = TestRelationAcceptAnySchema(StructType(Seq(
+      StructField("a", FloatType),
+      StructField("b", DoubleType))).toAttributes)
+    val query = UnresolvedRelation(Seq("t"))
+    val parsedPlan = byName(table, query)
+    assertNotResolved(parsedPlan)
+  }
+
   test("byName: basic behavior") {
     val query = TestRelation(table.schema.toAttributes)
 
@@ -579,9 +588,7 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
         Alias(Cast(a, DoubleType, Some(conf.sessionLocalTimeZone)), "x")(),
         Alias(Cast(b, DoubleType, Some(conf.sessionLocalTimeZone)), "y")()),
         query),
-      LessThanOrEqual(
-        AttributeReference("x", DoubleType, nullable = false)(x.exprId),
-        Literal(15.0d)))
+      LessThanOrEqual(x, Literal(15.0d)))
 
     assertNotResolved(parsedPlan)
     checkAnalysis(parsedPlan, expectedPlan)
@@ -589,7 +596,7 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
   }
 
   protected def testNotResolvedOverwriteByExpression(): Unit = {
-    val xRequiredTable = TestRelation(StructType(Seq(
+    val table = TestRelation(StructType(Seq(
       StructField("x", DoubleType, nullable = false),
       StructField("y", DoubleType))).toAttributes)
 
@@ -598,10 +605,19 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
       StructField("b", DoubleType))).toAttributes)
 
     // the write is resolved (checked above). this test plan is not because of the expression.
-    val parsedPlan = OverwriteByExpression.byPosition(xRequiredTable, query,
+    val parsedPlan = OverwriteByExpression.byPosition(table, query,
       LessThanOrEqual(UnresolvedAttribute(Seq("a")), Literal(15.0d)))
 
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq("cannot resolve", "`a`", "given input columns", "x, y"))
+
+    val tableAcceptAnySchema = TestRelationAcceptAnySchema(StructType(Seq(
+      StructField("x", DoubleType, nullable = false),
+      StructField("y", DoubleType))).toAttributes)
+
+    val parsedPlan2 = OverwriteByExpression.byPosition(tableAcceptAnySchema, query,
+      LessThanOrEqual(UnresolvedAttribute(Seq("a")), Literal(15.0d)))
+    assertNotResolved(parsedPlan2)
+    assertAnalysisError(parsedPlan2, Seq("cannot resolve", "`a`", "given input columns", "x, y"))
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
index 86a1f1fb58a07..46634c93148b1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
@@ -158,8 +158,8 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
 
     assertError(Min(Symbol("mapField")), "min does not support ordering on type")
     assertError(Max(Symbol("mapField")), "max does not support ordering on type")
-    assertError(Sum(Symbol("booleanField")), "requires (numeric or interval) type")
-    assertError(Average(Symbol("booleanField")), "requires (numeric or interval) type")
+    assertError(Sum(Symbol("booleanField")), "function sum requires numeric type")
+    assertError(Average(Symbol("booleanField")), "function average requires numeric type")
   }
 
   test("check types for others") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
index 5e66c038738a4..d3bd5d07a0932 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
@@ -241,4 +241,57 @@ class ResolveHintsSuite extends AnalysisTest {
       Project(testRelation.output, testRelation),
       caseSensitive = false)
   }
+
+  test("Supports multi-part table names for join strategy hint resolution") {
+    Seq(("MAPJOIN", BROADCAST),
+        ("MERGEJOIN", SHUFFLE_MERGE),
+        ("SHUFFLE_HASH", SHUFFLE_HASH),
+        ("SHUFFLE_REPLICATE_NL", SHUFFLE_REPLICATE_NL)).foreach { case (hintName, st) =>
+      // local temp table (single-part identifier case)
+      checkAnalysis(
+        UnresolvedHint(hintName, Seq("table", "table2"),
+          table("TaBlE").join(table("TaBlE2"))),
+        Join(
+          ResolvedHint(testRelation, HintInfo(strategy = Some(st))),
+          ResolvedHint(testRelation2, HintInfo(strategy = Some(st))),
+          Inner,
+          None,
+          JoinHint.NONE),
+        caseSensitive = false)
+
+      checkAnalysis(
+        UnresolvedHint(hintName, Seq("TaBlE", "table2"),
+          table("TaBlE").join(table("TaBlE2"))),
+        Join(
+          ResolvedHint(testRelation, HintInfo(strategy = Some(st))),
+          testRelation2,
+          Inner,
+          None,
+          JoinHint.NONE),
+        caseSensitive = true)
+
+      // global temp table (multi-part identifier case)
+      checkAnalysis(
+        UnresolvedHint(hintName, Seq("GlOBal_TeMP.table4", "table5"),
+          table("global_temp", "table4").join(table("global_temp", "table5"))),
+        Join(
+          ResolvedHint(testRelation4, HintInfo(strategy = Some(st))),
+          ResolvedHint(testRelation5, HintInfo(strategy = Some(st))),
+          Inner,
+          None,
+          JoinHint.NONE),
+        caseSensitive = false)
+
+      checkAnalysis(
+        UnresolvedHint(hintName, Seq("global_temp.TaBlE4", "table5"),
+          table("global_temp", "TaBlE4").join(table("global_temp", "TaBlE5"))),
+        Join(
+          ResolvedHint(testRelation4, HintInfo(strategy = Some(st))),
+          testRelation5,
+          Inner,
+          None,
+          JoinHint.NONE),
+        caseSensitive = true)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala
index e12e272aedffe..33b6029070938 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala
@@ -44,6 +44,8 @@ object TestRelations {
     AttributeReference("g", StringType)(),
     AttributeReference("h", MapType(IntegerType, IntegerType))())
 
+  val testRelation5 = LocalRelation(AttributeReference("i", StringType)())
+
   val nestedRelation = LocalRelation(
     AttributeReference("top", StructType(
       StructField("duplicateField", StringType) ::
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
index 0d6f9bcedb6a2..7b80de908fa08 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
@@ -21,15 +21,15 @@ import java.sql.Timestamp
 
 import org.apache.spark.sql.catalyst.analysis.TypeCoercion._
 import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.CalendarInterval
 
 class TypeCoercionSuite extends AnalysisTest {
+  import TypeCoercionSuite._
 
   // scalastyle:off line.size.limit
   // The following table shows all implicit data type conversions that are not visible to the user.
@@ -99,22 +99,6 @@ class TypeCoercionSuite extends AnalysisTest {
     case _ => Literal.create(null, dataType)
   }
 
-  val integralTypes: Seq[DataType] =
-    Seq(ByteType, ShortType, IntegerType, LongType)
-  val fractionalTypes: Seq[DataType] =
-    Seq(DoubleType, FloatType, DecimalType.SYSTEM_DEFAULT, DecimalType(10, 2))
-  val numericTypes: Seq[DataType] = integralTypes ++ fractionalTypes
-  val atomicTypes: Seq[DataType] =
-    numericTypes ++ Seq(BinaryType, BooleanType, StringType, DateType, TimestampType)
-  val complexTypes: Seq[DataType] =
-    Seq(ArrayType(IntegerType),
-      ArrayType(StringType),
-      MapType(StringType, StringType),
-      new StructType().add("a1", StringType),
-      new StructType().add("a1", StringType).add("a2", IntegerType))
-  val allTypes: Seq[DataType] =
-    atomicTypes ++ complexTypes ++ Seq(NullType, CalendarIntervalType)
-
   // Check whether the type `checkedType` can be cast to all the types in `castableTypes`,
   // but cannot be cast to the other types in `allTypes`.
   private def checkTypeCasting(checkedType: DataType, castableTypes: Seq[DataType]): Unit = {
@@ -467,11 +451,21 @@ class TypeCoercionSuite extends AnalysisTest {
       ArrayType(IntegerType, containsNull = false),
       Some(ArrayType(IntegerType, containsNull = true)))
 
+    widenTest(
+      ArrayType(NullType, containsNull = true),
+      ArrayType(IntegerType, containsNull = false),
+      Some(ArrayType(IntegerType, containsNull = true)))
+
     widenTest(
       MapType(IntegerType, StringType, valueContainsNull = true),
       MapType(IntegerType, StringType, valueContainsNull = false),
       Some(MapType(IntegerType, StringType, valueContainsNull = true)))
 
+    widenTest(
+      MapType(NullType, NullType, true),
+      MapType(IntegerType, StringType, false),
+      Some(MapType(IntegerType, StringType, true)))
+
     widenTest(
       new StructType()
         .add("arr", ArrayType(IntegerType, containsNull = true), nullable = false),
@@ -479,6 +473,31 @@ class TypeCoercionSuite extends AnalysisTest {
         .add("arr", ArrayType(IntegerType, containsNull = false), nullable = true),
       Some(new StructType()
         .add("arr", ArrayType(IntegerType, containsNull = true), nullable = true)))
+
+    widenTest(
+      new StructType()
+        .add("null", NullType, nullable = true),
+      new StructType()
+        .add("null", IntegerType, nullable = false),
+      Some(new StructType()
+        .add("null", IntegerType, nullable = true)))
+
+    widenTest(
+      ArrayType(NullType, containsNull = false),
+      ArrayType(IntegerType, containsNull = false),
+      Some(ArrayType(IntegerType, containsNull = false)))
+
+    widenTest(MapType(NullType, NullType, false),
+      MapType(IntegerType, StringType, false),
+      Some(MapType(IntegerType, StringType, false)))
+
+    widenTest(
+      new StructType()
+        .add("null", NullType, nullable = false),
+      new StructType()
+        .add("null", IntegerType, nullable = false),
+      Some(new StructType()
+        .add("null", IntegerType, nullable = false)))
   }
 
   test("wider common type for decimal and array") {
@@ -710,8 +729,6 @@ class TypeCoercionSuite extends AnalysisTest {
   }
 
   test("cast NullType for expressions that implement ExpectsInputTypes") {
-    import TypeCoercionSuite._
-
     ruleTest(TypeCoercion.ImplicitTypeCasts,
       AnyTypeUnaryExpression(Literal.create(null, NullType)),
       AnyTypeUnaryExpression(Literal.create(null, NullType)))
@@ -722,8 +739,6 @@ class TypeCoercionSuite extends AnalysisTest {
   }
 
   test("cast NullType for binary operators") {
-    import TypeCoercionSuite._
-
     ruleTest(TypeCoercion.ImplicitTypeCasts,
       AnyTypeBinaryOperator(Literal.create(null, NullType), Literal.create(null, NullType)),
       AnyTypeBinaryOperator(Literal.create(null, NullType), Literal.create(null, NullType)))
@@ -1401,6 +1416,21 @@ class TypeCoercionSuite extends AnalysisTest {
     }
   }
 
+  test("SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes") {
+    val t1 = LocalRelation(AttributeReference("v", DecimalType(10, 0))())
+    val t2 = LocalRelation(AttributeReference("v", DecimalType(11, 0))())
+    val p1 = t1.select(t1.output.head).as("p1")
+    val p2 = t2.select(t2.output.head).as("p2")
+    val union = p1.union(p2)
+    val wp1 = widenSetOperationTypes(union.select(p1.output.head, $"p2.v"))
+    assert(wp1.isInstanceOf[Project])
+    // The attribute `p1.output.head` should be replaced in the root `Project`.
+    assert(wp1.expressions.forall(_.find(_ == p1.output.head).isEmpty))
+    val wp2 = widenSetOperationTypes(Aggregate(Nil, sum(p1.output.head).as("v") :: Nil, union))
+    assert(wp2.isInstanceOf[Aggregate])
+    assert(wp2.missingInput.isEmpty)
+  }
+
   /**
    * There are rules that need to not fire before child expressions get resolved.
    * We use this test to make sure those rules do not fire early.
@@ -1525,11 +1555,69 @@ class TypeCoercionSuite extends AnalysisTest {
       Multiply(CaseWhen(Seq((EqualTo(1, 2), Cast(1, DecimalType(34, 24)))),
         Cast(100, DecimalType(34, 24))), Cast(1, IntegerType)))
   }
+
+  test("SPARK-31468: null types should be casted to decimal types in ImplicitTypeCasts") {
+    Seq(AnyTypeBinaryOperator(_, _), NumericTypeBinaryOperator(_, _)).foreach { binaryOp =>
+      // binaryOp(decimal, null) case
+      ruleTest(TypeCoercion.ImplicitTypeCasts,
+        binaryOp(Literal.create(null, DecimalType.SYSTEM_DEFAULT),
+          Literal.create(null, NullType)),
+        binaryOp(Literal.create(null, DecimalType.SYSTEM_DEFAULT),
+          Cast(Literal.create(null, NullType), DecimalType.SYSTEM_DEFAULT)))
+
+      // binaryOp(null, decimal) case
+      ruleTest(TypeCoercion.ImplicitTypeCasts,
+        binaryOp(Literal.create(null, NullType),
+          Literal.create(null, DecimalType.SYSTEM_DEFAULT)),
+        binaryOp(Cast(Literal.create(null, NullType), DecimalType.SYSTEM_DEFAULT),
+          Literal.create(null, DecimalType.SYSTEM_DEFAULT)))
+    }
+  }
+
+  test("SPARK-31761: byte, short and int should be cast to long for IntegralDivide's datatype") {
+    val rules = Seq(FunctionArgumentConversion, Division, ImplicitTypeCasts)
+    // Casts Byte to Long
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2.toByte, 1.toByte),
+      IntegralDivide(Cast(2.toByte, LongType), Cast(1.toByte, LongType)))
+    // Casts Short to Long
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2.toShort, 1.toShort),
+      IntegralDivide(Cast(2.toShort, LongType), Cast(1.toShort, LongType)))
+    // Casts Integer to Long
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2, 1),
+      IntegralDivide(Cast(2, LongType), Cast(1, LongType)))
+    // should not be any change for Long data types
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2L, 1L), IntegralDivide(2L, 1L))
+    // one of the operand is byte
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2L, 1.toByte),
+      IntegralDivide(2L, Cast(1.toByte, LongType)))
+    // one of the operand is short
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2.toShort, 1L),
+      IntegralDivide(Cast(2.toShort, LongType), 1L))
+    // one of the operand is int
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2, 1L),
+      IntegralDivide(Cast(2, LongType), 1L))
+  }
 }
 
 
 object TypeCoercionSuite {
 
+  val integralTypes: Seq[DataType] =
+    Seq(ByteType, ShortType, IntegerType, LongType)
+  val fractionalTypes: Seq[DataType] =
+    Seq(DoubleType, FloatType, DecimalType.SYSTEM_DEFAULT, DecimalType(10, 2))
+  val numericTypes: Seq[DataType] = integralTypes ++ fractionalTypes
+  val atomicTypes: Seq[DataType] =
+    numericTypes ++ Seq(BinaryType, BooleanType, StringType, DateType, TimestampType)
+  val complexTypes: Seq[DataType] =
+    Seq(ArrayType(IntegerType),
+      ArrayType(StringType),
+      MapType(StringType, StringType),
+      new StructType().add("a1", StringType),
+      new StructType().add("a1", StringType).add("a2", IntegerType))
+  val allTypes: Seq[DataType] =
+    atomicTypes ++ complexTypes ++ Seq(NullType, CalendarIntervalType)
+
   case class AnyTypeUnaryExpression(child: Expression)
     extends UnaryExpression with ExpectsInputTypes with Unevaluable {
     override def inputTypes: Seq[AbstractDataType] = Seq(AnyDataType)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
index 3ec6fdeedd4b8..c3bb7bfc80fcf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
@@ -694,7 +694,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
       OutputMode.Append(),
       expectFailure = expectFailure)
 
-    Seq(Inner, LeftOuter, RightOuter).map { joinType2 =>
+    Seq(Inner, LeftOuter, RightOuter).foreach { joinType2 =>
       testGlobalWatermarkLimit(
         s"streaming-stream $joinType2 after stream-stream $joinType join in Append mode",
         streamRelation.join(
@@ -740,7 +740,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
         isMapGroupsWithState = false, null, streamRelation).groupBy("*")(count("*")),
       OutputMode.Append())
 
-    Seq(Inner, LeftOuter, RightOuter).map { joinType =>
+    Seq(Inner, LeftOuter, RightOuter).foreach { joinType =>
       assertFailOnGlobalWatermarkLimit(
         s"stream-stream $joinType after FlatMapGroupsWithState in Append mode",
         streamRelation.join(
@@ -773,7 +773,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
       Deduplicate(Seq(attribute), streamRelation).groupBy("a")(count("*")),
       OutputMode.Append())
 
-    Seq(Inner, LeftOuter, RightOuter).map { joinType =>
+    Seq(Inner, LeftOuter, RightOuter).foreach { joinType =>
       assertPassOnGlobalWatermarkLimit(
         s"$joinType join after deduplicate in Append mode",
         streamRelation.join(Deduplicate(Seq(attribute), streamRelation), joinType = joinType,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
index 55712d0da518d..d310538e302de 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
@@ -408,8 +408,8 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac
       partitionColumnNames = Seq("partCol1", "partCol2"))
     catalog.createTable(table, ignoreIfExists = false)
 
-    val newLocationPart1 = newUriForDatabase()
-    val newLocationPart2 = newUriForDatabase()
+    val newLocationPart1 = newUriForPartition(Seq("p1=1", "p2=2"))
+    val newLocationPart2 = newUriForPartition(Seq("p1=3", "p2=4"))
 
     val partition1 =
       CatalogTablePartition(Map("partCol1" -> "1", "partCol2" -> "2"),
@@ -991,6 +991,11 @@ abstract class CatalogTestUtils {
 
   def newUriForDatabase(): URI = new URI(Utils.createTempDir().toURI.toString.stripSuffix("/"))
 
+  def newUriForPartition(parts: Seq[String]): URI = {
+    val path = parts.foldLeft(Utils.createTempDir())(new java.io.File(_, _))
+    new URI(path.toURI.toString.stripSuffix("/"))
+  }
+
   def newDb(name: String): CatalogDatabase = {
     CatalogDatabase(name, name + " description", newUriForDatabase(), Map.empty)
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
index 0d9e2f61e812a..4d88a8d7ee546 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
@@ -635,11 +635,11 @@ abstract class SessionCatalogSuite extends AnalysisTest {
       val view = View(desc = metadata, output = metadata.schema.toAttributes,
         child = CatalystSqlParser.parsePlan(metadata.viewText.get))
       comparePlans(catalog.lookupRelation(TableIdentifier("view1", Some("db3"))),
-        SubqueryAlias("view1", "db3", view))
+        SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view1"), view))
       // Look up a view using current database of the session catalog.
       catalog.setCurrentDatabase("db3")
       comparePlans(catalog.lookupRelation(TableIdentifier("view1")),
-        SubqueryAlias("view1", "db3", view))
+        SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view1"), view))
     }
   }
 
@@ -655,7 +655,7 @@ abstract class SessionCatalogSuite extends AnalysisTest {
       val view = View(desc = metadata, output = metadata.schema.toAttributes,
         child = CatalystSqlParser.parsePlan(metadata.viewText.get))
       comparePlans(catalog.lookupRelation(TableIdentifier("view2", Some("db3"))),
-        SubqueryAlias("view2", "db3", view))
+        SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view2"), view))
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
index ee73da33a1a90..b014eb92fae50 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.types._
 class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
 
   test("String fields types are inferred correctly from null types") {
-    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "GMT")
+    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(NullType, "") == NullType)
@@ -48,7 +48,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("String fields types are inferred correctly from other types") {
-    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "GMT")
+    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(LongType, "1.0") == DoubleType)
@@ -69,18 +69,18 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Timestamp field types are inferred correctly via custom data format") {
-    var options = new CSVOptions(Map("timestampFormat" -> "yyyy-mm"), false, "GMT")
+    var options = new CSVOptions(Map("timestampFormat" -> "yyyy-mm"), false, "UTC")
     var inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(TimestampType, "2015-08") == TimestampType)
 
-    options = new CSVOptions(Map("timestampFormat" -> "yyyy"), false, "GMT")
+    options = new CSVOptions(Map("timestampFormat" -> "yyyy"), false, "UTC")
     inferSchema = new CSVInferSchema(options)
     assert(inferSchema.inferField(TimestampType, "2015") == TimestampType)
   }
 
   test("Timestamp field types are inferred correctly from other types") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(IntegerType, "2015-08-20 14") == StringType)
@@ -89,7 +89,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Boolean fields types are inferred correctly from other types") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(LongType, "Fale") == StringType)
@@ -97,7 +97,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Type arrays are merged to highest common type") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(
@@ -112,14 +112,14 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Null fields are handled properly when a nullValue is specified") {
-    var options = new CSVOptions(Map("nullValue" -> "null"), false, "GMT")
+    var options = new CSVOptions(Map("nullValue" -> "null"), false, "UTC")
     var inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(NullType, "null") == NullType)
     assert(inferSchema.inferField(StringType, "null") == StringType)
     assert(inferSchema.inferField(LongType, "null") == LongType)
 
-    options = new CSVOptions(Map("nullValue" -> "\\N"), false, "GMT")
+    options = new CSVOptions(Map("nullValue" -> "\\N"), false, "UTC")
     inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(IntegerType, "\\N") == IntegerType)
@@ -130,7 +130,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Merging Nulltypes should yield Nulltype.") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     val mergedNullTypes = inferSchema.mergeRowTypes(Array(NullType), Array(NullType))
@@ -138,14 +138,14 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("SPARK-18433: Improve DataSource option keys to be more case-insensitive") {
-    val options = new CSVOptions(Map("TiMeStampFormat" -> "yyyy-mm"), false, "GMT")
+    val options = new CSVOptions(Map("TiMeStampFormat" -> "yyyy-mm"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(TimestampType, "2015-08") == TimestampType)
   }
 
   test("SPARK-18877: `inferField` on DecimalType should find a common type with `typeSoFar`") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     withSQLConf(SQLConf.LEGACY_ALLOW_NEGATIVE_SCALE_OF_DECIMAL_ENABLED.key -> "true") {
@@ -166,7 +166,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
 
   test("DoubleType should be inferred when user defined nan/inf are provided") {
     val options = new CSVOptions(Map("nanValue" -> "nan", "negativeInf" -> "-inf",
-      "positiveInf" -> "inf"), false, "GMT")
+      "positiveInf" -> "inf"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(NullType, "nan") == DoubleType)
@@ -179,7 +179,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
       val options = new CSVOptions(
         parameters = Map("locale" -> langTag, "inferSchema" -> "true", "sep" -> "|"),
         columnPruning = false,
-        defaultTimeZoneId = "GMT")
+        defaultTimeZoneId = "UTC")
       val inferSchema = new CSVInferSchema(options)
 
       val df = new DecimalFormat("", new DecimalFormatSymbols(Locale.forLanguageTag(langTag)))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
index 77a2ca7e4a828..474bb53c24564 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
@@ -27,6 +27,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.sources.{EqualTo, Filter, StringStartsWith}
 import org.apache.spark.sql.types._
@@ -42,7 +43,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
 
     stringValues.zip(decimalValues).foreach { case (strVal, decimalVal) =>
       val decimalValue = new BigDecimal(decimalVal.toString)
-      val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+      val options = new CSVOptions(Map.empty[String, String], false, "UTC")
       val parser = new UnivocityParser(StructType(Seq.empty), options)
       assert(parser.makeConverter("_1", decimalType).apply(strVal) ===
         Decimal(decimalValue, decimalType.precision, decimalType.scale))
@@ -56,14 +57,14 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     // Nullable field with nullValue option.
     types.foreach { t =>
       // Tests that a custom nullValue.
-      val nullValueOptions = new CSVOptions(Map("nullValue" -> "-"), false, "GMT")
+      val nullValueOptions = new CSVOptions(Map("nullValue" -> "-"), false, "UTC")
       var parser = new UnivocityParser(StructType(Seq.empty), nullValueOptions)
       val converter = parser.makeConverter("_1", t, nullable = true)
       assertNull(converter.apply("-"))
       assertNull(converter.apply(null))
 
       // Tests that the default nullValue is empty string.
-      val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+      val options = new CSVOptions(Map.empty[String, String], false, "UTC")
       parser = new UnivocityParser(StructType(Seq.empty), options)
       assertNull(parser.makeConverter("_1", t, nullable = true).apply(""))
     }
@@ -71,7 +72,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     // Not nullable field with nullValue option.
     types.foreach { t =>
       // Casts a null to not nullable field should throw an exception.
-      val options = new CSVOptions(Map("nullValue" -> "-"), false, "GMT")
+      val options = new CSVOptions(Map("nullValue" -> "-"), false, "UTC")
       val parser = new UnivocityParser(StructType(Seq.empty), options)
       val converter = parser.makeConverter("_1", t, nullable = false)
       var message = intercept[RuntimeException] {
@@ -87,7 +88,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     // If nullValue is different with empty string, then, empty string should not be casted into
     // null.
     Seq(true, false).foreach { b =>
-      val options = new CSVOptions(Map("nullValue" -> "null"), false, "GMT")
+      val options = new CSVOptions(Map("nullValue" -> "null"), false, "UTC")
       val parser = new UnivocityParser(StructType(Seq.empty), options)
       val converter = parser.makeConverter("_1", StringType, nullable = b)
       assert(converter.apply("") == UTF8String.fromString(""))
@@ -95,7 +96,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Throws exception for empty string with non null type") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val exception = intercept[RuntimeException]{
       parser.makeConverter("_1", IntegerType, nullable = false).apply("")
@@ -104,7 +105,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Types are cast correctly") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     var parser = new UnivocityParser(StructType(Seq.empty), options)
     assert(parser.makeConverter("_1", ByteType).apply("10") == 10)
     assert(parser.makeConverter("_1", ShortType).apply("10") == 10)
@@ -115,7 +116,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     assert(parser.makeConverter("_1", BooleanType).apply("true") == true)
 
     var timestampsOptions =
-      new CSVOptions(Map("timestampFormat" -> "dd/MM/yyyy HH:mm"), false, "GMT")
+      new CSVOptions(Map("timestampFormat" -> "dd/MM/yyyy HH:mm"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), timestampsOptions)
     val customTimestamp = "31/01/2015 00:00"
     var format = FastDateFormat.getInstance(
@@ -128,7 +129,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     assert(castedTimestamp == expectedTime * 1000L)
 
     val customDate = "31/01/2015"
-    val dateOptions = new CSVOptions(Map("dateFormat" -> "dd/MM/yyyy"), false, "GMT")
+    val dateOptions = new CSVOptions(Map("dateFormat" -> "dd/MM/yyyy"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), dateOptions)
     format = FastDateFormat.getInstance(
       dateOptions.dateFormat,
@@ -137,7 +138,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     val expectedDate = format.parse(customDate).getTime
     val castedDate = parser.makeConverter("_1", DateType, nullable = true)
         .apply(customDate)
-    assert(castedDate == DateTimeUtils.millisToDays(expectedDate, TimeZone.getTimeZone("GMT")))
+    assert(castedDate == DateTimeUtils.millisToDays(expectedDate, UTC))
 
     val timestamp = "2015-01-01 00:00:00"
     timestampsOptions = new CSVOptions(Map(
@@ -152,7 +153,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Throws exception for casting an invalid string to Float and Double Types") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val types = Seq(DoubleType, FloatType)
     val input = Seq("10u000", "abc", "1 2/3")
@@ -167,7 +168,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Float NaN values are parsed correctly") {
-    val options = new CSVOptions(Map("nanValue" -> "nn"), false, "GMT")
+    val options = new CSVOptions(Map("nanValue" -> "nn"), false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val floatVal: Float = parser.makeConverter(
       "_1", FloatType, nullable = true).apply("nn").asInstanceOf[Float]
@@ -178,7 +179,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Double NaN values are parsed correctly") {
-    val options = new CSVOptions(Map("nanValue" -> "-"), false, "GMT")
+    val options = new CSVOptions(Map("nanValue" -> "-"), false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val doubleVal: Double = parser.makeConverter(
       "_1", DoubleType, nullable = true).apply("-").asInstanceOf[Double]
@@ -187,14 +188,14 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Float infinite values can be parsed") {
-    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "GMT")
+    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "UTC")
     var parser = new UnivocityParser(StructType(Seq.empty), negativeInfOptions)
     val floatVal1 = parser.makeConverter(
       "_1", FloatType, nullable = true).apply("max").asInstanceOf[Float]
 
     assert(floatVal1 == Float.NegativeInfinity)
 
-    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "GMT")
+    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), positiveInfOptions)
     val floatVal2 = parser.makeConverter(
       "_1", FloatType, nullable = true).apply("max").asInstanceOf[Float]
@@ -203,14 +204,14 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Double infinite values can be parsed") {
-    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "GMT")
+    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "UTC")
     var parser = new UnivocityParser(StructType(Seq.empty), negativeInfOptions)
     val doubleVal1 = parser.makeConverter(
       "_1", DoubleType, nullable = true).apply("max").asInstanceOf[Double]
 
     assert(doubleVal1 == Double.NegativeInfinity)
 
-    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "GMT")
+    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), positiveInfOptions)
     val doubleVal2 = parser.makeConverter(
       "_1", DoubleType, nullable = true).apply("max").asInstanceOf[Double]
@@ -226,7 +227,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
       val df = new DecimalFormat("", new DecimalFormatSymbols(Locale.forLanguageTag(langTag)))
       val input = df.format(expected.toBigDecimal)
 
-      val options = new CSVOptions(Map("locale" -> langTag), false, "GMT")
+      val options = new CSVOptions(Map("locale" -> langTag), false, "UTC")
       val parser = new UnivocityParser(new StructType().add("d", decimalType), options)
 
       assert(parser.makeConverter("_1", decimalType).apply(input) === expected)
@@ -261,7 +262,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     val input = "name\t42"
     val expected = UTF8String.fromString(input)
 
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
 
     val convertedValue = parser.makeConverter("_1", StringBasedUDT, nullable = false).apply(input)
@@ -278,7 +279,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
         filters: Seq[Filter],
         expected: Option[InternalRow]): Unit = {
       Seq(false, true).foreach { columnPruning =>
-        val options = new CSVOptions(Map.empty[String, String], columnPruning, "GMT")
+        val options = new CSVOptions(Map.empty[String, String], columnPruning, "UTC")
         val parser = new UnivocityParser(dataSchema, requiredSchema, options, filters)
         val actual = parser.parse(input)
         assert(actual === expected)
@@ -317,4 +318,44 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     }.getMessage
     assert(errMsg2.contains("i does not exist"))
   }
+
+  test("SPARK-30960: parse date/timestamp string with legacy format") {
+    def check(parser: UnivocityParser): Unit = {
+      // The legacy format allows 1 or 2 chars for some fields.
+      assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45") ==
+        date(2020, 1, 12, 12, 3, 45, 0))
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12") ==
+        days(2020, 1, 12))
+      // The legacy format allows arbitrary length of second fraction.
+      assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45.1") ==
+        date(2020, 1, 12, 12, 3, 45, 100000))
+      assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45.1234") ==
+        date(2020, 1, 12, 12, 3, 45, 123400))
+      // The legacy format allow date string to end with T or space, with arbitrary string
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12T") ==
+        days(2020, 1, 12))
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12Txyz") ==
+        days(2020, 1, 12))
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12 ") ==
+        days(2020, 1, 12))
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12 xyz") ==
+        days(2020, 1, 12))
+      // The legacy format ignores the "GMT" from the string
+      assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45GMT") ==
+        date(2020, 1, 12, 12, 3, 45, 0))
+      assert(parser.makeConverter("t", TimestampType).apply("GMT2020-1-12 12:3:45") ==
+        date(2020, 1, 12, 12, 3, 45, 0))
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12GMT") ==
+        days(2020, 1, 12))
+      assert(parser.makeConverter("t", DateType).apply("GMT2020-1-12") ==
+        days(2020, 1, 12))
+    }
+
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
+    check(new UnivocityParser(StructType(Seq.empty), options))
+
+    val optionsWithPattern = new CSVOptions(
+      Map("timestampFormat" -> "invalid", "dateFormat" -> "invalid"), false, "UTC")
+    check(new UnivocityParser(StructType(Seq.empty), optionsWithPattern))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
index 53cb8bce0a52d..577814b9c6696 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
@@ -22,8 +22,9 @@ import scala.reflect.runtime.universe.TypeTag
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.PlanTest
-import org.apache.spark.sql.catalyst.util.GenericArrayData
+import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -42,22 +43,29 @@ case class NestedArrayClass(nestedArr: Array[ArrayClass])
 class EncoderResolutionSuite extends PlanTest {
   private val str = UTF8String.fromString("hello")
 
+  def testFromRow[T](
+      encoder: ExpressionEncoder[T],
+      attributes: Seq[Attribute],
+      row: InternalRow): Unit = {
+    encoder.resolveAndBind(attributes).createDeserializer().apply(row)
+  }
+
   test("real type doesn't match encoder schema but they are compatible: product") {
     val encoder = ExpressionEncoder[StringLongClass]
 
     // int type can be up cast to long type
     val attrs1 = Seq('a.string, 'b.int)
-    encoder.resolveAndBind(attrs1).fromRow(InternalRow(str, 1))
+    testFromRow(encoder, attrs1, InternalRow(str, 1))
 
     // int type can be up cast to string type
     val attrs2 = Seq('a.int, 'b.long)
-    encoder.resolveAndBind(attrs2).fromRow(InternalRow(1, 2L))
+    testFromRow(encoder, attrs2, InternalRow(1, 2L))
   }
 
   test("real type doesn't match encoder schema but they are compatible: nested product") {
     val encoder = ExpressionEncoder[ComplexClass]
     val attrs = Seq('a.int, 'b.struct('a.int, 'b.long))
-    encoder.resolveAndBind(attrs).fromRow(InternalRow(1, InternalRow(2, 3L)))
+    testFromRow(encoder, attrs, InternalRow(1, InternalRow(2, 3L)))
   }
 
   test("real type doesn't match encoder schema but they are compatible: tupled encoder") {
@@ -65,14 +73,14 @@ class EncoderResolutionSuite extends PlanTest {
       ExpressionEncoder[StringLongClass],
       ExpressionEncoder[Long])
     val attrs = Seq('a.struct('a.string, 'b.byte), 'b.int)
-    encoder.resolveAndBind(attrs).fromRow(InternalRow(InternalRow(str, 1.toByte), 2))
+    testFromRow(encoder, attrs, InternalRow(InternalRow(str, 1.toByte), 2))
   }
 
   test("real type doesn't match encoder schema but they are compatible: primitive array") {
     val encoder = ExpressionEncoder[PrimitiveArrayClass]
     val attrs = Seq('arr.array(IntegerType))
     val array = new GenericArrayData(Array(1, 2, 3))
-    encoder.resolveAndBind(attrs).fromRow(InternalRow(array))
+    testFromRow(encoder, attrs, InternalRow(array))
   }
 
   test("the real type is not compatible with encoder schema: primitive array") {
@@ -93,7 +101,7 @@ class EncoderResolutionSuite extends PlanTest {
     val encoder = ExpressionEncoder[ArrayClass]
     val attrs = Seq('arr.array(new StructType().add("a", "int").add("b", "int").add("c", "int")))
     val array = new GenericArrayData(Array(InternalRow(1, 2, 3)))
-    encoder.resolveAndBind(attrs).fromRow(InternalRow(array))
+    testFromRow(encoder, attrs, InternalRow(array))
   }
 
   test("real type doesn't match encoder schema but they are compatible: nested array") {
@@ -103,7 +111,7 @@ class EncoderResolutionSuite extends PlanTest {
     val attrs = Seq('nestedArr.array(et))
     val innerArr = new GenericArrayData(Array(InternalRow(1, 2, 3)))
     val outerArr = new GenericArrayData(Array(InternalRow(innerArr)))
-    encoder.resolveAndBind(attrs).fromRow(InternalRow(outerArr))
+    testFromRow(encoder, attrs, InternalRow(outerArr))
   }
 
   test("the real type is not compatible with encoder schema: non-array field") {
@@ -142,14 +150,14 @@ class EncoderResolutionSuite extends PlanTest {
     val attrs = 'a.array(IntegerType) :: Nil
 
     // It should pass analysis
-    val bound = encoder.resolveAndBind(attrs)
+    val fromRow = encoder.resolveAndBind(attrs).createDeserializer()
 
     // If no null values appear, it should work fine
-    bound.fromRow(InternalRow(new GenericArrayData(Array(1, 2))))
+    fromRow(InternalRow(new GenericArrayData(Array(1, 2))))
 
     // If there is null value, it should throw runtime exception
     val e = intercept[RuntimeException] {
-      bound.fromRow(InternalRow(new GenericArrayData(Array(1, null))))
+      fromRow(InternalRow(new GenericArrayData(Array(1, null))))
     }
     assert(e.getMessage.contains("Null value appeared in non-nullable field"))
   }
@@ -239,6 +247,13 @@ class EncoderResolutionSuite extends PlanTest {
        """.stripMargin.trim + " of the field in the target object")
   }
 
+  test("SPARK-31750: eliminate UpCast if child's dataType is DecimalType") {
+    val encoder = ExpressionEncoder[Seq[BigDecimal]]
+    val attr = Seq(AttributeReference("a", ArrayType(DecimalType(38, 0)))())
+    // Before SPARK-31750, it will fail because Decimal(38, 0) can not be casted to Decimal(38, 18)
+    testFromRow(encoder, attr, InternalRow(ArrayData.toArrayData(Array(Decimal(1.0)))))
+  }
+
   // test for leaf types
   castSuccess[Int, Long]
   castSuccess[java.sql.Date, java.sql.Timestamp]
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
index c1f1be3b30e4b..6a094d4aaddae 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.catalyst.util.ArrayData
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 import org.apache.spark.util.ClosureCleaner
 
 case class RepeatedStruct(s: Seq[PrimitiveData])
@@ -107,6 +107,8 @@ class UDTForCaseClass extends UserDefinedType[UDTCaseClass] {
   }
 }
 
+case class Bar(i: Any)
+case class Foo(i: Bar) extends AnyVal
 case class PrimitiveValueClass(wrapped: Int) extends AnyVal
 case class ReferenceValueClass(wrapped: ReferenceValueClass.Container) extends AnyVal
 object ReferenceValueClass {
@@ -207,9 +209,9 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
 
   productTest(
     OptionalData(Some(2), Some(2), Some(2), Some(2), Some(2), Some(2), Some(true),
-      Some(PrimitiveData(1, 1, 1, 1, 1, 1, true))))
+      Some(PrimitiveData(1, 1, 1, 1, 1, 1, true)), Some(new CalendarInterval(1, 2, 3))))
 
-  productTest(OptionalData(None, None, None, None, None, None, None, None))
+  productTest(OptionalData(None, None, None, None, None, None, None, None, None))
 
   encodeDecodeTest(Seq(Some(1), None), "Option in array")
   encodeDecodeTest(Map(1 -> Some(10L), 2 -> Some(20L), 3 -> None), "Option in map")
@@ -311,6 +313,13 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
 
   productTest(("UDT", new ExamplePoint(0.1, 0.2)))
 
+  test("AnyVal class with Any fields") {
+    val exception = intercept[UnsupportedOperationException](implicitly[ExpressionEncoder[Foo]])
+    val errorMsg = exception.getMessage
+    assert(errorMsg.contains("root class: \"org.apache.spark.sql.catalyst.encoders.Foo\""))
+    assert(errorMsg.contains("No Encoder found for Any"))
+  }
+
   test("nullable of encoder schema") {
     def checkNullable[T: ExpressionEncoder](nullable: Boolean*): Unit = {
       assert(implicitly[ExpressionEncoder[T]].schema.map(_.nullable) === nullable.toSeq)
@@ -360,14 +369,14 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
   }
 
   test("null check for map key: String") {
-    val encoder = ExpressionEncoder[Map[String, Int]]()
-    val e = intercept[RuntimeException](encoder.toRow(Map(("a", 1), (null, 2))))
+    val toRow = ExpressionEncoder[Map[String, Int]]().createSerializer()
+    val e = intercept[RuntimeException](toRow(Map(("a", 1), (null, 2))))
     assert(e.getMessage.contains("Cannot use null as map key"))
   }
 
   test("null check for map key: Integer") {
-    val encoder = ExpressionEncoder[Map[Integer, String]]()
-    val e = intercept[RuntimeException](encoder.toRow(Map((1, "a"), (null, "b"))))
+    val toRow = ExpressionEncoder[Map[Integer, String]]().createSerializer()
+    val e = intercept[RuntimeException](toRow(Map((1, "a"), (null, "b"))))
     assert(e.getMessage.contains("Cannot use null as map key"))
   }
 
@@ -427,10 +436,6 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
   testOverflowingBigNumeric(BigInt("9" * 100), "scala very large big int")
   testOverflowingBigNumeric(new BigInteger("9" * 100), "java very big int")
 
-  encodeDecodeTest("foo" -> 1L, "makeCopy") {
-    Encoders.product[(String, Long)].makeCopy.asInstanceOf[ExpressionEncoder[(String, Long)]]
-  }
-
   private def testOverflowingBigNumeric[T: TypeTag](bigNumeric: T, testName: String): Unit = {
     Seq(true, false).foreach { ansiEnabled =>
       testAndVerifyNotLeakingReflectionObjects(
@@ -441,12 +446,14 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
           // Need to construct Encoder here rather than implicitly resolving it
           // so that SQLConf changes are respected.
           val encoder = ExpressionEncoder[T]()
+          val toRow = encoder.createSerializer()
           if (!ansiEnabled) {
-            val convertedBack = encoder.resolveAndBind().fromRow(encoder.toRow(bigNumeric))
+            val fromRow = encoder.resolveAndBind().createDeserializer()
+            val convertedBack = fromRow(toRow(bigNumeric))
             assert(convertedBack === null)
           } else {
             val e = intercept[RuntimeException] {
-              encoder.toRow(bigNumeric)
+              toRow(bigNumeric)
             }
             assert(e.getMessage.contains("Error while encoding"))
             assert(e.getCause.getClass === classOf[ArithmeticException])
@@ -465,10 +472,10 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
       // Make sure encoder is serializable.
       ClosureCleaner.clean((s: String) => encoder.getClass.getName)
 
-      val row = encoder.toRow(input)
+      val row = encoder.createSerializer().apply(input)
       val schema = encoder.schema.toAttributes
       val boundEncoder = encoder.resolveAndBind()
-      val convertedBack = try boundEncoder.fromRow(row) catch {
+      val convertedBack = try boundEncoder.createDeserializer().apply(row) catch {
         case e: Exception =>
           fail(
            s"""Exception thrown while decoding
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
index 1a1cab823d4f3..fd24f058f357c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.encoders
 import scala.util.Random
 
 import org.apache.spark.sql.{RandomDataGenerator, Row}
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.plans.CodegenInterpretedPlanTest
 import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.internal.SQLConf
@@ -81,6 +82,18 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
   private val mapOfString = MapType(StringType, StringType)
   private val arrayOfUDT = ArrayType(new ExamplePointUDT, false)
 
+  private def toRow(encoder: ExpressionEncoder[Row], row: Row): InternalRow = {
+    encoder.createSerializer().apply(row)
+  }
+
+  private def fromRow(encoder: ExpressionEncoder[Row], row: InternalRow): Row = {
+    encoder.createDeserializer().apply(row)
+  }
+
+  private def roundTrip(encoder: ExpressionEncoder[Row], row: Row): Row = {
+    fromRow(encoder, toRow(encoder, row))
+  }
+
   encodeDecodeTest(
     new StructType()
       .add("null", NullType)
@@ -144,8 +157,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
     val catalystDecimal = Decimal("1234.5678")
 
     val input = Row(100, "test", 0.123, javaDecimal, scalaDecimal, catalystDecimal)
-    val row = encoder.toRow(input)
-    val convertedBack = encoder.fromRow(row)
+    val convertedBack = roundTrip(encoder, input)
     // Decimal will be converted back to Java BigDecimal when decoding.
     assert(convertedBack.getDecimal(3).compareTo(javaDecimal) == 0)
     assert(convertedBack.getDecimal(4).compareTo(scalaDecimal.bigDecimal) == 0)
@@ -157,7 +169,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
     val encoder = RowEncoder(schema).resolveAndBind()
     val decimal = Decimal("67123.45")
     val input = Row(decimal)
-    val row = encoder.toRow(input)
+    val row = toRow(encoder, input)
 
     assert(row.toSeq(schema).head == decimal)
   }
@@ -172,7 +184,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
       val encoder = RowEncoder(schema).resolveAndBind()
       intercept[Exception] {
-        encoder.toRow(row)
+        toRow(encoder, row)
       } match {
         case e: ArithmeticException =>
           assert(e.getMessage.contains("cannot be represented as Decimal"))
@@ -184,7 +196,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
 
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
       val encoder = RowEncoder(schema).resolveAndBind()
-      assert(encoder.fromRow(encoder.toRow(row)).get(0) == null)
+      assert(roundTrip(encoder, row).get(0) == null)
     }
   }
 
@@ -237,8 +249,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       Array(1.1.toFloat, 123.456.toFloat, Float.MaxValue),
       Array(11.1111, 123456.7890123, Double.MaxValue)
     )
-    val row = encoder.toRow(Row.fromSeq(input))
-    val convertedBack = encoder.fromRow(row)
+    val convertedBack = roundTrip(encoder, Row.fromSeq(input))
     input.zipWithIndex.map { case (array, index) =>
       assert(convertedBack.getSeq(index) === array)
     }
@@ -254,8 +265,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       Array(1, 2, null),
       Array(Array("abc", null), null),
       Array(Seq(Array(0L, null), null), null))
-    val row = encoder.toRow(input)
-    val convertedBack = encoder.fromRow(row)
+    val convertedBack = roundTrip(encoder, input)
     assert(convertedBack.getSeq(0) == Seq(1, 2, null))
     assert(convertedBack.getSeq(1) == Seq(Seq("abc", null), null))
     assert(convertedBack.getSeq(2) == Seq(Seq(Seq(0L, null), null), null))
@@ -264,7 +274,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
   test("RowEncoder should throw RuntimeException if input row object is null") {
     val schema = new StructType().add("int", IntegerType)
     val encoder = RowEncoder(schema)
-    val e = intercept[RuntimeException](encoder.toRow(null))
+    val e = intercept[RuntimeException](toRow(encoder, null))
     assert(e.getMessage.contains("Null value appeared in non-nullable field"))
     assert(e.getMessage.contains("top level Product or row object"))
   }
@@ -273,14 +283,14 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
     val e1 = intercept[RuntimeException] {
       val schema = new StructType().add("a", IntegerType)
       val encoder = RowEncoder(schema)
-      encoder.toRow(Row(1.toShort))
+      toRow(encoder, Row(1.toShort))
     }
     assert(e1.getMessage.contains("java.lang.Short is not a valid external type"))
 
     val e2 = intercept[RuntimeException] {
       val schema = new StructType().add("a", StringType)
       val encoder = RowEncoder(schema)
-      encoder.toRow(Row(1))
+      toRow(encoder, Row(1))
     }
     assert(e2.getMessage.contains("java.lang.Integer is not a valid external type"))
 
@@ -288,14 +298,14 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       val schema = new StructType().add("a",
         new StructType().add("b", IntegerType).add("c", StringType))
       val encoder = RowEncoder(schema)
-      encoder.toRow(Row(1 -> "a"))
+      toRow(encoder, Row(1 -> "a"))
     }
     assert(e3.getMessage.contains("scala.Tuple2 is not a valid external type"))
 
     val e4 = intercept[RuntimeException] {
       val schema = new StructType().add("a", ArrayType(TimestampType))
       val encoder = RowEncoder(schema)
-      encoder.toRow(Row(Array("a")))
+      toRow(encoder, Row(Array("a")))
     }
     assert(e4.getMessage.contains("java.lang.String is not a valid external type"))
   }
@@ -313,9 +323,9 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       val schema = new StructType().add("t", TimestampType)
       val encoder = RowEncoder(schema).resolveAndBind()
       val instant = java.time.Instant.parse("2019-02-26T16:56:00Z")
-      val row = encoder.toRow(Row(instant))
+      val row = toRow(encoder, Row(instant))
       assert(row.getLong(0) === DateTimeUtils.instantToMicros(instant))
-      val readback = encoder.fromRow(row)
+      val readback = fromRow(encoder, row)
       assert(readback.get(0) === instant)
     }
   }
@@ -325,9 +335,9 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       val schema = new StructType().add("d", DateType)
       val encoder = RowEncoder(schema).resolveAndBind()
       val localDate = java.time.LocalDate.parse("2019-02-27")
-      val row = encoder.toRow(Row(localDate))
+      val row = toRow(encoder, Row(localDate))
       assert(row.getLong(0) === DateTimeUtils.localDateToDays(localDate))
-      val readback = encoder.fromRow(row)
+      val readback = fromRow(encoder, row)
       assert(readback.get(0).equals(localDate))
     }
   }
@@ -367,24 +377,27 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
 
   private def encodeDecodeTest(schema: StructType): Unit = {
     test(s"encode/decode: ${schema.simpleString}") {
-      val encoder = RowEncoder(schema).resolveAndBind()
-      val inputGenerator = RandomDataGenerator.forType(schema, nullable = false).get
-
-      var input: Row = null
-      try {
-        for (_ <- 1 to 5) {
-          input = inputGenerator.apply().asInstanceOf[Row]
-          val row = encoder.toRow(input)
-          val convertedBack = encoder.fromRow(row)
-          assert(input == convertedBack)
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          val encoder = RowEncoder(schema).resolveAndBind()
+          val inputGenerator = RandomDataGenerator.forType(schema, nullable = false).get
+
+          var input: Row = null
+          try {
+            for (_ <- 1 to 5) {
+              input = inputGenerator.apply().asInstanceOf[Row]
+              val convertedBack = roundTrip(encoder, input)
+              assert(input == convertedBack)
+            }
+          } catch {
+            case e: Exception =>
+              fail(
+                s"""
+                   |schema: ${schema.simpleString}
+                   |input: ${input}
+                 """.stripMargin, e)
+          }
         }
-      } catch {
-        case e: Exception =>
-          fail(
-            s"""
-               |schema: ${schema.simpleString}
-               |input: ${input}
-             """.stripMargin, e)
       }
     }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
index b4a1ae20c3e11..f05598aeb5353 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
@@ -173,25 +173,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     }
   }
 
-  test("/ (Divide) for integral type") {
-    withSQLConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG.key -> "false") {
-      checkEvaluation(IntegralDivide(Literal(1.toByte), Literal(2.toByte)), 0.toByte)
-      checkEvaluation(IntegralDivide(Literal(1.toShort), Literal(2.toShort)), 0.toShort)
-      checkEvaluation(IntegralDivide(Literal(1), Literal(2)), 0)
-      checkEvaluation(IntegralDivide(Literal(1.toLong), Literal(2.toLong)), 0.toLong)
-      checkEvaluation(IntegralDivide(positiveShortLit, negativeShortLit), 0.toShort)
-      checkEvaluation(IntegralDivide(positiveIntLit, negativeIntLit), 0)
-      checkEvaluation(IntegralDivide(positiveLongLit, negativeLongLit), 0L)
-    }
-    withSQLConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG.key -> "true") {
-      checkEvaluation(IntegralDivide(Literal(1.toByte), Literal(2.toByte)), 0L)
-      checkEvaluation(IntegralDivide(Literal(1.toShort), Literal(2.toShort)), 0L)
-      checkEvaluation(IntegralDivide(Literal(1), Literal(2)), 0L)
-      checkEvaluation(IntegralDivide(Literal(1.toLong), Literal(2.toLong)), 0L)
-      checkEvaluation(IntegralDivide(positiveShortLit, negativeShortLit), 0L)
-      checkEvaluation(IntegralDivide(positiveIntLit, negativeIntLit), 0L)
-      checkEvaluation(IntegralDivide(positiveLongLit, negativeLongLit), 0L)
-    }
+  test("/ (Divide) for Long type") {
+    checkEvaluation(IntegralDivide(Literal(1.toLong), Literal(2.toLong)), 0L)
+    checkEvaluation(IntegralDivide(positiveLongLit, negativeLongLit), 0L)
   }
 
   test("% (Remainder)") {
@@ -407,33 +391,18 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
   }
 
   test("SPARK-28322: IntegralDivide supports decimal type") {
-    withSQLConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG.key -> "false") {
-      checkEvaluation(IntegralDivide(Literal(Decimal(1)), Literal(Decimal(2))), Decimal(0))
-      checkEvaluation(IntegralDivide(Literal(Decimal(2.4)), Literal(Decimal(1.1))), Decimal(2))
-      checkEvaluation(IntegralDivide(Literal(Decimal(1.2)), Literal(Decimal(1.1))), Decimal(1))
-      checkEvaluation(IntegralDivide(Literal(Decimal(0.2)), Literal(Decimal(0.0))), null)
-      checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
-        Literal(Decimal("99999999999999999999999999999999999")), Literal(Decimal(0.001)))),
-        BigDecimal("99999999999999999999999999999999999000"))
-      // overflow during promote precision
-      checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
-        Literal(Decimal("99999999999999999999999999999999999999")), Literal(Decimal(0.00001)))),
-        null)
-    }
-    withSQLConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG.key -> "true") {
-      checkEvaluation(IntegralDivide(Literal(Decimal(1)), Literal(Decimal(2))), 0L)
-      checkEvaluation(IntegralDivide(Literal(Decimal(2.4)), Literal(Decimal(1.1))), 2L)
-      checkEvaluation(IntegralDivide(Literal(Decimal(1.2)), Literal(Decimal(1.1))), 1L)
-      checkEvaluation(IntegralDivide(Literal(Decimal(0.2)), Literal(Decimal(0.0))), null)
-      // overflows long and so returns a wrong result
-      checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
-        Literal(Decimal("99999999999999999999999999999999999")), Literal(Decimal(0.001)))),
-        687399551400672280L)
-      // overflow during promote precision
-      checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
-        Literal(Decimal("99999999999999999999999999999999999999")), Literal(Decimal(0.00001)))),
-        null)
-    }
+    checkEvaluation(IntegralDivide(Literal(Decimal(1)), Literal(Decimal(2))), 0L)
+    checkEvaluation(IntegralDivide(Literal(Decimal(2.4)), Literal(Decimal(1.1))), 2L)
+    checkEvaluation(IntegralDivide(Literal(Decimal(1.2)), Literal(Decimal(1.1))), 1L)
+    checkEvaluation(IntegralDivide(Literal(Decimal(0.2)), Literal(Decimal(0.0))), null)
+    // overflows long and so returns a wrong result
+    checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
+      Literal(Decimal("99999999999999999999999999999999999")), Literal(Decimal(0.001)))),
+      687399551400672280L)
+    // overflow during promote precision
+    checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
+      Literal(Decimal("99999999999999999999999999999999999999")), Literal(Decimal(0.00001)))),
+      null)
   }
 
   test("SPARK-24598: overflow on long returns wrong result") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala
new file mode 100644
index 0000000000000..813a68f68451c
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.{caseInsensitiveResolution, caseSensitiveResolution}
+import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
+
+class AttributeResolutionSuite extends SparkFunSuite {
+  val resolver = caseInsensitiveResolution
+
+  test("basic attribute resolution with namespaces") {
+    val attrs = Seq(
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "ns2", "t1")),
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "ns2", "ns3", "t2")))
+
+    // Try to match attribute reference with name "a" with qualifier "ns1.ns2.t1".
+    Seq(Seq("t1", "a"), Seq("ns2", "t1", "a"), Seq("ns1", "ns2", "t1", "a")).foreach { nameParts =>
+      attrs.resolve(nameParts, resolver) match {
+        case Some(attr) => assert(attr.semanticEquals(attrs(0)))
+        case _ => fail()
+      }
+    }
+
+    // Non-matching cases
+    Seq(Seq("ns1", "ns2", "t1"), Seq("ns2", "a")).foreach { nameParts =>
+      assert(attrs.resolve(nameParts, resolver).isEmpty)
+    }
+  }
+
+  test("attribute resolution where table and attribute names are the same") {
+    val attrs = Seq(AttributeReference("t", IntegerType)(qualifier = Seq("ns1", "ns2", "t")))
+    // Matching cases
+    Seq(
+      Seq("t"), Seq("t", "t"), Seq("ns2", "t", "t"), Seq("ns1", "ns2", "t", "t")
+    ).foreach { nameParts =>
+      attrs.resolve(nameParts, resolver) match {
+        case Some(attr) => assert(attr.semanticEquals(attrs(0)))
+        case _ => fail()
+      }
+    }
+
+    // Non-matching case
+    assert(attrs.resolve(Seq("ns1", "ns2", "t"), resolver).isEmpty)
+  }
+
+  test("attribute resolution ambiguity at the attribute name level") {
+    val attrs = Seq(
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "t1")),
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "ns2", "t2")))
+
+    val ex = intercept[AnalysisException] {
+      attrs.resolve(Seq("a"), resolver)
+    }
+    assert(ex.getMessage.contains(
+      "Reference 'a' is ambiguous, could be: ns1.t1.a, ns1.ns2.t2.a."))
+  }
+
+  test("attribute resolution ambiguity at the qualifier level") {
+    val attrs = Seq(
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "t")),
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns2", "ns1", "t")))
+
+    val ex = intercept[AnalysisException] {
+      attrs.resolve(Seq("ns1", "t", "a"), resolver)
+    }
+    assert(ex.getMessage.contains(
+      "Reference 'ns1.t.a' is ambiguous, could be: ns1.t.a, ns2.ns1.t.a."))
+  }
+
+  test("attribute resolution with nested fields") {
+    val attrType = StructType(Seq(StructField("aa", IntegerType), StructField("bb", IntegerType)))
+    val attrs = Seq(AttributeReference("a", attrType)(qualifier = Seq("ns1", "t")))
+
+    val resolved = attrs.resolve(Seq("ns1", "t", "a", "aa"), resolver)
+    resolved match {
+      case Some(Alias(_, name)) => assert(name == "aa")
+      case _ => fail()
+    }
+
+    val ex = intercept[AnalysisException] {
+      attrs.resolve(Seq("ns1", "t", "a", "cc"), resolver)
+    }
+    assert(ex.getMessage.contains("No such struct field cc in aa, bb"))
+  }
+
+  test("attribute resolution with case insensitive resolver") {
+    val attrs = Seq(AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "t")))
+    attrs.resolve(Seq("Ns1", "T", "A"), caseInsensitiveResolution) match {
+      case Some(attr) => assert(attr.semanticEquals(attrs(0)) && attr.name == "A")
+      case _ => fail()
+    }
+  }
+
+  test("attribute resolution with case sensitive resolver") {
+    val attrs = Seq(AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "t")))
+    assert(attrs.resolve(Seq("Ns1", "T", "A"), caseSensitiveResolution).isEmpty)
+    assert(attrs.resolve(Seq("ns1", "t", "A"), caseSensitiveResolution).isEmpty)
+    attrs.resolve(Seq("ns1", "t", "a"), caseSensitiveResolution) match {
+      case Some(attr) => assert(attr.semanticEquals(attrs(0)))
+      case _ => fail()
+    }
+  }
+
+  test("attribute resolution should try to match the longest qualifier") {
+    // We have two attributes:
+    // 1) "a.b" where "a" is the name and "b" is the nested field.
+    // 2) "a.b.a" where "b" is the name, left-side "a" is the qualifier and the right-side "a"
+    //    is the nested field.
+    // When "a.b" is resolved, "b" is tried first as the name, so it is resolved to #2 attribute.
+    val a1Type = StructType(Seq(StructField("b", IntegerType)))
+    val a2Type = StructType(Seq(StructField("a", IntegerType)))
+    val attrs = Seq(
+      AttributeReference("a", a1Type)(),
+      AttributeReference("b", a2Type)(qualifier = Seq("a")))
+    attrs.resolve(Seq("a", "b"), resolver) match {
+      case Some(attr) => assert(attr.semanticEquals(attrs(1)))
+      case _ => fail()
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
index 88d4d460751b6..d8f3ad24246a3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
@@ -21,6 +21,7 @@ import java.sql.Timestamp
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckFailure
+import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.types.{IntegerType, StringType}
 
 /** A static class for testing purpose. */
@@ -101,6 +102,11 @@ class CallMethodViaReflectionSuite extends SparkFunSuite with ExpressionEvalHelp
     checkEvaluation(createExpr(staticClassName, "method4", 4, "four"), "m4four")
   }
 
+  test("escaping of class and method names") {
+    GenerateUnsafeProjection.generate(
+      CallMethodViaReflection(Seq(Literal("\"quote"), Literal("\"quote"), Literal(null))) :: Nil)
+  }
+
   private def createExpr(className: String, methodName: String, args: Any*) = {
     CallMethodViaReflection(
       Literal.create(className, StringType) +:
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
index 9802a6e5891b8..a043b4cbed1f1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
@@ -17,10 +17,12 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import java.util.TimeZone
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.plans.logical.Range
-import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
+import org.apache.spark.sql.types.{IntegerType, LongType, StructField, StructType}
 
 class CanonicalizeSuite extends SparkFunSuite {
 
@@ -79,4 +81,18 @@ class CanonicalizeSuite extends SparkFunSuite {
       0, Some("b2"))
     assert(fieldB1.semanticEquals(fieldB2))
   }
+
+  test("SPARK-30847: Take productPrefix into account in MurmurHash3.productHash") {
+    val range = Range(1, 1, 1, 1)
+    val addExpr = Add(range.output.head, Literal(1))
+    val subExpr = Subtract(range.output.head, Literal(1))
+    assert(addExpr.canonicalized.hashCode() != subExpr.canonicalized.hashCode())
+  }
+
+  test("SPARK-31515: Canonicalize Cast should consider the value of needTimeZone") {
+    val literal = Literal(1)
+    val cast = Cast(literal, LongType)
+    val castWithTimeZoneId = Cast(literal, LongType, Some(TimeZone.getDefault.getID))
+    assert(castWithTimeZoneId.semanticEquals(cast))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index ad66873c02518..08aa2aa68771e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.{Date, Timestamp}
 import java.util.{Calendar, TimeZone}
-import java.util.concurrent.TimeUnit._
 
 import scala.collection.parallel.immutable.ParVector
 
@@ -27,6 +26,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCoercion.numericPrecedence
+import org.apache.spark.sql.catalyst.analysis.TypeCoercionSuite
 import org.apache.spark.sql.catalyst.expressions.aggregate.{CollectList, CollectSet}
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
@@ -50,7 +50,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   protected def checkNullCast(from: DataType, to: DataType): Unit = {
-    checkEvaluation(cast(Literal.create(null, from), to, Option("GMT")), null)
+    checkEvaluation(cast(Literal.create(null, from), to, UTC_OPT), null)
   }
 
   test("null cast") {
@@ -112,13 +112,14 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("cast string to timestamp") {
-    new ParVector(ALL_TIMEZONES.toVector).foreach { tz =>
+    new ParVector(ALL_TIMEZONES.toVector).foreach { zid =>
       def checkCastStringToTimestamp(str: String, expected: Timestamp): Unit = {
-        checkEvaluation(cast(Literal(str), TimestampType, Option(tz.getID)), expected)
+        checkEvaluation(cast(Literal(str), TimestampType, Option(zid.getId)), expected)
       }
 
       checkCastStringToTimestamp("123", null)
 
+      val tz = TimeZone.getTimeZone(zid)
       var c = Calendar.getInstance(tz)
       c.set(2015, 0, 1, 0, 0, 0)
       c.set(Calendar.MILLISECOND, 0)
@@ -142,7 +143,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
 
       // If the string value includes timezone string, it represents the timestamp string
       // in the timezone regardless of the timeZoneId parameter.
-      c = Calendar.getInstance(TimeZone.getTimeZone("UTC"))
+      c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
       c.set(2015, 2, 18, 12, 3, 17)
       c.set(Calendar.MILLISECOND, 0)
       checkCastStringToTimestamp("2015-03-18T12:03:17Z", new Timestamp(c.getTimeInMillis))
@@ -173,7 +174,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
 
       // If the string value includes timezone string, it represents the timestamp string
       // in the timezone regardless of the timeZoneId parameter.
-      c = Calendar.getInstance(TimeZone.getTimeZone("UTC"))
+      c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
       c.set(2015, 2, 18, 12, 3, 17)
       c.set(Calendar.MILLISECOND, 456)
       checkCastStringToTimestamp("2015-03-18T12:03:17.456Z", new Timestamp(c.getTimeInMillis))
@@ -239,7 +240,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkCast(1.5, "1.5")
 
     checkEvaluation(cast(cast(1.toDouble, TimestampType), DoubleType), 1.toDouble)
-    checkEvaluation(cast(cast(1.toDouble, TimestampType), DoubleType), 1.toDouble)
   }
 
   test("cast from string") {
@@ -263,41 +263,41 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     val zts = sd + " 00:00:00"
     val sts = sd + " 00:00:02"
     val nts = sts + ".1"
-    val ts = withDefaultTimeZone(TimeZoneGMT)(Timestamp.valueOf(nts))
+    val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts))
 
     for (tz <- ALL_TIMEZONES) {
-      val timeZoneId = Option(tz.getID)
+      val timeZoneId = Option(tz.getId)
       var c = Calendar.getInstance(TimeZoneGMT)
       c.set(2015, 2, 8, 2, 30, 0)
       checkEvaluation(
         cast(cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
           TimestampType, timeZoneId),
-        MILLISECONDS.toMicros(c.getTimeInMillis))
+        fromMillis(c.getTimeInMillis))
       c = Calendar.getInstance(TimeZoneGMT)
       c.set(2015, 10, 1, 2, 30, 0)
       checkEvaluation(
         cast(cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
           TimestampType, timeZoneId),
-        MILLISECONDS.toMicros(c.getTimeInMillis))
+        fromMillis(c.getTimeInMillis))
     }
 
-    val gmtId = Option("GMT")
-
     checkEvaluation(cast("abdef", StringType), "abdef")
-    checkEvaluation(cast("abdef", TimestampType, gmtId), null)
+    checkEvaluation(cast("abdef", TimestampType, UTC_OPT), null)
     checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
 
     checkEvaluation(cast(cast(sd, DateType), StringType), sd)
     checkEvaluation(cast(cast(d, StringType), DateType), 0)
-    checkEvaluation(cast(cast(nts, TimestampType, gmtId), StringType, gmtId), nts)
+    checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType, UTC_OPT), nts)
     checkEvaluation(
-      cast(cast(ts, StringType, gmtId), TimestampType, gmtId),
+      cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
       DateTimeUtils.fromJavaTimestamp(ts))
 
     // all convert to string type to check
-    checkEvaluation(cast(cast(cast(nts, TimestampType, gmtId), DateType, gmtId), StringType), sd)
     checkEvaluation(
-      cast(cast(cast(ts, DateType, gmtId), TimestampType, gmtId), StringType, gmtId),
+      cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT), StringType),
+      sd)
+    checkEvaluation(
+      cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT), StringType, UTC_OPT),
       zts)
 
     checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef")
@@ -310,7 +310,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
         DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
       5.toShort)
     checkEvaluation(
-      cast(cast(cast(cast(cast(cast("5", TimestampType, gmtId), ByteType),
+      cast(cast(cast(cast(cast(cast("5", TimestampType, UTC_OPT), ByteType),
         DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
       null)
     checkEvaluation(cast(cast(cast(cast(cast(cast("5", DecimalType.SYSTEM_DEFAULT),
@@ -361,8 +361,9 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(cast(d, DecimalType(10, 2)), null)
     checkEvaluation(cast(d, StringType), "1970-01-01")
 
-    val gmtId = Option("GMT")
-    checkEvaluation(cast(cast(d, TimestampType, gmtId), StringType, gmtId), "1970-01-01 00:00:00")
+    checkEvaluation(
+      cast(cast(d, TimestampType, UTC_OPT), StringType, UTC_OPT),
+      "1970-01-01 00:00:00")
   }
 
   test("cast from timestamp") {
@@ -413,6 +414,14 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
       assert(ret.resolved)
       checkEvaluation(ret, Seq(null, true, false, null))
     }
+
+    {
+      val array = Literal.create(Seq.empty, ArrayType(NullType, containsNull = false))
+      val ret = cast(array, ArrayType(IntegerType, containsNull = false))
+      assert(ret.resolved)
+      checkEvaluation(ret, Seq.empty)
+    }
+
     {
       val ret = cast(array, ArrayType(BooleanType, containsNull = false))
       assert(ret.resolved === false)
@@ -1141,6 +1150,36 @@ class CastSuite extends CastSuiteBase {
     assert(Cast.canCast(set.dataType, ArrayType(StringType, false)))
   }
 
+  test("NullTypes should be able to cast to any complex types") {
+    assert(Cast.canCast(ArrayType(NullType, true), ArrayType(IntegerType, true)))
+    assert(Cast.canCast(ArrayType(NullType, false), ArrayType(IntegerType, true)))
+
+    assert(Cast.canCast(
+      MapType(NullType, NullType, true), MapType(IntegerType, IntegerType, true)))
+    assert(Cast.canCast(
+      MapType(NullType, NullType, false), MapType(IntegerType, IntegerType, true)))
+
+    assert(Cast.canCast(
+      StructType(StructField("a", NullType, true) :: Nil),
+      StructType(StructField("a", IntegerType, true) :: Nil)))
+    assert(Cast.canCast(
+      StructType(StructField("a", NullType, false) :: Nil),
+      StructType(StructField("a", IntegerType, true) :: Nil)))
+  }
+
+  test("SPARK-31227: Non-nullable null type should not coerce to nullable type") {
+    TypeCoercionSuite.allTypes.foreach { t =>
+      assert(Cast.canCast(ArrayType(NullType, false), ArrayType(t, false)))
+
+      assert(Cast.canCast(
+        MapType(NullType, NullType, false), MapType(t, t, false)))
+
+      assert(Cast.canCast(
+        StructType(StructField("a", NullType, false) :: Nil),
+        StructType(StructField("a", t, false) :: Nil)))
+    }
+  }
+
   test("Cast should output null for invalid strings when ANSI is not enabled.") {
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
       checkEvaluation(cast("abdef", DecimalType.USER_DEFAULT), null)
@@ -1259,6 +1298,18 @@ class CastSuite extends CastSuiteBase {
       }
     }
   }
+
+  test("cast a timestamp before the epoch 1970-01-01 00:00:00Z") {
+    withDefaultTimeZone(UTC) {
+      val negativeTs = Timestamp.valueOf("1900-05-05 18:34:56.1")
+      assert(negativeTs.getTime < 0)
+      val expectedSecs = Math.floorDiv(negativeTs.getTime, MILLIS_PER_SECOND)
+      checkEvaluation(cast(negativeTs, ByteType), expectedSecs.toByte)
+      checkEvaluation(cast(negativeTs, ShortType), expectedSecs.toShort)
+      checkEvaluation(cast(negativeTs, IntegerType), expectedSecs.toInt)
+      checkEvaluation(cast(negativeTs, LongType), expectedSecs)
+    }
+  }
 }
 
 /**
@@ -1281,22 +1332,38 @@ class AnsiCastSuite extends CastSuiteBase {
       val array = Literal.create(Seq("123", "true", "f", null),
         ArrayType(StringType, containsNull = true))
       checkExceptionInExpression[NumberFormatException](
-        cast(array, ArrayType(dataType, containsNull = true)), "invalid input")
+        cast(array, ArrayType(dataType, containsNull = true)),
+        "invalid input syntax for type numeric: true")
+      checkExceptionInExpression[NumberFormatException](
+        cast("string", dataType), "invalid input syntax for type numeric: string")
       checkExceptionInExpression[NumberFormatException](
-        cast("string", dataType), "invalid input")
+        cast("123-string", dataType), "invalid input syntax for type numeric: 123-string")
       checkExceptionInExpression[NumberFormatException](
-        cast("123-string", dataType), "invalid input")
+        cast("2020-07-19", dataType), "invalid input syntax for type numeric: 2020-07-19")
       checkExceptionInExpression[NumberFormatException](
-        cast("2020-07-19", dataType), "invalid input")
+        cast("1.23", dataType), "invalid input syntax for type numeric: 1.23")
     }
 
     Seq(DoubleType, FloatType, DecimalType.USER_DEFAULT).foreach { dataType =>
       checkExceptionInExpression[NumberFormatException](
-        cast("string", dataType), "invalid input")
+        cast("string", dataType), "invalid input syntax for type numeric: string")
       checkExceptionInExpression[NumberFormatException](
-        cast("123.000.00", dataType), "invalid input")
+        cast("123.000.00", dataType), "invalid input syntax for type numeric: 123.000.00")
       checkExceptionInExpression[NumberFormatException](
-        cast("abc.com", dataType), "invalid input")
+        cast("abc.com", dataType), "invalid input syntax for type numeric: abc.com")
+    }
+  }
+
+  test("cast a timestamp before the epoch 1970-01-01 00:00:00Z") {
+    def errMsg(t: String): String = s"Casting -2198208303900000 to $t causes overflow"
+    withDefaultTimeZone(UTC) {
+      val negativeTs = Timestamp.valueOf("1900-05-05 18:34:56.1")
+      assert(negativeTs.getTime < 0)
+      val expectedSecs = Math.floorDiv(negativeTs.getTime, MILLIS_PER_SECOND)
+      checkExceptionInExpression[ArithmeticException](cast(negativeTs, ByteType), errMsg("byte"))
+      checkExceptionInExpression[ArithmeticException](cast(negativeTs, ShortType), errMsg("short"))
+      checkExceptionInExpression[ArithmeticException](cast(negativeTs, IntegerType), errMsg("int"))
+      checkEvaluation(cast(negativeTs, LongType), expectedSecs)
     }
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
index b7628a373d0bd..f1de63adc3d9a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.Timestamp
 
+import scala.math.Ordering
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.metrics.source.CodegenMetrics
 import org.apache.spark.sql.Row
@@ -28,6 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.expressions.objects._
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, DateTimeUtils}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.LA
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.ThreadUtils
@@ -190,7 +193,7 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
     val expressions = Seq.fill(length) {
       ToUTCTimestamp(
         Literal.create(Timestamp.valueOf("2015-07-24 00:00:00"), TimestampType),
-        Literal.create("PST", StringType))
+        Literal.create(LA.getId, StringType))
     }
     val plan = GenerateMutableProjection.generate(expressions)
     val actual = plan(new GenericInternalRow(length)).toSeq(expressions.map(_.dataType))
@@ -207,7 +210,7 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
     val expressions = Seq.fill(length) {
       ToUTCTimestamp(
         Literal.create(Timestamp.valueOf("2017-10-10 00:00:00"), TimestampType),
-        Literal.create("PST", StringType))
+        Literal.create(LA.getId, StringType))
     }
     val plan = GenerateMutableProjection.generate(expressions)
     val actual = plan(new GenericInternalRow(length)).toSeq(expressions.map(_.dataType))
@@ -535,6 +538,26 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
     GenerateUnsafeProjection.generate(exprs, true)
     GenerateMutableProjection.generate(exprs, true)
   }
+
+  test("SPARK-32624: Use CodeGenerator.typeName() to fix byte[] compile issue") {
+    val ctx = new CodegenContext
+    val bytes = new Array[Byte](3)
+    val refTerm = ctx.addReferenceObj("bytes", bytes)
+    assert(refTerm == "((byte[]) references[0] /* bytes */)")
+  }
+
+  test("SPARK-32624: CodegenContext.addReferenceObj should work for nested Scala class") {
+    // emulate TypeUtils.getInterpretedOrdering(StringType)
+    val ctx = new CodegenContext
+    val comparator = implicitly[Ordering[UTF8String]]
+    val refTerm = ctx.addReferenceObj("comparator", comparator)
+
+    // Expecting result:
+    //   "((scala.math.LowPriorityOrderingImplicits$$anon$3) references[0] /* comparator */)"
+    // Using lenient assertions to be resilient to annonymous class numbering changes
+    assert(!refTerm.contains("null"))
+    assert(refTerm.contains("scala.math.LowPriorityOrderingImplicits$$anon$"))
+  }
 }
 
 case class HugeCodeIntExpression(value: Int) extends Expression {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 9e98e146c7a0e..7d278b6e9151b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -27,8 +27,9 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_DAY
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC
 import org.apache.spark.sql.catalyst.util.IntervalUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -74,6 +75,12 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     withSQLConf(SQLConf.LEGACY_SIZE_OF_NULL.key -> "false") {
       testSize(sizeOfNull = null)
     }
+    // size(null) should return null under ansi mode.
+    withSQLConf(
+      SQLConf.LEGACY_SIZE_OF_NULL.key -> "true",
+      SQLConf.ANSI_ENABLED.key -> "true") {
+      testSize(sizeOfNull = null)
+    }
   }
 
   test("MapKeys/MapValues") {
@@ -139,8 +146,12 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
       MapType(IntegerType, IntegerType, valueContainsNull = true))
     val mNull = Literal.create(null, MapType(StringType, StringType))
 
-    // overlapping maps should remove duplicated map keys w.r.t. last win policy.
-    checkEvaluation(MapConcat(Seq(m0, m1)), create_map("a" -> "4", "b" -> "2", "c" -> "3"))
+    checkExceptionInExpression[RuntimeException](
+      MapConcat(Seq(m0, m1)), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      // overlapping maps should remove duplicated map keys w.r.t. last win policy.
+      checkEvaluation(MapConcat(Seq(m0, m1)), create_map("a" -> "4", "b" -> "2", "c" -> "3"))
+    }
 
     // maps with no overlap
     checkEvaluation(MapConcat(Seq(m0, m2)),
@@ -272,8 +283,13 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(MapFromEntries(ai1), create_map(1 -> null, 2 -> 20, 3 -> null))
     checkEvaluation(MapFromEntries(ai2), Map.empty)
     checkEvaluation(MapFromEntries(ai3), null)
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(MapFromEntries(ai4), create_map(1 -> 20))
+
+    checkExceptionInExpression[RuntimeException](
+      MapFromEntries(ai4), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(MapFromEntries(ai4), create_map(1 -> 20))
+    }
     // Map key can't be null
     checkExceptionInExpression[RuntimeException](
       MapFromEntries(ai5),
@@ -294,8 +310,13 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(MapFromEntries(as1), create_map("a" -> null, "b" -> "bb", "c" -> null))
     checkEvaluation(MapFromEntries(as2), Map.empty)
     checkEvaluation(MapFromEntries(as3), null)
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(MapFromEntries(as4), create_map("a" -> "bb"))
+
+    checkExceptionInExpression[RuntimeException](
+      MapFromEntries(as4), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(MapFromEntries(as4), create_map("a" -> "bb"))
+    }
     // Map key can't be null
     checkExceptionInExpression[RuntimeException](
       MapFromEntries(as5),
@@ -825,20 +846,22 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
 
   test("Sequence on DST boundaries") {
     val timeZone = TimeZone.getTimeZone("Europe/Prague")
-    val dstOffset = timeZone.getDSTSavings
 
-    def noDST(t: Timestamp): Timestamp = new Timestamp(t.getTime - dstOffset)
+    def ts(s: String, noDST: Boolean = false): Long = {
+      val offset = if (noDST) timeZone.getDSTSavings else 0
+      DateTimeUtils.fromMillis(Timestamp.valueOf(s).getTime - offset)
+    }
 
-    DateTimeTestUtils.withDefaultTimeZone(timeZone) {
+    DateTimeTestUtils.withDefaultTimeZone(timeZone.toZoneId) {
       // Spring time change
       checkEvaluation(new Sequence(
         Literal(Timestamp.valueOf("2018-03-25 01:30:00")),
         Literal(Timestamp.valueOf("2018-03-25 03:30:00")),
         Literal(stringToInterval("interval 30 minutes"))),
         Seq(
-          Timestamp.valueOf("2018-03-25 01:30:00"),
-          Timestamp.valueOf("2018-03-25 03:00:00"),
-          Timestamp.valueOf("2018-03-25 03:30:00")))
+          ts("2018-03-25 01:30:00"),
+          ts("2018-03-25 03:00:00"),
+          ts("2018-03-25 03:30:00")))
 
       // Autumn time change
       checkEvaluation(new Sequence(
@@ -846,18 +869,18 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
         Literal(Timestamp.valueOf("2018-10-28 03:30:00")),
         Literal(stringToInterval("interval 30 minutes"))),
         Seq(
-          Timestamp.valueOf("2018-10-28 01:30:00"),
-          noDST(Timestamp.valueOf("2018-10-28 02:00:00")),
-          noDST(Timestamp.valueOf("2018-10-28 02:30:00")),
-          Timestamp.valueOf("2018-10-28 02:00:00"),
-          Timestamp.valueOf("2018-10-28 02:30:00"),
-          Timestamp.valueOf("2018-10-28 03:00:00"),
-          Timestamp.valueOf("2018-10-28 03:30:00")))
+          ts("2018-10-28 01:30:00"),
+          ts("2018-10-28 02:00:00", noDST = true),
+          ts("2018-10-28 02:30:00", noDST = true),
+          ts("2018-10-28 02:00:00"),
+          ts("2018-10-28 02:30:00"),
+          ts("2018-10-28 03:00:00"),
+          ts("2018-10-28 03:30:00")))
     }
   }
 
   test("Sequence of dates") {
-    DateTimeTestUtils.withDefaultTimeZone(TimeZone.getTimeZone("UTC")) {
+    DateTimeTestUtils.withDefaultTimeZone(UTC) {
       checkEvaluation(new Sequence(
         Literal(Date.valueOf("2018-01-01")),
         Literal(Date.valueOf("2018-01-05")),
@@ -1092,11 +1115,18 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     val a = AttributeReference("a", IntegerType, nullable = false)()
     val b = AttributeReference("b", IntegerType, nullable = true)()
     val array = CreateArray(a :: b :: Nil)
-    assert(!ElementAt(array, Literal(0)).nullable)
-    assert(ElementAt(array, Literal(1)).nullable)
-    assert(!ElementAt(array, Subtract(Literal(2), Literal(2))).nullable)
+    assert(!ElementAt(array, Literal(1)).nullable)
+    assert(!ElementAt(array, Literal(-2)).nullable)
+    assert(ElementAt(array, Literal(2)).nullable)
+    assert(ElementAt(array, Literal(-1)).nullable)
+    assert(!ElementAt(array, Subtract(Literal(2), Literal(1))).nullable)
     assert(ElementAt(array, AttributeReference("ordinal", IntegerType)()).nullable)
 
+    // CreateArray case invalid indices
+    assert(!ElementAt(array, Literal(0)).nullable)
+    assert(ElementAt(array, Literal(4)).nullable)
+    assert(ElementAt(array, Literal(-4)).nullable)
+
     // GetArrayStructFields case
     val f1 = StructField("a", IntegerType, nullable = false)
     val f2 = StructField("b", IntegerType, nullable = true)
@@ -1105,19 +1135,34 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     val inputArray1 = CreateArray(c :: Nil)
     val inputArray1ContainsNull = c.nullable
     val stArray1 = GetArrayStructFields(inputArray1, f1, 0, 2, inputArray1ContainsNull)
-    assert(!ElementAt(stArray1, Literal(0)).nullable)
+    assert(!ElementAt(stArray1, Literal(1)).nullable)
+    assert(!ElementAt(stArray1, Literal(-1)).nullable)
     val stArray2 = GetArrayStructFields(inputArray1, f2, 1, 2, inputArray1ContainsNull)
-    assert(ElementAt(stArray2, Literal(0)).nullable)
+    assert(ElementAt(stArray2, Literal(1)).nullable)
+    assert(ElementAt(stArray2, Literal(-1)).nullable)
 
     val d = AttributeReference("d", structType, nullable = true)()
     val inputArray2 = CreateArray(c :: d :: Nil)
     val inputArray2ContainsNull = c.nullable || d.nullable
     val stArray3 = GetArrayStructFields(inputArray2, f1, 0, 2, inputArray2ContainsNull)
-    assert(!ElementAt(stArray3, Literal(0)).nullable)
-    assert(ElementAt(stArray3, Literal(1)).nullable)
+    assert(!ElementAt(stArray3, Literal(1)).nullable)
+    assert(!ElementAt(stArray3, Literal(-2)).nullable)
+    assert(ElementAt(stArray3, Literal(2)).nullable)
+    assert(ElementAt(stArray3, Literal(-1)).nullable)
     val stArray4 = GetArrayStructFields(inputArray2, f2, 1, 2, inputArray2ContainsNull)
-    assert(ElementAt(stArray4, Literal(0)).nullable)
     assert(ElementAt(stArray4, Literal(1)).nullable)
+    assert(ElementAt(stArray4, Literal(-2)).nullable)
+    assert(ElementAt(stArray4, Literal(2)).nullable)
+    assert(ElementAt(stArray4, Literal(-1)).nullable)
+
+    // GetArrayStructFields case invalid indices
+    assert(!ElementAt(stArray3, Literal(0)).nullable)
+    assert(ElementAt(stArray3, Literal(4)).nullable)
+    assert(ElementAt(stArray3, Literal(-4)).nullable)
+
+    assert(ElementAt(stArray4, Literal(0)).nullable)
+    assert(ElementAt(stArray4, Literal(4)).nullable)
+    assert(ElementAt(stArray4, Literal(-4)).nullable)
   }
 
   test("Concat") {
@@ -1813,4 +1858,22 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(ArrayIntersect(empty, oneNull), Seq.empty)
     checkEvaluation(ArrayIntersect(oneNull, empty), Seq.empty)
   }
+
+  test("SPARK-31980: Start and end equal in month range") {
+    checkEvaluation(new Sequence(
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(stringToInterval("interval 1 day"))),
+      Seq(Date.valueOf("2018-01-01")))
+    checkEvaluation(new Sequence(
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(stringToInterval("interval 1 month"))),
+      Seq(Date.valueOf("2018-01-01")))
+    checkEvaluation(new Sequence(
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(stringToInterval("interval 1 year"))),
+      Seq(Date.valueOf("2018-01-01")))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
index 9039cd6451590..54eb07b51e9f3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
@@ -18,9 +18,12 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, UnresolvedExtractValue}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -158,6 +161,31 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(getArrayStructFields(nullArrayStruct, "a"), null)
   }
 
+  test("SPARK-32167: nullability of GetArrayStructFields") {
+    val resolver = SQLConf.get.resolver
+
+    val array1 = ArrayType(
+      new StructType().add("a", "int", nullable = true),
+      containsNull = false)
+    val data1 = Literal.create(Seq(Row(null)), array1)
+    val get1 = ExtractValue(data1, Literal("a"), resolver).asInstanceOf[GetArrayStructFields]
+    assert(get1.containsNull)
+
+    val array2 = ArrayType(
+      new StructType().add("a", "int", nullable = false),
+      containsNull = true)
+    val data2 = Literal.create(Seq(null), array2)
+    val get2 = ExtractValue(data2, Literal("a"), resolver).asInstanceOf[GetArrayStructFields]
+    assert(get2.containsNull)
+
+    val array3 = ArrayType(
+      new StructType().add("a", "int", nullable = false),
+      containsNull = false)
+    val data3 = Literal.create(Seq(Row(1)), array3)
+    val get3 = ExtractValue(data3, Literal("a"), resolver).asInstanceOf[GetArrayStructFields]
+    assert(!get3.containsNull)
+  }
+
   test("CreateArray") {
     val intSeq = Seq(5, 10, 15, 20, 25)
     val longSeq = intSeq.map(_.toLong)
@@ -216,10 +244,14 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
       CreateMap(interlace(strWithNull, intSeq.map(Literal(_)))),
       "Cannot use null as map key")
 
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(
-      CreateMap(Seq(Literal(1), Literal(2), Literal(1), Literal(3))),
-      create_map(1 -> 3))
+    checkExceptionInExpression[RuntimeException](
+      CreateMap(Seq(Literal(1), Literal(2), Literal(1), Literal(3))), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(
+        CreateMap(Seq(Literal(1), Literal(2), Literal(1), Literal(3))),
+        create_map(1 -> 3))
+    }
 
     // ArrayType map key and value
     val map = CreateMap(Seq(
@@ -281,12 +313,19 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
       MapFromArrays(intWithNullArray, strArray),
       "Cannot use null as map key")
 
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(
+    checkExceptionInExpression[RuntimeException](
       MapFromArrays(
         Literal.create(Seq(1, 1), ArrayType(IntegerType)),
         Literal.create(Seq(2, 3), ArrayType(IntegerType))),
-      create_map(1 -> 3))
+      "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(
+        MapFromArrays(
+          Literal.create(Seq(1, 1), ArrayType(IntegerType)),
+          Literal.create(Seq(2, 3), ArrayType(IntegerType))),
+        create_map(1 -> 3))
+    }
 
     // map key can't be map
     val arrayOfMap = Seq(create_map(1 -> "a", 2 -> "b"))
@@ -399,10 +438,14 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
     val m5 = Map("a" -> null)
     checkEvaluation(new StringToMap(s5), m5)
 
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(
-      new StringToMap(Literal("a:1,b:2,a:3")),
-      create_map("a" -> "3", "b" -> "2"))
+    checkExceptionInExpression[RuntimeException](
+      new StringToMap(Literal("a:1,b:2,a:3")), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(
+        new StringToMap(Literal("a:1,b:2,a:3")),
+        create_map("a" -> "3", "b" -> "2"))
+    }
 
     // arguments checking
     assert(new StringToMap(Literal("a:1,b:2,c:3")).checkInputDataTypes().isSuccess)
@@ -424,4 +467,18 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
     CreateNamedStruct(Seq("a", "x", "b", 2.0)).genCode(ctx)
     assert(ctx.inlinedMutableStates.isEmpty)
   }
+
+  test("SPARK-33338: semanticEquals should handle static GetMapValue correctly") {
+    val keys = new Array[UTF8String](1)
+    val values = new Array[UTF8String](1)
+    keys(0) = UTF8String.fromString("key")
+    values(0) = UTF8String.fromString("value")
+
+    val d1 = new ArrayBasedMapData(new GenericArrayData(keys), new GenericArrayData(values))
+    val d2 = new ArrayBasedMapData(new GenericArrayData(keys), new GenericArrayData(values))
+    val m1 = GetMapValue(Literal.create(d1, MapType(StringType, StringType)), Literal("a"))
+    val m2 = GetMapValue(Literal.create(d2, MapType(StringType, StringType)), Literal("a"))
+
+    assert(m1.semanticEquals(m2))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
index e623910e2efe1..16d78b0526503 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.text.SimpleDateFormat
-import java.util.{Calendar, Locale}
+import java.util.{Calendar, Locale, TimeZone}
 
 import org.scalatest.exceptions.TestFailedException
 
@@ -27,19 +27,18 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.plans.PlanTestBase
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{PST, UTC_OPT}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
 class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with PlanTestBase {
   val badCsv = "\u0000\u0000\u0000A\u0001AAA"
 
-  val gmtId = Option(DateTimeUtils.TimeZoneGMT.getID)
-
   test("from_csv") {
     val csvData = "1"
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal(csvData), gmtId),
+      CsvToStructs(schema, Map.empty, Literal(csvData), UTC_OPT),
       InternalRow(1)
     )
   }
@@ -48,17 +47,17 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
     val csvData = "---"
     val schema = StructType(StructField("a", DoubleType) :: Nil)
     checkEvaluation(
-      CsvToStructs(schema, Map("mode" -> PermissiveMode.name), Literal(csvData), gmtId),
+      CsvToStructs(schema, Map("mode" -> PermissiveMode.name), Literal(csvData), UTC_OPT),
       InternalRow(null))
 
     // Default mode is Permissive
-    checkEvaluation(CsvToStructs(schema, Map.empty, Literal(csvData), gmtId), InternalRow(null))
+    checkEvaluation(CsvToStructs(schema, Map.empty, Literal(csvData), UTC_OPT), InternalRow(null))
   }
 
   test("from_csv null input column") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal.create(null, StringType), gmtId),
+      CsvToStructs(schema, Map.empty, Literal.create(null, StringType), UTC_OPT),
       null
     )
   }
@@ -66,7 +65,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
   test("from_csv bad UTF-8") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal(badCsv), gmtId),
+      CsvToStructs(schema, Map.empty, Literal(badCsv), UTC_OPT),
       InternalRow(null))
   }
 
@@ -78,20 +77,20 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
     c.set(2016, 0, 1, 0, 0, 0)
     c.set(Calendar.MILLISECOND, 123)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal(csvData1), gmtId),
+      CsvToStructs(schema, Map.empty, Literal(csvData1), UTC_OPT),
       InternalRow(c.getTimeInMillis * 1000L)
     )
     // The result doesn't change because the CSV string includes timezone string ("Z" here),
     // which means the string represents the timestamp string in the timezone regardless of
     // the timeZoneId parameter.
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal(csvData1), Option("PST")),
+      CsvToStructs(schema, Map.empty, Literal(csvData1), Option(PST.getId)),
       InternalRow(c.getTimeInMillis * 1000L)
     )
 
     val csvData2 = "2016-01-01T00:00:00"
-    for (tz <- DateTimeTestUtils.outstandingTimezones) {
-      c = Calendar.getInstance(tz)
+    for (zid <- DateTimeTestUtils.outstandingZoneIds) {
+      c = Calendar.getInstance(TimeZone.getTimeZone(zid))
       c.set(2016, 0, 1, 0, 0, 0)
       c.set(Calendar.MILLISECOND, 0)
       checkEvaluation(
@@ -99,16 +98,16 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
           schema,
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss"),
           Literal(csvData2),
-          Option(tz.getID)),
+          Option(zid.getId)),
         InternalRow(c.getTimeInMillis * 1000L)
       )
       checkEvaluation(
         CsvToStructs(
           schema,
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-            DateTimeUtils.TIMEZONE_OPTION -> tz.getID),
+            DateTimeUtils.TIMEZONE_OPTION -> zid.getId),
           Literal(csvData2),
-          gmtId),
+          UTC_OPT),
         InternalRow(c.getTimeInMillis * 1000L)
       )
     }
@@ -117,7 +116,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
   test("from_csv empty input column") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal.create(" ", StringType), gmtId),
+      CsvToStructs(schema, Map.empty, Literal.create(" ", StringType), UTC_OPT),
       InternalRow(null)
     )
   }
@@ -129,7 +128,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
       .add("b", StringType, nullable = false)
       .add("c", StringType, nullable = false)
     val output = InternalRow(1L, null, UTF8String.fromString("foo"))
-    val expr = CsvToStructs(csvSchema, Map.empty, Literal.create(input, StringType), gmtId)
+    val expr = CsvToStructs(csvSchema, Map.empty, Literal.create(input, StringType), UTC_OPT)
     checkEvaluation(expr, output)
     val schema = expr.dataType
     val schemaToCompare = csvSchema.asNullable
@@ -142,7 +141,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
       .add("a", IntegerType)
       .add("b", IntegerType)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal.create("1"), gmtId),
+      CsvToStructs(schema, Map.empty, Literal.create("1"), UTC_OPT),
       InternalRow(1, null)
     )
   }
@@ -152,7 +151,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
     val schema = StructType(StructField("a", DoubleType) :: Nil)
     val exception = intercept[TestFailedException] {
       checkEvaluation(
-        CsvToStructs(schema, Map("mode" -> DropMalformedMode.name), Literal(csvData), gmtId),
+        CsvToStructs(schema, Map("mode" -> DropMalformedMode.name), Literal(csvData), UTC_OPT),
         InternalRow(null))
     }.getCause
     assert(exception.getMessage.contains("from_csv() doesn't support the DROPMALFORMED mode"))
@@ -171,14 +170,14 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
   test("to_csv - struct") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val struct = Literal.create(create_row(1), schema)
-    checkEvaluation(StructsToCsv(Map.empty, struct, gmtId), "1")
+    checkEvaluation(StructsToCsv(Map.empty, struct, UTC_OPT), "1")
   }
 
   test("to_csv null input column") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val struct = Literal.create(null, schema)
     checkEvaluation(
-      StructsToCsv(Map.empty, struct, gmtId),
+      StructsToCsv(Map.empty, struct, UTC_OPT),
       null
     )
   }
@@ -190,24 +189,24 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
     c.set(Calendar.MILLISECOND, 0)
     val struct = Literal.create(create_row(c.getTimeInMillis * 1000L), schema)
 
-    checkEvaluation(StructsToCsv(Map.empty, struct, gmtId), "2016-01-01T00:00:00.000Z")
+    checkEvaluation(StructsToCsv(Map.empty, struct, UTC_OPT), "2016-01-01T00:00:00.000Z")
     checkEvaluation(
-      StructsToCsv(Map.empty, struct, Option("PST")), "2015-12-31T16:00:00.000-08:00")
+      StructsToCsv(Map.empty, struct, Option(PST.getId)), "2015-12-31T16:00:00.000-08:00")
 
     checkEvaluation(
       StructsToCsv(
         Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-          DateTimeUtils.TIMEZONE_OPTION -> gmtId.get),
+          DateTimeUtils.TIMEZONE_OPTION -> UTC_OPT.get),
         struct,
-        gmtId),
+        UTC_OPT),
       "2016-01-01T00:00:00"
     )
     checkEvaluation(
       StructsToCsv(
         Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-          DateTimeUtils.TIMEZONE_OPTION -> "PST"),
+          DateTimeUtils.TIMEZONE_OPTION -> PST.getId),
         struct,
-        gmtId),
+        UTC_OPT),
       "2015-12-31T16:00:00"
     )
   }
@@ -223,7 +222,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
       val options = Map("dateFormat" -> dateFormat, "locale" -> langTag)
 
       checkEvaluation(
-        CsvToStructs(schema, options, Literal.create(dateStr), gmtId),
+        CsvToStructs(schema, options, Literal.create(dateStr), UTC_OPT),
         InternalRow(17836)) // number of days from 1970-01-01
     }
   }
@@ -234,7 +233,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
         schema = StructType.fromDDL("i int, _unparsed boolean"),
         options = Map("columnNameOfCorruptRecord" -> "_unparsed"),
         child = Literal.create("a"),
-        timeZoneId = gmtId),
+        timeZoneId = UTC_OPT),
       expectedErrMsg = "The field for corrupt records must be string type and nullable")
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 274d0beebd300..df417544dd15c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -19,18 +19,17 @@ package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.{Date, Timestamp}
 import java.text.SimpleDateFormat
-import java.time.{Instant, LocalDate, LocalDateTime, ZoneId, ZoneOffset}
+import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
 import java.util.{Calendar, Locale, TimeZone}
-import java.util.concurrent.TimeUnit
 import java.util.concurrent.TimeUnit._
 
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils, TimestampFormatter}
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND
+import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimeZoneGMT
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
@@ -38,16 +37,12 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   import IntegralLiteralTestUtils._
 
-  val TimeZonePST = TimeZone.getTimeZone("PST")
-  val TimeZoneJST = TimeZone.getTimeZone("JST")
-
-  val gmtId = Option(TimeZoneGMT.getID)
-  val pstId = Option(TimeZonePST.getID)
-  val jstId = Option(TimeZoneJST.getID)
+  private val PST_OPT = Option(PST.getId)
+  private val JST_OPT = Option(JST.getId)
 
   def toMillis(timestamp: String): Long = {
-    val tf = TimestampFormatter("yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC)
-    TimeUnit.MICROSECONDS.toMillis(tf.parse(timestamp))
+    val tf = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC, isParsing = true)
+    DateTimeUtils.toMillis(tf.parse(timestamp))
   }
   val date = "2015-04-08 13:10:15"
   val d = new Date(toMillis(date))
@@ -55,13 +50,13 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   val ts = new Timestamp(toMillis(time))
 
   test("datetime function current_date") {
-    val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis(), TimeZoneGMT)
-    val cd = CurrentDate(gmtId).eval(EmptyRow).asInstanceOf[Int]
-    val d1 = DateTimeUtils.millisToDays(System.currentTimeMillis(), TimeZoneGMT)
+    val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis(), UTC)
+    val cd = CurrentDate(UTC_OPT).eval(EmptyRow).asInstanceOf[Int]
+    val d1 = DateTimeUtils.millisToDays(System.currentTimeMillis(), UTC)
     assert(d0 <= cd && cd <= d1 && d1 - d0 <= 1)
 
-    val cdjst = CurrentDate(jstId).eval(EmptyRow).asInstanceOf[Int]
-    val cdpst = CurrentDate(pstId).eval(EmptyRow).asInstanceOf[Int]
+    val cdjst = CurrentDate(JST_OPT).eval(EmptyRow).asInstanceOf[Int]
+    val cdpst = CurrentDate(PST_OPT).eval(EmptyRow).asInstanceOf[Int]
     assert(cdpst <= cd && cd <= cdjst)
   }
 
@@ -93,8 +88,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("Year") {
     checkEvaluation(Year(Literal.create(null, DateType)), null)
     checkEvaluation(Year(Literal(d)), 2015)
-    checkEvaluation(Year(Cast(Literal(date), DateType, gmtId)), 2015)
-    checkEvaluation(Year(Cast(Literal(ts), DateType, gmtId)), 2013)
+    checkEvaluation(Year(Cast(Literal(date), DateType, UTC_OPT)), 2015)
+    checkEvaluation(Year(Cast(Literal(ts), DateType, UTC_OPT)), 2013)
 
     val c = Calendar.getInstance()
     (2000 to 2002).foreach { y =>
@@ -115,8 +110,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("Quarter") {
     checkEvaluation(Quarter(Literal.create(null, DateType)), null)
     checkEvaluation(Quarter(Literal(d)), 2)
-    checkEvaluation(Quarter(Cast(Literal(date), DateType, gmtId)), 2)
-    checkEvaluation(Quarter(Cast(Literal(ts), DateType, gmtId)), 4)
+    checkEvaluation(Quarter(Cast(Literal(date), DateType, UTC_OPT)), 2)
+    checkEvaluation(Quarter(Cast(Literal(ts), DateType, UTC_OPT)), 4)
 
     val c = Calendar.getInstance()
     (2003 to 2004).foreach { y =>
@@ -138,8 +133,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("Month") {
     checkEvaluation(Month(Literal.create(null, DateType)), null)
     checkEvaluation(Month(Literal(d)), 4)
-    checkEvaluation(Month(Cast(Literal(date), DateType, gmtId)), 4)
-    checkEvaluation(Month(Cast(Literal(ts), DateType, gmtId)), 11)
+    checkEvaluation(Month(Cast(Literal(date), DateType, UTC_OPT)), 4)
+    checkEvaluation(Month(Cast(Literal(ts), DateType, UTC_OPT)), 11)
 
     checkEvaluation(Month(Cast(Literal("1582-04-28 13:10:15"), DateType)), 4)
     checkEvaluation(Month(Cast(Literal("1582-10-04 13:10:15"), DateType)), 10)
@@ -163,8 +158,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(DayOfMonth(Cast(Literal("2000-02-29"), DateType)), 29)
     checkEvaluation(DayOfMonth(Literal.create(null, DateType)), null)
     checkEvaluation(DayOfMonth(Literal(d)), 8)
-    checkEvaluation(DayOfMonth(Cast(Literal(date), DateType, gmtId)), 8)
-    checkEvaluation(DayOfMonth(Cast(Literal(ts), DateType, gmtId)), 8)
+    checkEvaluation(DayOfMonth(Cast(Literal(date), DateType, UTC_OPT)), 8)
+    checkEvaluation(DayOfMonth(Cast(Literal(ts), DateType, UTC_OPT)), 8)
 
     checkEvaluation(DayOfMonth(Cast(Literal("1582-04-28 13:10:15"), DateType)), 28)
     checkEvaluation(DayOfMonth(Cast(Literal("1582-10-15 13:10:15"), DateType)), 15)
@@ -183,16 +178,16 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("Seconds") {
-    assert(Second(Literal.create(null, DateType), gmtId).resolved === false)
-    assert(Second(Cast(Literal(d), TimestampType, gmtId), gmtId).resolved )
-    checkEvaluation(Second(Cast(Literal(d), TimestampType, gmtId), gmtId), 0)
-    checkEvaluation(Second(Cast(Literal(date), TimestampType, gmtId), gmtId), 15)
-    checkEvaluation(Second(Literal(ts), gmtId), 15)
+    assert(Second(Literal.create(null, DateType), UTC_OPT).resolved === false)
+    assert(Second(Cast(Literal(d), TimestampType, UTC_OPT), UTC_OPT).resolved )
+    checkEvaluation(Second(Cast(Literal(d), TimestampType, UTC_OPT), UTC_OPT), 0)
+    checkEvaluation(Second(Cast(Literal(date), TimestampType, UTC_OPT), UTC_OPT), 15)
+    checkEvaluation(Second(Literal(ts), UTC_OPT), 15)
 
     val c = Calendar.getInstance()
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-      val timeZoneId = Option(tz.getID)
-      c.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      c.setTimeZone(TimeZone.getTimeZone(zid))
       (0 to 60 by 5).foreach { s =>
         c.set(2015, 18, 3, 3, 5, s)
         checkEvaluation(
@@ -207,10 +202,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("DayOfWeek") {
     checkEvaluation(DayOfWeek(Literal.create(null, DateType)), null)
     checkEvaluation(DayOfWeek(Literal(d)), Calendar.WEDNESDAY)
-    checkEvaluation(DayOfWeek(Cast(Literal(date), DateType, gmtId)),
+    checkEvaluation(DayOfWeek(Cast(Literal(date), DateType, UTC_OPT)),
       Calendar.WEDNESDAY)
-    checkEvaluation(DayOfWeek(Cast(Literal(ts), DateType, gmtId)), Calendar.FRIDAY)
-    checkEvaluation(DayOfWeek(Cast(Literal("2011-05-06"), DateType, gmtId)), Calendar.FRIDAY)
+    checkEvaluation(DayOfWeek(Cast(Literal(ts), DateType, UTC_OPT)), Calendar.FRIDAY)
+    checkEvaluation(DayOfWeek(Cast(Literal("2011-05-06"), DateType, UTC_OPT)), Calendar.FRIDAY)
     checkEvaluation(DayOfWeek(Literal(new Date(toMillis("2017-05-27 13:10:15")))),
       Calendar.SATURDAY)
     checkEvaluation(DayOfWeek(Literal(new Date(toMillis("1582-10-15 13:10:15")))),
@@ -221,9 +216,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("WeekDay") {
     checkEvaluation(WeekDay(Literal.create(null, DateType)), null)
     checkEvaluation(WeekDay(Literal(d)), 2)
-    checkEvaluation(WeekDay(Cast(Literal(date), DateType, gmtId)), 2)
-    checkEvaluation(WeekDay(Cast(Literal(ts), DateType, gmtId)), 4)
-    checkEvaluation(WeekDay(Cast(Literal("2011-05-06"), DateType, gmtId)), 4)
+    checkEvaluation(WeekDay(Cast(Literal(date), DateType, UTC_OPT)), 2)
+    checkEvaluation(WeekDay(Cast(Literal(ts), DateType, UTC_OPT)), 4)
+    checkEvaluation(WeekDay(Cast(Literal("2011-05-06"), DateType, UTC_OPT)), 4)
     checkEvaluation(WeekDay(Literal(new Date(toMillis("2017-05-27 13:10:15")))), 5)
     checkEvaluation(WeekDay(Literal(new Date(toMillis("1582-10-15 13:10:15")))), 4)
     checkConsistencyBetweenInterpretedAndCodegen(WeekDay, DateType)
@@ -232,63 +227,71 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("WeekOfYear") {
     checkEvaluation(WeekOfYear(Literal.create(null, DateType)), null)
     checkEvaluation(WeekOfYear(Literal(d)), 15)
-    checkEvaluation(WeekOfYear(Cast(Literal(date), DateType, gmtId)), 15)
-    checkEvaluation(WeekOfYear(Cast(Literal(ts), DateType, gmtId)), 45)
-    checkEvaluation(WeekOfYear(Cast(Literal("2011-05-06"), DateType, gmtId)), 18)
-    checkEvaluation(WeekOfYear(Cast(Literal("1582-10-15 13:10:15"), DateType, gmtId)), 41)
-    checkEvaluation(WeekOfYear(Cast(Literal("1582-10-04 13:10:15"), DateType, gmtId)), 40)
+    checkEvaluation(WeekOfYear(Cast(Literal(date), DateType, UTC_OPT)), 15)
+    checkEvaluation(WeekOfYear(Cast(Literal(ts), DateType, UTC_OPT)), 45)
+    checkEvaluation(WeekOfYear(Cast(Literal("2011-05-06"), DateType, UTC_OPT)), 18)
+    checkEvaluation(WeekOfYear(Cast(Literal("1582-10-15 13:10:15"), DateType, UTC_OPT)), 41)
+    checkEvaluation(WeekOfYear(Cast(Literal("1582-10-04 13:10:15"), DateType, UTC_OPT)), 40)
     checkConsistencyBetweenInterpretedAndCodegen(WeekOfYear, DateType)
   }
 
   test("DateFormat") {
-    checkEvaluation(
-      DateFormatClass(Literal.create(null, TimestampType), Literal("y"), gmtId),
-      null)
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
-      Literal.create(null, StringType), gmtId), null)
-
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
-      Literal("y"), gmtId), "2015")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), gmtId), "2013")
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
-      Literal("H"), gmtId), "0")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), gmtId), "13")
-
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, pstId),
-      Literal("y"), pstId), "2015")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), pstId), "2013")
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, pstId),
-      Literal("H"), pstId), "0")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), pstId), "5")
-
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, jstId),
-      Literal("y"), jstId), "2015")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), jstId), "2013")
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, jstId),
-      Literal("H"), jstId), "0")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), jstId), "22")
-
-    // SPARK-28072 The codegen path should work
-    checkEvaluation(
-      expression = DateFormatClass(
-        BoundReference(ordinal = 0, dataType = TimestampType, nullable = true),
-        BoundReference(ordinal = 1, dataType = StringType, nullable = true),
-        jstId),
-      expected = "22",
-      inputRow = InternalRow(DateTimeUtils.fromJavaTimestamp(ts), UTF8String.fromString("H")))
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+        checkEvaluation(
+          DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT),
+          null)
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT),
+          Literal.create(null, StringType), UTC_OPT), null)
+
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT),
+          Literal("y"), UTC_OPT), "2015")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013")
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT),
+          Literal("H"), UTC_OPT), "0")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13")
+
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT),
+          Literal("y"), PST_OPT), "2015")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013")
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT),
+          Literal("H"), PST_OPT), "0")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5")
+
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT),
+          Literal("y"), JST_OPT), "2015")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013")
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT),
+          Literal("H"), JST_OPT), "0")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22")
+
+        // Test escaping of format
+        GenerateUnsafeProjection.generate(
+          DateFormatClass(Literal(ts), Literal("\""), JST_OPT) :: Nil)
+
+        // SPARK-28072 The codegen path should work
+        checkEvaluation(
+          expression = DateFormatClass(
+            BoundReference(ordinal = 0, dataType = TimestampType, nullable = true),
+            BoundReference(ordinal = 1, dataType = StringType, nullable = true),
+            JST_OPT),
+          expected = "22",
+          inputRow = InternalRow(DateTimeUtils.fromJavaTimestamp(ts), UTF8String.fromString("H")))
+      }
+    }
   }
 
   test("Hour") {
-    assert(Hour(Literal.create(null, DateType), gmtId).resolved === false)
-    assert(Hour(Literal(ts), gmtId).resolved)
-    checkEvaluation(Hour(Cast(Literal(d), TimestampType, gmtId), gmtId), 0)
-    checkEvaluation(Hour(Cast(Literal(date), TimestampType, gmtId), gmtId), 13)
-    checkEvaluation(Hour(Literal(ts), gmtId), 13)
+    assert(Hour(Literal.create(null, DateType), UTC_OPT).resolved === false)
+    assert(Hour(Literal(ts), UTC_OPT).resolved)
+    checkEvaluation(Hour(Cast(Literal(d), TimestampType, UTC_OPT), UTC_OPT), 0)
+    checkEvaluation(Hour(Cast(Literal(date), TimestampType, UTC_OPT), UTC_OPT), 13)
+    checkEvaluation(Hour(Literal(ts), UTC_OPT), 13)
 
     val c = Calendar.getInstance()
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-      val timeZoneId = Option(tz.getID)
-      c.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      c.setTimeZone(TimeZone.getTimeZone(zid))
       (0 to 24 by 6).foreach { h =>
         (0 to 60 by 30).foreach { m =>
           (0 to 60 by 30).foreach { s =>
@@ -305,17 +308,17 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("Minute") {
-    assert(Minute(Literal.create(null, DateType), gmtId).resolved === false)
-    assert(Minute(Literal(ts), gmtId).resolved)
-    checkEvaluation(Minute(Cast(Literal(d), TimestampType, gmtId), gmtId), 0)
+    assert(Minute(Literal.create(null, DateType), UTC_OPT).resolved === false)
+    assert(Minute(Literal(ts), UTC_OPT).resolved)
+    checkEvaluation(Minute(Cast(Literal(d), TimestampType, UTC_OPT), UTC_OPT), 0)
     checkEvaluation(
-      Minute(Cast(Literal(date), TimestampType, gmtId), gmtId), 10)
-    checkEvaluation(Minute(Literal(ts), gmtId), 10)
+      Minute(Cast(Literal(date), TimestampType, UTC_OPT), UTC_OPT), 10)
+    checkEvaluation(Minute(Literal(ts), UTC_OPT), 10)
 
     val c = Calendar.getInstance()
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-      val timeZoneId = Option(tz.getID)
-      c.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      c.setTimeZone(TimeZone.getTimeZone(zid))
       (0 to 60 by 5).foreach { m =>
         (0 to 60 by 15).foreach { s =>
           c.set(2015, 18, 3, 3, m, s)
@@ -356,6 +359,40 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkConsistencyBetweenInterpretedAndCodegen(DateAdd, DateType, IntegerType)
   }
 
+  test("date add interval") {
+    val d = Date.valueOf("2016-02-28")
+    Seq("true", "false") foreach { flag =>
+      withSQLConf((SQLConf.ANSI_ENABLED.key, flag)) {
+        checkEvaluation(
+          DateAddInterval(Literal(d), Literal(new CalendarInterval(0, 1, 0))),
+          DateTimeUtils.fromJavaDate(Date.valueOf("2016-02-29")))
+        checkEvaluation(
+          DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 0))),
+          DateTimeUtils.fromJavaDate(Date.valueOf("2016-03-29")))
+        checkEvaluation(DateAddInterval(Literal(d), Literal.create(null, CalendarIntervalType)),
+          null)
+        checkEvaluation(DateAddInterval(Literal.create(null, DateType),
+          Literal(new CalendarInterval(1, 1, 0))),
+          null)
+      }
+    }
+
+    withSQLConf((SQLConf.ANSI_ENABLED.key, "true")) {
+      checkExceptionInExpression[IllegalArgumentException](
+        DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25 * MICROS_PER_HOUR))),
+        "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date")
+    }
+
+    withSQLConf((SQLConf.ANSI_ENABLED.key, "false")) {
+      checkEvaluation(
+        DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25))),
+        DateTimeUtils.fromJavaDate(Date.valueOf("2016-03-29")))
+      checkEvaluation(
+        DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25 * MICROS_PER_HOUR))),
+        DateTimeUtils.fromJavaDate(Date.valueOf("2016-03-30")))
+    }
+  }
+
   test("date_sub") {
     checkEvaluation(
       DateSub(Literal(Date.valueOf("2015-01-01")), Literal(1.toByte)),
@@ -385,9 +422,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("time_add") {
     val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-      val timeZoneId = Option(tz.getID)
-      sdf.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      sdf.setTimeZone(TimeZone.getTimeZone(zid))
 
       checkEvaluation(
         TimeAdd(
@@ -423,9 +460,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("time_sub") {
     val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-      val timeZoneId = Option(tz.getID)
-      sdf.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      sdf.setTimeZone(TimeZone.getTimeZone(zid))
 
       checkEvaluation(
         TimeSub(
@@ -494,7 +531,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     // Valid range of DateType is [0001-01-01, 9999-12-31]
     val maxMonthInterval = 10000 * 12
     checkEvaluation(
-      AddMonths(Literal(Date.valueOf("0001-01-01")), Literal(maxMonthInterval)), 2933261)
+      AddMonths(Literal(LocalDate.parse("0001-01-01")), Literal(maxMonthInterval)),
+      LocalDate.of(10001, 1, 1).toEpochDay.toInt)
     checkEvaluation(
       AddMonths(Literal(Date.valueOf("9999-12-31")), Literal(-1 * maxMonthInterval)), -719529)
     // Test evaluation results between Interpreted mode and Codegen mode
@@ -508,9 +546,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("months_between") {
     val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-      val timeZoneId = Option(tz.getID)
-      sdf.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      sdf.setTimeZone(TimeZone.getTimeZone(zid))
 
       checkEvaluation(
         MonthsBetween(
@@ -602,267 +640,308 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(NextDay(Literal.create(null, DateType), Literal("xx")), null)
     checkEvaluation(
       NextDay(Literal(Date.valueOf("2015-07-23")), Literal.create(null, StringType)), null)
+    // Test escaping of dayOfWeek
+    GenerateUnsafeProjection.generate(
+      NextDay(Literal(Date.valueOf("2015-07-23")), Literal("\"quote")) :: Nil)
+  }
+
+  private def testTruncDate(input: Date, fmt: String, expected: Date): Unit = {
+    checkEvaluation(TruncDate(Literal.create(input, DateType), Literal.create(fmt, StringType)),
+      expected)
+    checkEvaluation(
+      TruncDate(Literal.create(input, DateType), NonFoldableLiteral.create(fmt, StringType)),
+      expected)
   }
 
   test("TruncDate") {
-    def testTrunc(input: Date, fmt: String, expected: Date): Unit = {
-      checkEvaluation(TruncDate(Literal.create(input, DateType), Literal.create(fmt, StringType)),
-        expected)
-      checkEvaluation(
-        TruncDate(Literal.create(input, DateType), NonFoldableLiteral.create(fmt, StringType)),
-        expected)
-    }
     val date = Date.valueOf("2015-07-22")
     Seq("yyyy", "YYYY", "year", "YEAR", "yy", "YY").foreach { fmt =>
-      testTrunc(date, fmt, Date.valueOf("2015-01-01"))
+      testTruncDate(date, fmt, Date.valueOf("2015-01-01"))
     }
     Seq("month", "MONTH", "mon", "MON", "mm", "MM").foreach { fmt =>
-      testTrunc(date, fmt, Date.valueOf("2015-07-01"))
+      testTruncDate(date, fmt, Date.valueOf("2015-07-01"))
     }
-    testTrunc(date, "DD", null)
-    testTrunc(date, "SECOND", null)
-    testTrunc(date, "HOUR", null)
-    testTrunc(date, null, null)
-    testTrunc(null, "MON", null)
-    testTrunc(null, null, null)
-
-    testTrunc(Date.valueOf("2000-03-08"), "decade", Date.valueOf("2000-01-01"))
-    testTrunc(Date.valueOf("2000-03-08"), "century", Date.valueOf("1901-01-01"))
+    testTruncDate(date, "DD", null)
+    testTruncDate(date, "SECOND", null)
+    testTruncDate(date, "HOUR", null)
+    testTruncDate(null, "MON", null)
+    // Test escaping of format
+    GenerateUnsafeProjection.generate(TruncDate(Literal(0, DateType), Literal("\"quote")) :: Nil)
+
   }
 
-  test("TruncTimestamp") {
-    def testTrunc(input: Timestamp, fmt: String, expected: Timestamp): Unit = {
-      checkEvaluation(
-        TruncTimestamp(Literal.create(fmt, StringType), Literal.create(input, TimestampType)),
-        expected)
-      checkEvaluation(
-        TruncTimestamp(
-          NonFoldableLiteral.create(fmt, StringType), Literal.create(input, TimestampType)),
-        expected)
-    }
+  private def testTruncTimestamp(input: Timestamp, fmt: String, expected: Timestamp): Unit = {
+    checkEvaluation(
+      TruncTimestamp(Literal.create(fmt, StringType), Literal.create(input, TimestampType)),
+      expected)
+    checkEvaluation(
+      TruncTimestamp(
+        NonFoldableLiteral.create(fmt, StringType), Literal.create(input, TimestampType)),
+      expected)
+  }
 
-    withDefaultTimeZone(TimeZoneGMT) {
+  test("TruncTimestamp") {
+    withDefaultTimeZone(UTC) {
       val inputDate = Timestamp.valueOf("2015-07-22 05:30:06")
 
       Seq("yyyy", "YYYY", "year", "YEAR", "yy", "YY").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-01-01 00:00:00"))
       }
 
       Seq("month", "MONTH", "mon", "MON", "mm", "MM").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-01 00:00:00"))
       }
 
       Seq("DAY", "day", "DD", "dd").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-22 00:00:00"))
       }
 
       Seq("HOUR", "hour").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-22 05:00:00"))
       }
 
       Seq("MINUTE", "minute").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-22 05:30:00"))
       }
 
       Seq("SECOND", "second").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-22 05:30:06"))
       }
 
       Seq("WEEK", "week").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-20 00:00:00"))
       }
 
       Seq("QUARTER", "quarter").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-01 00:00:00"))
       }
 
-      testTrunc(inputDate, "INVALID", null)
-      testTrunc(inputDate, null, null)
-      testTrunc(null, "MON", null)
-      testTrunc(null, null, null)
+      testTruncTimestamp(null, "MON", null)
+    }
+  }
 
-      testTrunc(Timestamp.valueOf("2000-03-08 11:12:13"), "decade",
-        Timestamp.valueOf("2000-01-01 00:00:00"))
-      testTrunc(Timestamp.valueOf("2000-03-08 11:12:13"), "century",
-        Timestamp.valueOf("1901-01-01 00:00:00"))
+  test("unsupported fmt fields for trunc/date_trunc results null") {
+    Seq("INVALID", "decade", "century", "millennium", "whatever", null).foreach { field =>
+      testTruncDate(Date.valueOf("2000-03-08"), field, null)
+      testTruncDate(null, field, null)
+      testTruncTimestamp(Timestamp.valueOf("2000-03-08 11:12:13"), field, null)
+      testTruncTimestamp(null, field, null)
     }
   }
 
   test("from_unixtime") {
-    val fmt1 = "yyyy-MM-dd HH:mm:ss"
-    val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
-    val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
-    val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-      val timeZoneId = Option(tz.getID)
-      sdf1.setTimeZone(tz)
-      sdf2.setTimeZone(tz)
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+        val fmt1 = "yyyy-MM-dd HH:mm:ss"
+        val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
+        val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+        val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
+        for (zid <- outstandingZoneIds) {
+          val timeZoneId = Option(zid.getId)
+          val tz = TimeZone.getTimeZone(zid)
+          sdf1.setTimeZone(tz)
+          sdf2.setTimeZone(tz)
 
-      checkEvaluation(
-        FromUnixTime(Literal(0L), Literal(fmt1), timeZoneId),
-        sdf1.format(new Timestamp(0)))
-      checkEvaluation(FromUnixTime(
-        Literal(1000L), Literal(fmt1), timeZoneId),
-        sdf1.format(new Timestamp(1000000)))
-      checkEvaluation(
-        FromUnixTime(Literal(-1000L), Literal(fmt2), timeZoneId),
-        sdf2.format(new Timestamp(-1000000)))
-      checkEvaluation(
-        FromUnixTime(Literal.create(null, LongType), Literal.create(null, StringType), timeZoneId),
-        null)
-      checkEvaluation(
-        FromUnixTime(Literal.create(null, LongType), Literal(fmt1), timeZoneId),
-        null)
-      checkEvaluation(
-        FromUnixTime(Literal(1000L), Literal.create(null, StringType), timeZoneId),
-        null)
-      checkEvaluation(
-        FromUnixTime(Literal(0L), Literal("not a valid format"), timeZoneId), null)
+          checkEvaluation(
+            FromUnixTime(Literal(0L), Literal(fmt1), timeZoneId),
+            sdf1.format(new Timestamp(0)))
+          checkEvaluation(FromUnixTime(
+            Literal(1000L), Literal(fmt1), timeZoneId),
+            sdf1.format(new Timestamp(1000000)))
+          checkEvaluation(
+            FromUnixTime(Literal(-1000L), Literal(fmt2), timeZoneId),
+            sdf2.format(new Timestamp(-1000000)))
+          checkEvaluation(
+            FromUnixTime(
+              Literal.create(null, LongType),
+              Literal.create(null, StringType), timeZoneId),
+            null)
+          checkEvaluation(
+            FromUnixTime(Literal.create(null, LongType), Literal(fmt1), timeZoneId),
+            null)
+          checkEvaluation(
+            FromUnixTime(Literal(1000L), Literal.create(null, StringType), timeZoneId),
+            null)
+          checkEvaluation(
+            FromUnixTime(Literal(0L), Literal("not a valid format"), timeZoneId), null)
 
-      // SPARK-28072 The codegen path for non-literal input should also work
-      checkEvaluation(
-        expression = FromUnixTime(
-          BoundReference(ordinal = 0, dataType = LongType, nullable = true),
-          BoundReference(ordinal = 1, dataType = StringType, nullable = true),
-          timeZoneId),
-        expected = UTF8String.fromString(sdf1.format(new Timestamp(0))),
-        inputRow = InternalRow(0L, UTF8String.fromString(fmt1)))
+          // SPARK-28072 The codegen path for non-literal input should also work
+          checkEvaluation(
+            expression = FromUnixTime(
+              BoundReference(ordinal = 0, dataType = LongType, nullable = true),
+              BoundReference(ordinal = 1, dataType = StringType, nullable = true),
+              timeZoneId),
+            expected = UTF8String.fromString(sdf1.format(new Timestamp(0))),
+            inputRow = InternalRow(0L, UTF8String.fromString(fmt1)))
+        }
+      }
     }
+    // Test escaping of format
+    GenerateUnsafeProjection.generate(FromUnixTime(Literal(0L), Literal("\"quote"), UTC_OPT) :: Nil)
   }
 
   test("unix_timestamp") {
-    val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
-    val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
-    val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
-    val fmt3 = "yy-MM-dd"
-    val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
-    sdf3.setTimeZone(TimeZoneGMT)
-
-    withDefaultTimeZone(TimeZoneGMT) {
-      for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-        val timeZoneId = Option(tz.getID)
-        sdf1.setTimeZone(tz)
-        sdf2.setTimeZone(tz)
-
-        val date1 = Date.valueOf("2015-07-24")
-        checkEvaluation(UnixTimestamp(
-          Literal(sdf1.format(new Timestamp(0))), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId), 0L)
-        checkEvaluation(UnixTimestamp(
-          Literal(sdf1.format(new Timestamp(1000000))), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
-          1000L)
-        checkEvaluation(
-          UnixTimestamp(
-            Literal(new Timestamp(1000000)), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
-          1000L)
-        checkEvaluation(
-          UnixTimestamp(Literal(date1), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
-        checkEvaluation(
-          UnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))), Literal(fmt2), timeZoneId),
-          -1000L)
-        checkEvaluation(UnixTimestamp(
-          Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
-            DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz)))
-        val t1 = UnixTimestamp(
-          CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
-        val t2 = UnixTimestamp(
-          CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
-        assert(t2 - t1 <= 1)
-        checkEvaluation(
-          UnixTimestamp(
-            Literal.create(null, DateType), Literal.create(null, StringType), timeZoneId),
-          null)
-        checkEvaluation(
-          UnixTimestamp(Literal.create(null, DateType), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
-          null)
-        checkEvaluation(
-          UnixTimestamp(Literal(date1), Literal.create(null, StringType), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
-        checkEvaluation(
-          UnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null)
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+        val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
+        val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+        val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
+        val fmt3 = "yy-MM-dd"
+        val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
+        sdf3.setTimeZone(TimeZone.getTimeZone(UTC))
+
+        withDefaultTimeZone(UTC) {
+          for (zid <- outstandingZoneIds) {
+            val timeZoneId = Option(zid.getId)
+            val tz = TimeZone.getTimeZone(zid)
+            sdf1.setTimeZone(tz)
+            sdf2.setTimeZone(tz)
+
+            val date1 = Date.valueOf("2015-07-24")
+            checkEvaluation(UnixTimestamp(
+              Literal(sdf1.format(new Timestamp(0))),
+              Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId), 0L)
+            checkEvaluation(UnixTimestamp(
+              Literal(sdf1.format(new Timestamp(1000000))),
+              Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
+              1000L)
+            checkEvaluation(
+              UnixTimestamp(
+                Literal(new Timestamp(1000000)), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
+              1000L)
+            checkEvaluation(
+              UnixTimestamp(Literal(date1), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
+              MILLISECONDS.toSeconds(
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
+            checkEvaluation(
+              UnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))),
+                Literal(fmt2), timeZoneId),
+              -1000L)
+            checkEvaluation(UnixTimestamp(
+              Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
+              MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
+                DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz.toZoneId)))
+            val t1 = UnixTimestamp(
+              CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
+            val t2 = UnixTimestamp(
+              CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
+            assert(t2 - t1 <= 1)
+            checkEvaluation(
+              UnixTimestamp(
+                Literal.create(null, DateType), Literal.create(null, StringType), timeZoneId),
+              null)
+            checkEvaluation(
+              UnixTimestamp(
+                Literal.create(null, DateType),
+                Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
+              null)
+            checkEvaluation(
+              UnixTimestamp(Literal(date1), Literal.create(null, StringType), timeZoneId),
+              MILLISECONDS.toSeconds(
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
+            checkEvaluation(
+              UnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null)
+          }
+        }
       }
     }
+    // Test escaping of format
+    GenerateUnsafeProjection.generate(
+      UnixTimestamp(Literal("2015-07-24"), Literal("\"quote"), UTC_OPT) :: Nil)
   }
 
   test("to_unix_timestamp") {
-    val fmt1 = "yyyy-MM-dd HH:mm:ss"
-    val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
-    val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
-    val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
-    val fmt3 = "yy-MM-dd"
-    val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
-    sdf3.setTimeZone(TimeZoneGMT)
-
-    withDefaultTimeZone(TimeZoneGMT) {
-      for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-        val timeZoneId = Option(tz.getID)
-        sdf1.setTimeZone(tz)
-        sdf2.setTimeZone(tz)
-
-        val date1 = Date.valueOf("2015-07-24")
-        checkEvaluation(ToUnixTimestamp(
-          Literal(sdf1.format(new Timestamp(0))), Literal(fmt1), timeZoneId), 0L)
-        checkEvaluation(ToUnixTimestamp(
-          Literal(sdf1.format(new Timestamp(1000000))), Literal(fmt1), timeZoneId),
-          1000L)
-        checkEvaluation(ToUnixTimestamp(
-          Literal(new Timestamp(1000000)), Literal(fmt1)),
-          1000L)
-        checkEvaluation(
-          ToUnixTimestamp(Literal(date1), Literal(fmt1), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
-        checkEvaluation(
-          ToUnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))), Literal(fmt2), timeZoneId),
-          -1000L)
-        checkEvaluation(ToUnixTimestamp(
-          Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
-            DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz)))
-        val t1 = ToUnixTimestamp(
-          CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
-        val t2 = ToUnixTimestamp(
-          CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
-        assert(t2 - t1 <= 1)
-        checkEvaluation(ToUnixTimestamp(
-          Literal.create(null, DateType), Literal.create(null, StringType), timeZoneId), null)
-        checkEvaluation(
-          ToUnixTimestamp(
-            Literal.create(null, DateType), Literal(fmt1), timeZoneId),
-          null)
-        checkEvaluation(ToUnixTimestamp(
-          Literal(date1), Literal.create(null, StringType), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
-        checkEvaluation(
-          ToUnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null)
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+        val fmt1 = "yyyy-MM-dd HH:mm:ss"
+        val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
+        val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+        val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
+        val fmt3 = "yy-MM-dd"
+        val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
+        sdf3.setTimeZone(TimeZone.getTimeZone(UTC))
+
+        withDefaultTimeZone(UTC) {
+          for (zid <- outstandingZoneIds) {
+            val timeZoneId = Option(zid.getId)
+            val tz = TimeZone.getTimeZone(zid)
+            sdf1.setTimeZone(tz)
+            sdf2.setTimeZone(tz)
+
+            val date1 = Date.valueOf("2015-07-24")
+            checkEvaluation(ToUnixTimestamp(
+              Literal(sdf1.format(new Timestamp(0))), Literal(fmt1), timeZoneId), 0L)
+            checkEvaluation(ToUnixTimestamp(
+              Literal(sdf1.format(new Timestamp(1000000))), Literal(fmt1), timeZoneId),
+              1000L)
+            checkEvaluation(ToUnixTimestamp(
+              Literal(new Timestamp(1000000)), Literal(fmt1)),
+              1000L)
+            checkEvaluation(
+              ToUnixTimestamp(Literal(date1), Literal(fmt1), timeZoneId),
+              MILLISECONDS.toSeconds(
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), zid)))
+            checkEvaluation(
+              ToUnixTimestamp(
+                Literal(sdf2.format(new Timestamp(-1000000))),
+                Literal(fmt2), timeZoneId),
+              -1000L)
+            checkEvaluation(ToUnixTimestamp(
+              Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
+              MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
+                DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), zid)))
+            val t1 = ToUnixTimestamp(
+              CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
+            val t2 = ToUnixTimestamp(
+              CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
+            assert(t2 - t1 <= 1)
+            checkEvaluation(ToUnixTimestamp(
+              Literal.create(null, DateType), Literal.create(null, StringType), timeZoneId), null)
+            checkEvaluation(
+              ToUnixTimestamp(
+                Literal.create(null, DateType), Literal(fmt1), timeZoneId),
+              null)
+            checkEvaluation(ToUnixTimestamp(
+              Literal(date1), Literal.create(null, StringType), timeZoneId),
+              MILLISECONDS.toSeconds(
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), zid)))
+            checkEvaluation(
+              ToUnixTimestamp(
+                Literal("2015-07-24"),
+                Literal("not a valid format"), timeZoneId), null)
 
-        // SPARK-28072 The codegen path for non-literal input should also work
-        checkEvaluation(
-          expression = ToUnixTimestamp(
-            BoundReference(ordinal = 0, dataType = StringType, nullable = true),
-            BoundReference(ordinal = 1, dataType = StringType, nullable = true),
-            timeZoneId),
-          expected = 0L,
-          inputRow = InternalRow(
-            UTF8String.fromString(sdf1.format(new Timestamp(0))), UTF8String.fromString(fmt1)))
+            // SPARK-28072 The codegen path for non-literal input should also work
+            checkEvaluation(
+              expression = ToUnixTimestamp(
+                BoundReference(ordinal = 0, dataType = StringType, nullable = true),
+                BoundReference(ordinal = 1, dataType = StringType, nullable = true),
+                timeZoneId),
+              expected = 0L,
+              inputRow = InternalRow(
+                UTF8String.fromString(sdf1.format(new Timestamp(0))), UTF8String.fromString(fmt1)))
+          }
+        }
       }
     }
+    // Test escaping of format
+    GenerateUnsafeProjection.generate(
+      ToUnixTimestamp(Literal("2015-07-24"), Literal("\"quote"), UTC_OPT) :: Nil)
   }
 
   test("datediff") {
@@ -892,8 +971,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
           NonFoldableLiteral.create(tz, StringType)),
         if (expected != null) Timestamp.valueOf(expected) else null)
     }
-    test("2015-07-24 00:00:00", "PST", "2015-07-24 07:00:00")
-    test("2015-01-24 00:00:00", "PST", "2015-01-24 08:00:00")
+    test("2015-07-24 00:00:00", LA.getId, "2015-07-24 07:00:00")
+    test("2015-01-24 00:00:00", LA.getId, "2015-01-24 08:00:00")
     test(null, "UTC", null)
     test("2015-07-24 00:00:00", null, null)
     test(null, null, null)
@@ -923,8 +1002,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
           NonFoldableLiteral.create(tz, StringType)),
         if (expected != null) Timestamp.valueOf(expected) else null)
     }
-    test("2015-07-24 00:00:00", "PST", "2015-07-23 17:00:00")
-    test("2015-01-24 00:00:00", "PST", "2015-01-23 16:00:00")
+    test("2015-07-24 00:00:00", LA.getId, "2015-07-23 17:00:00")
+    test("2015-01-24 00:00:00", LA.getId, "2015-01-23 16:00:00")
     test(null, "UTC", null)
     test("2015-07-24 00:00:00", null, null)
     test(null, null, null)
@@ -985,100 +1064,28 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(makeTimestampExpr, Timestamp.valueOf("2019-08-12 00:00:58.000001"))
   }
 
-  test("millennium") {
-    val date = MakeDate(Literal(2019), Literal(1), Literal(1))
-    checkEvaluation(Millennium(date), 3)
-    checkEvaluation(Millennium(date.copy(year = Literal(2001))), 3)
-    checkEvaluation(Millennium(date.copy(year = Literal(2000))), 2)
-    checkEvaluation(Millennium(date.copy(year = Literal(1001), day = Literal(28))), 2)
-    checkEvaluation(Millennium(date.copy(year = Literal(1))), 1)
-    checkEvaluation(Millennium(date.copy(year = Literal(-1))), -1)
-    checkEvaluation(Millennium(date.copy(year = Literal(-100), month = Literal(12))), -1)
-    checkEvaluation(Millennium(date.copy(year = Literal(-2019))), -3)
-  }
-
-  test("century") {
-    val date = MakeDate(Literal(2019), Literal(1), Literal(1))
-    checkEvaluation(Century(date), 21)
-    checkEvaluation(Century(date.copy(year = Literal(2001))), 21)
-    checkEvaluation(Century(date.copy(year = Literal(2000))), 20)
-    checkEvaluation(Century(date.copy(year = Literal(1001), day = Literal(28))), 11)
-    checkEvaluation(Century(date.copy(year = Literal(1))), 1)
-    checkEvaluation(Century(date.copy(year = Literal(-1))), -1)
-    checkEvaluation(Century(date.copy(year = Literal(-100), month = Literal(12))), -2)
-    checkEvaluation(Century(date.copy(year = Literal(-2019))), -21)
-  }
-
-  test("decade") {
-    val date = MakeDate(Literal(2019), Literal(8), Literal(8))
-    checkEvaluation(Decade(date), 201)
-    checkEvaluation(Decade(date.copy(year = Literal(2011))), 201)
-    checkEvaluation(Decade(date.copy(year = Literal(2010))), 201)
-    checkEvaluation(Decade(date.copy(year = Literal(2009))), 200)
-    checkEvaluation(Decade(date.copy(year = Literal(10))), 1)
-    checkEvaluation(Decade(date.copy(year = Literal(1))), 0)
-    checkEvaluation(Decade(date.copy(year = Literal(-1))), -1)
-    checkEvaluation(Decade(date.copy(year = Literal(-10))), -1)
-    checkEvaluation(Decade(date.copy(year = Literal(-11))), -2)
-    checkEvaluation(Decade(date.copy(year = Literal(-2019))), -202)
-  }
-
-  test("milliseconds and microseconds") {
-    outstandingTimezonesIds.foreach { timezone =>
-      var timestamp = MakeTimestamp(Literal(2019), Literal(8), Literal(10),
-        Literal(0), Literal(0), Literal(Decimal(BigDecimal(10.123456789), 8, 6)),
-        Some(Literal(timezone)))
-
-      checkEvaluation(Milliseconds(timestamp), Decimal(BigDecimal(10123.457), 8, 3))
-      checkEvaluation(Microseconds(timestamp), 10123457)
-
-      timestamp = timestamp.copy(sec = Literal(Decimal(0.0, 8, 6)))
-      checkEvaluation(Milliseconds(timestamp), Decimal(0, 8, 3))
-      checkEvaluation(Microseconds(timestamp), 0)
-
-      timestamp = timestamp.copy(sec = Literal(Decimal(BigDecimal(59.999999), 8, 6)))
-      checkEvaluation(Milliseconds(timestamp), Decimal(BigDecimal(59999.999), 8, 3))
-      checkEvaluation(Microseconds(timestamp), 59999999)
-
-      timestamp = timestamp.copy(sec = Literal(Decimal(BigDecimal(60.0), 8, 6)))
-      checkEvaluation(Milliseconds(timestamp), Decimal(0, 8, 3))
-      checkEvaluation(Microseconds(timestamp), 0)
-    }
-  }
-
-  test("epoch") {
-    val zoneId = ZoneId.systemDefault()
-    val nanos = 123456000
-    val timestamp = Epoch(MakeTimestamp(
-      Literal(2019), Literal(8), Literal(9), Literal(0), Literal(0),
-      Literal(Decimal(nanos / NANOS_PER_SECOND.toDouble, 8, 6)),
-      Some(Literal(zoneId.getId))))
-    val instant = LocalDateTime.of(2019, 8, 9, 0, 0, 0, nanos)
-      .atZone(zoneId).toInstant
-    val expected = Decimal(BigDecimal(nanos) / NANOS_PER_SECOND +
-      instant.getEpochSecond +
-      zoneId.getRules.getOffset(instant).getTotalSeconds)
-    checkEvaluation(timestamp, expected)
-  }
-
   test("ISO 8601 week-numbering year") {
-    checkEvaluation(IsoYear(MakeDate(Literal(2006), Literal(1), Literal(1))), 2005)
-    checkEvaluation(IsoYear(MakeDate(Literal(2006), Literal(1), Literal(2))), 2006)
+    checkEvaluation(YearOfWeek(MakeDate(Literal(2006), Literal(1), Literal(1))), 2005)
+    checkEvaluation(YearOfWeek(MakeDate(Literal(2006), Literal(1), Literal(2))), 2006)
   }
 
   test("extract the seconds part with fraction from timestamps") {
     outstandingTimezonesIds.foreach { timezone =>
       val timestamp = MakeTimestamp(Literal(2019), Literal(8), Literal(10),
         Literal(0), Literal(0), Literal(Decimal(10.123456, 8, 6)),
-        Some(Literal(timezone)))
+        Some(Literal(timezone)), Some(timezone))
+      def secFrac(ts: MakeTimestamp): SecondWithFraction = SecondWithFraction(ts, Some(timezone))
 
-      checkEvaluation(SecondWithFraction(timestamp), Decimal(10.123456, 8, 6))
+      checkEvaluation(secFrac(timestamp), Decimal(10.123456, 8, 6))
       checkEvaluation(
-        SecondWithFraction(timestamp.copy(sec = Literal(Decimal(59000001, 8, 6)))),
+        secFrac(timestamp.copy(sec = Literal(Decimal(59000001, 8, 6)))),
         Decimal(59000001, 8, 6))
       checkEvaluation(
-        SecondWithFraction(timestamp.copy(sec = Literal(Decimal(1, 8, 6)))),
+        secFrac(timestamp.copy(sec = Literal(Decimal(1, 8, 6)))),
         Decimal(0.000001, 8, 6))
+      checkEvaluation(
+        secFrac(timestamp.copy(year = Literal(10))),
+        Decimal(10.123456, 8, 6))
     }
   }
 
@@ -1119,4 +1126,64 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         Literal(LocalDate.of(1, 1, 1))),
       IntervalUtils.stringToInterval(UTF8String.fromString("interval 9999 years")))
   }
+
+  test("to_timestamp exception mode") {
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") {
+      checkEvaluation(
+        GetTimestamp(
+          Literal("2020-01-27T20:06:11.847-0800"),
+          Literal("yyyy-MM-dd'T'HH:mm:ss.SSSz")), 1580184371847000L)
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "corrected") {
+      checkEvaluation(
+        GetTimestamp(
+          Literal("2020-01-27T20:06:11.847-0800"),
+          Literal("yyyy-MM-dd'T'HH:mm:ss.SSSz")), null)
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "exception") {
+      checkExceptionInExpression[SparkUpgradeException](
+        GetTimestamp(
+          Literal("2020-01-27T20:06:11.847-0800"),
+          Literal("yyyy-MM-dd'T'HH:mm:ss.SSSz")), "Fail to parse")
+    }
+  }
+
+  test("Disable week-based date fields and quarter fields for parsing") {
+
+    def checkSparkUpgrade(c: Char): Unit = {
+      checkExceptionInExpression[SparkUpgradeException](
+        new ParseToTimestamp(Literal("1"), Literal(c.toString)).child, "3.0")
+      checkExceptionInExpression[SparkUpgradeException](
+        new ParseToDate(Literal("1"), Literal(c.toString)).child, "3.0")
+      checkExceptionInExpression[SparkUpgradeException](
+        ToUnixTimestamp(Literal("1"), Literal(c.toString)), "3.0")
+      checkExceptionInExpression[SparkUpgradeException](
+        UnixTimestamp(Literal("1"), Literal(c.toString)), "3.0")
+    }
+
+    def checkNullify(c: Char): Unit = {
+      checkEvaluation(new ParseToTimestamp(Literal("1"), Literal(c.toString)).child, null)
+      checkEvaluation(new ParseToDate(Literal("1"), Literal(c.toString)).child, null)
+      checkEvaluation(ToUnixTimestamp(Literal("1"), Literal(c.toString)), null)
+      checkEvaluation(UnixTimestamp(Literal("1"), Literal(c.toString)), null)
+    }
+
+    Seq('Y', 'W', 'w', 'E', 'u', 'F').foreach { l =>
+      checkSparkUpgrade(l)
+    }
+
+    Seq('q', 'Q').foreach { l =>
+      checkNullify(l)
+    }
+  }
+
+
+  test("SPARK-31896: Handle am-pm timestamp parsing when hour is missing") {
+    checkEvaluation(
+      new ParseToTimestamp(Literal("PM"), Literal("a")).child,
+      Timestamp.valueOf("1970-01-01 12:00:00.0"))
+    checkEvaluation(
+      new ParseToTimestamp(Literal("11:11 PM"), Literal("mm:ss a")).child,
+      Timestamp.valueOf("1970-01-01 12:11:11.0"))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala
index 492d97ba9d524..ab0ef22bb746f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala
@@ -176,5 +176,9 @@ class ExpressionSQLBuilderSuite extends SparkFunSuite {
       TimeSub('a, interval),
       "`a` - INTERVAL '1 hours'"
     )
+    checkSQL(
+      DateAddInterval('a, interval),
+      "`a` + INTERVAL '1 hours'"
+    )
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
index e6cf979649c83..af6e5a3f35ee1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
@@ -554,28 +554,14 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       .add("arrayOfString", arrayOfString)
       .add("arrayOfArrayOfString", ArrayType(arrayOfString))
       .add("arrayOfArrayOfInt", ArrayType(ArrayType(IntegerType)))
-      .add("arrayOfMap", ArrayType(mapOfString))
       .add("arrayOfStruct", ArrayType(structOfString))
       .add("arrayOfUDT", arrayOfUDT))
 
-  testHash(
-    new StructType()
-      .add("mapOfIntAndString", MapType(IntegerType, StringType))
-      .add("mapOfStringAndArray", MapType(StringType, arrayOfString))
-      .add("mapOfArrayAndInt", MapType(arrayOfString, IntegerType))
-      .add("mapOfArray", MapType(arrayOfString, arrayOfString))
-      .add("mapOfStringAndStruct", MapType(StringType, structOfString))
-      .add("mapOfStructAndString", MapType(structOfString, StringType))
-      .add("mapOfStruct", MapType(structOfString, structOfString)))
-
   testHash(
     new StructType()
       .add("structOfString", structOfString)
       .add("structOfStructOfString", new StructType().add("struct", structOfString))
       .add("structOfArray", new StructType().add("array", arrayOfString))
-      .add("structOfMap", new StructType().add("map", mapOfString))
-      .add("structOfArrayAndMap",
-        new StructType().add("array", arrayOfString).add("map", mapOfString))
       .add("structOfUDT", structOfUDT))
 
   test("hive-hash for decimal") {
@@ -713,11 +699,11 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   private def testHash(inputSchema: StructType): Unit = {
     val inputGenerator = RandomDataGenerator.forType(inputSchema, nullable = false).get
-    val encoder = RowEncoder(inputSchema)
+    val toRow = RowEncoder(inputSchema).createSerializer()
     val seed = scala.util.Random.nextInt()
     test(s"murmur3/xxHash64/hive hash: ${inputSchema.simpleString}") {
       for (_ <- 1 to 10) {
-        val input = encoder.toRow(inputGenerator.apply().asInstanceOf[Row]).asInstanceOf[UnsafeRow]
+        val input = toRow(inputGenerator.apply().asInstanceOf[Row]).asInstanceOf[UnsafeRow]
         val literals = input.toSeq(inputSchema).zip(inputSchema.map(_.dataType)).map {
           case (value, dt) => Literal.create(value, dt)
         }
@@ -731,7 +717,7 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     val longSeed = Math.abs(seed).toLong + Integer.MAX_VALUE.toLong
     test(s"SPARK-30633: xxHash64 with long seed: ${inputSchema.simpleString}") {
       for (_ <- 1 to 10) {
-        val input = encoder.toRow(inputGenerator.apply().asInstanceOf[Row]).asInstanceOf[UnsafeRow]
+        val input = toRow(inputGenerator.apply().asInstanceOf[Row]).asInstanceOf[UnsafeRow]
         val literals = input.toSeq(inputSchema).zip(inputSchema.map(_.dataType)).map {
           case (value, dt) => Literal.create(value, dt)
         }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
index e7b713840b884..c07f06eac08ea 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
@@ -465,8 +465,13 @@ class HigherOrderFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(
       transformKeys(transformKeys(ai0, plusOne), plusValue),
       create_map(3 -> 1, 5 -> 2, 7 -> 3, 9 -> 4))
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(transformKeys(ai0, modKey), create_map(1 -> 4, 2 -> 2, 0 -> 3))
+
+    checkExceptionInExpression[RuntimeException](
+      transformKeys(ai0, modKey), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(transformKeys(ai0, modKey), create_map(1 -> 4, 2 -> 2, 0 -> 3))
+    }
     checkEvaluation(transformKeys(ai1, plusOne), Map.empty[Int, Int])
     checkEvaluation(transformKeys(ai1, plusOne), Map.empty[Int, Int])
     checkEvaluation(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
index d31a0e210552d..6b7be4f1609a5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
@@ -21,6 +21,7 @@ import scala.language.implicitConversions
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
 import org.apache.spark.sql.catalyst.util.IntervalUtils.{safeStringToInterval, stringToInterval}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.Decimal
@@ -33,42 +34,6 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     Literal(stringToInterval( "interval " + s))
   }
 
-  test("millenniums") {
-    checkEvaluation(ExtractIntervalMillenniums("0 years"), 0)
-    checkEvaluation(ExtractIntervalMillenniums("9999 years"), 9)
-    checkEvaluation(ExtractIntervalMillenniums("1000 years"), 1)
-    checkEvaluation(ExtractIntervalMillenniums("-2000 years"), -2)
-    // Microseconds part must not be taken into account
-    checkEvaluation(ExtractIntervalMillenniums("999 years 400 days"), 0)
-    // Millennium must be taken from years and months
-    checkEvaluation(ExtractIntervalMillenniums("999 years 12 months"), 1)
-    checkEvaluation(ExtractIntervalMillenniums("1000 years -1 months"), 0)
-  }
-
-  test("centuries") {
-    checkEvaluation(ExtractIntervalCenturies("0 years"), 0)
-    checkEvaluation(ExtractIntervalCenturies("9999 years"), 99)
-    checkEvaluation(ExtractIntervalCenturies("1000 years"), 10)
-    checkEvaluation(ExtractIntervalCenturies("-2000 years"), -20)
-    // Microseconds part must not be taken into account
-    checkEvaluation(ExtractIntervalCenturies("99 years 400 days"), 0)
-    // Century must be taken from years and months
-    checkEvaluation(ExtractIntervalCenturies("99 years 12 months"), 1)
-    checkEvaluation(ExtractIntervalCenturies("100 years -1 months"), 0)
-  }
-
-  test("decades") {
-    checkEvaluation(ExtractIntervalDecades("0 years"), 0)
-    checkEvaluation(ExtractIntervalDecades("9999 years"), 999)
-    checkEvaluation(ExtractIntervalDecades("1000 years"), 100)
-    checkEvaluation(ExtractIntervalDecades("-2000 years"), -200)
-    // Microseconds part must not be taken into account
-    checkEvaluation(ExtractIntervalDecades("9 years 400 days"), 0)
-    // Decade must be taken from years and months
-    checkEvaluation(ExtractIntervalDecades("9 years 12 months"), 1)
-    checkEvaluation(ExtractIntervalDecades("10 years -1 months"), 0)
-  }
-
   test("years") {
     checkEvaluation(ExtractIntervalYears("0 years"), 0)
     checkEvaluation(ExtractIntervalYears("9999 years"), 9999)
@@ -81,19 +46,6 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(ExtractIntervalYears("10 years -1 months"), 9)
   }
 
-  test("quarters") {
-    checkEvaluation(ExtractIntervalQuarters("0 months"), 1.toByte)
-    checkEvaluation(ExtractIntervalQuarters("1 months"), 1.toByte)
-    checkEvaluation(ExtractIntervalQuarters("-1 months"), 1.toByte)
-    checkEvaluation(ExtractIntervalQuarters("2 months"), 1.toByte)
-    checkEvaluation(ExtractIntervalQuarters("-2 months"), 1.toByte)
-    checkEvaluation(ExtractIntervalQuarters("1 years -1 months"), 4.toByte)
-    checkEvaluation(ExtractIntervalQuarters("-1 years 1 months"), -2.toByte)
-    checkEvaluation(ExtractIntervalQuarters("2 years 3 months"), 2.toByte)
-    checkEvaluation(ExtractIntervalQuarters("-2 years -3 months"), 0.toByte)
-    checkEvaluation(ExtractIntervalQuarters("9999 years"), 1.toByte)
-  }
-
   test("months") {
     checkEvaluation(ExtractIntervalMonths("0 year"), 0.toByte)
     for (m <- -24 to 24) {
@@ -116,6 +68,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     // Years and months must not be taken into account
     checkEvaluation(ExtractIntervalDays("100 year 10 months 5 days"), 5)
     checkEvaluation(ExtractIntervalDays(largeInterval), 31)
+    checkEvaluation(ExtractIntervalDays("25 hours"), 1)
   }
 
   test("hours") {
@@ -129,6 +82,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     // Minutes should be taken into account
     checkEvaluation(ExtractIntervalHours("10 hours 100 minutes"), 11L)
     checkEvaluation(ExtractIntervalHours(largeInterval), 11L)
+    checkEvaluation(ExtractIntervalHours("25 hours"), 1L)
+
   }
 
   test("minutes") {
@@ -157,52 +112,17 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(ExtractIntervalSeconds("61 seconds 1 microseconds"), Decimal(1000001, 8, 6))
   }
 
-  test("milliseconds") {
-    checkEvaluation(ExtractIntervalMilliseconds("0 milliseconds"), Decimal(0, 8, 3))
-    checkEvaluation(ExtractIntervalMilliseconds("1 milliseconds"), Decimal(1.0, 8, 3))
-    checkEvaluation(ExtractIntervalMilliseconds("-1 milliseconds"), Decimal(-1.0, 8, 3))
-    checkEvaluation(
-      ExtractIntervalMilliseconds("1 second 999 milliseconds"),
-      Decimal(1999.0, 8, 3))
-    checkEvaluation(
-      ExtractIntervalMilliseconds("999 milliseconds 1 microsecond"),
-      Decimal(999.001, 8, 3))
-    checkEvaluation(
-      ExtractIntervalMilliseconds("-1 second -999 milliseconds"),
-      Decimal(-1999.0, 8, 3))
-    // Years and months must not be taken into account
-    checkEvaluation(ExtractIntervalMilliseconds("100 year 1 millisecond"), Decimal(1.0, 8, 3))
-    checkEvaluation(ExtractIntervalMilliseconds(largeInterval), Decimal(59999.999, 8, 3))
-  }
-
-  test("microseconds") {
-    checkEvaluation(ExtractIntervalMicroseconds("0 microseconds"), 0L)
-    checkEvaluation(ExtractIntervalMicroseconds("1 microseconds"), 1L)
-    checkEvaluation(ExtractIntervalMicroseconds("-1 microseconds"), -1L)
-    checkEvaluation(ExtractIntervalMicroseconds("1 second 999 microseconds"), 1000999L)
-    checkEvaluation(ExtractIntervalMicroseconds("999 milliseconds 1 microseconds"), 999001L)
-    checkEvaluation(ExtractIntervalMicroseconds("-1 second -999 microseconds"), -1000999L)
-    // Years and months must not be taken into account
-    checkEvaluation(ExtractIntervalMicroseconds("11 year 1 microseconds"), 1L)
-    checkEvaluation(ExtractIntervalMicroseconds(largeInterval), 59999999L)
-  }
-
-  test("epoch") {
-    checkEvaluation(ExtractIntervalEpoch("0 months"), Decimal(0.0, 18, 6))
-    checkEvaluation(ExtractIntervalEpoch("10000 years"), Decimal(315576000000.0, 18, 6))
-    checkEvaluation(ExtractIntervalEpoch("1 year"), Decimal(31557600.0, 18, 6))
-    checkEvaluation(ExtractIntervalEpoch("-1 year"), Decimal(-31557600.0, 18, 6))
-    checkEvaluation(
-      ExtractIntervalEpoch("1 second 1 millisecond 1 microsecond"),
-      Decimal(1.001001, 18, 6))
-  }
-
   test("multiply") {
-    def check(interval: String, num: Double, expected: String): Unit = {
-      val expr = MultiplyInterval(Literal(stringToInterval(interval)), Literal(num))
+    def check(
+        interval: String,
+        num: Double,
+        expected: String,
+        isAnsi: Option[Boolean] = None): Unit = {
       val expectedRes = safeStringToInterval(expected)
-      Seq("true", "false").foreach { v =>
+      val configs = if (isAnsi.isEmpty) Seq("true", "false") else isAnsi.map(_.toString).toSeq
+      configs.foreach { v =>
         withSQLConf(SQLConf.ANSI_ENABLED.key -> v) {
+          val expr = MultiplyInterval(Literal(stringToInterval(interval)), Literal(num))
           if (expectedRes == null) {
             checkExceptionInExpression[ArithmeticException](expr, expected)
           } else {
@@ -219,17 +139,23 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     check("1 year 1 second", 0.5, "6 months 500 milliseconds")
     check("-100 years -1 millisecond", 0.5, "-50 years -500 microseconds")
     check("2 months 4 seconds", -0.5, "-1 months -2 seconds")
-    check("1 month 2 microseconds", 1.5, "1 months 15 days 3 microseconds")
-    check("2 months", Int.MaxValue, "integer overflow")
+    check("1 month 2 microseconds", 1.5, "1 months 3 microseconds")
+    check("2 months", Int.MaxValue, "integer overflow", Some(true))
+    check("2 months", Int.MaxValue, Int.MaxValue + " months", Some(false))
   }
 
   test("divide") {
-    def check(interval: String, num: Double, expected: String): Unit = {
-      val expr = DivideInterval(Literal(stringToInterval(interval)), Literal(num))
+    def check(
+        interval: String,
+        num: Double,
+        expected: String,
+        isAnsi: Option[Boolean] = None): Unit = {
       val expectedRes = safeStringToInterval(expected)
-      Seq("true", "false").foreach { v =>
+      val configs = if (isAnsi.isEmpty) Seq("true", "false") else isAnsi.map(_.toString).toSeq
+      configs.foreach { v =>
         withSQLConf(SQLConf.ANSI_ENABLED.key -> v) {
-          if (expectedRes == null) {
+          val expr = DivideInterval(Literal(stringToInterval(interval)), Literal(num))
+          if (expected != null && expectedRes == null) {
             checkExceptionInExpression[ArithmeticException](expr, expected)
           } else {
             checkEvaluation(expr, expectedRes)
@@ -244,9 +170,11 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     check("6 years -7 seconds", 3, "2 years -2.333333 seconds")
     check("2 years -8 seconds", 0.5, "4 years -16 seconds")
     check("-1 month 2 microseconds", -0.25, "4 months -8 microseconds")
-    check("1 month 3 microsecond", 1.5, "20 days 2 microseconds")
-    check("1 second", 0, "divide by zero")
-    check(s"${Int.MaxValue} months", 0.9, "integer overflow")
+    check("1 month 3 microsecond", 1.5, "2 microseconds")
+    check("1 second", 0, "divide by zero", Some(true))
+    check("1 second", 0, null, Some(false))
+    check(s"${Int.MaxValue} months", 0.9, "integer overflow", Some(true))
+    check(s"${Int.MaxValue} months", 0.9, Int.MaxValue + " months", Some(false))
   }
 
   test("make interval") {
@@ -260,9 +188,10 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         seconds: Int = 0,
         millis: Int = 0,
         micros: Int = 0): Unit = {
-      val secFrac = seconds * MICROS_PER_SECOND + millis * MICROS_PER_MILLIS + micros
+      val secFrac = DateTimeTestUtils.secFrac(seconds, millis, micros)
       val intervalExpr = MakeInterval(Literal(years), Literal(months), Literal(weeks),
-        Literal(days), Literal(hours), Literal(minutes), Literal(Decimal(secFrac, 8, 6)))
+        Literal(days), Literal(hours), Literal(minutes),
+        Literal(Decimal(secFrac, Decimal.MAX_LONG_DIGITS, 6)))
       val totalMonths = years * MONTHS_PER_YEAR + months
       val totalDays = weeks * DAYS_PER_WEEK + days
       val totalMicros = secFrac + minutes * MICROS_PER_MINUTE + hours * MICROS_PER_HOUR
@@ -278,5 +207,11 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     check(years = 10000, micros = -1)
     check(-9999, -11, 0, -31, -23, -59, -59, -999, -999)
     check(years = -10000, micros = 1)
+    check(
+      hours = Int.MaxValue,
+      minutes = Int.MaxValue,
+      seconds = Int.MaxValue,
+      millis = Int.MaxValue,
+      micros = Int.MaxValue)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
index 3693531f47610..90c4d8f789660 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
@@ -18,15 +18,17 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.text.{DecimalFormat, DecimalFormatSymbols, SimpleDateFormat}
-import java.util.{Calendar, Locale}
+import java.util.{Calendar, Locale, TimeZone}
 
 import org.scalatest.exceptions.TestFailedException
 
 import org.apache.spark.{SparkException, SparkFunSuite}
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.plans.PlanTestBase
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{PST, UTC, UTC_OPT}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
@@ -48,6 +50,10 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
    in Jackson's JsonFactory.createParser(byte[]) due to RFC-4627 encoding detection */
   val badJson = "\u0000\u0000\u0000A\u0001AAA"
 
+  test("get_json_object escaping") {
+    GenerateUnsafeProjection.generate(GetJsonObject(Literal("\"quote"), Literal("\"quote")) :: Nil)
+  }
+
   test("$.store.bicycle") {
     checkEvaluation(
       GetJsonObject(Literal(json), Literal("$.store.bicycle")),
@@ -264,6 +270,11 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     assert(jt.eval(null).toSeq.head === expected)
   }
 
+  test("json_tuple escaping") {
+    GenerateUnsafeProjection.generate(
+      JsonTuple(Literal("\"quote") ::  Literal("\"quote") :: Nil) :: Nil)
+  }
+
   test("json_tuple - hive key 1") {
     checkJsonTuple(
       JsonTuple(
@@ -395,13 +406,17 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       InternalRow(UTF8String.fromString("1"), null, UTF8String.fromString("1")))
   }
 
-  val gmtId = Option(DateTimeUtils.TimeZoneGMT.getID)
+  test("from_json escaping") {
+    val schema = StructType(StructField("\"quote", IntegerType) :: Nil)
+    GenerateUnsafeProjection.generate(
+      JsonToStructs(schema, Map.empty, Literal("\"quote"), UTC_OPT) :: Nil)
+  }
 
   test("from_json") {
     val jsonData = """{"a": 1}"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal(jsonData), gmtId),
+      JsonToStructs(schema, Map.empty, Literal(jsonData), UTC_OPT),
       InternalRow(1)
     )
   }
@@ -410,13 +425,13 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val jsonData = """{"a" 1}"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal(jsonData), gmtId),
+      JsonToStructs(schema, Map.empty, Literal(jsonData), UTC_OPT),
       InternalRow(null)
     )
 
     val exception = intercept[TestFailedException] {
       checkEvaluation(
-        JsonToStructs(schema, Map("mode" -> FailFastMode.name), Literal(jsonData), gmtId),
+        JsonToStructs(schema, Map("mode" -> FailFastMode.name), Literal(jsonData), UTC_OPT),
         InternalRow(null)
       )
     }.getCause
@@ -429,35 +444,35 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val input = """[{"a": 1}, {"a": 2}]"""
     val schema = ArrayType(StructType(StructField("a", IntegerType) :: Nil))
     val output = InternalRow(1) :: InternalRow(2) :: Nil
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=object, schema=array, output=array of single row") {
     val input = """{"a": 1}"""
     val schema = ArrayType(StructType(StructField("a", IntegerType) :: Nil))
     val output = InternalRow(1) :: Nil
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=empty array, schema=array, output=empty array") {
     val input = "[ ]"
     val schema = ArrayType(StructType(StructField("a", IntegerType) :: Nil))
     val output = Nil
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=empty object, schema=array, output=array of single row with null") {
     val input = "{ }"
     val schema = ArrayType(StructType(StructField("a", IntegerType) :: Nil))
     val output = InternalRow(null) :: Nil
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=array of single object, schema=struct, output=single row") {
     val input = """[{"a": 1}]"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val output = InternalRow(null)
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=array, schema=struct, output=single row") {
@@ -466,27 +481,27 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val schema = new StructType().add("a", IntegerType).add(corrupted, StringType)
     val output = InternalRow(null, UTF8String.fromString(input))
     val options = Map("columnNameOfCorruptRecord" -> corrupted)
-    checkEvaluation(JsonToStructs(schema, options, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, options, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=empty array, schema=struct, output=single row with null") {
     val input = """[]"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val output = InternalRow(null)
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=empty object, schema=struct, output=single row with null") {
     val input = """{  }"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val output = InternalRow(null)
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json null input column") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal.create(null, StringType), gmtId),
+      JsonToStructs(schema, Map.empty, Literal.create(null, StringType), UTC_OPT),
       null
     )
   }
@@ -494,7 +509,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
   test("SPARK-20549: from_json bad UTF-8") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal(badJson), gmtId),
+      JsonToStructs(schema, Map.empty, Literal(badJson), UTC_OPT),
       InternalRow(null))
   }
 
@@ -502,24 +517,24 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val schema = StructType(StructField("t", TimestampType) :: Nil)
 
     val jsonData1 = """{"t": "2016-01-01T00:00:00.123Z"}"""
-    var c = Calendar.getInstance(DateTimeUtils.TimeZoneGMT)
+    var c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
     c.set(2016, 0, 1, 0, 0, 0)
     c.set(Calendar.MILLISECOND, 123)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal(jsonData1), gmtId),
+      JsonToStructs(schema, Map.empty, Literal(jsonData1), UTC_OPT),
       InternalRow(c.getTimeInMillis * 1000L)
     )
     // The result doesn't change because the json string includes timezone string ("Z" here),
     // which means the string represents the timestamp string in the timezone regardless of
     // the timeZoneId parameter.
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal(jsonData1), Option("PST")),
+      JsonToStructs(schema, Map.empty, Literal(jsonData1), Option(PST.getId)),
       InternalRow(c.getTimeInMillis * 1000L)
     )
 
     val jsonData2 = """{"t": "2016-01-01T00:00:00"}"""
-    for (tz <- DateTimeTestUtils.outstandingTimezones) {
-      c = Calendar.getInstance(tz)
+    for (zid <- DateTimeTestUtils.outstandingZoneIds) {
+      c = Calendar.getInstance(TimeZone.getTimeZone(zid))
       c.set(2016, 0, 1, 0, 0, 0)
       c.set(Calendar.MILLISECOND, 0)
       checkEvaluation(
@@ -527,16 +542,16 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
           schema,
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss"),
           Literal(jsonData2),
-          Option(tz.getID)),
+          Option(zid.getId)),
         InternalRow(c.getTimeInMillis * 1000L)
       )
       checkEvaluation(
         JsonToStructs(
           schema,
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-            DateTimeUtils.TIMEZONE_OPTION -> tz.getID),
+            DateTimeUtils.TIMEZONE_OPTION -> zid.getId),
           Literal(jsonData2),
-          gmtId),
+          UTC_OPT),
         InternalRow(c.getTimeInMillis * 1000L)
       )
     }
@@ -545,16 +560,23 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
   test("SPARK-19543: from_json empty input column") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal.create(" ", StringType), gmtId),
+      JsonToStructs(schema, Map.empty, Literal.create(" ", StringType), UTC_OPT),
       null
     )
   }
 
+  test("to_json escaping") {
+    val schema = StructType(StructField("\"quote", IntegerType) :: Nil)
+    val struct = Literal.create(create_row(1), schema)
+    GenerateUnsafeProjection.generate(
+      StructsToJson(Map.empty, struct, UTC_OPT) :: Nil)
+  }
+
   test("to_json - struct") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val struct = Literal.create(create_row(1), schema)
     checkEvaluation(
-      StructsToJson(Map.empty, struct, gmtId),
+      StructsToJson(Map.empty, struct, UTC_OPT),
       """{"a":1}"""
     )
   }
@@ -564,7 +586,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val input = new GenericArrayData(InternalRow(1) :: InternalRow(2) :: Nil)
     val output = """[{"a":1},{"a":2}]"""
     checkEvaluation(
-      StructsToJson(Map.empty, Literal.create(input, inputSchema), gmtId),
+      StructsToJson(Map.empty, Literal.create(input, inputSchema), UTC_OPT),
       output)
   }
 
@@ -573,7 +595,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val input = new GenericArrayData(InternalRow(null) :: Nil)
     val output = """[{}]"""
     checkEvaluation(
-      StructsToJson(Map.empty, Literal.create(input, inputSchema), gmtId),
+      StructsToJson(Map.empty, Literal.create(input, inputSchema), UTC_OPT),
       output)
   }
 
@@ -582,7 +604,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val input = new GenericArrayData(Nil)
     val output = """[]"""
     checkEvaluation(
-      StructsToJson(Map.empty, Literal.create(input, inputSchema), gmtId),
+      StructsToJson(Map.empty, Literal.create(input, inputSchema), UTC_OPT),
       output)
   }
 
@@ -590,41 +612,41 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val struct = Literal.create(null, schema)
     checkEvaluation(
-      StructsToJson(Map.empty, struct, gmtId),
+      StructsToJson(Map.empty, struct, UTC_OPT),
       null
     )
   }
 
   test("to_json with timestamp") {
     val schema = StructType(StructField("t", TimestampType) :: Nil)
-    val c = Calendar.getInstance(DateTimeUtils.TimeZoneGMT)
+    val c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
     c.set(2016, 0, 1, 0, 0, 0)
     c.set(Calendar.MILLISECOND, 0)
     val struct = Literal.create(create_row(c.getTimeInMillis * 1000L), schema)
 
     checkEvaluation(
-      StructsToJson(Map.empty, struct, gmtId),
+      StructsToJson(Map.empty, struct, UTC_OPT),
       """{"t":"2016-01-01T00:00:00.000Z"}"""
     )
     checkEvaluation(
-      StructsToJson(Map.empty, struct, Option("PST")),
+      StructsToJson(Map.empty, struct, Option(PST.getId)),
       """{"t":"2015-12-31T16:00:00.000-08:00"}"""
     )
 
     checkEvaluation(
       StructsToJson(
         Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-          DateTimeUtils.TIMEZONE_OPTION -> gmtId.get),
+          DateTimeUtils.TIMEZONE_OPTION -> UTC_OPT.get),
         struct,
-        gmtId),
+        UTC_OPT),
       """{"t":"2016-01-01T00:00:00"}"""
     )
     checkEvaluation(
       StructsToJson(
         Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-          DateTimeUtils.TIMEZONE_OPTION -> "PST"),
+          DateTimeUtils.TIMEZONE_OPTION -> PST.getId),
         struct,
-        gmtId),
+        UTC_OPT),
       """{"t":"2015-12-31T16:00:00"}"""
     )
   }
@@ -665,7 +687,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       ArrayBasedMapData(Map(UTF8String.fromString("b") -> 2)) :: Nil)
     val output = """[{"a":1},{"b":2}]"""
     checkEvaluation(
-      StructsToJson(Map.empty, Literal(input, inputSchema), gmtId),
+      StructsToJson(Map.empty, Literal(input, inputSchema), UTC_OPT),
       output)
   }
 
@@ -674,7 +696,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val input = new GenericArrayData(ArrayBasedMapData(Map(UTF8String.fromString("a") -> 1)) :: Nil)
     val output = """[{"a":1}]"""
     checkEvaluation(
-      StructsToJson(Map.empty, Literal.create(input, inputSchema), gmtId),
+      StructsToJson(Map.empty, Literal.create(input, inputSchema), UTC_OPT),
       output)
   }
 
@@ -704,7 +726,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       .add("b", StringType, nullable = false)
       .add("c", StringType, nullable = false)
     val output = InternalRow(1L, null, UTF8String.fromString("foo"))
-    val expr = JsonToStructs(jsonSchema, Map.empty, Literal.create(input, StringType), gmtId)
+    val expr = JsonToStructs(jsonSchema, Map.empty, Literal.create(input, StringType), UTC_OPT)
     checkEvaluation(expr, output)
     val schema = expr.dataType
     val schemaToCompare = jsonSchema.asNullable
@@ -737,7 +759,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       val options = Map("dateFormat" -> dateFormat, "locale" -> langTag)
 
       checkEvaluation(
-        JsonToStructs(schema, options, Literal.create(dateStr), gmtId),
+        JsonToStructs(schema, options, Literal.create(dateStr), UTC_OPT),
         InternalRow(17836)) // number of days from 1970-01-01
     }
   }
@@ -748,7 +770,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
         schema = StructType.fromDDL("i int, _unparsed boolean"),
         options = Map("columnNameOfCorruptRecord" -> "_unparsed"),
         child = Literal.create("""{"i":"a"}"""),
-        timeZoneId = gmtId),
+        timeZoneId = UTC_OPT),
       expectedErrMsg = "The field for corrupt records must be string type and nullable")
   }
 
@@ -770,7 +792,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       val (expected, input) = decimalInput(langTag)
 
       checkEvaluation(
-        JsonToStructs(schema, options, Literal.create(input), gmtId),
+        JsonToStructs(schema, options, Literal.create(input), UTC_OPT),
         InternalRow(expected))
     }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
index 4714635a3370b..bb86135021b91 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
@@ -209,6 +209,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
     }
     checkMapLiteral(Map("a" -> 1, "b" -> 2, "c" -> 3))
     checkMapLiteral(Map("1" -> 1.0, "2" -> 2.0, "3" -> 3.0))
+    assert(Literal.create(Map("a" -> 1)).toString === "map(keys: [a], values: [1])")
   }
 
   test("struct") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
index d92eb01b69bf0..c8e3b0e157319 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
@@ -68,16 +68,27 @@ object LiteralGenerator {
   lazy val longLiteralGen: Gen[Literal] =
     for { l <- Arbitrary.arbLong.arbitrary } yield Literal.create(l, LongType)
 
+  // The floatLiteralGen and doubleLiteralGen will 50% of the time yield arbitrary values
+  // and 50% of the time will yield some special values that are more likely to reveal
+  // corner cases. This behavior is similar to the integral value generators.
   lazy val floatLiteralGen: Gen[Literal] =
     for {
-      f <- Gen.chooseNum(Float.MinValue / 2, Float.MaxValue / 2,
-        Float.NaN, Float.PositiveInfinity, Float.NegativeInfinity)
+      f <- Gen.oneOf(
+        Gen.oneOf(
+          Float.NaN, Float.PositiveInfinity, Float.NegativeInfinity, Float.MinPositiveValue,
+          Float.MaxValue, -Float.MaxValue, 0.0f, -0.0f, 1.0f, -1.0f),
+        Arbitrary.arbFloat.arbitrary
+      )
     } yield Literal.create(f, FloatType)
 
   lazy val doubleLiteralGen: Gen[Literal] =
     for {
-      f <- Gen.chooseNum(Double.MinValue / 2, Double.MaxValue / 2,
-        Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity)
+      f <- Gen.oneOf(
+        Gen.oneOf(
+          Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity, Double.MinPositiveValue,
+          Double.MaxValue, -Double.MaxValue, 0.0, -0.0, 1.0, -1.0),
+        Arbitrary.arbDouble.arbitrary
+      )
     } yield Literal.create(f, DoubleType)
 
   // TODO cache the generated data
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
index ef7764dba1e9e..c40149368b055 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
@@ -445,8 +445,8 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     testTypes.foreach { dt =>
       genSchema(dt).map { schema =>
         val row = RandomDataGenerator.randomRow(random, schema)
-        val rowConverter = RowEncoder(schema)
-        val internalRow = rowConverter.toRow(row)
+        val toRow = RowEncoder(schema).createSerializer()
+        val internalRow = toRow(row)
         val lambda = LambdaVariable("dummy", schema(0).dataType, schema(0).nullable, id = 0)
         checkEvaluationWithoutCodegen(lambda, internalRow.get(0, schema(0).dataType), internalRow)
       }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala
index 94e251d90bcfa..448890223c879 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala
@@ -106,7 +106,7 @@ class OrderingSuite extends SparkFunSuite with ExpressionEvalHelper {
           StructField("a", dataType, nullable = true) ::
             StructField("b", dataType, nullable = true) :: Nil)
         val maybeDataGenerator = RandomDataGenerator.forType(rowType, nullable = false)
-        assume(maybeDataGenerator.isDefined)
+        assert(maybeDataGenerator.isDefined)
         val randGenerator = maybeDataGenerator.get
         val toCatalyst = CatalystTypeConverters.createToCatalystConverter(rowType)
         for (_ <- 1 to 50) {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
index 67a41e7cc2767..6f75623dc59ae 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
@@ -522,37 +522,9 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
     assert(expression == expected)
   }
 
-  val row0 = create_row(null)
-  val row1 = create_row(false)
-  val row2 = create_row(true)
-
-  test("istrue and isnottrue") {
-    checkEvaluation(IsTrue(Literal.create(null, BooleanType)), false, row0)
-    checkEvaluation(IsNotTrue(Literal.create(null, BooleanType)), true, row0)
-    checkEvaluation(IsTrue(Literal.create(false, BooleanType)), false, row1)
-    checkEvaluation(IsNotTrue(Literal.create(false, BooleanType)), true, row1)
-    checkEvaluation(IsTrue(Literal.create(true, BooleanType)), true, row2)
-    checkEvaluation(IsNotTrue(Literal.create(true, BooleanType)), false, row2)
-    IsTrue(Literal.create(null, IntegerType)).checkInputDataTypes() match {
-      case TypeCheckResult.TypeCheckFailure(msg) =>
-        assert(msg.contains("argument 1 requires boolean type"))
-    }
-  }
-
-  test("isfalse and isnotfalse") {
-    checkEvaluation(IsFalse(Literal.create(null, BooleanType)), false, row0)
-    checkEvaluation(IsNotFalse(Literal.create(null, BooleanType)), true, row0)
-    checkEvaluation(IsFalse(Literal.create(false, BooleanType)), true, row1)
-    checkEvaluation(IsNotFalse(Literal.create(false, BooleanType)), false, row1)
-    checkEvaluation(IsFalse(Literal.create(true, BooleanType)), false, row2)
-    checkEvaluation(IsNotFalse(Literal.create(true, BooleanType)), true, row2)
-    IsFalse(Literal.create(null, IntegerType)).checkInputDataTypes() match {
-      case TypeCheckResult.TypeCheckFailure(msg) =>
-        assert(msg.contains("argument 1 requires boolean type"))
-    }
-  }
-
   test("isunknown and isnotunknown") {
+    val row0 = create_row(null)
+
     checkEvaluation(IsUnknown(Literal.create(null, BooleanType)), true, row0)
     checkEvaluation(IsNotUnknown(Literal.create(null, BooleanType)), false, row0)
     IsUnknown(Literal.create(null, IntegerType)).checkInputDataTypes() match {
@@ -566,4 +538,110 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
     val inSet = InSet(BoundReference(0, IntegerType, true), Set.empty)
     checkEvaluation(inSet, false, row)
   }
+
+  test("SPARK-32764: compare special double/float values") {
+    checkEvaluation(EqualTo(Literal(Double.NaN), Literal(Double.NaN)), true)
+    checkEvaluation(EqualTo(Literal(Double.NaN), Literal(Double.PositiveInfinity)), false)
+    checkEvaluation(EqualTo(Literal(0.0D), Literal(-0.0D)), true)
+    checkEvaluation(GreaterThan(Literal(Double.NaN), Literal(Double.PositiveInfinity)), true)
+    checkEvaluation(GreaterThan(Literal(Double.NaN), Literal(Double.NaN)), false)
+    checkEvaluation(GreaterThan(Literal(0.0D), Literal(-0.0D)), false)
+
+    checkEvaluation(EqualTo(Literal(Float.NaN), Literal(Float.NaN)), true)
+    checkEvaluation(EqualTo(Literal(Float.NaN), Literal(Float.PositiveInfinity)), false)
+    checkEvaluation(EqualTo(Literal(0.0F), Literal(-0.0F)), true)
+    checkEvaluation(GreaterThan(Literal(Float.NaN), Literal(Float.PositiveInfinity)), true)
+    checkEvaluation(GreaterThan(Literal(Float.NaN), Literal(Float.NaN)), false)
+    checkEvaluation(GreaterThan(Literal(0.0F), Literal(-0.0F)), false)
+  }
+
+  test("SPARK-32110: compare special double/float values in array") {
+    def createUnsafeDoubleArray(d: Double): Literal = {
+      Literal(UnsafeArrayData.fromPrimitiveArray(Array(d)), ArrayType(DoubleType))
+    }
+    def createSafeDoubleArray(d: Double): Literal = {
+      Literal(new GenericArrayData(Array(d)), ArrayType(DoubleType))
+    }
+    def createUnsafeFloatArray(d: Double): Literal = {
+      Literal(UnsafeArrayData.fromPrimitiveArray(Array(d.toFloat)), ArrayType(FloatType))
+    }
+    def createSafeFloatArray(d: Double): Literal = {
+      Literal(new GenericArrayData(Array(d.toFloat)), ArrayType(FloatType))
+    }
+    def checkExpr(
+        exprBuilder: (Expression, Expression) => Expression,
+        left: Double,
+        right: Double,
+        expected: Any): Unit = {
+      // test double
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleArray(left), createUnsafeDoubleArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleArray(left), createSafeDoubleArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeDoubleArray(left), createSafeDoubleArray(right)), expected)
+      // test float
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatArray(left), createUnsafeFloatArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatArray(left), createSafeFloatArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeFloatArray(left), createSafeFloatArray(right)), expected)
+    }
+
+    checkExpr(EqualTo, Double.NaN, Double.NaN, true)
+    checkExpr(EqualTo, Double.NaN, Double.PositiveInfinity, false)
+    checkExpr(EqualTo, 0.0, -0.0, true)
+    checkExpr(GreaterThan, Double.NaN, Double.PositiveInfinity, true)
+    checkExpr(GreaterThan, Double.NaN, Double.NaN, false)
+    checkExpr(GreaterThan, 0.0, -0.0, false)
+  }
+
+  test("SPARK-32110: compare special double/float values in struct") {
+    def createUnsafeDoubleRow(d: Double): Literal = {
+      val dt = new StructType().add("d", "double")
+      val converter = UnsafeProjection.create(dt)
+      val unsafeRow = converter.apply(InternalRow(d))
+      Literal(unsafeRow, dt)
+    }
+    def createSafeDoubleRow(d: Double): Literal = {
+      Literal(InternalRow(d), new StructType().add("d", "double"))
+    }
+    def createUnsafeFloatRow(d: Double): Literal = {
+      val dt = new StructType().add("f", "float")
+      val converter = UnsafeProjection.create(dt)
+      val unsafeRow = converter.apply(InternalRow(d.toFloat))
+      Literal(unsafeRow, dt)
+    }
+    def createSafeFloatRow(d: Double): Literal = {
+      Literal(InternalRow(d.toFloat), new StructType().add("f", "float"))
+    }
+    def checkExpr(
+        exprBuilder: (Expression, Expression) => Expression,
+        left: Double,
+        right: Double,
+        expected: Any): Unit = {
+      // test double
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleRow(left), createUnsafeDoubleRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleRow(left), createSafeDoubleRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeDoubleRow(left), createSafeDoubleRow(right)), expected)
+      // test float
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatRow(left), createUnsafeFloatRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatRow(left), createSafeFloatRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeFloatRow(left), createSafeFloatRow(right)), expected)
+    }
+
+    checkExpr(EqualTo, Double.NaN, Double.NaN, true)
+    checkExpr(EqualTo, Double.NaN, Double.PositiveInfinity, false)
+    checkExpr(EqualTo, 0.0, -0.0, true)
+    checkExpr(GreaterThan, Double.NaN, Double.PositiveInfinity, true)
+    checkExpr(GreaterThan, Double.NaN, Double.NaN, false)
+    checkExpr(GreaterThan, 0.0, -0.0, false)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
index 2c8794f083dbb..cab81f85fda06 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
+import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.types.StringType
 
 /**
@@ -255,6 +256,10 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
     val nonNullExpr = RegExpReplace(Literal("100-200"), Literal("(\\d+)"), Literal("num"))
     checkEvaluation(nonNullExpr, "num-num", row1)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      RegExpReplace(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
 
   test("SPARK-22570: RegExpReplace should not create a lot of global variables") {
@@ -293,6 +298,28 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
     val nonNullExpr = RegExpExtract(Literal("100-200"), Literal("(\\d+)-(\\d+)"), Literal(1))
     checkEvaluation(nonNullExpr, "100", row1)
+
+    // invalid group index
+    val row8 = create_row("100-200", "(\\d+)-(\\d+)", 3)
+    val row9 = create_row("100-200", "(\\d+).*", 2)
+    val row10 = create_row("100-200", "\\d+", 1)
+    val row11 = create_row("100-200", "(\\d+)-(\\d+)", -1)
+    val row12 = create_row("100-200", "\\d+", -1)
+
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row8, "Regex group count is 2, but the specified group index is 3")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row9, "Regex group count is 1, but the specified group index is 2")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row10, "Regex group count is 0, but the specified group index is 1")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row11, "The specified group index cannot be less than zero")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row12, "The specified group index cannot be less than zero")
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      RegExpExtract(Literal("\"quote"), Literal("\"quote"), Literal(1)) :: Nil)
   }
 
   test("SPLIT") {
@@ -315,6 +342,18 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       StringSplit(s1, s2, -1), Seq("aa", "bb", "cc"), row1)
     checkEvaluation(StringSplit(s1, s2, -1), null, row2)
     checkEvaluation(StringSplit(s1, s2, -1), null, row3)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      StringSplit(Literal("\"quote"), Literal("\"quote"), Literal(-1)) :: Nil)
   }
 
+  test("SPARK-30759: cache initialization for literal patterns") {
+    val expr = "A" like Literal.create("a", StringType)
+    expr.eval()
+    val cache = expr.getClass.getSuperclass
+      .getDeclaredFields.filter(_.getName.endsWith("cache")).head
+    cache.setAccessible(true)
+    assert(cache.get(expr).asInstanceOf[java.util.regex.Pattern].pattern().contains("a"))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
index c5ffc381b58e2..836b2eaa642a3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import java.util.Locale
 
 import org.apache.spark.{SparkException, SparkFunSuite}
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DecimalType, IntegerType, StringType}
@@ -27,10 +28,12 @@ import org.apache.spark.sql.types.{DecimalType, IntegerType, StringType}
 class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("basic") {
-    val intUdf = ScalaUDF((i: Int) => i + 1, IntegerType, Literal(1) :: Nil, true :: Nil)
+    val intUdf = ScalaUDF((i: Int) => i + 1, IntegerType, Literal(1) :: Nil,
+      Option(ExpressionEncoder[Int]()) :: Nil)
     checkEvaluation(intUdf, 2)
 
-    val stringUdf = ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil, false :: Nil)
+    val stringUdf = ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil,
+      Option(ExpressionEncoder[String]()) :: Nil)
     checkEvaluation(stringUdf, "ax")
   }
 
@@ -39,7 +42,7 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
       (s: String) => s.toLowerCase(Locale.ROOT),
       StringType,
       Literal.create(null, StringType) :: Nil,
-      false :: Nil)
+      Option(ExpressionEncoder[String]()) :: Nil)
 
     val e1 = intercept[SparkException](udf.eval())
     assert(e1.getMessage.contains("Failed to execute user defined function"))
@@ -52,7 +55,8 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("SPARK-22695: ScalaUDF should not use global variables") {
     val ctx = new CodegenContext
-    ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil, false :: Nil).genCode(ctx)
+    ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil,
+      Option(ExpressionEncoder[String]()) :: Nil).genCode(ctx)
     assert(ctx.inlinedMutableStates.isEmpty)
   }
 
@@ -61,7 +65,8 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
       val udf = ScalaUDF(
         (a: java.math.BigDecimal) => a.multiply(new java.math.BigDecimal(100)),
         DecimalType.SYSTEM_DEFAULT,
-        Literal(BigDecimal("12345678901234567890.123")) :: Nil, false :: Nil)
+        Literal(BigDecimal("12345678901234567890.123")) :: Nil,
+        Option(ExpressionEncoder[java.math.BigDecimal]()) :: Nil)
       val e1 = intercept[ArithmeticException](udf.eval())
       assert(e1.getMessage.contains("cannot be represented as Decimal"))
       val e2 = intercept[SparkException] {
@@ -73,7 +78,8 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
       val udf = ScalaUDF(
         (a: java.math.BigDecimal) => a.multiply(new java.math.BigDecimal(100)),
         DecimalType.SYSTEM_DEFAULT,
-        Literal(BigDecimal("12345678901234567890.123")) :: Nil, false :: Nil)
+        Literal(BigDecimal("12345678901234567890.123")) :: Nil,
+        Option(ExpressionEncoder[java.math.BigDecimal]()) :: Nil)
       checkEvaluation(udf, null)
     }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
index 3c826e812b5cc..76d6890cc8f6f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
@@ -254,13 +254,13 @@ class SelectedFieldSuite extends AnalysisTest {
     StructField("col3", ArrayType(StructType(
       StructField("field1", StructType(
         StructField("subfield1", IntegerType, nullable = false) :: Nil))
-        :: Nil), containsNull = false), nullable = false)
+        :: Nil), containsNull = true), nullable = false)
   }
 
   testSelect(arrayWithStructAndMap, "col3.field2['foo'] as foo") {
     StructField("col3", ArrayType(StructType(
       StructField("field2", MapType(StringType, IntegerType, valueContainsNull = false))
-        :: Nil), containsNull = false), nullable = false)
+        :: Nil), containsNull = true), nullable = false)
   }
 
   //  |-- col1: string (nullable = false)
@@ -471,7 +471,7 @@ class SelectedFieldSuite extends AnalysisTest {
   testSelect(mapWithArrayOfStructKey, "map_keys(col2)[0].field1 as foo") {
     StructField("col2", MapType(
       ArrayType(StructType(
-        StructField("field1", StringType) :: Nil), containsNull = false),
+        StructField("field1", StringType) :: Nil), containsNull = true),
       ArrayType(StructType(
         StructField("field3", StructType(
           StructField("subfield3", IntegerType) ::
@@ -482,7 +482,7 @@ class SelectedFieldSuite extends AnalysisTest {
     StructField("col2", MapType(
       ArrayType(StructType(
         StructField("field2", StructType(
-          StructField("subfield1", IntegerType) :: Nil)) :: Nil), containsNull = false),
+          StructField("subfield1", IntegerType) :: Nil)) :: Nil), containsNull = true),
       ArrayType(StructType(
         StructField("field3", StructType(
           StructField("subfield3", IntegerType) ::
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala
index f2696849d7753..4e6976f76ea5f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.Timestamp
-import java.util.TimeZone
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.types._
@@ -35,15 +34,7 @@ class SortOrderExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     val l1 = Literal.create(20132983L, LongType)
     val l2 = Literal.create(-20132983L, LongType)
     val millis = 1524954911000L
-    // Explicitly choose a time zone, since Date objects can create different values depending on
-    // local time zone of the machine on which the test is running
-    val oldDefaultTZ = TimeZone.getDefault
-    val d1 = try {
-      TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-      Literal.create(new java.sql.Date(millis), DateType)
-    } finally {
-      TimeZone.setDefault(oldDefaultTZ)
-    }
+    val d1 = Literal.create(new java.sql.Date(millis), DateType)
     val t1 = Literal.create(new Timestamp(millis), TimestampType)
     val f1 = Literal.create(0.7788229f, FloatType)
     val f2 = Literal.create(-0.7788229f, FloatType)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index 4308f98d6969a..967ccc42c632d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.types._
 
 class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
@@ -155,6 +156,11 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(c1 endsWith "b", false, row)
     checkEvaluation(c2 endsWith "b", null, row)
     checkEvaluation(c1 endsWith Literal.create(null, StringType), null, row)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(Contains(Literal("\"quote"), Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(EndsWith(Literal("\"quote"), Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(StartsWith(Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
 
   test("Substring") {
@@ -230,6 +236,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       "xample",
       row)
 
+    // Substring with from negative position with negative length
+    checkEvaluation(Substring(s, Literal.create(-1207959552, IntegerType),
+      Literal.create(-1207959552, IntegerType)), "", row)
+
     val s_notNull = 'a.string.notNull.at(0)
 
     assert(Substring(s, Literal.create(0, IntegerType), Literal.create(2, IntegerType)).nullable)
@@ -352,6 +362,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Decode(b, Literal("utf-8")), null, create_row(null))
     checkEvaluation(Decode(Literal.create(null, BinaryType), Literal("utf-8")), null)
     checkEvaluation(Decode(b, Literal.create(null, StringType)), null, create_row(null))
+
+    // Test escaping of charset
+    GenerateUnsafeProjection.generate(Encode(a, Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(Decode(b, Literal("\"quote")) :: Nil)
   }
 
   test("initcap unit test") {
@@ -379,6 +393,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Levenshtein(Literal("千世"), Literal("fog")), 3)
     checkEvaluation(Levenshtein(Literal("世界千世"), Literal("大a界b")), 4)
     // scalastyle:on
+
+    // Test escaping of arguments:
+    GenerateUnsafeProjection.generate(Levenshtein(Literal("\"quotea"), Literal("\"quoteb")) :: Nil)
   }
 
   test("soundex unit test") {
@@ -560,6 +577,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringTrim(Literal("a"), Literal.create(null, StringType)), null)
     checkEvaluation(StringTrim(Literal.create(null, StringType), Literal("a")), null)
 
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(StringTrim(Literal("\"quote"), Literal("\"quote")) :: Nil)
+
     checkEvaluation(StringTrim(Literal("yxTomxx"), Literal("xyz")), "Tom")
     checkEvaluation(StringTrim(Literal("xxxbarxxx"), Literal("x")), "bar")
   }
@@ -587,6 +607,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringTrimLeft(Literal.create(null, StringType), Literal("a")), null)
     checkEvaluation(StringTrimLeft(Literal("a"), Literal.create(null, StringType)), null)
 
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      StringTrimLeft(Literal("\"quote"), Literal("\"quote")) :: Nil)
+
     checkEvaluation(StringTrimLeft(Literal("zzzytest"), Literal("xyz")), "test")
     checkEvaluation(StringTrimLeft(Literal("zzzytestxyz"), Literal("xyz")), "testxyz")
     checkEvaluation(StringTrimLeft(Literal("xyxXxyLAST WORD"), Literal("xy")), "XxyLAST WORD")
@@ -616,6 +640,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringTrimRight(Literal("a"), Literal.create(null, StringType)), null)
     checkEvaluation(StringTrimRight(Literal.create(null, StringType), Literal("a")), null)
 
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      StringTrimRight(Literal("\"quote"), Literal("\"quote")) :: Nil)
+
     checkEvaluation(StringTrimRight(Literal("testxxzx"), Literal("xyz")), "test")
     checkEvaluation(StringTrimRight(Literal("xyztestxxzx"), Literal("xyz")), "xyztest")
     checkEvaluation(StringTrimRight(Literal("TURNERyxXxy"), Literal("xy")), "TURNERyxX")
@@ -632,6 +660,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       FormatString(Literal("aa%d%s"), Literal.create(null, IntegerType), "cc"), "aanullcc")
     checkEvaluation(
       FormatString(Literal("aa%d%s"), 12, Literal.create(null, StringType)), "aa12null")
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(FormatString(Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
 
   test("SPARK-22603: FormatString should not generate codes beyond 64KB") {
@@ -662,6 +693,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringInstr(s1, s2), 1, create_row("花花世界", "花"))
     checkEvaluation(StringInstr(s1, s2), 0, create_row("花花世界", "小"))
     // scalastyle:on
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(StringInstr(Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
 
   test("LOCATE") {
@@ -718,6 +752,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringRPad(s1, s2, s3), null, row3)
     checkEvaluation(StringRPad(s1, s2, s3), null, row4)
     checkEvaluation(StringRPad(s1, s2, s3), null, row5)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(StringLPad(Literal("\"quote"), s2, Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(StringRPad(Literal("\"quote"), s2, Literal("\"quote")) :: Nil)
     checkEvaluation(StringRPad(Literal("hi"), Literal(5)), "hi   ")
     checkEvaluation(StringRPad(Literal("hi"), Literal(1)), "h")
   }
@@ -732,6 +770,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringRepeat(Literal("hi"), Literal(-1)), "", row1)
     checkEvaluation(StringRepeat(s1, s2), "hihi", row1)
     checkEvaluation(StringRepeat(s1, s2), null, row2)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(StringRepeat(Literal("\"quote"), Literal(2)) :: Nil)
   }
 
   test("REVERSE") {
@@ -897,6 +938,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     assert(ParseUrl(Seq(Literal("1"), Literal(2))).checkInputDataTypes().isFailure)
     assert(ParseUrl(Seq(Literal(1), Literal("2"))).checkInputDataTypes().isFailure)
     assert(ParseUrl(Seq(Literal("1"), Literal("2"), Literal(3))).checkInputDataTypes().isFailure)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(ParseUrl(Seq(Literal("\"quote"), Literal("\"quote"))) :: Nil)
   }
 
   test("Sentences") {
@@ -919,5 +963,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       answer)
     checkEvaluation(Sentences("Hi there! The price was $1,234.56.... But, not now.", "XXX", "YYY"),
       answer)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      Sentences(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/LastTestSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/FirstLastTestSuite.scala
similarity index 84%
rename from sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/LastTestSuite.scala
rename to sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/FirstLastTestSuite.scala
index ba36bc074e154..bb6672e1046da 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/LastTestSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/FirstLastTestSuite.scala
@@ -17,14 +17,15 @@
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Literal}
 import org.apache.spark.sql.types.IntegerType
 
-class LastTestSuite extends SparkFunSuite {
+class FirstLastTestSuite extends SparkFunSuite {
   val input = AttributeReference("input", IntegerType, nullable = true)()
-  val evaluator = DeclarativeAggregateEvaluator(Last(input, Literal(false)), Seq(input))
-  val evaluatorIgnoreNulls = DeclarativeAggregateEvaluator(Last(input, Literal(true)), Seq(input))
+  val evaluator = DeclarativeAggregateEvaluator(Last(input, false), Seq(input))
+  val evaluatorIgnoreNulls = DeclarativeAggregateEvaluator(Last(input, true), Seq(input))
 
   test("empty buffer") {
     assert(evaluator.initialize() === InternalRow(null, false))
@@ -106,4 +107,15 @@ class LastTestSuite extends SparkFunSuite {
     val m1 = evaluatorIgnoreNulls.merge(p1, p2)
     assert(evaluatorIgnoreNulls.eval(m1) === InternalRow(1))
   }
+
+  test("SPARK-32344: correct error handling for a type mismatch") {
+    val msg1 = intercept[AnalysisException] {
+      new First(input, Literal(1, IntegerType))
+    }.getMessage
+    assert(msg1.contains("The second argument in first should be a boolean literal"))
+    val msg2 = intercept[AnalysisException] {
+      new Last(input, Literal(1, IntegerType))
+    }.getMessage
+    assert(msg2.contains("The second argument in last should be a boolean literal"))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala
index 98fd04c9cca91..1afccea5aef15 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
+import java.lang.{Double => JDouble}
 import java.util.Random
 
 import scala.collection.mutable
@@ -24,7 +25,7 @@ import scala.collection.mutable
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{BoundReference, SpecificInternalRow}
-import org.apache.spark.sql.types.{DataType, IntegerType}
+import org.apache.spark.sql.types.{DataType, DoubleType, IntegerType}
 
 class HyperLogLogPlusPlusSuite extends SparkFunSuite {
 
@@ -153,4 +154,25 @@ class HyperLogLogPlusPlusSuite extends SparkFunSuite {
     // Check if the buffers are equal.
     assert(buffer2 == buffer1a, "Buffers should be equal")
   }
+
+  test("SPARK-32110: add 0.0 and -0.0") {
+    val (hll, input, buffer) = createEstimator(0.05, DoubleType)
+    input.setDouble(0, 0.0)
+    hll.update(buffer, input)
+    input.setDouble(0, -0.0)
+    hll.update(buffer, input)
+    evaluateEstimate(hll, buffer, 1);
+  }
+
+  test("SPARK-32110: add NaN") {
+    val (hll, input, buffer) = createEstimator(0.05, DoubleType)
+    input.setDouble(0, Double.NaN)
+    hll.update(buffer, input)
+    val specialNaN = JDouble.longBitsToDouble(0x7ff1234512345678L)
+    assert(JDouble.isNaN(specialNaN))
+    assert(JDouble.doubleToRawLongBits(Double.NaN) != JDouble.doubleToRawLongBits(specialNaN))
+    input.setDouble(0, specialNaN)
+    hll.update(buffer, input)
+    evaluateEstimate(hll, buffer, 1);
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
index 55569b6f2933e..67e3bc69543e8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
@@ -37,6 +37,18 @@ class CodeBlockSuite extends SparkFunSuite {
     assert(code.asInstanceOf[CodeBlock].blockInputs === Seq(value))
   }
 
+  test("Code parts should be treated for escapes, but string inputs shouldn't be") {
+    val strlit = raw"\\"
+    val code = code"""String s = "foo\\bar" + "$strlit";"""
+
+    val builtin = s"""String s = "foo\\bar" + "$strlit";"""
+
+    val expected = raw"""String s = "foo\bar" + "\\";"""
+
+    assert(builtin == expected)
+    assert(code.asInstanceOf[CodeBlock].toString == expected)
+  }
+
   test("Block.stripMargin") {
     val isNull = JavaCode.isNullVariable("expr1_isNull")
     val value = JavaCode.variable("expr1", IntegerType)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala
index 81e29931e41dd..dd67a61015e72 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala
@@ -17,13 +17,15 @@
 
 package org.apache.spark.sql.catalyst.expressions.codegen
 
+import java.time.{LocalDateTime, ZoneOffset}
+
 import scala.util.Random
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.RandomDataGenerator
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.catalyst.expressions.{JoinedRow, UnsafeProjection, UnsafeRow}
+import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, JoinedRow, UnsafeProjection, UnsafeRow}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -58,8 +60,8 @@ class GenerateUnsafeRowJoinerSuite extends SparkFunSuite {
   test("rows with all empty int arrays") {
     val schema = StructType(Seq(
       StructField("f1", ArrayType(IntegerType)), StructField("f2", ArrayType(IntegerType))))
-    val emptyIntArray =
-      ExpressionEncoder[Array[Int]]().resolveAndBind().toRow(Array.emptyIntArray).getArray(0)
+    val toRow = ExpressionEncoder[Array[Int]]().resolveAndBind().createSerializer()
+    val emptyIntArray = toRow(Array.emptyIntArray).getArray(0)
     val row: UnsafeRow = UnsafeProjection.create(schema).apply(
       InternalRow(emptyIntArray, emptyIntArray))
     testConcat(schema, row, schema, row)
@@ -99,6 +101,23 @@ class GenerateUnsafeRowJoinerSuite extends SparkFunSuite {
     testConcatOnce(N, N, variable)
   }
 
+  test("SPARK-30993: UserDefinedType matched to fixed length SQL type shouldn't be corrupted") {
+    val schema1 = new StructType(Array(
+      StructField("date", new WrappedDateTimeUDT),
+      StructField("s", StringType),
+      StructField("i", IntegerType)))
+    val proj1 = UnsafeProjection.create(schema1.fields.map(_.dataType))
+    val intRow1 = new GenericInternalRow(Array[Any](
+      LocalDateTime.now().toEpochSecond(ZoneOffset.UTC),
+      UTF8String.fromString("hello"), 1))
+
+    val schema2 = new StructType(Array(StructField("i", IntegerType)))
+    val proj2 = UnsafeProjection.create(schema2.fields.map(_.dataType))
+    val intRow2 = new GenericInternalRow(Array[Any](2))
+
+    testConcat(schema1, proj1.apply(intRow1), schema2, proj2.apply(intRow2))
+  }
+
   private def testConcat(numFields1: Int, numFields2: Int, candidateTypes: Seq[DataType]): Unit = {
     for (i <- 0 until 10) {
       testConcatOnce(numFields1, numFields2, candidateTypes)
@@ -204,3 +223,23 @@ class GenerateUnsafeRowJoinerSuite extends SparkFunSuite {
   }
 
 }
+
+private[sql] case class WrappedDateTime(dt: LocalDateTime)
+
+private[sql] class WrappedDateTimeUDT extends UserDefinedType[WrappedDateTime] {
+  override def sqlType: DataType = LongType
+
+  override def serialize(obj: WrappedDateTime): Long = {
+    obj.dt.toEpochSecond(ZoneOffset.UTC)
+  }
+
+  def deserialize(datum: Any): WrappedDateTime = datum match {
+    case value: Long =>
+      val v = LocalDateTime.ofEpochSecond(value, 0, ZoneOffset.UTC)
+      WrappedDateTime(v)
+  }
+
+  override def userClass: Class[WrappedDateTime] = classOf[WrappedDateTime]
+
+  private[spark] override def asNullable: WrappedDateTimeUDT = this
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
index a48e61861c158..8290b38e33934 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
@@ -17,10 +17,9 @@
 
 package org.apache.spark.sql.catalyst.json
 
-import com.fasterxml.jackson.core.JsonFactory
-
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
@@ -36,50 +35,76 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
     assert(inferSchema.inferField(parser) === expectedType)
   }
 
-  def checkTimestampType(pattern: String, json: String): Unit = {
-    checkType(Map("timestampFormat" -> pattern), json, TimestampType)
+  def checkTimestampType(pattern: String, json: String, inferTimestamp: Boolean): Unit = {
+    checkType(
+      Map("timestampFormat" -> pattern, "inferTimestamp" -> inferTimestamp.toString),
+      json,
+      if (inferTimestamp) TimestampType else StringType)
   }
 
   test("inferring timestamp type") {
-    checkTimestampType("yyyy", """{"a": "2018"}""")
-    checkTimestampType("yyyy=MM", """{"a": "2018=12"}""")
-    checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""")
-    checkTimestampType(
-      "yyyy-MM-dd'T'HH:mm:ss.SSS",
-      """{"a": "2018-12-02T21:04:00.123"}""")
-    checkTimestampType(
-      "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX",
-      """{"a": "2018-12-02T21:04:00.123567+01:00"}""")
+    Seq(true, false).foreach { inferTimestamp =>
+      Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+        withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+          checkTimestampType("yyyy", """{"a": "2018"}""", inferTimestamp)
+          checkTimestampType("yyyy=MM", """{"a": "2018=12"}""", inferTimestamp)
+          checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""", inferTimestamp)
+          checkTimestampType(
+            "yyyy-MM-dd'T'HH:mm:ss.SSS",
+            """{"a": "2018-12-02T21:04:00.123"}""",
+            inferTimestamp)
+          checkTimestampType(
+            "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX",
+            """{"a": "2018-12-02T21:04:00.123567+01:00"}""",
+            inferTimestamp)
+        }
+      }
+    }
   }
 
   test("prefer decimals over timestamps") {
-    checkType(
-      options = Map(
-        "prefersDecimal" -> "true",
-        "timestampFormat" -> "yyyyMMdd.HHmmssSSS"
-      ),
-      json = """{"a": "20181202.210400123"}""",
-      dt = DecimalType(17, 9)
-    )
+    Seq("legacy", "corrected").foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParser) {
+        checkType(
+          options = Map(
+            "prefersDecimal" -> "true",
+            "timestampFormat" -> "yyyyMMdd.HHmmssSSS"
+          ),
+          json = """{"a": "20181202.210400123"}""",
+          dt = DecimalType(17, 9)
+        )
+      }
+    }
   }
 
   test("skip decimal type inferring") {
-    checkType(
-      options = Map(
-        "prefersDecimal" -> "false",
-        "timestampFormat" -> "yyyyMMdd.HHmmssSSS"
-      ),
-      json = """{"a": "20181202.210400123"}""",
-      dt = TimestampType
-    )
+    Seq(true, false).foreach { inferTimestamp =>
+      Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+        withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+          checkType(
+            options = Map(
+              "prefersDecimal" -> "false",
+              "timestampFormat" -> "yyyyMMdd.HHmmssSSS",
+              "inferTimestamp" -> inferTimestamp.toString
+            ),
+            json = """{"a": "20181202.210400123"}""",
+            dt = if (inferTimestamp) TimestampType else StringType
+          )
+        }
+      }
+    }
   }
 
   test("fallback to string type") {
-    checkType(
-      options = Map("timestampFormat" -> "yyyy,MM,dd.HHmmssSSS"),
-      json = """{"a": "20181202.210400123"}""",
-      dt = StringType
-    )
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+        checkType(
+          options = Map("timestampFormat" -> "yyyy,MM,dd.HHmmssSSS"),
+          json = """{"a": "20181202.210400123"}""",
+          dt = StringType
+        )
+      }
+    }
   }
 
   test("disable timestamp inferring") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
index 9c71cc8e0d291..c02691848c8f0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
@@ -119,9 +119,7 @@ class BinaryComparisonSimplificationSuite extends PlanTest with PredicateHelper
 
     // GetStructField with different names are semantically equal; thus, `EqualTo(fieldA1, fieldA2)`
     // will be optimized to `TrueLiteral` by `SimplifyBinaryComparison`.
-    val originalQuery = nonNullableRelation
-        .where(EqualTo(fieldA1, fieldA2))
-        .analyze
+    val originalQuery = nonNullableRelation.where(EqualTo(fieldA1, fieldA2))
 
     val optimized = Optimize.execute(originalQuery)
     val correctAnswer = nonNullableRelation.analyze
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
index d9a6fbf81de91..e34f141412ca6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
@@ -22,6 +22,7 @@ import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry}
 import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
@@ -96,12 +97,34 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
-  test("remove redundant order by") {
+  test("SPARK-33183: remove consecutive no-op sorts") {
+    val plan = testRelation.orderBy().orderBy().orderBy()
+    val optimized = Optimize.execute(plan.analyze)
+    val correctAnswer = testRelation.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: remove redundant sort by") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc_nullsFirst)
-    val unnecessaryReordered = orderedPlan.limit(2).select('a).orderBy('a.asc, 'b.desc_nullsFirst)
+    val unnecessaryReordered = orderedPlan.limit(2).select('a).sortBy('a.asc, 'b.desc_nullsFirst)
     val optimized = Optimize.execute(unnecessaryReordered.analyze)
     val correctAnswer = orderedPlan.limit(2).select('a).analyze
-    comparePlans(Optimize.execute(optimized), correctAnswer)
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: remove all redundant local sorts") {
+    val orderedPlan = testRelation.sortBy('a.asc).orderBy('a.asc).sortBy('a.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = testRelation.orderBy('a.asc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: should not remove global sort") {
+    val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc_nullsFirst)
+    val reordered = orderedPlan.limit(2).select('a).orderBy('a.asc, 'b.desc_nullsFirst)
+    val optimized = Optimize.execute(reordered.analyze)
+    val correctAnswer = reordered.analyze
+    comparePlans(optimized, correctAnswer)
   }
 
   test("do not remove sort if the order is different") {
@@ -112,22 +135,39 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
-  test("filters don't affect order") {
+  test("SPARK-33183: remove top level local sort with filter operators") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc)
-    val filteredAndReordered = orderedPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.where('a > Literal(10)).sortBy('a.asc, 'b.desc)
     val optimized = Optimize.execute(filteredAndReordered.analyze)
     val correctAnswer = orderedPlan.where('a > Literal(10)).analyze
     comparePlans(optimized, correctAnswer)
   }
 
-  test("limits don't affect order") {
+  test("SPARK-33183: keep top level global sort with filter operators") {
+    val projectPlan = testRelation.select('a, 'b)
+    val orderedPlan = projectPlan.orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc)
+    val optimized = Optimize.execute(filteredAndReordered.analyze)
+    val correctAnswer = projectPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: limits should not affect order for local sort") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc)
-    val filteredAndReordered = orderedPlan.limit(Literal(10)).orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.limit(Literal(10)).sortBy('a.asc, 'b.desc)
     val optimized = Optimize.execute(filteredAndReordered.analyze)
     val correctAnswer = orderedPlan.limit(Literal(10)).analyze
     comparePlans(optimized, correctAnswer)
   }
 
+  test("SPARK-33183: should not remove global sort with limit operators") {
+    val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.limit(Literal(10)).orderBy('a.asc, 'b.desc)
+    val optimized = Optimize.execute(filteredAndReordered.analyze)
+    val correctAnswer = filteredAndReordered.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
   test("different sorts are not simplified if limit is in between") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('b.desc).limit(Literal(10))
       .orderBy('a.asc)
@@ -136,11 +176,11 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
-  test("range is already sorted") {
+  test("SPARK-33183: should not remove global sort with range operator") {
     val inputPlan = Range(1L, 1000L, 1, 10)
     val orderedPlan = inputPlan.orderBy('id.asc)
     val optimized = Optimize.execute(orderedPlan.analyze)
-    val correctAnswer = inputPlan.analyze
+    val correctAnswer = orderedPlan.analyze
     comparePlans(optimized, correctAnswer)
 
     val reversedPlan = inputPlan.orderBy('id.desc)
@@ -151,10 +191,18 @@ class EliminateSortsSuite extends PlanTest {
     val negativeStepInputPlan = Range(10L, 1L, -1, 10)
     val negativeStepOrderedPlan = negativeStepInputPlan.orderBy('id.desc)
     val negativeStepOptimized = Optimize.execute(negativeStepOrderedPlan.analyze)
-    val negativeStepCorrectAnswer = negativeStepInputPlan.analyze
+    val negativeStepCorrectAnswer = negativeStepOrderedPlan.analyze
     comparePlans(negativeStepOptimized, negativeStepCorrectAnswer)
   }
 
+  test("SPARK-33183: remove local sort with range operator") {
+    val inputPlan = Range(1L, 1000L, 1, 10)
+    val orderedPlan = inputPlan.sortBy('id.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = inputPlan.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
   test("sort should not be removed when there is a node which doesn't guarantee any order") {
     val orderedPlan = testRelation.select('a, 'b)
     val groupedAndResorted = orderedPlan.groupBy('a)(sum('a)).orderBy('a.asc)
@@ -244,7 +292,8 @@ class EliminateSortsSuite extends PlanTest {
   }
 
   test("should not remove orderBy in groupBy clause with ScalaUDF as aggs") {
-    val scalaUdf = ScalaUDF((s: Int) => s, IntegerType, 'a :: Nil, true :: Nil)
+    val scalaUdf = ScalaUDF((s: Int) => s, IntegerType, 'a :: Nil,
+      Option(ExpressionEncoder[Int]()) :: Nil)
     val projectPlan = testRelation.select('a, 'b)
     val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc)
     val groupByPlan = orderByPlan.groupBy('a)(scalaUdf)
@@ -282,6 +331,15 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
+  test("SPARK-32318: should not remove orderBy in distribute statement") {
+    val projectPlan = testRelation.select('a, 'b)
+    val orderByPlan = projectPlan.orderBy('b.desc)
+    val distributedPlan = orderByPlan.distribute('a)(1)
+    val optimized = Optimize.execute(distributedPlan.analyze)
+    val correctAnswer = distributedPlan.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
   test("should not remove orderBy in left join clause if there is an outer limit") {
     val projectPlan = testRelation.select('a, 'b)
     val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc)
@@ -308,4 +366,39 @@ class EliminateSortsSuite extends PlanTest {
     val correctAnswer = PushDownOptimizer.execute(noOrderByPlan.analyze)
     comparePlans(optimized, correctAnswer)
   }
+
+  test("SPARK-33183: remove consecutive global sorts with the same ordering") {
+    Seq(
+      (testRelation.orderBy('a.asc).orderBy('a.asc), testRelation.orderBy('a.asc)),
+      (testRelation.orderBy('a.asc, 'b.desc).orderBy('a.asc), testRelation.orderBy('a.asc))
+    ).foreach { case (ordered, answer) =>
+      val optimized = Optimize.execute(ordered.analyze)
+      comparePlans(optimized, answer.analyze)
+    }
+  }
+
+  test("SPARK-33183: remove consecutive local sorts with the same ordering") {
+    val orderedPlan = testRelation.sortBy('a.asc).sortBy('a.asc).sortBy('a.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = testRelation.sortBy('a.asc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: remove consecutive local sorts with different ordering") {
+    val orderedPlan = testRelation.sortBy('b.asc).sortBy('a.desc).sortBy('a.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = testRelation.sortBy('a.asc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: should keep global sort when child is a local sort with the same ordering") {
+    val correctAnswer = testRelation.orderBy('a.asc).analyze
+    Seq(
+      testRelation.sortBy('a.asc).orderBy('a.asc),
+      testRelation.orderBy('a.asc).sortBy('a.asc).orderBy('a.asc)
+    ).foreach { ordered =>
+      val optimized = Optimize.execute(ordered.analyze)
+      comparePlans(optimized, correctAnswer)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
index 70e29dca46e9e..50de3a436bc33 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.types.{BooleanType, IntegerType}
+import org.apache.spark.sql.types.IntegerType
 import org.apache.spark.unsafe.types.CalendarInterval
 
 class FilterPushdownSuite extends PlanTest {
@@ -664,14 +664,14 @@ class FilterPushdownSuite extends PlanTest {
     val generator = Explode('c_arr)
     val originalQuery = {
       testRelationWithArrayType
-        .generate(generator, alias = Some("arr"))
+        .generate(generator, alias = Some("arr"), outputNames = Seq("c"))
         .where(('b >= 5) && ('c > 6))
     }
     val optimized = Optimize.execute(originalQuery.analyze)
     val referenceResult = {
       testRelationWithArrayType
         .where('b >= 5)
-        .generate(generator, alias = Some("arr"))
+        .generate(generator, alias = Some("arr"), outputNames = Seq("c"))
         .where('c > 6).analyze
     }
 
@@ -1135,49 +1135,32 @@ class FilterPushdownSuite extends PlanTest {
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer)
   }
 
-  test("join condition pushdown: deterministic and non-deterministic") {
-    val x = testRelation.subquery('x)
-    val y = testRelation.subquery('y)
-
-    // Verify that all conditions except the watermark touching condition are pushed down
-    // by the optimizer and others are not.
-    val originalQuery = x.join(y, condition = Some("x.a".attr === 5 && "y.a".attr === 5 &&
-      "x.a".attr === Rand(10) && "y.b".attr === 5))
-    val correctAnswer =
-      x.where("x.a".attr === 5).join(y.where("y.a".attr === 5 && "y.b".attr === 5),
-        condition = Some("x.a".attr === Rand(10)))
-
-    // CheckAnalysis will ensure nondeterministic expressions not appear in join condition.
-    // TODO support nondeterministic expressions in join condition.
-    comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
-      checkAnalysis = false)
-  }
-
   test("watermark pushdown: no pushdown on watermark attribute #1") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation(attrA, 'b.timestamp, attrC)
 
     // Verify that all conditions except the watermark touching condition are pushed down
     // by the optimizer and others are not.
-    val originalQuery = EventTimeWatermark('b, interval, testRelation)
-      .where('a === 5 && 'b === 10 && 'c === 5)
+    val originalQuery = EventTimeWatermark('b, interval, relation)
+      .where('a === 5 && 'b === new java.sql.Timestamp(0) && 'c === 5)
     val correctAnswer = EventTimeWatermark(
-      'b, interval, testRelation.where('a === 5 && 'c === 5))
-      .where('b === 10)
+      'b, interval, relation.where('a === 5 && 'c === 5))
+      .where('b === new java.sql.Timestamp(0))
 
-    comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
-      checkAnalysis = false)
+    comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze)
   }
 
   test("watermark pushdown: no pushdown for nondeterministic filter") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation(attrA, attrB, 'c.timestamp)
 
     // Verify that all conditions except the watermark touching condition are pushed down
     // by the optimizer and others are not.
-    val originalQuery = EventTimeWatermark('c, interval, testRelation)
-      .where('a === 5 && 'b === Rand(10) && 'c === 5)
+    val originalQuery = EventTimeWatermark('c, interval, relation)
+      .where('a === 5 && 'b === Rand(10) && 'c === new java.sql.Timestamp(0))
     val correctAnswer = EventTimeWatermark(
-      'c, interval, testRelation.where('a === 5))
-      .where('b === Rand(10) && 'c === 5)
+      'c, interval, relation.where('a === 5))
+      .where('b === Rand(10) && 'c === new java.sql.Timestamp(0))
 
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
       checkAnalysis = false)
@@ -1185,13 +1168,14 @@ class FilterPushdownSuite extends PlanTest {
 
   test("watermark pushdown: full pushdown") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation(attrA, attrB, 'c.timestamp)
 
     // Verify that all conditions except the watermark touching condition are pushed down
     // by the optimizer and others are not.
-    val originalQuery = EventTimeWatermark('c, interval, testRelation)
+    val originalQuery = EventTimeWatermark('c, interval, relation)
       .where('a === 5 && 'b === 10)
     val correctAnswer = EventTimeWatermark(
-      'c, interval, testRelation.where('a === 5 && 'b === 10))
+      'c, interval, relation.where('a === 5 && 'b === 10))
 
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
       checkAnalysis = false)
@@ -1199,11 +1183,12 @@ class FilterPushdownSuite extends PlanTest {
 
   test("watermark pushdown: no pushdown on watermark attribute #2") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation('a.timestamp, attrB, attrC)
 
-    val originalQuery = EventTimeWatermark('a, interval, testRelation)
-      .where('a === 5 && 'b === 10)
+    val originalQuery = EventTimeWatermark('a, interval, relation)
+      .where('a === new java.sql.Timestamp(0) && 'b === 10)
     val correctAnswer = EventTimeWatermark(
-      'a, interval, testRelation.where('b === 10)).where('a === 5)
+      'a, interval, relation.where('b === 10)).where('a === new java.sql.Timestamp(0))
 
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
       checkAnalysis = false)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
index 0d48ecb31cfa4..59dfd3a7932bd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
@@ -180,4 +180,16 @@ class FoldablePropagationSuite extends PlanTest {
       .select((Literal(1) + 3).as('res)).analyze
     comparePlans(optimized, correctAnswer)
   }
+
+  test("SPARK-32635: Replace references with foldables coming only from the node's children") {
+    val leftExpression = 'a.int
+    val left = LocalRelation(leftExpression).select('a)
+    val rightExpression = Alias(Literal(2), "a")(leftExpression.exprId)
+    val right = LocalRelation('b.int).select('b, rightExpression).select('b)
+    val join = left.join(right, joinType = LeftOuter, condition = Some('b === 'a))
+
+    val query = join.analyze
+    val optimized = Optimize.execute(query)
+    comparePlans(optimized, query)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
index f7755000b3f39..75fe3dd4062a7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.optimizer
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap}
+import org.apache.spark.sql.catalyst.optimizer.JoinReorderDP.JoinPlan
 import org.apache.spark.sql.catalyst.plans.{Cross, Inner, PlanTest}
 import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LocalRelation, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
@@ -307,9 +308,9 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
     val originalPlan2 =
       t1.join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
         .hint("broadcast")
-        .join(t4, Inner, Some(nameToAttr("t4.v-1-10") === nameToAttr("t3.v-1-100")))
+        .join(t4, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t4.k-1-2")))
         .hint("broadcast")
-        .join(t3, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t4.k-1-2")))
+        .join(t3, Inner, Some(nameToAttr("t4.v-1-10") === nameToAttr("t3.v-1-100")))
 
     assertEqualPlans(originalPlan2, originalPlan2)
 
@@ -385,4 +386,18 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
   private def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
     plans.map(_.output).reduce(_ ++ _)
   }
+
+  test("SPARK-33935: betterThan should be consistent") {
+    val plan1 = JoinPlan(null, null, null, Cost(300, 80))
+    val plan2 = JoinPlan(null, null, null, Cost(500, 30))
+
+    // cost1 = 300*0.7 + 80*0.3 = 234
+    // cost2 = 500*0.7 + 30*0.3 = 359
+
+    assert(!plan1.betterThan(plan1, conf))
+    assert(!plan2.betterThan(plan2, conf))
+
+    assert(plan1.betterThan(plan2, conf))
+    assert(!plan2.betterThan(plan1, conf))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala
index 436f62e4225c8..1812dce0da426 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala
@@ -116,4 +116,52 @@ class LikeSimplificationSuite extends PlanTest {
     val optimized2 = Optimize.execute(originalQuery2.analyze)
     comparePlans(optimized2, originalQuery2.analyze)
   }
+
+  test("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") {
+    val originalQuery1 =
+      testRelation
+        .where(('a like "abc%") || ('a like "\\abc%"))
+    val optimized1 = Optimize.execute(originalQuery1.analyze)
+    val correctAnswer1 = testRelation
+      .where(StartsWith('a, "abc") || ('a like "\\abc%"))
+      .analyze
+    comparePlans(optimized1, correctAnswer1)
+
+    val originalQuery2 =
+      testRelation
+        .where(('a like "%xyz") || ('a like "%xyz\\"))
+    val optimized2 = Optimize.execute(originalQuery2.analyze)
+    val correctAnswer2 = testRelation
+      .where(EndsWith('a, "xyz") || ('a like "%xyz\\"))
+      .analyze
+    comparePlans(optimized2, correctAnswer2)
+
+    val originalQuery3 =
+      testRelation
+        .where(('a like ("@bc%def", '@')) || ('a like "abc%def"))
+    val optimized3 = Optimize.execute(originalQuery3.analyze)
+    val correctAnswer3 = testRelation
+      .where(('a like ("@bc%def", '@')) ||
+        (Length('a) >= 6 && (StartsWith('a, "abc") && EndsWith('a, "def"))))
+      .analyze
+    comparePlans(optimized3, correctAnswer3)
+
+    val originalQuery4 =
+      testRelation
+        .where(('a like "%mn%") || ('a like ("%mn%", '%')))
+    val optimized4 = Optimize.execute(originalQuery4.analyze)
+    val correctAnswer4 = testRelation
+      .where(Contains('a, "mn") || ('a like ("%mn%", '%')))
+      .analyze
+    comparePlans(optimized4, correctAnswer4)
+
+    val originalQuery5 =
+      testRelation
+        .where(('a like "abc") || ('a like ("abbc", 'b')))
+    val optimized5 = Optimize.execute(originalQuery5.analyze)
+    val correctAnswer5 = testRelation
+      .where(('a === "abc") || ('a like ("abbc", 'b')))
+      .analyze
+    comparePlans(optimized5, correctAnswer5)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
index 2351d8321c5f3..a9267379e12fd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
@@ -215,12 +215,7 @@ class NestedColumnAliasingSuite extends SchemaPruningTest {
 
     val optimized = Optimize.execute(query)
 
-    val expected = nestedRelation
-      .select(GetStructField('a, 0, Some("b")))
-      .limit(5)
-      .analyze
-
-    comparePlans(optimized, expected)
+    comparePlans(optimized, query)
   }
 
   test("nested field pruning for getting struct field in array of struct") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala
index 0dd4d6a245f5c..6d7c4c3c7e9d2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala
@@ -22,8 +22,9 @@ import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.objects.Invoke
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
@@ -107,4 +108,34 @@ class ObjectSerializerPruningSuite extends PlanTest {
       comparePlans(optimized, expected)
     }
   }
+
+  test("SPARK-32652: Prune nested serializers: RowEncoder") {
+    withSQLConf(SQLConf.SERIALIZER_NESTED_SCHEMA_PRUNING_ENABLED.key -> "true") {
+      val testRelation = LocalRelation('i.struct(StructType.fromDDL("a int, b string")), 'j.int)
+      val rowEncoder = RowEncoder(new StructType()
+        .add("i", new StructType().add("a", "int").add("b", "string"))
+        .add("j", "int"))
+      val serializerObject = CatalystSerde.serialize(
+        CatalystSerde.deserialize(testRelation)(rowEncoder))(rowEncoder)
+      val query = serializerObject.select($"i.a")
+      val optimized = Optimize.execute(query.analyze)
+
+      val prunedSerializer = serializerObject.serializer.head.transformDown {
+        case CreateNamedStruct(children) => CreateNamedStruct(children.take(2))
+      }.transformUp {
+        // Aligns null literal in `If` expression to make it resolvable.
+        case i @ If(invoke: Invoke, Literal(null, dt), ser) if invoke.functionName == "isNullAt" &&
+            !dt.sameType(ser.dataType) =>
+          i.copy(trueValue = Literal(null, ser.dataType))
+      }.asInstanceOf[NamedExpression]
+
+      // `name` in `GetStructField` affects `comparePlans`. Maybe we can ignore
+      // `name` in `GetStructField.equals`?
+      val expected = serializerObject.copy(serializer = Seq(prunedSerializer))
+        .select($"i.a").analyze.transformAllExpressions {
+        case g: GetStructField => g.copy(name = None)
+      }
+      comparePlans(optimized, expected)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala
index d3b0a0e7d9347..db22121e08601 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala
@@ -49,12 +49,19 @@ class OptimizerLoggingSuite extends PlanTest {
       case event => Seq(
         "Applying Rule",
         "Result of Batch",
-        "has no effect").exists(event.getRenderedMessage().contains)
+        "has no effect",
+        "Metrics of Executed Rules").exists(event.getRenderedMessage().contains)
     }
     val logMessages = events.map(_.getRenderedMessage)
     assert(expectedRulesOrBatches.forall
     (ruleOrBatch => logMessages.exists(_.contains(ruleOrBatch))))
     assert(events.forall(_.getLevel == expectedLevel))
+    val expectedMetrics = Seq(
+      "Total number of runs: 7",
+      "Total time:",
+      "Total number of effective runs: 3",
+      "Total time of effective runs:")
+    assert(expectedMetrics.forall(metrics => logMessages.exists(_.contains(metrics))))
   }
 
   test("test log level") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
index 9c7d4c7d8d233..535be12af5161 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
@@ -221,4 +221,10 @@ class PropagateEmptyRelationSuite extends PlanTest {
     val optimized = Optimize.execute(query.analyze)
     assert(optimized.resolved)
   }
+
+  test("should not optimize away limit if streaming") {
+    val query = LocalRelation(Nil, Nil, isStreaming = true).limit(1).analyze
+    val optimized = Optimize.execute(query)
+    comparePlans(optimized, query)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
index 2d86d5a97e769..191edc81034a5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
@@ -91,9 +91,8 @@ class PullupCorrelatedPredicatesSuite extends PlanTest {
         .select(max('d))
     val scalarSubquery =
       testRelation
-        .where(ScalarSubquery(subPlan))
+        .where(ScalarSubquery(subPlan) === 1)
         .select('a).analyze
-    assert(scalarSubquery.resolved)
 
     val optimized = Optimize.execute(scalarSubquery)
     val doubleOptimized = Optimize.execute(optimized)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
index b692c3fee53c7..eb52c5b74772c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
@@ -50,10 +51,10 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
   }
 
   test("Not expected type - replaceNullWithFalse") {
-    val e = intercept[IllegalArgumentException] {
+    val e = intercept[AnalysisException] {
       testFilter(originalCond = Literal(null, IntegerType), expectedCond = FalseLiteral)
     }.getMessage
-    assert(e.contains("but got the type `int` in `CAST(NULL AS INT)"))
+    assert(e.contains("'CAST(NULL AS INT)' of type int is not a boolean"))
   }
 
   test("replace null in branches of If") {
@@ -324,7 +325,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
       testProjection(originalExpr = expr, expectedExpr = expr)
     }
     withSQLConf(SQLConf.LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC.key -> "false") {
-      testHigherOrderFunc('a, ArrayExists, Seq(lv('e)))
+      testHigherOrderFunc('a, ArrayExists.apply, Seq(lv('e)))
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
index 7b3f5b084b015..0ccf8aea660b2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
@@ -42,7 +42,8 @@ class SimplifyCastsSuite extends PlanTest {
 
   test("nullable element to non-nullable element array cast") {
     val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
-    val plan = input.select('a.cast(ArrayType(IntegerType, false)).as("casted")).analyze
+    val attr = input.output.head
+    val plan = input.select(attr.cast(ArrayType(IntegerType, false)).as("casted"))
     val optimized = Optimize.execute(plan)
     // Though cast from `ArrayType(IntegerType, true)` to `ArrayType(IntegerType, false)` is not
     // allowed, here we just ensure that `SimplifyCasts` rule respect the plan.
@@ -60,8 +61,9 @@ class SimplifyCastsSuite extends PlanTest {
 
   test("nullable value map to non-nullable value map cast") {
     val input = LocalRelation('m.map(MapType(StringType, StringType, true)))
-    val plan = input.select('m.cast(MapType(StringType, StringType, false))
-      .as("casted")).analyze
+    val attr = input.output.head
+    val plan = input.select(attr.cast(MapType(StringType, StringType, false))
+      .as("casted"))
     val optimized = Optimize.execute(plan)
     // Though cast from `MapType(StringType, StringType, true)` to
     // `MapType(StringType, StringType, false)` is not allowed, here we just ensure that
@@ -69,4 +71,3 @@ class SimplifyCastsSuite extends PlanTest {
     comparePlans(optimized, plan, checkAnalysis = false)
   }
 }
-
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
index 8ad7c12020b82..7b4ef54c627e2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
@@ -18,13 +18,14 @@
 package org.apache.spark.sql.catalyst.optimizer
 
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.types.{IntegerType, NullType}
+import org.apache.spark.sql.types.{BooleanType, IntegerType}
 
 
 class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
@@ -34,20 +35,18 @@ class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
       BooleanSimplification, ConstantFolding, SimplifyConditionals) :: Nil
   }
 
+  private val relation = LocalRelation('a.int, 'b.int, 'c.boolean)
+
   protected def assertEquivalent(e1: Expression, e2: Expression): Unit = {
-    val correctAnswer = Project(Alias(e2, "out")() :: Nil, OneRowRelation()).analyze
-    val actual = Optimize.execute(Project(Alias(e1, "out")() :: Nil, OneRowRelation()).analyze)
+    val correctAnswer = Project(Alias(e2, "out")() :: Nil, relation).analyze
+    val actual = Optimize.execute(Project(Alias(e1, "out")() :: Nil, relation).analyze)
     comparePlans(actual, correctAnswer)
   }
 
   private val trueBranch = (TrueLiteral, Literal(5))
   private val normalBranch = (NonFoldableLiteral(true), Literal(10))
   private val unreachableBranch = (FalseLiteral, Literal(20))
-  private val nullBranch = (Literal.create(null, NullType), Literal(30))
-
-  val isNotNullCond = IsNotNull(UnresolvedAttribute(Seq("a")))
-  val isNullCond = IsNull(UnresolvedAttribute("b"))
-  val notCond = Not(UnresolvedAttribute("c"))
+  private val nullBranch = (Literal.create(null, BooleanType), Literal(30))
 
   test("simplify if") {
     assertEquivalent(
@@ -59,7 +58,7 @@ class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
       Literal(20))
 
     assertEquivalent(
-      If(Literal.create(null, NullType), Literal(10), Literal(20)),
+      If(Literal.create(null, BooleanType), Literal(10), Literal(20)),
       Literal(20))
   }
 
@@ -127,9 +126,9 @@ class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
   test("simplify CaseWhen if all the outputs are semantic equivalence") {
     // When the conditions in `CaseWhen` are all deterministic, `CaseWhen` can be removed.
     assertEquivalent(
-      CaseWhen((isNotNullCond, Subtract(Literal(3), Literal(2))) ::
-        (isNullCond, Literal(1)) ::
-        (notCond, Add(Literal(6), Literal(-5))) ::
+      CaseWhen(('a.isNotNull, Subtract(Literal(3), Literal(2))) ::
+        ('b.isNull, Literal(1)) ::
+        (!'c, Add(Literal(6), Literal(-5))) ::
         Nil,
         Add(Literal(2), Literal(-1))),
       Literal(1)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
index 5b8e59ae7cb31..b7cf383732923 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
@@ -293,12 +293,12 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
           (nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
 
     val expected =
-      f1.join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
-        .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk")))
-        .join(t3.join(t4, Inner, Some(nameToAttr("t3_c1") === nameToAttr("t4_c1"))), Inner,
-          Some(nameToAttr("t3_c1") === nameToAttr("t4_c1")))
+      t3.join(t4, Inner, Some(nameToAttr("t3_c1") === nameToAttr("t4_c1")))
         .join(t1.join(t2, Inner, Some(nameToAttr("t1_c1") === nameToAttr("t2_c1"))), Inner,
           Some(nameToAttr("t1_c2") === nameToAttr("t4_c2")))
+        .join(f1
+          .join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
+          .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk"))))
         .select(outputsOf(d1, t1, t2, t3, t4, f1, d2): _*)
 
     assertEqualPlans(query, expected)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
index 10e970d534c49..17dc958841c48 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
@@ -226,7 +226,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
     assertEqualPlans(query, expected)
   }
 
-  test("Test 3:  Star join on a subset of dimensions since join column is not unique") {
+  test("Test 3: Star join on a subset of dimensions since join column is not unique") {
     // Star join:
     //   (=)  (=)
     // d1 - f1 - d2
@@ -254,9 +254,9 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
     val expected =
       f1.join(d1.where(nameToAttr("d1_c2") === 2), Inner,
           Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk1")))
-        .join(d3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
-        .join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
-        .join(s3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("s3_c2")))
+        .join(d3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk1")))
+        .join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_c4")))
+        .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
         .select(outputsOf(d1, f1, d2, s3, d3): _*)
 
     assertEqualPlans(query, expected)
@@ -316,20 +316,23 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
     // Positional join reordering: d3_ns, f1, d1, d2, s3
     // Star join reordering: empty
 
+    val d3_pk1 = d3_ns.output.find(_.name == "d3_pk1").get
+    val d3_fk1 = d3_ns.output.find(_.name == "d3_fk1").get
+
     val query =
       d3_ns.join(f1).join(d1).join(d2).join(s3)
         .where((nameToAttr("f1_fk2") === nameToAttr("d2_pk1")) &&
           (nameToAttr("d2_c2") === 2) &&
           (nameToAttr("f1_fk1") === nameToAttr("d1_pk1")) &&
-          (nameToAttr("f1_fk3") === nameToAttr("d3_pk1")) &&
-          (nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+          (nameToAttr("f1_fk3") === d3_pk1) &&
+          (d3_fk1 === nameToAttr("s3_pk1")))
 
     val equivQuery =
-      d3_ns.join(f1, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk1")))
+      d3_ns.join(f1, Inner, Some(nameToAttr("f1_fk3") === d3_pk1))
         .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk1")))
         .join(d2.where(nameToAttr("d2_c2") === 2), Inner,
           Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
-        .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+        .join(s3, Inner, Some(d3_fk1 === nameToAttr("s3_pk1")))
 
     assertEqualPlans(query, equivQuery)
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 56d52571d1cc3..55c3880f54ef5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.expressions.{EqualTo, Literal}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition.{after, first}
 import org.apache.spark.sql.connector.expressions.{ApplyTransform, BucketTransform, DaysTransform, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, Transform, YearsTransform}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType, TimestampType}
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -49,26 +50,6 @@ class DDLParserSuite extends AnalysisTest {
     comparePlans(parsePlan(sql), expected, checkAnalysis = false)
   }
 
-  test("SPARK-30098: create table without provider should " +
-    "use default data source under non-legacy mode") {
-    val createSql = "CREATE TABLE my_tab(a INT COMMENT 'test', b STRING)"
-    val defaultProvider = conf.defaultDataSourceName
-    val expectedPlan = CreateTableStatement(
-      Seq("my_tab"),
-      new StructType()
-        .add("a", IntegerType, nullable = true, "test")
-        .add("b", StringType),
-      Seq.empty[Transform],
-      None,
-      Map.empty[String, String],
-      defaultProvider,
-      Map.empty[String, String],
-      None,
-      None,
-      false)
-    parseCompare(createSql, expectedPlan)
-  }
-
   test("create/replace table using - schema") {
     val createSql = "CREATE TABLE my_tab(a INT COMMENT 'test', b STRING NOT NULL) USING parquet"
     val replaceSql = "REPLACE TABLE my_tab(a INT COMMENT 'test', b STRING NOT NULL) USING parquet"
@@ -80,7 +61,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq.empty[Transform],
       None,
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -103,7 +84,7 @@ class DDLParserSuite extends AnalysisTest {
         Seq.empty[Transform],
         None,
         Map.empty[String, String],
-        "parquet",
+        Some("parquet"),
         Map.empty[String, String],
         None,
         None),
@@ -123,7 +104,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq(IdentityTransform(FieldReference("a"))),
       None,
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -177,7 +158,7 @@ class DDLParserSuite extends AnalysisTest {
           LiteralValue(34, IntegerType)))),
       None,
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -199,7 +180,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq.empty[Transform],
       Some(BucketSpec(5, Seq("a"), Seq("b"))),
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -217,7 +198,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq.empty[Transform],
       None,
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       Some("abc"))
@@ -237,7 +218,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq.empty[Transform],
       None,
       Map("test" -> "test"),
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -255,7 +236,7 @@ class DDLParserSuite extends AnalysisTest {
         Seq.empty[Transform],
         None,
         Map.empty[String, String],
-        "parquet",
+        Some("parquet"),
         Map.empty[String, String],
         Some("/tmp/file"),
         None)
@@ -273,7 +254,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq.empty[Transform],
       None,
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -334,7 +315,7 @@ class DDLParserSuite extends AnalysisTest {
           Seq.empty[Transform],
           Option.empty[BucketSpec],
           Map.empty[String, String],
-          "json",
+          Some("json"),
           Map("a" -> "1", "b" -> "0.1", "c" -> "true"),
           None,
           None),
@@ -389,7 +370,7 @@ class DDLParserSuite extends AnalysisTest {
         Seq.empty[Transform],
         None,
         Map("p1" -> "v1", "p2" -> "v2"),
-        "parquet",
+        Some("parquet"),
         Map.empty[String, String],
         Some("/user/external/page_view"),
         Some("This is the staging page view table"))
@@ -646,17 +627,18 @@ class DDLParserSuite extends AnalysisTest {
         Some(first())))
   }
 
-  test("alter table: update column type, comment and position") {
-    comparePlans(
-      parsePlan("ALTER TABLE table_name CHANGE COLUMN a.b.c " +
-        "TYPE bigint COMMENT 'new comment' AFTER d"),
-      AlterTableAlterColumnStatement(
-        Seq("table_name"),
-        Seq("a", "b", "c"),
-        Some(LongType),
-        None,
-        Some("new comment"),
-        Some(after("d"))))
+  test("alter table: mutiple property changes are not allowed") {
+    intercept[ParseException] {
+      parsePlan("ALTER TABLE table_name ALTER COLUMN a.b.c " +
+        "TYPE bigint COMMENT 'new comment'")}
+
+    intercept[ParseException] {
+      parsePlan("ALTER TABLE table_name ALTER COLUMN a.b.c " +
+        "TYPE bigint COMMENT AFTER d")}
+
+    intercept[ParseException] {
+      parsePlan("ALTER TABLE table_name ALTER COLUMN a.b.c " +
+        "TYPE bigint COMMENT 'new comment' AFTER d")}
   }
 
   test("alter table: SET/DROP NOT NULL") {
@@ -1143,10 +1125,66 @@ class DDLParserSuite extends AnalysisTest {
     assert(exc.getMessage.contains("There should be at most 1 'WHEN NOT MATCHED' clause."))
   }
 
+  test("merge into table: the first matched clause must have a condition if there's a second") {
+    val exc = intercept[ParseException] {
+      parsePlan(
+        """
+          |MERGE INTO testcat1.ns1.ns2.tbl AS target
+          |USING testcat2.ns1.ns2.tbl AS source
+          |ON target.col1 = source.col1
+          |WHEN MATCHED THEN UPDATE SET target.col2 = source.col2
+          |WHEN MATCHED THEN DELETE
+          |WHEN NOT MATCHED AND (target.col2='insert')
+          |THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)
+        """.stripMargin)
+    }
+
+    assert(exc.getMessage.contains("the first MATCHED clause must have a condition"))
+  }
+
+  test("merge into table: there must be a when (not) matched condition") {
+    val exc = intercept[ParseException] {
+      parsePlan(
+        """
+          |MERGE INTO testcat1.ns1.ns2.tbl AS target
+          |USING testcat2.ns1.ns2.tbl AS source
+          |ON target.col1 = source.col1
+        """.stripMargin)
+    }
+
+    assert(exc.getMessage.contains("There must be at least one WHEN clause in a MERGE statement"))
+  }
+
+  test("merge into table: there can be only a single use DELETE or UPDATE") {
+    Seq("UPDATE SET *", "DELETE").foreach { op =>
+      val exc = intercept[ParseException] {
+        parsePlan(
+          s"""
+             |MERGE INTO testcat1.ns1.ns2.tbl AS target
+             |USING testcat2.ns1.ns2.tbl AS source
+             |ON target.col1 = source.col1
+             |WHEN MATCHED AND (target.col2='delete') THEN $op
+             |WHEN MATCHED THEN $op
+             |WHEN NOT MATCHED AND (target.col2='insert')
+             |THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)
+           """.stripMargin)
+      }
+
+      assert(exc.getMessage.contains(
+        "UPDATE and DELETE can appear at most once in MATCHED clauses"))
+    }
+  }
+
   test("show tables") {
     comparePlans(
       parsePlan("SHOW TABLES"),
       ShowTables(UnresolvedNamespace(Seq.empty[String]), None))
+    comparePlans(
+      parsePlan("SHOW TABLES '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW TABLES LIKE '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
     comparePlans(
       parsePlan("SHOW TABLES FROM testcat.ns1.ns2.tbl"),
       ShowTables(UnresolvedNamespace(Seq("testcat", "ns1", "ns2", "tbl")), None))
@@ -1154,8 +1192,11 @@ class DDLParserSuite extends AnalysisTest {
       parsePlan("SHOW TABLES IN testcat.ns1.ns2.tbl"),
       ShowTables(UnresolvedNamespace(Seq("testcat", "ns1", "ns2", "tbl")), None))
     comparePlans(
-      parsePlan("SHOW TABLES IN tbl LIKE '*dog*'"),
-      ShowTables(UnresolvedNamespace(Seq("tbl")), Some("*dog*")))
+      parsePlan("SHOW TABLES IN ns1 '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW TABLES IN ns1 LIKE '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
   }
 
   test("show table extended") {
@@ -1183,6 +1224,30 @@ class DDLParserSuite extends AnalysisTest {
         Some(Map("ds" -> "2008-04-09"))))
   }
 
+  test("show views") {
+    comparePlans(
+      parsePlan("SHOW VIEWS"),
+      ShowViews(UnresolvedNamespace(Seq.empty[String]), None))
+    comparePlans(
+      parsePlan("SHOW VIEWS '*test*'"),
+      ShowViews(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW VIEWS LIKE '*test*'"),
+      ShowViews(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW VIEWS FROM testcat.ns1.ns2.tbl"),
+      ShowViews(UnresolvedNamespace(Seq("testcat", "ns1", "ns2", "tbl")), None))
+    comparePlans(
+      parsePlan("SHOW VIEWS IN testcat.ns1.ns2.tbl"),
+      ShowViews(UnresolvedNamespace(Seq("testcat", "ns1", "ns2", "tbl")), None))
+    comparePlans(
+      parsePlan("SHOW VIEWS IN ns1 '*test*'"),
+      ShowViews(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW VIEWS IN ns1 LIKE '*test*'"),
+      ShowViews(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
+  }
+
   test("create namespace -- backward compatibility with DATABASE/DBPROPERTIES") {
     val expected = CreateNamespaceStatement(
       Seq("a", "b", "c"),
@@ -1448,7 +1513,7 @@ class DDLParserSuite extends AnalysisTest {
       AnalyzeColumnStatement(Seq("a", "b", "c"), None, allColumns = true))
 
     intercept("ANALYZE TABLE a.b.c COMPUTE STATISTICS FOR ALL COLUMNS key, value",
-      "mismatched input 'key' expecting <EOF>")
+      "mismatched input 'key' expecting {<EOF>, ';'}")
     intercept("ANALYZE TABLE a.b.c COMPUTE STATISTICS FOR ALL",
       "missing 'COLUMNS' at '<EOF>'")
   }
@@ -1899,11 +1964,11 @@ class DDLParserSuite extends AnalysisTest {
   test("SHOW TBLPROPERTIES table") {
     comparePlans(
       parsePlan("SHOW TBLPROPERTIES a.b.c"),
-      ShowTableProperties(UnresolvedTable(Seq("a", "b", "c")), None))
+      ShowTableProperties(UnresolvedTableOrView(Seq("a", "b", "c")), None))
 
     comparePlans(
       parsePlan("SHOW TBLPROPERTIES a.b.c('propKey1')"),
-      ShowTableProperties(UnresolvedTable(Seq("a", "b", "c")), Some("propKey1")))
+      ShowTableProperties(UnresolvedTableOrView(Seq("a", "b", "c")), Some("propKey1")))
   }
 
   test("DESCRIBE FUNCTION") {
@@ -2008,7 +2073,7 @@ class DDLParserSuite extends AnalysisTest {
       partitioning: Seq[Transform],
       bucketSpec: Option[BucketSpec],
       properties: Map[String, String],
-      provider: String,
+      provider: Option[String],
       options: Map[String, String],
       location: Option[String],
       comment: Option[String])
@@ -2084,4 +2149,21 @@ class DDLParserSuite extends AnalysisTest {
       parsePlan("COMMENT ON TABLE a.b.c IS 'xYz'"),
       CommentOnTable(UnresolvedTable(Seq("a", "b", "c")), "xYz"))
   }
+
+  // TODO: ignored by SPARK-31707, restore the test after create table syntax unification
+  ignore("create table - without using") {
+    val sql = "CREATE TABLE 1m.2g(a INT)"
+    val expectedTableSpec = TableSpec(
+      Seq("1m", "2g"),
+      Some(new StructType().add("a", IntegerType)),
+      Seq.empty[Transform],
+      None,
+      Map.empty[String, String],
+      None,
+      Map.empty[String, String],
+      None,
+      None)
+
+    testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 85efc6accf01f..c3430a4a0ecae 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -27,7 +27,6 @@ import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, _}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{First, Last}
 import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, IntervalUtils}
-import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -457,6 +456,11 @@ class ExpressionParserSuite extends AnalysisTest {
     assertEqual(
       "-interval '3 month 1 hour'",
       UnaryMinus(Literal(IntervalUtils.stringToInterval("interval 3 month 1 hour"))))
+    val intervalStrWithAllUnits = "1 year 3 months 2 weeks 2 days 1 hour 3 minutes 2 seconds " +
+      "100 millisecond 200 microseconds"
+    assertEqual(
+      s"interval '$intervalStrWithAllUnits'",
+      Literal(IntervalUtils.stringToInterval(intervalStrWithAllUnits)))
 
     // Binary.
     assertEqual("X'A'", Literal(Array(0x0a).map(_.toByte)))
@@ -681,13 +685,13 @@ class ExpressionParserSuite extends AnalysisTest {
       Literal(new CalendarInterval(
         0,
         0,
-        -13 * MICROS_PER_SECOND - 123 * MICROS_PER_MILLIS - 456)))
+        DateTimeTestUtils.secFrac(-13, -123, -456))))
     checkIntervals(
       "13.123456 second",
       Literal(new CalendarInterval(
         0,
         0,
-        13 * MICROS_PER_SECOND + 123 * MICROS_PER_MILLIS + 456)))
+        DateTimeTestUtils.secFrac(13, 123, 456))))
     checkIntervals("1.001 second",
       Literal(IntervalUtils.stringToInterval("1 second 1 millisecond")))
 
@@ -695,7 +699,7 @@ class ExpressionParserSuite extends AnalysisTest {
     intercept("interval 10 nanoseconds", "invalid unit 'nanoseconds'")
 
     // Year-Month intervals.
-    val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0")
+    val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0", "\t -1-2\t")
     yearMonthValues.foreach { value =>
       val result = Literal(IntervalUtils.fromYearMonthString(value))
       checkIntervals(s"'$value' year to month", result)
@@ -708,7 +712,8 @@ class ExpressionParserSuite extends AnalysisTest {
       "10 9:8:7.123456789",
       "1 0:0:0",
       "-1 0:0:0",
-      "1 0:0:1")
+      "1 0:0:1",
+      "\t 1 0:0:1 ")
     datTimeValues.foreach { value =>
       val result = Literal(IntervalUtils.fromDayTimeString(value))
       checkIntervals(s"'$value' day to second", result)
@@ -765,26 +770,17 @@ class ExpressionParserSuite extends AnalysisTest {
   }
 
   test("SPARK-19526 Support ignore nulls keywords for first and last") {
-    assertEqual("first(a ignore nulls)", First('a, Literal(true)).toAggregateExpression())
-    assertEqual("first(a)", First('a, Literal(false)).toAggregateExpression())
-    assertEqual("last(a ignore nulls)", Last('a, Literal(true)).toAggregateExpression())
-    assertEqual("last(a)", Last('a, Literal(false)).toAggregateExpression())
-  }
-
-  test("Support respect nulls keywords for first_value and last_value") {
-    assertEqual("first_value(a ignore nulls)", First('a, Literal(true)).toAggregateExpression())
-    assertEqual("first_value(a respect nulls)", First('a, Literal(false)).toAggregateExpression())
-    assertEqual("first_value(a)", First('a, Literal(false)).toAggregateExpression())
-    assertEqual("last_value(a ignore nulls)", Last('a, Literal(true)).toAggregateExpression())
-    assertEqual("last_value(a respect nulls)", Last('a, Literal(false)).toAggregateExpression())
-    assertEqual("last_value(a)", Last('a, Literal(false)).toAggregateExpression())
+    assertEqual("first(a ignore nulls)", First('a, true).toAggregateExpression())
+    assertEqual("first(a)", First('a, false).toAggregateExpression())
+    assertEqual("last(a ignore nulls)", Last('a, true).toAggregateExpression())
+    assertEqual("last(a)", Last('a, false).toAggregateExpression())
   }
 
   test("timestamp literals") {
-    DateTimeTestUtils.outstandingTimezones.foreach { timeZone =>
-      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone.getID) {
+    DateTimeTestUtils.outstandingZoneIds.foreach { zid =>
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId) {
         def toMicros(time: LocalDateTime): Long = {
-          val seconds = time.atZone(timeZone.toZoneId).toInstant.getEpochSecond
+          val seconds = time.atZone(zid).toInstant.getEpochSecond
           TimeUnit.SECONDS.toMicros(seconds)
         }
         assertEval(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
index 875096f615241..515ba2a076fa8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
@@ -55,6 +55,16 @@ class PlanParserSuite extends AnalysisTest {
     With(plan, ctes)
   }
 
+  test("single comment case one") {
+    val plan = table("a").select(star())
+    assertEqual("-- single comment\nSELECT * FROM a", plan)
+  }
+
+  test("single comment case two") {
+    val plan = table("a").select(star())
+    assertEqual("-- single comment\\\nwith line continuity\nSELECT * FROM a", plan)
+  }
+
   test("case insensitive") {
     val plan = table("a").select(star())
     assertEqual("sELEct * FroM a", plan)
@@ -110,7 +120,7 @@ class PlanParserSuite extends AnalysisTest {
     assertEqual("select a, b from db.c where x < 1", table("db", "c").where('x < 1).select('a, 'b))
     assertEqual(
       "select a, b from db.c having x < 1",
-      table("db", "c").groupBy()('a, 'b).where('x < 1))
+      table("db", "c").having()('a, 'b)('x < 1))
     assertEqual("select distinct a, b from db.c", Distinct(table("db", "c").select('a, 'b)))
     assertEqual("select all a, b from db.c", table("db", "c").select('a, 'b))
     assertEqual("select from tbl", OneRowRelation().select('from.as("tbl")))
@@ -476,8 +486,7 @@ class PlanParserSuite extends AnalysisTest {
     assertEqual(
       "select g from t group by g having a > (select b from s)",
       table("t")
-        .groupBy('g)('g)
-        .where('a > ScalarSubquery(table("s").select('b))))
+        .having('g)('g)('a > ScalarSubquery(table("s").select('b))))
   }
 
   test("table reference") {
@@ -922,4 +931,11 @@ class PlanParserSuite extends AnalysisTest {
       "WITH t(x) AS (SELECT c FROM a) SELECT * FROM t",
       cte(table("t").select(star()), "t" -> ((table("a").select('c), Seq("x")))))
   }
+
+  test("statement containing terminal semicolons") {
+    assertEqual("select 1;", OneRowRelation().select(1))
+    assertEqual("select a, b;", OneRowRelation().select('a, 'b))
+    assertEqual("select a, b from db.c;;;", table("db", "c").select('a, 'b))
+    assertEqual("select a, b from db.c; ;;  ;", table("db", "c").select('a, 'b))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index 43244b3c0a57d..f037ce7b9e793 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -17,11 +17,10 @@
 package org.apache.spark.sql.catalyst.parser
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.{SQLKeywordUtils, TableIdentifier}
 import org.apache.spark.sql.internal.SQLConf
 
-class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
+class TableIdentifierParserSuite extends SparkFunSuite with SQLKeywordUtils {
   import CatalystSqlParser._
 
   // Add "$elem$", "$value$" & "$key$"
@@ -254,6 +253,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "utctimestamp",
     "values",
     "view",
+    "views",
     "while",
     "with",
     "work",
@@ -284,337 +284,6 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "where",
     "with")
 
-  // All the keywords in `docs/sql-keywords.md` are listed below:
-  val allCandidateKeywords = Set(
-    "add",
-    "after",
-    "all",
-    "alter",
-    "analyze",
-    "and",
-    "anti",
-    "any",
-    "archive",
-    "array",
-    "as",
-    "asc",
-    "at",
-    "authorization",
-    "between",
-    "both",
-    "bucket",
-    "buckets",
-    "by",
-    "cache",
-    "cascade",
-    "case",
-    "cast",
-    "change",
-    "check",
-    "clear",
-    "cluster",
-    "clustered",
-    "codegen",
-    "collate",
-    "collection",
-    "column",
-    "columns",
-    "comment",
-    "commit",
-    "compact",
-    "compactions",
-    "compute",
-    "concatenate",
-    "constraint",
-    "cost",
-    "create",
-    "cross",
-    "cube",
-    "current",
-    "current_date",
-    "current_time",
-    "current_timestamp",
-    "current_user",
-    "data",
-    "database",
-    "databases",
-    "day",
-    "dbproperties",
-    "defined",
-    "delete",
-    "delimited",
-    "desc",
-    "describe",
-    "dfs",
-    "directories",
-    "directory",
-    "distinct",
-    "distribute",
-    "div",
-    "drop",
-    "else",
-    "end",
-    "escape",
-    "escaped",
-    "except",
-    "exchange",
-    "exists",
-    "explain",
-    "export",
-    "extended",
-    "external",
-    "extract",
-    "false",
-    "fetch",
-    "fields",
-    "fileformat",
-    "first",
-    "first_value",
-    "following",
-    "for",
-    "foreign",
-    "format",
-    "formatted",
-    "from",
-    "full",
-    "function",
-    "functions",
-    "global",
-    "grant",
-    "group",
-    "grouping",
-    "having",
-    "hour",
-    "if",
-    "ignore",
-    "import",
-    "in",
-    "index",
-    "indexes",
-    "inner",
-    "inpath",
-    "inputformat",
-    "insert",
-    "intersect",
-    "interval",
-    "into",
-    "is",
-    "items",
-    "join",
-    "keys",
-    "last",
-    "last_value",
-    "lateral",
-    "lazy",
-    "leading",
-    "left",
-    "like",
-    "limit",
-    "lines",
-    "list",
-    "load",
-    "local",
-    "location",
-    "lock",
-    "locks",
-    "logical",
-    "macro",
-    "map",
-    "minus",
-    "minute",
-    "month",
-    "msck",
-    "namespaces",
-    "natural",
-    "no",
-    "not",
-    "null",
-    "nulls",
-    "of",
-    "on",
-    "only",
-    "option",
-    "options",
-    "or",
-    "order",
-    "out",
-    "outer",
-    "outputformat",
-    "over",
-    "overlaps",
-    "overlay",
-    "overwrite",
-    "partition",
-    "partitioned",
-    "partitions",
-    "percent",
-    "pivot",
-    "placing",
-    "position",
-    "preceding",
-    "primary",
-    "principals",
-    "purge",
-    "query",
-    "range",
-    "recordreader",
-    "recordwriter",
-    "recover",
-    "reduce",
-    "references",
-    "refresh",
-    "rename",
-    "repair",
-    "replace",
-    "reset",
-    "respect",
-    "restrict",
-    "revoke",
-    "right",
-    "rlike",
-    "role",
-    "roles",
-    "rollback",
-    "rollup",
-    "row",
-    "rows",
-    "schema",
-    "second",
-    "select",
-    "semi",
-    "separated",
-    "serde",
-    "serdeproperties",
-    "session_user",
-    "set",
-    "sets",
-    "show",
-    "skewed",
-    "some",
-    "sort",
-    "sorted",
-    "start",
-    "statistics",
-    "stored",
-    "stratify",
-    "struct",
-    "substr",
-    "substring",
-    "table",
-    "tables",
-    "tablesample",
-    "tblproperties",
-    "temporary",
-    "terminated",
-    "then",
-    "to",
-    "touch",
-    "trailing",
-    "transaction",
-    "transactions",
-    "transform",
-    "true",
-    "truncate",
-    "unarchive",
-    "unbounded",
-    "uncache",
-    "union",
-    "unique",
-    "unknown",
-    "unlock",
-    "unset",
-    "use",
-    "user",
-    "using",
-    "values",
-    "view",
-    "when",
-    "where",
-    "window",
-    "with",
-    "year")
-
-  val reservedKeywordsInAnsiMode = Set(
-    "all",
-    "and",
-    "anti",
-    "any",
-    "as",
-    "authorization",
-    "both",
-    "case",
-    "cast",
-    "check",
-    "collate",
-    "column",
-    "constraint",
-    "create",
-    "cross",
-    "current_date",
-    "current_time",
-    "current_timestamp",
-    "current_user",
-    "day",
-    "distinct",
-    "else",
-    "end",
-    "escape",
-    "except",
-    "false",
-    "fetch",
-    "first_value",
-    "for",
-    "foreign",
-    "from",
-    "full",
-    "grant",
-    "group",
-    "having",
-    "hour",
-    "in",
-    "inner",
-    "intersect",
-    "into",
-    "join",
-    "is",
-    "last_value",
-    "leading",
-    "left",
-    "minute",
-    "month",
-    "natural",
-    "not",
-    "null",
-    "on",
-    "only",
-    "or",
-    "order",
-    "outer",
-    "overlaps",
-    "primary",
-    "references",
-    "right",
-    "select",
-    "semi",
-    "session_user",
-    "minus",
-    "second",
-    "some",
-    "table",
-    "then",
-    "to",
-    "trailing",
-    "union",
-    "unique",
-    "unknown",
-    "user",
-    "using",
-    "when",
-    "where",
-    "with",
-    "year")
-
-  val nonReservedKeywordsInAnsiMode = allCandidateKeywords -- reservedKeywordsInAnsiMode
 
   test("table identifier") {
     // Regular names.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
index d96f8086a3e93..404c8895c4d11 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
@@ -20,9 +20,11 @@ package org.apache.spark.sql.catalyst.plans
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.dsl.plans
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ListQuery, Literal, NamedExpression}
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, Project, Union}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, Expression, ListQuery, Literal, NamedExpression}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LocalRelation, LogicalPlan, Project, Union}
+import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin}
 import org.apache.spark.sql.types.IntegerType
 
@@ -31,7 +33,7 @@ class QueryPlanSuite extends SparkFunSuite {
   test("origin remains the same after mapExpressions (SPARK-23823)") {
     CurrentOrigin.setPosition(0, 0)
     val column = AttributeReference("column", IntegerType)(NamedExpression.newExprId)
-    val query = plans.DslLogicalPlan(plans.table("table")).select(column)
+    val query = DslLogicalPlan(table("table")).select(column)
     CurrentOrigin.reset()
 
     val mappedQuery = query mapExpressions {
@@ -78,9 +80,25 @@ class QueryPlanSuite extends SparkFunSuite {
 
     val countRelationsInPlan = plan.collect({ case _: UnresolvedRelation => 1 }).sum
     val countRelationsInPlanAndSubqueries =
-      plan.collectInPlanAndSubqueries({ case _: UnresolvedRelation => 1 }).sum
+      plan.collectWithSubqueries({ case _: UnresolvedRelation => 1 }).sum
 
     assert(countRelationsInPlan == 2)
     assert(countRelationsInPlanAndSubqueries == 5)
   }
+
+  test("SPARK-33035: consecutive attribute updates in parent plan nodes") {
+    val testRule = new Rule[LogicalPlan] {
+      override def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithNewOutput {
+        case p @ Project(projList, _) =>
+          // Assigns new `ExprId`s for output references
+          val newPlan = p.copy(projectList = projList.map { ne => Alias(ne, ne.name)() })
+          val attrMapping = p.output.zip(newPlan.output)
+          newPlan -> attrMapping
+      }
+    }
+
+    val t = LocalRelation('a.int, 'b.int)
+    val plan = t.select($"a", $"b").select($"a", $"b").select($"a", $"b").analyze
+    assert(testRule(plan).resolved)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
index d213743946e76..5deab79b9f67d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
@@ -21,6 +21,8 @@ import java.time.ZoneId
 
 import scala.util.control.NonFatal
 
+import org.scalatest.Assertions.fail
+
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.getZoneId
@@ -83,4 +85,11 @@ trait SQLHelper {
       }
     }
   }
+
+  protected lazy val sparkHome: String = {
+    if (!(sys.props.contains("spark.test.home") || sys.env.contains("SPARK_HOME"))) {
+      fail("spark.test.home or SPARK_HOME is not set.")
+    }
+    sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala
index 6c5a2b247fc23..cdfc863cc0212 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala
@@ -551,4 +551,26 @@ class JoinEstimationSuite extends StatsEstimationTestBase {
       attributeStats = AttributeMap(Nil))
     assert(join.stats == expectedStats)
   }
+
+  test("SPARK-33018 Fix estimate statistics issue if child has 0 bytes") {
+    case class MyStatsTestPlan(
+        outputList: Seq[Attribute],
+        sizeInBytes: BigInt) extends LeafNode {
+      override def output: Seq[Attribute] = outputList
+      override def computeStats(): Statistics = Statistics(sizeInBytes = sizeInBytes)
+    }
+
+    val left = MyStatsTestPlan(
+      outputList = Seq("key-1-2", "key-2-4").map(nameToAttr),
+      sizeInBytes = BigInt(100))
+
+    val right = MyStatsTestPlan(
+      outputList = Seq("key-1-2", "key-2-3").map(nameToAttr),
+      sizeInBytes = BigInt(0))
+
+    val join = Join(left, right, LeftOuter,
+      Some(EqualTo(nameToAttr("key-2-4"), nameToAttr("key-2-3"))), JoinHint.NONE)
+
+    assert(join.stats == Statistics(sizeInBytes = 100))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala
index 9dceca59f5b87..0a27e31b3c9f6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala
@@ -26,17 +26,20 @@ import org.apache.spark.sql.types.{IntegerType, StringType}
 
 trait StatsEstimationTestBase extends SparkFunSuite {
 
-  var originalValue: Boolean = false
+  var originalCBOValue: Boolean = false
+  var originalPlanStatsValue: Boolean = false
 
   override def beforeAll(): Unit = {
     super.beforeAll()
     // Enable stats estimation based on CBO.
-    originalValue = SQLConf.get.getConf(SQLConf.CBO_ENABLED)
+    originalCBOValue = SQLConf.get.getConf(SQLConf.CBO_ENABLED)
+    originalPlanStatsValue = SQLConf.get.getConf(SQLConf.PLAN_STATS_ENABLED)
     SQLConf.get.setConf(SQLConf.CBO_ENABLED, true)
   }
 
   override def afterAll(): Unit = {
-    SQLConf.get.setConf(SQLConf.CBO_ENABLED, originalValue)
+    SQLConf.get.setConf(SQLConf.CBO_ENABLED, originalCBOValue)
+    SQLConf.get.setConf(SQLConf.PLAN_STATS_ENABLED, originalPlanStatsValue)
     super.afterAll()
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index 0e094bc06b05f..8ee64ed20a2a6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -31,6 +31,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.{AliasIdentifier, FunctionIdentifier, InternalRow, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.dsl.expressions.DslString
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.catalyst.plans.{LeftOuter, NaturalJoin, SQLHelper}
@@ -433,10 +434,11 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
 
     // Converts AliasIdentifier to JSON
     assertJSON(
-      AliasIdentifier("alias"),
+      AliasIdentifier("alias", Seq("ns1", "ns2")),
       JObject(
         "product-class" -> JString(classOf[AliasIdentifier].getName),
-          "identifier" -> "alias"))
+          "name" -> "alias",
+          "qualifier" -> "[ns1, ns2]"))
 
     // Converts SubqueryAlias to JSON
     assertJSON(
@@ -445,8 +447,9 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
         JObject(
           "class" -> classOf[SubqueryAlias].getName,
           "num-children" -> 1,
-          "name" -> JObject("product-class" -> JString(classOf[AliasIdentifier].getName),
-            "identifier" -> "t1"),
+          "identifier" -> JObject("product-class" -> JString(classOf[AliasIdentifier].getName),
+            "name" -> "t1",
+            "qualifier" -> JArray(Nil)),
           "child" -> 0),
         JObject(
           "class" -> classOf[JsonTestTreeNode].getName,
@@ -592,7 +595,8 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("toJSON should not throws java.lang.StackOverflowError") {
-    val udf = ScalaUDF(SelfReferenceUDF(), BooleanType, Seq("col1".attr), false :: Nil)
+    val udf = ScalaUDF(SelfReferenceUDF(), BooleanType, Seq("col1".attr),
+      Option(ExpressionEncoder[String]()) :: Nil)
     // Should not throw java.lang.StackOverflowError
     udf.toJSON
   }
@@ -730,4 +734,30 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
     assertDifferentInstance(leaf, leafCloned)
     assert(leaf.child.eq(leafCloned.asInstanceOf[FakeLeafPlan].child))
   }
+
+  object MalformedClassObject extends Serializable {
+    case class MalformedNameExpression(child: Expression) extends TaggingExpression
+  }
+
+  test("SPARK-32999: TreeNode.nodeName should not throw malformed class name error") {
+    val testTriggersExpectedError = try {
+      classOf[MalformedClassObject.MalformedNameExpression].getSimpleName
+      false
+    } catch {
+      case ex: java.lang.InternalError if ex.getMessage.contains("Malformed class name") =>
+        true
+      case ex: Throwable => throw ex
+    }
+    // This test case only applies on older JDK versions (e.g. JDK8u), and doesn't trigger the
+    // issue on newer JDK versions (e.g. JDK11u).
+    assume(testTriggersExpectedError, "the test case didn't trigger malformed class name error")
+
+    val expr = MalformedClassObject.MalformedNameExpression(Literal(1))
+    try {
+      expr.nodeName
+    } catch {
+      case ex: java.lang.InternalError if ex.getMessage.contains("Malformed class name") =>
+        fail("TreeNode.nodeName should not throw malformed class name error")
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
index 8509bce177129..6e07cd5d6415d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
@@ -20,10 +20,12 @@ package org.apache.spark.sql.catalyst.util
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{UnsafeArrayData, UnsafeRow}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{ArrayType, BinaryType, IntegerType, StructType}
 import org.apache.spark.unsafe.Platform
 
-class ArrayBasedMapBuilderSuite extends SparkFunSuite {
+class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
 
   test("basic") {
     val builder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
@@ -42,34 +44,49 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite {
     assert(e.getMessage.contains("Cannot use null as map key"))
   }
 
-  test("remove duplicated keys with last wins policy") {
+  test("fail while duplicated keys detected") {
     val builder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
     builder.put(1, 1)
-    builder.put(2, 2)
-    builder.put(1, 2)
-    val map = builder.build()
-    assert(map.numElements() == 2)
-    assert(ArrayBasedMapData.toScalaMap(map) == Map(1 -> 2, 2 -> 2))
+    val e = intercept[RuntimeException](builder.put(1, 2))
+    assert(e.getMessage.contains("Duplicate map key 1 was found"))
+  }
+
+  test("remove duplicated keys with last wins policy") {
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      val builder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
+      builder.put(1, 1)
+      builder.put(2, 2)
+      builder.put(1, 2)
+      val map = builder.build()
+      assert(map.numElements() == 2)
+      assert(ArrayBasedMapData.toScalaMap(map) == Map(1 -> 2, 2 -> 2))
+    }
   }
 
-  test("binary type key") {
+  test("binary type key with duplication") {
     val builder = new ArrayBasedMapBuilder(BinaryType, IntegerType)
     builder.put(Array(1.toByte), 1)
     builder.put(Array(2.toByte), 2)
-    builder.put(Array(1.toByte), 3)
-    val map = builder.build()
-    assert(map.numElements() == 2)
-    val entries = ArrayBasedMapData.toScalaMap(map).iterator.toSeq
-    assert(entries(0)._1.asInstanceOf[Array[Byte]].toSeq == Seq(1))
-    assert(entries(0)._2 == 3)
-    assert(entries(1)._1.asInstanceOf[Array[Byte]].toSeq == Seq(2))
-    assert(entries(1)._2 == 2)
+    val e = intercept[RuntimeException](builder.put(Array(1.toByte), 3))
+    // By default duplicated map key fails the query.
+    assert(e.getMessage.contains("Duplicate map key"))
+
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      val builder = new ArrayBasedMapBuilder(BinaryType, IntegerType)
+      builder.put(Array(1.toByte), 1)
+      builder.put(Array(2.toByte), 2)
+      builder.put(Array(1.toByte), 3)
+      val map = builder.build()
+      assert(map.numElements() == 2)
+      val entries = ArrayBasedMapData.toScalaMap(map).iterator.toSeq
+      assert(entries(0)._1.asInstanceOf[Array[Byte]].toSeq == Seq(1))
+      assert(entries(0)._2 == 3)
+      assert(entries(1)._1.asInstanceOf[Array[Byte]].toSeq == Seq(2))
+      assert(entries(1)._2 == 2)
+    }
   }
 
-  test("struct type key") {
-    val builder = new ArrayBasedMapBuilder(new StructType().add("i", "int"), IntegerType)
-    builder.put(InternalRow(1), 1)
-    builder.put(InternalRow(2), 2)
+  test("struct type key with duplication") {
     val unsafeRow = {
       val row = new UnsafeRow(1)
       val bytes = new Array[Byte](16)
@@ -77,16 +94,26 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite {
       row.setInt(0, 1)
       row
     }
-    builder.put(unsafeRow, 3)
-    val map = builder.build()
-    assert(map.numElements() == 2)
-    assert(ArrayBasedMapData.toScalaMap(map) == Map(InternalRow(1) -> 3, InternalRow(2) -> 2))
+
+    val builder = new ArrayBasedMapBuilder(new StructType().add("i", "int"), IntegerType)
+    builder.put(InternalRow(1), 1)
+    builder.put(InternalRow(2), 2)
+    val e = intercept[RuntimeException](builder.put(unsafeRow, 3))
+    // By default duplicated map key fails the query.
+    assert(e.getMessage.contains("Duplicate map key"))
+
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      val builder = new ArrayBasedMapBuilder(new StructType().add("i", "int"), IntegerType)
+      builder.put(InternalRow(1), 1)
+      builder.put(InternalRow(2), 2)
+      builder.put(unsafeRow, 3)
+      val map = builder.build()
+      assert(map.numElements() == 2)
+      assert(ArrayBasedMapData.toScalaMap(map) == Map(InternalRow(1) -> 3, InternalRow(2) -> 2))
+    }
   }
 
-  test("array type key") {
-    val builder = new ArrayBasedMapBuilder(ArrayType(IntegerType), IntegerType)
-    builder.put(new GenericArrayData(Seq(1, 1)), 1)
-    builder.put(new GenericArrayData(Seq(2, 2)), 2)
+  test("array type key with duplication") {
     val unsafeArray = {
       val array = new UnsafeArrayData()
       val bytes = new Array[Byte](24)
@@ -96,10 +123,23 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite {
       array.setInt(1, 1)
       array
     }
-    builder.put(unsafeArray, 3)
-    val map = builder.build()
-    assert(map.numElements() == 2)
-    assert(ArrayBasedMapData.toScalaMap(map) ==
-      Map(new GenericArrayData(Seq(1, 1)) -> 3, new GenericArrayData(Seq(2, 2)) -> 2))
+
+    val builder = new ArrayBasedMapBuilder(ArrayType(IntegerType), IntegerType)
+    builder.put(new GenericArrayData(Seq(1, 1)), 1)
+    builder.put(new GenericArrayData(Seq(2, 2)), 2)
+    val e = intercept[RuntimeException](builder.put(unsafeArray, 3))
+    // By default duplicated map key fails the query.
+    assert(e.getMessage.contains("Duplicate map key"))
+
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      val builder = new ArrayBasedMapBuilder(ArrayType(IntegerType), IntegerType)
+      builder.put(new GenericArrayData(Seq(1, 1)), 1)
+      builder.put(new GenericArrayData(Seq(2, 2)), 2)
+      builder.put(unsafeArray, 3)
+      val map = builder.build()
+      assert(map.numElements() == 2)
+      assert(ArrayBasedMapData.toScalaMap(map) ==
+        Map(new GenericArrayData(Seq(1, 1)) -> 3, new GenericArrayData(Seq(2, 2)) -> 2))
+    }
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
index da71e3a4d53e2..1e430351b5137 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
@@ -73,8 +73,8 @@ class ArrayDataIndexedSeqSuite extends SparkFunSuite {
     arrayTypes.foreach { dt =>
       val schema = StructType(StructField("col_1", dt, nullable = false) :: Nil)
       val row = RandomDataGenerator.randomRow(random, schema)
-      val rowConverter = RowEncoder(schema)
-      val internalRow = rowConverter.toRow(row)
+      val toRow = RowEncoder(schema).createSerializer()
+      val internalRow = toRow(row)
 
       val unsafeRowConverter = UnsafeProjection.create(schema)
       val safeRowConverter = SafeProjection.create(schema)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala
index a8bb1d0afdb87..db8b7403cb60e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala
@@ -44,4 +44,12 @@ class CaseInsensitiveMapSuite extends SparkFunSuite {
     assert(m == Map("a" -> "b", "foo" -> "bar", "x" -> "y"))
     shouldBeSerializable(m)
   }
+
+  test("SPARK-32377: CaseInsensitiveMap should be deterministic for addition") {
+    var m = CaseInsensitiveMap(Map.empty[String, String])
+    Seq(("paTh", "1"), ("PATH", "2"), ("Path", "3"), ("patH", "4"), ("path", "5")).foreach { kv =>
+      m = (m + kv).asInstanceOf[CaseInsensitiveMap[String]]
+      assert(m.get("path").contains(kv._2))
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
new file mode 100644
index 0000000000000..0a29d94dd984d
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.time.{DateTimeException, LocalDate}
+
+import org.apache.spark.SparkUpgradeException
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
+
+class DateFormatterSuite extends DatetimeFormatterSuite {
+
+  override def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit = {
+    DateFormatter(pattern, UTC, isParsing)
+  }
+
+  override protected def useDateFormatter: Boolean = true
+
+  test("parsing dates") {
+    outstandingTimezonesIds.foreach { timeZone =>
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+        val formatter = DateFormatter(getZoneId(timeZone))
+        val daysSinceEpoch = formatter.parse("2018-12-02")
+        assert(daysSinceEpoch === 17867)
+      }
+    }
+  }
+
+  test("format dates") {
+    outstandingTimezonesIds.foreach { timeZone =>
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+        val formatter = DateFormatter(getZoneId(timeZone))
+        val (days, expected) = (17867, "2018-12-02")
+        val date = formatter.format(days)
+        assert(date === expected)
+        assert(formatter.format(daysToLocalDate(days)) === expected)
+        assert(formatter.format(toJavaDate(days)) === expected)
+      }
+    }
+  }
+
+  test("roundtrip date -> days -> date") {
+    LegacyBehaviorPolicy.values.foreach { parserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> parserPolicy.toString) {
+        LegacyDateFormats.values.foreach { legacyFormat =>
+          Seq(
+            "0050-01-01",
+            "0953-02-02",
+            "1423-03-08",
+            "1582-10-15",
+            "1969-12-31",
+            "1972-08-25",
+            "1975-09-26",
+            "2018-12-12",
+            "2038-01-01",
+            "5010-11-17").foreach { date =>
+            outstandingTimezonesIds.foreach { timeZone =>
+              withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+                val formatter = DateFormatter(
+                  DateFormatter.defaultPattern,
+                  getZoneId(timeZone),
+                  DateFormatter.defaultLocale,
+                  legacyFormat,
+                  isParsing = false)
+                val days = formatter.parse(date)
+                assert(date === formatter.format(days))
+                assert(date === formatter.format(daysToLocalDate(days)))
+                assert(date === formatter.format(toJavaDate(days)))
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("roundtrip days -> date -> days") {
+    LegacyBehaviorPolicy.values.foreach { parserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> parserPolicy.toString) {
+        LegacyDateFormats.values.foreach { legacyFormat =>
+          Seq(
+            -701265,
+            -371419,
+            -199722,
+            -1,
+            0,
+            967,
+            2094,
+            17877,
+            24837,
+            1110657).foreach { days =>
+            outstandingTimezonesIds.foreach { timeZone =>
+              withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+                val formatter = DateFormatter(
+                  DateFormatter.defaultPattern,
+                  getZoneId(timeZone),
+                  DateFormatter.defaultLocale,
+                  legacyFormat,
+                  isParsing = false)
+                val date = formatter.format(days)
+                val parsed = formatter.parse(date)
+                assert(days === parsed)
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("parsing date without explicit day") {
+    val formatter = DateFormatter("yyyy MMM", UTC)
+    val daysSinceEpoch = formatter.parse("2018 Dec")
+    assert(daysSinceEpoch === days(2018, 12, 1))
+  }
+
+  test("formatting negative years with default pattern") {
+    val epochDays = days(-99, 1, 1)
+    assert(DateFormatter(UTC).format(epochDays) === "-0099-01-01")
+  }
+
+  test("special date values") {
+    testSpecialDatetimeValues { zoneId =>
+      val formatter = DateFormatter(zoneId)
+
+      assert(formatter.parse("EPOCH") === 0)
+      val today = localDateToDays(LocalDate.now(zoneId))
+      assert(formatter.parse("Yesterday") === today - 1)
+      assert(formatter.parse("now") === today)
+      assert(formatter.parse("today ") === today)
+      assert(formatter.parse("tomorrow UTC") === today + 1)
+    }
+  }
+
+  test("SPARK-30958: parse date with negative year") {
+    val formatter1 = DateFormatter("yyyy-MM-dd", UTC)
+    assert(formatter1.parse("-1234-02-22") === days(-1234, 2, 22))
+
+    def assertParsingError(f: => Unit): Unit = {
+      intercept[Exception](f) match {
+        case e: SparkUpgradeException =>
+          assert(e.getCause.isInstanceOf[DateTimeException])
+        case e =>
+          assert(e.isInstanceOf[DateTimeException])
+      }
+    }
+
+    // "yyyy" with "G" can't parse negative year or year 0000.
+    val formatter2 = DateFormatter("G yyyy-MM-dd", UTC)
+    assertParsingError(formatter2.parse("BC -1234-02-22"))
+    assertParsingError(formatter2.parse("AD 0000-02-22"))
+
+    assert(formatter2.parse("BC 1234-02-22") === days(-1233, 2, 22))
+    assert(formatter2.parse("AD 1234-02-22") === days(1234, 2, 22))
+  }
+
+  test("SPARK-31557: rebasing in legacy formatters/parsers") {
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
+      LegacyDateFormats.values.foreach { legacyFormat =>
+        outstandingTimezonesIds.foreach { timeZone =>
+          withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+            val formatter = DateFormatter(
+              DateFormatter.defaultPattern,
+              getZoneId(timeZone),
+              DateFormatter.defaultLocale,
+              legacyFormat,
+              isParsing = false)
+            assert(LocalDate.ofEpochDay(formatter.parse("1000-01-01")) === LocalDate.of(1000, 1, 1))
+            assert(formatter.format(LocalDate.of(1000, 1, 1)) === "1000-01-01")
+            assert(formatter.format(localDateToDays(LocalDate.of(1000, 1, 1))) === "1000-01-01")
+            assert(formatter.format(java.sql.Date.valueOf("1000-01-01")) === "1000-01-01")
+          }
+        }
+      }
+    }
+  }
+
+  test("missing date fields") {
+    val formatter = DateFormatter("HH", UTC)
+    val daysSinceEpoch = formatter.parse("20")
+    assert(daysSinceEpoch === days(1970, 1, 1))
+  }
+
+  test("missing year field with invalid date") {
+    val formatter = DateFormatter("MM-dd", UTC)
+    // The date parser in 2.4 accepts 1970-02-29 and turn it into 1970-03-01, so we should get a
+    // SparkUpgradeException here.
+    intercept[SparkUpgradeException](formatter.parse("02-29"))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
new file mode 100644
index 0000000000000..0b15e49af0210
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._
+
+class DateTimeFormatterHelperSuite extends SparkFunSuite {
+
+  private def convertIncompatiblePattern(pattern: String): String = {
+    DateTimeFormatterHelper.convertIncompatiblePattern(pattern, isParsing = false)
+  }
+
+  test("check incompatible pattern") {
+    assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz")
+      === "uuuu-MM-dd'T'HH:mm:ss.SSSz")
+    assert(convertIncompatiblePattern("yyyy-MM'y contains in quoted text'HH:mm:ss")
+      === "uuuu-MM'y contains in quoted text'HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'HH:mm:ss")
+      === "uuuu-MM'u contains in quoted text'HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'''''HH:mm:ss")
+      === "uuuu-MM'u contains in quoted text'''''HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz G")
+      === "yyyy-MM-dd'T'HH:mm:ss.SSSz G")
+    weekBasedLetters.foreach { l =>
+      val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G"))
+      assert(e.getMessage.contains("week-based"))
+    }
+    unsupportedLetters.foreach { l =>
+      val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G"))
+      assert(e.getMessage === s"Illegal pattern character: $l")
+    }
+    unsupportedLettersForParsing.foreach { l =>
+      val e = intercept[IllegalArgumentException] {
+        DateTimeFormatterHelper.convertIncompatiblePattern(s"$l", isParsing = true)
+      }
+      assert(e.getMessage === s"Illegal pattern character: $l")
+      assert(convertIncompatiblePattern(s"$l").nonEmpty)
+    }
+    unsupportedPatternLengths.foreach { style =>
+      val e1 = intercept[IllegalArgumentException] {
+        convertIncompatiblePattern(s"yyyy-MM-dd $style")
+      }
+      assert(e1.getMessage === s"Too many pattern letters: ${style.head}")
+      val e2 = intercept[IllegalArgumentException] {
+        convertIncompatiblePattern(s"yyyy-MM-dd $style${style.head}")
+      }
+      assert(e2.getMessage === s"Too many pattern letters: ${style.head}")
+    }
+    assert(convertIncompatiblePattern("yyyy-MM-dd EEEE") === "uuuu-MM-dd EEEE")
+    assert(convertIncompatiblePattern("yyyy-MM-dd'e'HH:mm:ss") === "uuuu-MM-dd'e'HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM-dd'T'") === "uuuu-MM-dd'T'")
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
index 4dfeb85c74f93..66aef1b4b6cb0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
@@ -17,43 +17,55 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId}
+import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId, ZoneOffset}
 import java.util.TimeZone
 import java.util.concurrent.TimeUnit
 
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimeZoneUTC
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.getZoneId
 
 /**
  * Helper functions for testing date and time functionality.
  */
 object DateTimeTestUtils {
 
-  val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone)
+  val CEST = getZoneId("+02:00")
+  val CET = getZoneId("+01:00")
+  val JST = getZoneId("+09:00")
+  val LA = getZoneId("America/Los_Angeles")
+  val MIT = getZoneId("-09:30")
+  val PST = getZoneId("-08:00")
+  val UTC = getZoneId("+00:00")
+
+  val UTC_OPT = Option("UTC")
+
+  val ALL_TIMEZONES: Seq[ZoneId] = ZoneId.getAvailableZoneIds.asScala.map(getZoneId).toSeq
 
   val outstandingTimezonesIds: Seq[String] = Seq(
     "UTC",
-    "PST",
-    "CET",
+    PST.getId,
+    CET.getId,
     "Africa/Dakar",
-    "America/Los_Angeles",
+    LA.getId,
     "Antarctica/Vostok",
     "Asia/Hong_Kong",
     "Europe/Amsterdam")
-  val outstandingTimezones: Seq[TimeZone] = outstandingTimezonesIds.map(TimeZone.getTimeZone)
-  val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(DateTimeUtils.getZoneId)
+  val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(getZoneId)
 
-  def withDefaultTimeZone[T](newDefaultTimeZone: TimeZone)(block: => T): T = {
-    val originalDefaultTimeZone = TimeZone.getDefault
+  def withDefaultTimeZone[T](newDefaultTimeZone: ZoneId)(block: => T): T = {
+    val originalDefaultTimeZone = ZoneId.systemDefault()
     try {
-      TimeZone.setDefault(newDefaultTimeZone)
+      TimeZone.setDefault(TimeZone.getTimeZone(newDefaultTimeZone))
       block
     } finally {
-      TimeZone.setDefault(originalDefaultTimeZone)
+      TimeZone.setDefault(TimeZone.getTimeZone(originalDefaultTimeZone))
     }
   }
 
-  def localDateTimeToMicros(localDateTime: LocalDateTime, tz: TimeZone): Long = {
-    val instant = localDateTime.atZone(tz.toZoneId).toInstant
+  def localDateTimeToMicros(localDateTime: LocalDateTime, zoneId: ZoneId): Long = {
+    val instant = localDateTime.atZone(zoneId).toInstant
     DateTimeUtils.instantToMicros(instant)
   }
 
@@ -66,22 +78,18 @@ object DateTimeTestUtils {
       minute: Byte = 0,
       sec: Byte = 0,
       micros: Int = 0,
-      tz: TimeZone = TimeZoneUTC): Long = {
+      zid: ZoneId = ZoneOffset.UTC): Long = {
     val nanos = TimeUnit.MICROSECONDS.toNanos(micros).toInt
     val localDateTime = LocalDateTime.of(year, month, day, hour, minute, sec, nanos)
-    localDateTimeToMicros(localDateTime, tz)
+    localDateTimeToMicros(localDateTime, zid)
   }
 
   // Returns number of days since epoch for the given date
   def days(
       year: Int,
       month: Byte = 1,
-      day: Byte = 1,
-      hour: Byte = 0,
-      minute: Byte = 0,
-      sec: Byte = 0): Int = {
-    val micros = date(year, month, day, hour, minute, sec)
-    TimeUnit.MICROSECONDS.toDays(micros).toInt
+      day: Byte = 1): Int = {
+    LocalDate.of(year, month, day).toEpochDay.toInt
   }
 
   // Returns microseconds since epoch for current date and give time
@@ -90,11 +98,18 @@ object DateTimeTestUtils {
       minute: Byte = 0,
       sec: Byte = 0,
       micros: Int = 0,
-      tz: TimeZone = TimeZoneUTC): Long = {
+      zid: ZoneId = ZoneOffset.UTC): Long = {
     val nanos = TimeUnit.MICROSECONDS.toNanos(micros).toInt
-    val localDate = LocalDate.now(tz.toZoneId)
+    val localDate = LocalDate.now(zid)
     val localTime = LocalTime.of(hour, minute, sec, nanos)
     val localDateTime = LocalDateTime.of(localDate, localTime)
-    localDateTimeToMicros(localDateTime, tz)
+    localDateTimeToMicros(localDateTime, zid)
+  }
+
+  def secFrac(seconds: Int, milliseconds: Int, microseconds: Int): Long = {
+    var result: Long = microseconds
+    result = Math.addExact(result, Math.multiplyExact(milliseconds, MICROS_PER_MILLIS))
+    result = Math.addExact(result, Math.multiplyExact(seconds, MICROS_PER_SECOND))
+    result
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index cabcd3007d1c0..f92d65c937928 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -19,8 +19,8 @@ package org.apache.spark.sql.catalyst.util
 
 import java.sql.{Date, Timestamp}
 import java.text.SimpleDateFormat
-import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneId, ZoneOffset}
-import java.util.{Locale, TimeZone}
+import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneId}
+import java.util.Locale
 import java.util.concurrent.TimeUnit
 
 import org.scalatest.Matchers
@@ -30,11 +30,10 @@ import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
-import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
 class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
 
-  val TimeZonePST = TimeZone.getTimeZone("PST")
   private def defaultZoneId = ZoneId.systemDefault()
 
   test("nanoseconds truncation") {
@@ -122,7 +121,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     checkFromToJavaDate(new Date(df2.parse("1776-07-04 18:30:00 UTC").getTime))
   }
 
-  private def toDate(s: String, zoneId: ZoneId = ZoneOffset.UTC): Option[SQLDate] = {
+  private def toDate(s: String, zoneId: ZoneId = UTC): Option[SQLDate] = {
     stringToDate(UTF8String.fromString(s), zoneId)
   }
 
@@ -155,91 +154,101 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("string to timestamp") {
-    for (tz <- ALL_TIMEZONES) {
+    for (zid <- ALL_TIMEZONES) {
       def checkStringToTimestamp(str: String, expected: Option[Long]): Unit = {
-        assert(toTimestamp(str, tz.toZoneId) === expected)
+        assert(toTimestamp(str, zid) === expected)
       }
 
-      checkStringToTimestamp("1969-12-31 16:00:00", Option(date(1969, 12, 31, 16, tz = tz)))
-      checkStringToTimestamp("0001", Option(date(1, 1, 1, 0, tz = tz)))
-      checkStringToTimestamp("2015-03", Option(date(2015, 3, 1, tz = tz)))
+      checkStringToTimestamp("1969-12-31 16:00:00", Option(date(1969, 12, 31, 16, zid = zid)))
+      checkStringToTimestamp("0001", Option(date(1, 1, 1, 0, zid = zid)))
+      checkStringToTimestamp("2015-03", Option(date(2015, 3, 1, zid = zid)))
       Seq("2015-03-18", "2015-03-18 ", " 2015-03-18", " 2015-03-18 ", "2015-03-18T").foreach { s =>
-        checkStringToTimestamp(s, Option(date(2015, 3, 18, tz = tz)))
+        checkStringToTimestamp(s, Option(date(2015, 3, 18, zid = zid)))
       }
 
-      var expected = Option(date(2015, 3, 18, 12, 3, 17, tz = tz))
+      var expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zid))
       checkStringToTimestamp("2015-03-18 12:03:17", expected)
       checkStringToTimestamp("2015-03-18T12:03:17", expected)
 
       // If the string value includes timezone string, it represents the timestamp string
       // in the timezone regardless of the tz parameter.
-      var timeZone = TimeZone.getTimeZone("GMT-13:53")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, tz = timeZone))
+      var zoneId = getZoneId("-13:53")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17-13:53", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17GMT-13:53", expected)
 
-      timeZone = TimeZone.getTimeZone("UTC")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, tz = timeZone))
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = UTC))
       checkStringToTimestamp("2015-03-18T12:03:17Z", expected)
       checkStringToTimestamp("2015-03-18 12:03:17Z", expected)
+      checkStringToTimestamp("2015-03-18 12:03:17UTC", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT-01:00")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, tz = timeZone))
+      zoneId = getZoneId("-01:00")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17-1:0", expected)
       checkStringToTimestamp("2015-03-18T12:03:17-01:00", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17GMT-01:00", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, tz = timeZone))
+      zoneId = getZoneId("+07:30")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17+07:30", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17 GMT+07:30", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:03")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, tz = timeZone))
+      zoneId = getZoneId("+07:03")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17+07:03", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17GMT+07:03", expected)
 
       // tests for the string including milliseconds.
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, tz = tz))
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zid))
       checkStringToTimestamp("2015-03-18 12:03:17.123", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123", expected)
 
       // If the string value includes timezone string, it represents the timestamp string
       // in the timezone regardless of the tz parameter.
-      timeZone = TimeZone.getTimeZone("UTC")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 456000, tz = timeZone))
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 456000, zid = UTC))
       checkStringToTimestamp("2015-03-18T12:03:17.456Z", expected)
       checkStringToTimestamp("2015-03-18 12:03:17.456Z", expected)
+      checkStringToTimestamp("2015-03-18 12:03:17.456 UTC", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT-01:00")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, tz = timeZone))
+      zoneId = getZoneId("-01:00")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123-1:0", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123-01:00", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123 GMT-01:00", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, tz = timeZone))
+      zoneId = getZoneId("+07:30")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123+07:30", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123 GMT+07:30", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, tz = timeZone))
+      zoneId = getZoneId("+07:30")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123+07:30", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123GMT+07:30", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123121, tz = timeZone))
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123121, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123121+7:30", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123121 GMT+0730", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123120, tz = timeZone))
+      zoneId = getZoneId("+07:30")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123120, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.12312+7:30", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.12312 UT+07:30", expected)
 
-      expected = Option(time(18, 12, 15, tz = tz))
+      expected = Option(time(18, 12, 15, zid = zid))
       checkStringToTimestamp("18:12:15", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(time(18, 12, 15, 123120, tz = timeZone))
+      zoneId = getZoneId("+07:30")
+      expected = Option(time(18, 12, 15, 123120, zid = zoneId))
       checkStringToTimestamp("T18:12:15.12312+7:30", expected)
+      checkStringToTimestamp("T18:12:15.12312 UTC+07:30", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(time(18, 12, 15, 123120, tz = timeZone))
+      zoneId = getZoneId("+07:30")
+      expected = Option(time(18, 12, 15, 123120, zid = zoneId))
       checkStringToTimestamp("18:12:15.12312+7:30", expected)
+      checkStringToTimestamp("18:12:15.12312 GMT+07:30", expected)
 
-      expected = Option(date(2011, 5, 6, 7, 8, 9, 100000, tz = tz))
+      expected = Option(date(2011, 5, 6, 7, 8, 9, 100000, zid = zid))
       checkStringToTimestamp("2011-05-06 07:08:09.1000", expected)
 
       checkStringToTimestamp("238", None)
@@ -261,10 +270,14 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       checkStringToTimestamp("1999 08", None)
 
       // Truncating the fractional seconds
-      timeZone = TimeZone.getTimeZone("GMT+00:00")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123456, tz = timeZone))
-      checkStringToTimestamp(
-        "2015-03-18T12:03:17.123456789+0:00", expected)
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123456, zid = UTC))
+      checkStringToTimestamp("2015-03-18T12:03:17.123456789+0:00", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123456789 UTC+0", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123456789GMT+00:00", expected)
+
+      zoneId = getZoneId("Europe/Moscow")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123456, zid = zoneId))
+      checkStringToTimestamp("2015-03-18T12:03:17.123456 Europe/Moscow", expected)
     }
   }
 
@@ -286,32 +299,38 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("hours") {
-    var input = date(2015, 3, 18, 13, 2, 11, 0, TimeZonePST)
-    assert(getHours(input, TimeZonePST) === 13)
-    assert(getHours(input, TimeZoneGMT) === 20)
-    input = date(2015, 12, 8, 2, 7, 9, 0, TimeZonePST)
-    assert(getHours(input, TimeZonePST) === 2)
-    assert(getHours(input, TimeZoneGMT) === 10)
+    var input = date(2015, 3, 18, 13, 2, 11, 0, LA)
+    assert(getHours(input, LA) === 13)
+    assert(getHours(input, UTC) === 20)
+    input = date(2015, 12, 8, 2, 7, 9, 0, LA)
+    assert(getHours(input, LA) === 2)
+    assert(getHours(input, UTC) === 10)
+    input = date(10, 1, 1, 0, 0, 0, 0, LA)
+    assert(getHours(input, LA) === 0)
   }
 
   test("minutes") {
-    var input = date(2015, 3, 18, 13, 2, 11, 0, TimeZonePST)
-    assert(getMinutes(input, TimeZonePST) === 2)
-    assert(getMinutes(input, TimeZoneGMT) === 2)
-    assert(getMinutes(input, TimeZone.getTimeZone("Australia/North")) === 32)
-    input = date(2015, 3, 8, 2, 7, 9, 0, TimeZonePST)
-    assert(getMinutes(input, TimeZonePST) === 7)
-    assert(getMinutes(input, TimeZoneGMT) === 7)
-    assert(getMinutes(input, TimeZone.getTimeZone("Australia/North")) === 37)
+    var input = date(2015, 3, 18, 13, 2, 11, 0, LA)
+    assert(getMinutes(input, LA) === 2)
+    assert(getMinutes(input, UTC) === 2)
+    assert(getMinutes(input, getZoneId("Australia/North")) === 32)
+    input = date(2015, 3, 8, 2, 7, 9, 0, LA)
+    assert(getMinutes(input, LA) === 7)
+    assert(getMinutes(input, UTC) === 7)
+    assert(getMinutes(input, getZoneId("Australia/North")) === 37)
+    input = date(10, 1, 1, 0, 0, 0, 0, LA)
+    assert(getMinutes(input, LA) === 0)
   }
 
   test("seconds") {
-    var input = date(2015, 3, 18, 13, 2, 11, 0, TimeZonePST)
-    assert(getSeconds(input, TimeZonePST) === 11)
-    assert(getSeconds(input, TimeZoneGMT) === 11)
-    input = date(2015, 3, 8, 2, 7, 9, 0, TimeZonePST)
-    assert(getSeconds(input, TimeZonePST) === 9)
-    assert(getSeconds(input, TimeZoneGMT) === 9)
+    var input = date(2015, 3, 18, 13, 2, 11, 0, LA)
+    assert(getSeconds(input, LA) === 11)
+    assert(getSeconds(input, UTC) === 11)
+    input = date(2015, 3, 8, 2, 7, 9, 0, LA)
+    assert(getSeconds(input, LA) === 9)
+    assert(getSeconds(input, UTC) === 9)
+    input = date(10, 1, 1, 0, 0, 0, 0, LA)
+    assert(getSeconds(input, LA) === 0)
   }
 
   test("hours / minutes / seconds") {
@@ -367,68 +386,76 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("date add months") {
-    val input = days(1997, 2, 28, 10, 30)
+    val input = days(1997, 2, 28)
     assert(dateAddMonths(input, 36) === days(2000, 2, 28))
     assert(dateAddMonths(input, -13) === days(1996, 1, 28))
   }
 
+  test("date add interval with day precision") {
+    val input = days(1997, 2, 28)
+    assert(dateAddInterval(input, new CalendarInterval(36, 0, 0)) === days(2000, 2, 28))
+    assert(dateAddInterval(input, new CalendarInterval(36, 47, 0)) === days(2000, 4, 15))
+    assert(dateAddInterval(input, new CalendarInterval(-13, 0, 0)) === days(1996, 1, 28))
+    intercept[IllegalArgumentException](dateAddInterval(input, new CalendarInterval(36, 47, 1)))
+  }
+
   test("timestamp add months") {
     val ts1 = date(1997, 2, 28, 10, 30, 0)
     val ts2 = date(2000, 2, 28, 10, 30, 0, 123000)
     assert(timestampAddInterval(ts1, 36, 0, 123000, defaultZoneId) === ts2)
 
-    val ts3 = date(1997, 2, 27, 16, 0, 0, 0, TimeZonePST)
-    val ts4 = date(2000, 2, 27, 16, 0, 0, 123000, TimeZonePST)
-    val ts5 = date(2000, 2, 28, 0, 0, 0, 123000, TimeZoneGMT)
-    assert(timestampAddInterval(ts3, 36, 0, 123000, TimeZonePST.toZoneId) === ts4)
-    assert(timestampAddInterval(ts3, 36, 0, 123000, TimeZoneGMT.toZoneId) === ts5)
+    val ts3 = date(1997, 2, 27, 16, 0, 0, 0, LA)
+    val ts4 = date(2000, 2, 27, 16, 0, 0, 123000, LA)
+    val ts5 = date(2000, 2, 28, 0, 0, 0, 123000, UTC)
+    assert(timestampAddInterval(ts3, 36, 0, 123000, LA) === ts4)
+    assert(timestampAddInterval(ts3, 36, 0, 123000, UTC) === ts5)
   }
 
   test("timestamp add days") {
     // 2019-3-9 is the end of Pacific Standard Time
-    val ts1 = date(2019, 3, 9, 12, 0, 0, 123000, TimeZonePST)
+    val ts1 = date(2019, 3, 9, 12, 0, 0, 123000, LA)
     // 2019-3-10 is the start of Pacific Daylight Time
-    val ts2 = date(2019, 3, 10, 12, 0, 0, 123000, TimeZonePST)
-    val ts3 = date(2019, 5, 9, 12, 0, 0, 123000, TimeZonePST)
-    val ts4 = date(2019, 5, 10, 12, 0, 0, 123000, TimeZonePST)
+    val ts2 = date(2019, 3, 10, 12, 0, 0, 123000, LA)
+    val ts3 = date(2019, 5, 9, 12, 0, 0, 123000, LA)
+    val ts4 = date(2019, 5, 10, 12, 0, 0, 123000, LA)
     // 2019-11-2 is the end of Pacific Daylight Time
-    val ts5 = date(2019, 11, 2, 12, 0, 0, 123000, TimeZonePST)
+    val ts5 = date(2019, 11, 2, 12, 0, 0, 123000, LA)
     // 2019-11-3 is the start of Pacific Standard Time
-    val ts6 = date(2019, 11, 3, 12, 0, 0, 123000, TimeZonePST)
+    val ts6 = date(2019, 11, 3, 12, 0, 0, 123000, LA)
 
     // transit from Pacific Standard Time to Pacific Daylight Time
     assert(timestampAddInterval(
-      ts1, 0, 0, 23 * MICROS_PER_HOUR, TimeZonePST.toZoneId) === ts2)
-    assert(timestampAddInterval(ts1, 0, 1, 0, TimeZonePST.toZoneId) === ts2)
+      ts1, 0, 0, 23 * MICROS_PER_HOUR, LA) === ts2)
+    assert(timestampAddInterval(ts1, 0, 1, 0, LA) === ts2)
     // just a normal day
     assert(timestampAddInterval(
-      ts3, 0, 0, 24 * MICROS_PER_HOUR, TimeZonePST.toZoneId) === ts4)
-    assert(timestampAddInterval(ts3, 0, 1, 0, TimeZonePST.toZoneId) === ts4)
+      ts3, 0, 0, 24 * MICROS_PER_HOUR, LA) === ts4)
+    assert(timestampAddInterval(ts3, 0, 1, 0, LA) === ts4)
     // transit from Pacific Daylight Time to Pacific Standard Time
     assert(timestampAddInterval(
-      ts5, 0, 0, 25 * MICROS_PER_HOUR, TimeZonePST.toZoneId) === ts6)
-    assert(timestampAddInterval(ts5, 0, 1, 0, TimeZonePST.toZoneId) === ts6)
+      ts5, 0, 0, 25 * MICROS_PER_HOUR, LA) === ts6)
+    assert(timestampAddInterval(ts5, 0, 1, 0, LA) === ts6)
   }
 
   test("monthsBetween") {
     val date1 = date(1997, 2, 28, 10, 30, 0)
     var date2 = date(1996, 10, 30)
-    assert(monthsBetween(date1, date2, true, TimeZoneUTC) === 3.94959677)
-    assert(monthsBetween(date1, date2, false, TimeZoneUTC) === 3.9495967741935485)
+    assert(monthsBetween(date1, date2, true, UTC) === 3.94959677)
+    assert(monthsBetween(date1, date2, false, UTC) === 3.9495967741935485)
     Seq(true, false).foreach { roundOff =>
       date2 = date(2000, 2, 28)
-      assert(monthsBetween(date1, date2, roundOff, TimeZoneUTC) === -36)
+      assert(monthsBetween(date1, date2, roundOff, UTC) === -36)
       date2 = date(2000, 2, 29)
-      assert(monthsBetween(date1, date2, roundOff, TimeZoneUTC) === -36)
+      assert(monthsBetween(date1, date2, roundOff, UTC) === -36)
       date2 = date(1996, 3, 31)
-      assert(monthsBetween(date1, date2, roundOff, TimeZoneUTC) === 11)
+      assert(monthsBetween(date1, date2, roundOff, UTC) === 11)
     }
 
-    val date3 = date(2000, 2, 28, 16, tz = TimeZonePST)
-    val date4 = date(1997, 2, 28, 16, tz = TimeZonePST)
-    assert(monthsBetween(date3, date4, true, TimeZonePST) === 36.0)
-    assert(monthsBetween(date3, date4, true, TimeZoneGMT) === 35.90322581)
-    assert(monthsBetween(date3, date4, false, TimeZoneGMT) === 35.903225806451616)
+    val date3 = date(2000, 2, 28, 16, zid = LA)
+    val date4 = date(1997, 2, 28, 16, zid = LA)
+    assert(monthsBetween(date3, date4, true, LA) === 36.0)
+    assert(monthsBetween(date3, date4, true, UTC) === 35.90322581)
+    assert(monthsBetween(date3, date4, false, UTC) === 35.903225806451616)
   }
 
   test("from UTC timestamp") {
@@ -439,19 +466,19 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     for (tz <- ALL_TIMEZONES) {
       withDefaultTimeZone(tz) {
         test("2011-12-25 09:00:00.123456", "UTC", "2011-12-25 09:00:00.123456")
-        test("2011-12-25 09:00:00.123456", "JST", "2011-12-25 18:00:00.123456")
-        test("2011-12-25 09:00:00.123456", "PST", "2011-12-25 01:00:00.123456")
+        test("2011-12-25 09:00:00.123456", JST.getId, "2011-12-25 18:00:00.123456")
+        test("2011-12-25 09:00:00.123456", LA.getId, "2011-12-25 01:00:00.123456")
         test("2011-12-25 09:00:00.123456", "Asia/Shanghai", "2011-12-25 17:00:00.123456")
       }
     }
 
-    withDefaultTimeZone(TimeZone.getTimeZone("PST")) {
+    withDefaultTimeZone(LA) {
       // Daylight Saving Time
-      test("2016-03-13 09:59:59.0", "PST", "2016-03-13 01:59:59.0")
-      test("2016-03-13 10:00:00.0", "PST", "2016-03-13 03:00:00.0")
-      test("2016-11-06 08:59:59.0", "PST", "2016-11-06 01:59:59.0")
-      test("2016-11-06 09:00:00.0", "PST", "2016-11-06 01:00:00.0")
-      test("2016-11-06 10:00:00.0", "PST", "2016-11-06 02:00:00.0")
+      test("2016-03-13 09:59:59.0", LA.getId, "2016-03-13 01:59:59.0")
+      test("2016-03-13 10:00:00.0", LA.getId, "2016-03-13 03:00:00.0")
+      test("2016-11-06 08:59:59.0", LA.getId, "2016-11-06 01:59:59.0")
+      test("2016-11-06 09:00:00.0", LA.getId, "2016-11-06 01:00:00.0")
+      test("2016-11-06 10:00:00.0", LA.getId, "2016-11-06 02:00:00.0")
     }
   }
 
@@ -461,26 +488,25 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
         === expected)
     }
 
-    for (tz <- ALL_TIMEZONES) {
-      withDefaultTimeZone(tz) {
+    for (zid <- ALL_TIMEZONES) {
+      withDefaultTimeZone(zid) {
         test("2011-12-25 09:00:00.123456", "UTC", "2011-12-25 09:00:00.123456")
-        test("2011-12-25 18:00:00.123456", "JST", "2011-12-25 09:00:00.123456")
-        test("2011-12-25 01:00:00.123456", "PST", "2011-12-25 09:00:00.123456")
+        test("2011-12-25 18:00:00.123456", JST.getId, "2011-12-25 09:00:00.123456")
+        test("2011-12-25 01:00:00.123456", LA.getId, "2011-12-25 09:00:00.123456")
         test("2011-12-25 17:00:00.123456", "Asia/Shanghai", "2011-12-25 09:00:00.123456")
       }
     }
 
-    withDefaultTimeZone(TimeZone.getTimeZone("PST")) {
+    withDefaultTimeZone(LA) {
+      val tz = LA.getId
       // Daylight Saving Time
-      test("2016-03-13 01:59:59", "PST", "2016-03-13 09:59:59.0")
-      // 2016-03-13 02:00:00 PST does not exists
-      test("2016-03-13 02:00:00", "PST", "2016-03-13 10:00:00.0")
-      test("2016-03-13 03:00:00", "PST", "2016-03-13 10:00:00.0")
-      test("2016-11-06 00:59:59", "PST", "2016-11-06 07:59:59.0")
-      // 2016-11-06 01:00:00 PST could be 2016-11-06 08:00:00 UTC or 2016-11-06 09:00:00 UTC
-      test("2016-11-06 01:00:00", "PST", "2016-11-06 09:00:00.0")
-      test("2016-11-06 01:59:59", "PST", "2016-11-06 09:59:59.0")
-      test("2016-11-06 02:00:00", "PST", "2016-11-06 10:00:00.0")
+      test("2016-03-13 01:59:59", tz, "2016-03-13 09:59:59.0")
+      test("2016-03-13 02:00:00", tz, "2016-03-13 10:00:00.0")
+      test("2016-03-13 03:00:00", tz, "2016-03-13 10:00:00.0")
+      test("2016-11-06 00:59:59", tz, "2016-11-06 07:59:59.0")
+      test("2016-11-06 01:00:00", tz, "2016-11-06 09:00:00.0")
+      test("2016-11-06 01:59:59", tz, "2016-11-06 09:59:59.0")
+      test("2016-11-06 02:00:00", tz, "2016-11-06 10:00:00.0")
     }
   }
 
@@ -490,18 +516,32 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     assert(time == None)
   }
 
-  test("truncTimestamp") {
-    def testTrunc(
-        level: Int,
-        expected: String,
-        inputTS: SQLTimestamp,
-        timezone: TimeZone = DateTimeUtils.defaultTimeZone()): Unit = {
-      val truncated =
-        DateTimeUtils.truncTimestamp(inputTS, level, timezone)
-      val expectedTS = toTimestamp(expected, defaultZoneId)
-      assert(truncated === expectedTS.get)
+  def testTrunc(
+      level: Int,
+      expected: String,
+      inputTS: Long,
+      zoneId: ZoneId = defaultZoneId): Unit = {
+    val truncated = DateTimeUtils.truncTimestamp(inputTS, level, zoneId)
+    val expectedTS = toTimestamp(expected, defaultZoneId)
+    assert(truncated === expectedTS.get)
+  }
+
+  test("SPARK-33404: test truncTimestamp when time zone offset from UTC has a " +
+    "granularity of seconds") {
+    for (zid <- ALL_TIMEZONES) {
+      withDefaultTimeZone(zid) {
+        val inputTS = DateTimeUtils.stringToTimestamp(
+          UTF8String.fromString("1769-10-17T17:10:02.123456"), defaultZoneId)
+        testTrunc(DateTimeUtils.TRUNC_TO_MINUTE, "1769-10-17T17:10:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_SECOND, "1769-10-17T17:10:02", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MILLISECOND, "1769-10-17T17:10:02.123", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MICROSECOND, "1769-10-17T17:10:02.123456",
+          inputTS.get, zid)
+      }
     }
+  }
 
+  test("truncTimestamp") {
     val defaultInputTS = DateTimeUtils.stringToTimestamp(
       UTF8String.fromString("2015-03-05T09:32:05.359123"), defaultZoneId)
     val defaultInputTS1 = DateTimeUtils.stringToTimestamp(
@@ -529,12 +569,9 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-04-01T00:00:00", defaultInputTS2.get)
     testTrunc(DateTimeUtils.TRUNC_TO_MICROSECOND, "2015-03-05T09:32:05.359123", defaultInputTS.get)
     testTrunc(DateTimeUtils.TRUNC_TO_MILLISECOND, "2015-03-05T09:32:05.359", defaultInputTS.get)
-    testTrunc(DateTimeUtils.TRUNC_TO_DECADE, "2010-01-01", defaultInputTS.get)
-    testTrunc(DateTimeUtils.TRUNC_TO_CENTURY, "2001-01-01", defaultInputTS.get)
-    testTrunc(DateTimeUtils.TRUNC_TO_MILLENNIUM, "2001-01-01", defaultInputTS.get)
 
-    for (tz <- ALL_TIMEZONES) {
-      withDefaultTimeZone(tz) {
+    for (zid <- ALL_TIMEZONES) {
+      withDefaultTimeZone(zid) {
         val inputTS = DateTimeUtils.stringToTimestamp(
           UTF8String.fromString("2015-03-05T09:32:05.359"), defaultZoneId)
         val inputTS1 = DateTimeUtils.stringToTimestamp(
@@ -548,38 +585,35 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
         val inputTS5 = DateTimeUtils.stringToTimestamp(
           UTF8String.fromString("1999-03-29T01:02:03.456789"), defaultZoneId)
 
-        testTrunc(DateTimeUtils.TRUNC_TO_YEAR, "2015-01-01T00:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_MONTH, "2015-03-01T00:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_DAY, "2015-03-05T00:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_HOUR, "2015-03-05T09:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_MINUTE, "2015-03-05T09:32:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_SECOND, "2015-03-05T09:32:05", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-02T00:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS1.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS2.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS3.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-23T00:00:00", inputTS4.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-01-01T00:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-01-01T00:00:00", inputTS1.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-04-01T00:00:00", inputTS2.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_DECADE, "1990-01-01", inputTS5.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_CENTURY, "1901-01-01", inputTS5.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_MILLENNIUM, "2001-01-01", inputTS.get, tz)
+        testTrunc(DateTimeUtils.TRUNC_TO_YEAR, "2015-01-01T00:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MONTH, "2015-03-01T00:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_DAY, "2015-03-05T00:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_HOUR, "2015-03-05T09:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MINUTE, "2015-03-05T09:32:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_SECOND, "2015-03-05T09:32:05", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-02T00:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS1.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS2.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS3.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-23T00:00:00", inputTS4.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-01-01T00:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-01-01T00:00:00", inputTS1.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-04-01T00:00:00", inputTS2.get, zid)
       }
     }
   }
 
   test("daysToMillis and millisToDays") {
-    val input = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, 16, tz = TimeZonePST))
-    assert(millisToDays(input, TimeZonePST) === 16800)
-    assert(millisToDays(input, TimeZoneGMT) === 16801)
-    assert(millisToDays(-1 * MILLIS_PER_DAY + 1, TimeZoneGMT) == -1)
+    val input = toMillis(date(2015, 12, 31, 16, zid = LA))
+    assert(millisToDays(input, LA) === 16800)
+    assert(millisToDays(input, UTC) === 16801)
+    assert(millisToDays(-1 * MILLIS_PER_DAY + 1, UTC) == -1)
 
-    var expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, tz = TimeZonePST))
-    assert(daysToMillis(16800, TimeZonePST) === expected)
+    var expected = toMillis(date(2015, 12, 31, zid = LA))
+    assert(daysToMillis(16800, LA) === expected)
 
-    expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, tz = TimeZoneGMT))
-    assert(daysToMillis(16800, TimeZoneGMT) === expected)
+    expected = toMillis(date(2015, 12, 31, zid = UTC))
+    assert(daysToMillis(16800, UTC) === expected)
 
     // There are some days are skipped entirely in some timezone, skip them here.
     val skipped_days = Map[String, Set[Int]](
@@ -589,13 +623,15 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       "Pacific/Fakaofo" -> Set(15338),
       "Pacific/Kiritimati" -> Set(9130, 9131),
       "Pacific/Kwajalein" -> Set(8632, 8633, 8634),
-      "MIT" -> Set(15338))
-    for (tz <- ALL_TIMEZONES) {
-      val skipped = skipped_days.getOrElse(tz.getID, Set.empty)
-      (-20000 to 20000).foreach { d =>
+      MIT.getId -> Set(15338))
+    for (zid <- ALL_TIMEZONES) {
+      val skipped = skipped_days.getOrElse(zid.getId, Set.empty)
+      val testingData = Seq(-20000, 20000) ++
+        (1 to 1000).map(_ => (math.random() * 40000 - 20000).toInt)
+      testingData.foreach { d =>
         if (!skipped.contains(d)) {
-          assert(millisToDays(daysToMillis(d, tz), tz) === d,
-            s"Round trip of ${d} did not work in tz ${tz}")
+          assert(millisToDays(daysToMillis(d, zid), zid) === d,
+            s"Round trip of $d did not work in tz ${zid.getId}")
         }
       }
     }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
new file mode 100644
index 0000000000000..b78facd963338
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.time.DateTimeException
+
+import org.scalatest.Matchers
+
+import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{date, UTC}
+
+trait DatetimeFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
+  import DateTimeFormatterHelper._
+  import LegacyDateFormats._
+  def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit
+
+  private def dateFormatter(
+      pattern: String, ldf: LegacyDateFormat = FAST_DATE_FORMAT): DateFormatter = {
+    DateFormatter(pattern, UTC, DateFormatter.defaultLocale, ldf, isParsing = true)
+  }
+
+  private def timestampFormatter(
+      pattern: String, ldf: LegacyDateFormat = SIMPLE_DATE_FORMAT): TimestampFormatter = {
+    TimestampFormatter(pattern, UTC, legacyFormat = ldf, isParsing = true)
+  }
+
+  protected def useDateFormatter: Boolean
+
+  private def assertEqual(pattern: String, datetimeStr: String, expected: Long): Unit = {
+    if (useDateFormatter) {
+      assert(dateFormatter(pattern).parse(datetimeStr) ===
+        DateTimeUtils.microsToEpochDays(expected, UTC))
+    } else {
+      assert(timestampFormatter(pattern).parse(datetimeStr) === expected)
+    }
+  }
+
+  private def assertError(pattern: String, datetimeStr: String, expectedMsg: String): Unit = {
+    if (useDateFormatter) {
+      LegacyDateFormats.values.foreach { ldf =>
+        // The legacy DateFormatter is always lenient by default
+        val e = intercept[SparkUpgradeException](dateFormatter(pattern, ldf).parse(datetimeStr))
+        assert(e.getCause.getMessage.contains(expectedMsg))
+      }
+    } else {
+      // In strict mode, the legacy TimestampFormatter fails too
+      val e = intercept[DateTimeException](timestampFormatter(pattern).parse(datetimeStr))
+      assert(e.getMessage.contains(expectedMsg))
+      // In lenient mode, the legacy TimestampFormatter does not fail
+      Seq(FAST_DATE_FORMAT, LENIENT_SIMPLE_DATE_FORMAT).foreach { ldf =>
+        val e = intercept[SparkUpgradeException] {
+          timestampFormatter(pattern, ldf).parse(datetimeStr)
+        }
+        assert(e.getCause.getMessage.contains(expectedMsg))
+      }
+    }
+  }
+
+  test("explicitly forbidden datetime patterns") {
+
+    Seq(true, false).foreach { isParsing =>
+      // not support by the legacy one too
+      val unsupportedBoth = Seq("QQQQQ", "qqqqq", "eeeee", "A", "c", "n", "N", "p", "e")
+      unsupportedBoth.foreach { pattern =>
+        intercept[IllegalArgumentException](checkFormatterCreation(pattern, isParsing))
+      }
+      // supported by the legacy one, then we will suggest users with SparkUpgradeException
+      ((weekBasedLetters ++ unsupportedLetters).map(_.toString)
+        ++ unsupportedPatternLengths -- unsupportedBoth).foreach {
+        pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, isParsing))
+      }
+    }
+
+    // not support by the legacy one too
+    val unsupportedBoth = Seq("q", "Q")
+    unsupportedBoth.foreach { pattern =>
+      intercept[IllegalArgumentException](checkFormatterCreation(pattern, true))
+    }
+    // supported by the legacy one, then we will suggest users with SparkUpgradeException
+    (unsupportedLettersForParsing.map(_.toString) -- unsupportedBoth).foreach {
+      pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, true))
+    }
+  }
+
+  test("SPARK-31939: Fix Parsing day of year when year field pattern is missing") {
+    // resolved to queryable LocaleDate or fail directly
+    assertEqual("yyyy-dd-DD", "2020-29-60", date(2020, 2, 29))
+    assertError("yyyy-dd-DD", "2020-02-60",
+      "Field DayOfMonth 29 differs from DayOfMonth 2 derived from 2020-02-29")
+    assertEqual("yyyy-MM-DD", "2020-02-60", date(2020, 2, 29))
+    assertError("yyyy-MM-DD", "2020-03-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 2020-02-29")
+    assertEqual("yyyy-MM-dd-DD", "2020-02-29-60", date(2020, 2, 29))
+    assertError("yyyy-MM-dd-DD", "2020-03-01-60",
+      "Field DayOfYear 61 differs from DayOfYear 60 derived from 2020-03-01")
+    assertEqual("yyyy-DDD", "2020-366", date(2020, 12, 31))
+    assertError("yyyy-DDD", "2019-366",
+      "Invalid date 'DayOfYear 366' as '2019' is not a leap year")
+
+    // unresolved and need to check manually(SPARK-31939 fixed)
+    assertEqual("DDD", "365", date(1970, 12, 31))
+    assertError("DDD", "366",
+      "Invalid date 'DayOfYear 366' as '1970' is not a leap year")
+    assertEqual("MM-DD", "03-60", date(1970, 3))
+    assertError("MM-DD", "02-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 1970-03-01")
+    assertEqual("MM-dd-DD", "02-28-59", date(1970, 2, 28))
+    assertError("MM-dd-DD", "02-28-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 1970-03-01")
+    assertError("MM-dd-DD", "02-28-58",
+      "Field DayOfMonth 28 differs from DayOfMonth 27 derived from 1970-02-27")
+    assertEqual("dd-DD", "28-59", date(1970, 2, 28))
+    assertError("dd-DD", "27-59",
+      "Field DayOfMonth 27 differs from DayOfMonth 28 derived from 1970-02-28")
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index 514804cbda16c..1858378edc894 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.fromMillis
 import org.apache.spark.sql.catalyst.util.IntervalUtils._
 import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit._
 import org.apache.spark.sql.internal.SQLConf
@@ -34,27 +35,17 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     assert(safeStringToInterval(UTF8String.fromString(input)) === expected)
   }
 
-  private def checkFromStringWithFunc(
-      input: String,
-      months: Int,
-      days: Int,
-      us: Long,
-      func: CalendarInterval => CalendarInterval): Unit = {
-    val expected = new CalendarInterval(months, days, us)
-    assert(func(stringToInterval(UTF8String.fromString(input))) === expected)
-    assert(func(safeStringToInterval(UTF8String.fromString(input))) === expected)
+  private def checkFromInvalidString(input: String, errorMsg: String): Unit = {
+    failFuncWithInvalidInput(input, errorMsg, s => stringToInterval(UTF8String.fromString(s)))
+    assert(safeStringToInterval(UTF8String.fromString(input)) === null)
   }
 
-  private def checkFromInvalidString(input: String, errorMsg: String): Unit = {
-    try {
-      stringToInterval(UTF8String.fromString(input))
-      fail("Expected to throw an exception for the invalid input")
-    } catch {
-      case e: IllegalArgumentException =>
-        val msg = e.getMessage
-        assert(msg.contains(errorMsg))
+  private def failFuncWithInvalidInput(
+      input: String, errorMsg: String, converter: String => CalendarInterval): Unit = {
+    withClue("Expected to throw an exception for the invalid input") {
+      val e = intercept[IllegalArgumentException](converter(input))
+      assert(e.getMessage.contains(errorMsg))
     }
-    assert(safeStringToInterval(UTF8String.fromString(input)) === null)
   }
 
   private def testSingleUnit(
@@ -76,7 +67,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     testSingleUnit("HouR", 3, 0, 0, 3 * MICROS_PER_HOUR)
     testSingleUnit("MiNuTe", 3, 0, 0, 3 * MICROS_PER_MINUTE)
     testSingleUnit("Second", 3, 0, 0, 3 * MICROS_PER_SECOND)
-    testSingleUnit("MilliSecond", 3, 0, 0, 3 * MICROS_PER_MILLIS)
+    testSingleUnit("MilliSecond", 3, 0, 0, fromMillis(3))
     testSingleUnit("MicroSecond", 3, 0, 0, 3)
 
     checkFromInvalidString(null, "cannot be null")
@@ -86,6 +77,18 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     }
   }
 
+  test("string to interval: interval with dangling parts should not results null") {
+    checkFromInvalidString("+", "expect a number after '+' but hit EOL")
+    checkFromInvalidString("-", "expect a number after '-' but hit EOL")
+    checkFromInvalidString("+ 2", "expect a unit name after '2' but hit EOL")
+    checkFromInvalidString("- 1", "expect a unit name after '1' but hit EOL")
+    checkFromInvalidString("1", "expect a unit name after '1' but hit EOL")
+    checkFromInvalidString("1.2", "expect a unit name after '1.2' but hit EOL")
+    checkFromInvalidString("1 day 2", "expect a unit name after '2' but hit EOL")
+    checkFromInvalidString("1 day 2.2", "expect a unit name after '2.2' but hit EOL")
+    checkFromInvalidString("1 day -", "expect a number after '-' but hit EOL")
+    checkFromInvalidString("-.", "expect a unit name after '-.' but hit EOL")
+  }
 
   test("string to interval: multiple units") {
     Seq(
@@ -144,22 +147,18 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     assert(fromYearMonthString("99-10") === new CalendarInterval(99 * 12 + 10, 0, 0L))
     assert(fromYearMonthString("+99-10") === new CalendarInterval(99 * 12 + 10, 0, 0L))
     assert(fromYearMonthString("-8-10") === new CalendarInterval(-8 * 12 - 10, 0, 0L))
-
-    try {
-      fromYearMonthString("99-15")
-      fail("Expected to throw an exception for the invalid input")
-    } catch {
-      case e: IllegalArgumentException =>
-        assert(e.getMessage.contains("month 15 outside range"))
-    }
-
-    try {
-      fromYearMonthString("9a9-15")
-      fail("Expected to throw an exception for the invalid input")
-    } catch {
-      case e: IllegalArgumentException =>
-        assert(e.getMessage.contains("Interval string does not match year-month format"))
-    }
+    failFuncWithInvalidInput("99-15", "month 15 outside range", fromYearMonthString)
+    failFuncWithInvalidInput("9a9-15", "Interval string does not match year-month format",
+      fromYearMonthString)
+
+    // whitespaces
+    assert(fromYearMonthString("99-10 ") === new CalendarInterval(99 * 12 + 10, 0, 0L))
+    assert(fromYearMonthString("+99-10\t") === new CalendarInterval(99 * 12 + 10, 0, 0L))
+    assert(fromYearMonthString("\t\t-8-10\t") === new CalendarInterval(-8 * 12 - 10, 0, 0L))
+    failFuncWithInvalidInput("99\t-15", "Interval string does not match year-month format",
+      fromYearMonthString)
+    failFuncWithInvalidInput("-\t99-15", "Interval string does not match year-month format",
+      fromYearMonthString)
   }
 
   test("from day-time string - legacy") {
@@ -175,32 +174,13 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
         new CalendarInterval(
           0,
           10,
-          12 * MICROS_PER_MINUTE + 888 * MICROS_PER_MILLIS))
+          12 * MICROS_PER_MINUTE + fromMillis(888)))
       assert(fromDayTimeString("-3 0:0:0") === new CalendarInterval(0, -3, 0L))
 
-      try {
-        fromDayTimeString("5 30:12:20")
-        fail("Expected to throw an exception for the invalid input")
-      } catch {
-        case e: IllegalArgumentException =>
-          assert(e.getMessage.contains("hour 30 outside range"))
-      }
-
-      try {
-        fromDayTimeString("5 30-12")
-        fail("Expected to throw an exception for the invalid input")
-      } catch {
-        case e: IllegalArgumentException =>
-          assert(e.getMessage.contains("must match day-time format"))
-      }
-
-      try {
-        fromDayTimeString("5 1:12:20", HOUR, MICROSECOND)
-        fail("Expected to throw an exception for the invalid convention type")
-      } catch {
-        case e: IllegalArgumentException =>
-          assert(e.getMessage.contains("Cannot support (interval"))
-      }
+      failFuncWithInvalidInput("5 30:12:20", "hour 30 outside range", fromDayTimeString)
+      failFuncWithInvalidInput("5 30-12", "must match day-time format", fromDayTimeString)
+      failFuncWithInvalidInput("5 1:12:20", "Cannot support (interval",
+        s => fromDayTimeString(s, HOUR, MICROSECOND))
     }
   }
 
@@ -214,13 +194,10 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     assert(duration("1 microsecond", TimeUnit.MICROSECONDS, 30) === 1)
     assert(duration("1 month -30 days", TimeUnit.DAYS, 31) === 1)
 
-    try {
+    val e = intercept[ArithmeticException] {
       duration(Integer.MAX_VALUE + " month", TimeUnit.SECONDS, 31)
-      fail("Expected to throw an exception for the invalid input")
-    } catch {
-      case e: ArithmeticException =>
-        assert(e.getMessage.contains("overflow"))
     }
+    assert(e.getMessage.contains("overflow"))
   }
 
   test("negative interval") {
@@ -267,41 +244,49 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("multiply by num") {
-    var interval = new CalendarInterval(0, 0, 0)
-    assert(interval === multiplyExact(interval, 0))
-    interval = new CalendarInterval(123, 456, 789)
-    assert(new CalendarInterval(123 * 42, 456 * 42, 789 * 42) === multiplyExact(interval, 42))
-    interval = new CalendarInterval(-123, -456, -789)
-    assert(new CalendarInterval(-123 * 42, -456 * 42, -789 * 42) === multiplyExact(interval, 42))
-    assert(new CalendarInterval(1, 22, 12 * MICROS_PER_HOUR) ===
-      multiplyExact(new CalendarInterval(1, 5, 0), 1.5))
-    assert(new CalendarInterval(2, 14, 12 * MICROS_PER_HOUR) ===
-      multiplyExact(new CalendarInterval(2, 2, 2 * MICROS_PER_HOUR), 1.2))
-
-    try {
-      multiplyExact(new CalendarInterval(2, 0, 0), Integer.MAX_VALUE)
-      fail("Expected to throw an exception on months overflow")
-    } catch {
-      case e: ArithmeticException => assert(e.getMessage.contains("overflow"))
+    Seq[(CalendarInterval, Double) => CalendarInterval](multiply, multiplyExact).foreach { func =>
+      var interval = new CalendarInterval(0, 0, 0)
+      assert(interval === func(interval, 0))
+      interval = new CalendarInterval(123, 456, 789)
+      assert(new CalendarInterval(123 * 42, 456 * 42, 789 * 42) === func(interval, 42))
+      interval = new CalendarInterval(-123, -456, -789)
+      assert(new CalendarInterval(-123 * 42, -456 * 42, -789 * 42) === func(interval, 42))
+      interval = new CalendarInterval(1, 5, 0)
+      assert(new CalendarInterval(1, 7, 12 * MICROS_PER_HOUR) === func(interval, 1.5))
+      interval = new CalendarInterval(2, 2, 2 * MICROS_PER_HOUR)
+      assert(new CalendarInterval(2, 2, 12 * MICROS_PER_HOUR) === func(interval, 1.2))
     }
+
+    val interval = new CalendarInterval(2, 0, 0)
+    assert(multiply(interval, Integer.MAX_VALUE) === new CalendarInterval(Int.MaxValue, 0, 0))
+
+    val e = intercept[ArithmeticException](multiplyExact(interval, Integer.MAX_VALUE))
+    assert(e.getMessage.contains("overflow"))
   }
 
   test("divide by num") {
-    var interval = new CalendarInterval(0, 0, 0)
-    assert(interval === divideExact(interval, 10))
-    interval = new CalendarInterval(1, 3, 30 * MICROS_PER_SECOND)
-    assert(new CalendarInterval(0, 16, 12 * MICROS_PER_HOUR + 15 * MICROS_PER_SECOND) ===
-      divideExact(interval, 2))
-    assert(new CalendarInterval(2, 6, MICROS_PER_MINUTE) === divideExact(interval, 0.5))
-    interval = new CalendarInterval(-1, 0, -30 * MICROS_PER_SECOND)
-    assert(new CalendarInterval(0, -15, -15 * MICROS_PER_SECOND) === divideExact(interval, 2))
-    assert(new CalendarInterval(-2, 0, -1 * MICROS_PER_MINUTE) === divideExact(interval, 0.5))
-    try {
-      divideExact(new CalendarInterval(123, 456, 789), 0)
-      fail("Expected to throw an exception on divide by zero")
-    } catch {
-      case e: ArithmeticException => assert(e.getMessage.contains("divide by zero"))
+    Seq[(CalendarInterval, Double) => CalendarInterval](divide, divideExact).foreach { func =>
+      var interval = new CalendarInterval(0, 0, 0)
+      assert(interval === func(interval, 10))
+      interval = new CalendarInterval(1, 3, 30 * MICROS_PER_SECOND)
+      assert(new CalendarInterval(0, 1, 12 * MICROS_PER_HOUR + 15 * MICROS_PER_SECOND) ===
+        func(interval, 2))
+      assert(new CalendarInterval(2, 6, MICROS_PER_MINUTE) === func(interval, 0.5))
+      interval = new CalendarInterval(-1, 0, -30 * MICROS_PER_SECOND)
+      assert(new CalendarInterval(0, 0, -15 * MICROS_PER_SECOND) === func(interval, 2))
+      assert(new CalendarInterval(-2, 0, -MICROS_PER_MINUTE) === func(interval, 0.5))
     }
+
+    var interval = new CalendarInterval(Int.MaxValue, Int.MaxValue, 0)
+    assert(divide(interval, 0.9) === new CalendarInterval(Int.MaxValue, Int.MaxValue,
+      ((Int.MaxValue / 9.0) * MICROS_PER_DAY).round))
+    val e1 = intercept[ArithmeticException](divideExact(interval, 0.9))
+    assert(e1.getMessage.contains("integer overflow"))
+
+    interval = new CalendarInterval(123, 456, 789)
+    assert(divide(interval, 0) === null)
+    val e2 = intercept[ArithmeticException](divideExact(interval, 0))
+    assert(e2.getMessage.contains("divide by zero"))
   }
 
   test("from day-time string") {
@@ -311,18 +296,8 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
         assert(fromDayTimeString(input, from, to) === safeStringToInterval(expectedUtf8))
       }
     }
-    def checkFail(
-        input: String,
-        from: IntervalUnit,
-        to: IntervalUnit,
-        errMsg: String): Unit = {
-      try {
-        fromDayTimeString(input, from, to)
-        fail("Expected to throw an exception for the invalid input")
-      } catch {
-        case e: IllegalArgumentException =>
-          assert(e.getMessage.contains(errMsg))
-      }
+    def checkFail(input: String, from: IntervalUnit, to: IntervalUnit, errMsg: String): Unit = {
+      failFuncWithInvalidInput(input, errMsg, s => fromDayTimeString(s, from, to))
     }
 
     check("12:40", HOUR, MINUTE, "12 hours 40 minutes")
@@ -359,6 +334,11 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     checkFail("5 30:12:20", DAY, SECOND, "hour 30 outside range")
     checkFail("5 30-12", DAY, SECOND, "must match day-time format")
     checkFail("5 1:12:20", HOUR, MICROSECOND, "Cannot support (interval")
+
+    // whitespaces
+    check("\t +5 12:40\t ", DAY, MINUTE, "5 days 12 hours 40 minutes")
+    checkFail("+5\t 12:40", DAY, MINUTE, "must match day-time format")
+
   }
 
   test("interval overflow check") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
new file mode 100644
index 0000000000000..971bad1cd957e
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
+import java.util.TimeZone
+
+import org.scalatest.Matchers
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.catalyst.util.RebaseDateTime._
+
+class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
+
+  private def toJulianMicros(ts: Timestamp): Long = {
+    val julianMicros = fromMillis(ts.getTime) +
+      ((ts.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS)
+    julianMicros
+  }
+  private def parseToJulianMicros(s: String): Long = toJulianMicros(Timestamp.valueOf(s))
+
+  private def toGregorianMicros(ldt: LocalDateTime, zoneId: ZoneId): Long = {
+    instantToMicros(ldt.atZone(zoneId).toInstant)
+  }
+  private def parseToGregMicros(s: String, zoneId: ZoneId): Long = {
+    toGregorianMicros(LocalDateTime.parse(s), zoneId)
+  }
+
+  test("rebase julian to/from gregorian micros") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "0001-01-01 01:02:03.654321",
+          "1000-01-01 03:02:01.123456",
+          "1582-10-04 00:00:00.000000",
+          "1582-10-15 00:00:00.999999", // Gregorian cutover day
+          "1883-11-10 00:00:00.000000", // America/Los_Angeles -7:52:58 zone offset
+          "1883-11-20 00:00:00.000000", // America/Los_Angeles -08:00 zone offset
+          "1969-12-31 11:22:33.000100",
+          "1970-01-01 00:00:00.000001", // The epoch day
+          "2020-03-14 09:33:01.500000").foreach { ts =>
+          withClue(s"time zone = ${zid.getId} ts = $ts") {
+            val julianMicros = parseToJulianMicros(ts)
+            val gregMicros = parseToGregMicros(ts.replace(' ', 'T'), zid)
+
+            assert(rebaseJulianToGregorianMicros(julianMicros) === gregMicros)
+            assert(rebaseGregorianToJulianMicros(gregMicros) === julianMicros)
+          }
+        }
+      }
+    }
+  }
+
+  // millisToDays() and fromJavaDate() are taken from Spark 2.4
+  private def millisToDaysLegacy(millisUtc: Long, timeZone: TimeZone): Int = {
+    val millisLocal = millisUtc + timeZone.getOffset(millisUtc)
+    Math.floor(millisLocal.toDouble / MILLIS_PER_DAY).toInt
+  }
+  private def fromJavaDateLegacy(date: Date): Int = {
+    millisToDaysLegacy(date.getTime, defaultTimeZone())
+  }
+
+  test("rebase gregorian to/from julian days") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "0001-01-01",
+          "1000-01-01",
+          "1582-10-04",
+          "1582-10-15", // Gregorian cutover day
+          "1883-11-10", // America/Los_Angeles -7:52:58 zone offset
+          "1883-11-20", // America/Los_Angeles -08:00 zone offset
+          "1969-12-31",
+          "1970-01-01", // The epoch day
+          "2020-03-14").foreach { date =>
+          val julianDays = fromJavaDateLegacy(Date.valueOf(date))
+          val gregorianDays = localDateToDays(LocalDate.parse(date))
+
+          assert(rebaseGregorianToJulianDays(gregorianDays) === julianDays)
+          assert(rebaseJulianToGregorianDays(julianDays) === gregorianDays)
+        }
+      }
+    }
+  }
+
+  test("rebase julian to gregorian date for leap years") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "1000-02-29" -> "1000-03-01",
+          "1600-02-29" -> "1600-02-29",
+          "1700-02-29" -> "1700-03-01",
+          "2000-02-29" -> "2000-02-29").foreach { case (julianDate, gregDate) =>
+          withClue(s"tz = ${zid.getId} julian date = $julianDate greg date = $gregDate") {
+            val date = Date.valueOf(julianDate)
+            val julianDays = fromJavaDateLegacy(date)
+            val gregorianDays = localDateToDays(LocalDate.parse(gregDate))
+
+            assert(rebaseJulianToGregorianDays(julianDays) === gregorianDays)
+          }
+        }
+      }
+    }
+  }
+
+  test("rebase julian to gregorian timestamp for leap years") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "1000-02-29 01:02:03.123456" -> "1000-03-01T01:02:03.123456",
+          "1600-02-29 11:12:13.654321" -> "1600-02-29T11:12:13.654321",
+          "1700-02-29 21:22:23.000001" -> "1700-03-01T21:22:23.000001",
+          "2000-02-29 00:00:00.999999" -> "2000-02-29T00:00:00.999999"
+        ).foreach { case (julianTs, gregTs) =>
+          withClue(s"tz = ${zid.getId} julian ts = $julianTs greg ts = $gregTs") {
+            val julianMicros = parseToJulianMicros(julianTs)
+            val gregorianMicros = parseToGregMicros(gregTs, zid)
+
+            assert(rebaseJulianToGregorianMicros(julianMicros) === gregorianMicros)
+          }
+        }
+      }
+    }
+  }
+
+  test("optimization of days rebasing - Gregorian to Julian") {
+    val start = localDateToDays(LocalDate.of(1, 1, 1))
+    val end = localDateToDays(LocalDate.of(2030, 1, 1))
+
+    var days = start
+    while (days < end) {
+      assert(rebaseGregorianToJulianDays(days) === localRebaseGregorianToJulianDays(days))
+      days += 1
+    }
+  }
+
+  test("optimization of days rebasing - Julian to Gregorian") {
+    val start = rebaseGregorianToJulianDays(
+      localDateToDays(LocalDate.of(1, 1, 1)))
+    val end = rebaseGregorianToJulianDays(
+      localDateToDays(LocalDate.of(2030, 1, 1)))
+
+    var days = start
+    while (days < end) {
+      assert(rebaseJulianToGregorianDays(days) === localRebaseJulianToGregorianDays(days))
+      days += 1
+    }
+  }
+
+  test("SPARK-31328: rebasing overlapped timestamps during daylight saving time") {
+    Seq(
+      LA.getId -> Seq("2019-11-03T08:00:00Z", "2019-11-03T08:30:00Z", "2019-11-03T09:00:00Z"),
+      "Europe/Amsterdam" ->
+        Seq("2019-10-27T00:00:00Z", "2019-10-27T00:30:00Z", "2019-10-27T01:00:00Z")
+    ).foreach { case (tz, ts) =>
+      withDefaultTimeZone(getZoneId(tz)) {
+        ts.foreach { str =>
+          val micros = instantToMicros(Instant.parse(str))
+          assert(rebaseGregorianToJulianMicros(micros) === micros)
+          assert(rebaseJulianToGregorianMicros(micros) === micros)
+        }
+      }
+    }
+  }
+
+  test("validate rebase records in JSON files") {
+    Seq(
+      "gregorian-julian-rebase-micros.json",
+      "julian-gregorian-rebase-micros.json").foreach { json =>
+      withClue(s"JSON file = $json") {
+        val rebaseRecords = loadRebaseRecords(json)
+        rebaseRecords.foreach { case (_, rebaseRecord) =>
+          assert(rebaseRecord.switches.size === rebaseRecord.diffs.size)
+          // Check ascending order of switches values
+          assert(rebaseRecord.switches.toSeq === rebaseRecord.switches.sorted.toSeq)
+        }
+      }
+    }
+  }
+
+  test("optimization of micros rebasing - Gregorian to Julian") {
+    outstandingZoneIds.foreach { zid =>
+      withClue(s"zone id = $zid") {
+        withDefaultTimeZone(zid) {
+          val start = instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0)
+            .atZone(zid)
+            .toInstant)
+          val end = instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0)
+            .atZone(zid)
+            .toInstant)
+          var micros = start
+          do {
+            val rebased = rebaseGregorianToJulianMicros(zid, micros)
+            val rebasedAndOptimized = rebaseGregorianToJulianMicros(micros)
+            assert(rebasedAndOptimized === rebased)
+            micros += (MICROS_PER_DAY * 30 * (0.5 + Math.random())).toLong
+          } while (micros <= end)
+        }
+      }
+    }
+  }
+
+  test("optimization of micros rebasing - Julian to Gregorian") {
+    outstandingZoneIds.foreach { zid =>
+      withClue(s"zone id = $zid") {
+        withDefaultTimeZone(zid) {
+          val start = rebaseGregorianToJulianMicros(
+            instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0).atZone(zid).toInstant))
+          val end = rebaseGregorianToJulianMicros(
+            instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0).atZone(zid).toInstant))
+          var micros = start
+          do {
+            val rebased = rebaseJulianToGregorianMicros(zid, micros)
+            val rebasedAndOptimized = rebaseJulianToGregorianMicros(micros)
+            assert(rebasedAndOptimized === rebased)
+            micros += (MICROS_PER_DAY * 30 * (0.5 + Math.random())).toLong
+          } while (micros <= end)
+        }
+      }
+    }
+  }
+
+  private def generateRebaseJson(
+      adjustFunc: Long => Long,
+      rebaseFunc: (ZoneId, Long) => Long,
+      dir: String,
+      fileName: String): Unit = {
+    import java.nio.file.{Files, Paths}
+    import java.nio.file.StandardOpenOption
+
+    import scala.collection.mutable.ArrayBuffer
+
+    import com.fasterxml.jackson.databind.ObjectMapper
+    import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper}
+
+    case class RebaseRecord(tz: String, switches: Array[Long], diffs: Array[Long])
+
+    val result = new ArrayBuffer[RebaseRecord]()
+    ALL_TIMEZONES
+      .sortBy(_.getId)
+      .foreach { zid =>
+      withDefaultTimeZone(zid) {
+        val start = adjustFunc(instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0)
+          .atZone(zid)
+          .toInstant))
+        // sun.util.calendar.ZoneInfo resolves DST after 2037 year incorrectly.
+        // See https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8073446
+        val end = adjustFunc(instantToMicros(LocalDateTime.of(2037, 1, 1, 0, 0, 0)
+          .atZone(zid)
+          .toInstant))
+
+        var micros = start
+        var diff = Long.MaxValue
+        val maxStep = DAYS_PER_WEEK * MICROS_PER_DAY
+        var step: Long = MICROS_PER_SECOND
+        val switches = new ArrayBuffer[Long]()
+        val diffs = new ArrayBuffer[Long]()
+        while (micros < end) {
+          val rebased = rebaseFunc(zid, micros)
+          val curDiff = rebased - micros
+          if (curDiff != diff) {
+            if (step > MICROS_PER_SECOND) {
+              micros -= step
+              step = (Math.max(MICROS_PER_SECOND, step / 2) / MICROS_PER_SECOND) * MICROS_PER_SECOND
+            } else {
+              diff = curDiff
+              step = maxStep
+              assert(diff % MICROS_PER_SECOND == 0)
+              diffs.append(diff / MICROS_PER_SECOND)
+              assert(micros % MICROS_PER_SECOND == 0)
+              switches.append(micros / MICROS_PER_SECOND)
+            }
+          }
+          micros += step
+        }
+        result.append(RebaseRecord(zid.getId, switches.toArray, diffs.toArray))
+      }
+    }
+    val mapper = (new ObjectMapper() with ScalaObjectMapper)
+      .registerModule(DefaultScalaModule)
+      .writerWithDefaultPrettyPrinter()
+    mapper.writeValue(
+      Files.newOutputStream(
+        Paths.get(dir, fileName),
+        StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING),
+      result.toArray)
+  }
+
+  ignore("generate 'gregorian-julian-rebase-micros.json'") {
+    generateRebaseJson(
+      adjustFunc = identity[Long],
+      rebaseFunc = rebaseGregorianToJulianMicros,
+      dir = "/Users/maximgekk/tmp",
+      fileName = "gregorian-julian-rebase-micros.json")
+  }
+
+  ignore("generate 'julian-gregorian-rebase-micros.json'") {
+    generateRebaseJson(
+      adjustFunc = rebaseGregorianToJulianMicros,
+      rebaseFunc = rebaseJulianToGregorianMicros,
+      dir = "/Users/maximgekk/tmp",
+      fileName = "julian-gregorian-rebase-micros.json")
+  }
+
+  test("rebase gregorian to/from julian days - BCE era") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          (-1100, 1, 1),
+          (-1044, 3, 5),
+          (-44, 3, 5)).foreach { case (year, month, day) =>
+          val julianDays = fromJavaDateLegacy(new Date(year - 1900, month - 1, day))
+          val gregorianDays = localDateToDays(LocalDate.of(year, month, day))
+
+          assert(rebaseGregorianToJulianDays(gregorianDays) === julianDays)
+          assert(rebaseJulianToGregorianDays(julianDays) === gregorianDays)
+        }
+      }
+    }
+  }
+
+  test("rebase gregorian to/from julian micros - BCE era") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          (-1100, 1, 1, 1, 2, 3, 0),
+          (-1044, 3, 5, 0, 0, 0, 123456000),
+          (-44, 3, 5, 23, 59, 59, 99999000)
+        ).foreach { case (year, month, day, hour, minute, second, nanos) =>
+          val julianMicros = toJulianMicros(new Timestamp(
+            year - 1900, month - 1, day, hour, minute, second, nanos))
+          val gregorianMicros = toGregorianMicros(LocalDateTime.of(
+            year, month, day, hour, minute, second, nanos), zid)
+
+          assert(rebaseGregorianToJulianMicros(gregorianMicros) === julianMicros)
+          assert(rebaseJulianToGregorianMicros(julianMicros) === gregorianMicros)
+        }
+      }
+    }
+  }
+
+  test("rebase not-existed dates in the hybrid calendar") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "1582-10-04" -> "1582-10-04",
+          "1582-10-05" -> "1582-10-15", "1582-10-06" -> "1582-10-15", "1582-10-07" -> "1582-10-15",
+          "1582-10-08" -> "1582-10-15", "1582-10-09" -> "1582-10-15", "1582-10-11" -> "1582-10-15",
+          "1582-10-12" -> "1582-10-15", "1582-10-13" -> "1582-10-15", "1582-10-14" -> "1582-10-15",
+          "1582-10-15" -> "1582-10-15").foreach { case (gregDate, hybridDate) =>
+          withClue(s"tz = ${zid.getId} greg date = $gregDate hybrid date = $hybridDate ") {
+            val date = Date.valueOf(hybridDate)
+            val hybridDays = fromJavaDateLegacy(date)
+            val gregorianDays = localDateToDays(LocalDate.parse(gregDate))
+
+            assert(localRebaseGregorianToJulianDays(gregorianDays) === hybridDays)
+            assert(rebaseGregorianToJulianDays(gregorianDays) === hybridDays)
+          }
+        }
+      }
+    }
+  }
+
+  test("rebase not-existed timestamps in the hybrid calendar") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "1582-10-04T23:59:59.999999" -> "1582-10-04 23:59:59.999999",
+          "1582-10-05T00:00:00.000000" -> "1582-10-15 00:00:00.000000",
+          "1582-10-06T01:02:03.000001" -> "1582-10-15 01:02:03.000001",
+          "1582-10-07T00:00:00.000000" -> "1582-10-15 00:00:00.000000",
+          "1582-10-08T23:59:59.999999" -> "1582-10-15 23:59:59.999999",
+          "1582-10-09T23:59:59.001001" -> "1582-10-15 23:59:59.001001",
+          "1582-10-10T00:11:22.334455" -> "1582-10-15 00:11:22.334455",
+          "1582-10-11T11:12:13.111111" -> "1582-10-15 11:12:13.111111",
+          "1582-10-12T10:11:12.131415" -> "1582-10-15 10:11:12.131415",
+          "1582-10-13T00:00:00.123321" -> "1582-10-15 00:00:00.123321",
+          "1582-10-14T23:59:59.999999" -> "1582-10-15 23:59:59.999999",
+          "1582-10-15T00:00:00.000000" -> "1582-10-15 00:00:00.000000"
+        ).foreach { case (gregTs, hybridTs) =>
+          withClue(s"tz = ${zid.getId} greg ts = $gregTs hybrid ts = $hybridTs") {
+            val hybridMicros = parseToJulianMicros(hybridTs)
+            val gregorianMicros = parseToGregMicros(gregTs, zid)
+
+            assert(rebaseGregorianToJulianMicros(zid, gregorianMicros) === hybridMicros)
+            assert(rebaseGregorianToJulianMicros(gregorianMicros) === hybridMicros)
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-31959: JST -> HKT at Asia/Hong_Kong in 1945") {
+    // The 'Asia/Hong_Kong' time zone switched from 'Japan Standard Time' (JST = UTC+9)
+    // to 'Hong Kong Time' (HKT = UTC+8). After Sunday, 18 November, 1945 01:59:59 AM,
+    // clocks were moved backward to become Sunday, 18 November, 1945 01:00:00 AM.
+    // In this way, the overlap happened w/o Daylight Saving Time.
+    val hkZid = getZoneId("Asia/Hong_Kong")
+    withDefaultTimeZone(hkZid) {
+      var expected = "1945-11-18 01:30:00.0"
+      var ldt = LocalDateTime.of(1945, 11, 18, 1, 30, 0)
+      var earlierMicros = instantToMicros(ldt.atZone(hkZid).withEarlierOffsetAtOverlap().toInstant)
+      var laterMicros = instantToMicros(ldt.atZone(hkZid).withLaterOffsetAtOverlap().toInstant)
+      var overlapInterval = MICROS_PER_HOUR
+      if (earlierMicros + overlapInterval != laterMicros) {
+        // Old JDK might have an outdated time zone database.
+        // See https://bugs.openjdk.java.net/browse/JDK-8228469: "Hong Kong ... Its 1945 transition
+        // from JST to HKT was on 11-18 at 02:00, not 09-15 at 00:00"
+        expected = "1945-09-14 23:30:00.0"
+        ldt = LocalDateTime.of(1945, 9, 14, 23, 30, 0)
+        earlierMicros = instantToMicros(ldt.atZone(hkZid).withEarlierOffsetAtOverlap().toInstant)
+        laterMicros = instantToMicros(ldt.atZone(hkZid).withLaterOffsetAtOverlap().toInstant)
+        // If time zone db doesn't have overlapping at all, set the overlap interval to zero.
+        overlapInterval = laterMicros - earlierMicros
+      }
+      val rebasedEarlierMicros = rebaseGregorianToJulianMicros(hkZid, earlierMicros)
+      val rebasedLaterMicros = rebaseGregorianToJulianMicros(hkZid, laterMicros)
+      def toTsStr(micros: Long): String = toJavaTimestamp(micros).toString
+      assert(toTsStr(rebasedEarlierMicros) === expected)
+      assert(toTsStr(rebasedLaterMicros) === expected)
+      assert(rebasedEarlierMicros + overlapInterval === rebasedLaterMicros)
+      // Check optimized rebasing
+      assert(rebaseGregorianToJulianMicros(earlierMicros) === rebasedEarlierMicros)
+      assert(rebaseGregorianToJulianMicros(laterMicros) === rebasedLaterMicros)
+      // Check reverse rebasing
+      assert(rebaseJulianToGregorianMicros(rebasedEarlierMicros) === earlierMicros)
+      assert(rebaseJulianToGregorianMicros(rebasedLaterMicros) === laterMicros)
+      // Check reverse not-optimized rebasing
+      assert(rebaseJulianToGregorianMicros(hkZid, rebasedEarlierMicros) === earlierMicros)
+      assert(rebaseJulianToGregorianMicros(hkZid, rebasedLaterMicros) === laterMicros)
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala
new file mode 100644
index 0000000000000..6fe774e8afcb6
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.lang.{Double => JDouble, Float => JFloat}
+
+import org.apache.spark.SparkFunSuite
+
+class SQLOrderingUtilSuite extends SparkFunSuite {
+
+  test("compareDoublesSQL") {
+    def shouldMatchDefaultOrder(a: Double, b: Double): Unit = {
+      assert(SQLOrderingUtil.compareDoubles(a, b) === JDouble.compare(a, b))
+      assert(SQLOrderingUtil.compareDoubles(b, a) === JDouble.compare(b, a))
+    }
+    shouldMatchDefaultOrder(0d, 0d)
+    shouldMatchDefaultOrder(0d, 1d)
+    shouldMatchDefaultOrder(-1d, 1d)
+    shouldMatchDefaultOrder(Double.MinValue, Double.MaxValue)
+
+    val specialNaN = JDouble.longBitsToDouble(0x7ff1234512345678L)
+    assert(JDouble.isNaN(specialNaN))
+    assert(JDouble.doubleToRawLongBits(Double.NaN) != JDouble.doubleToRawLongBits(specialNaN))
+
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, Double.NaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, specialNaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, Double.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(specialNaN, Double.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, Double.NegativeInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.PositiveInfinity, Double.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NegativeInfinity, Double.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(0.0d, -0.0d) === 0)
+    assert(SQLOrderingUtil.compareDoubles(-0.0d, 0.0d) === 0)
+  }
+
+  test("compareFloatsSQL") {
+    def shouldMatchDefaultOrder(a: Float, b: Float): Unit = {
+      assert(SQLOrderingUtil.compareFloats(a, b) === JFloat.compare(a, b))
+      assert(SQLOrderingUtil.compareFloats(b, a) === JFloat.compare(b, a))
+    }
+    shouldMatchDefaultOrder(0f, 0f)
+    shouldMatchDefaultOrder(0f, 1f)
+    shouldMatchDefaultOrder(-1f, 1f)
+    shouldMatchDefaultOrder(Float.MinValue, Float.MaxValue)
+
+    val specialNaN = JFloat.intBitsToFloat(-6966608)
+    assert(JFloat.isNaN(specialNaN))
+    assert(JFloat.floatToRawIntBits(Float.NaN) != JFloat.floatToRawIntBits(specialNaN))
+
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, Float.NaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, specialNaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, Float.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(specialNaN, Float.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, Float.NegativeInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.PositiveInfinity, Float.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NegativeInfinity, Float.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(0.0f, -0.0f) === 0)
+    assert(SQLOrderingUtil.compareDoubles(-0.0f, 0.0f) === 0)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala
index 67bc4bc81cb92..c68e89fc8a5ca 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala
@@ -18,9 +18,11 @@
 package org.apache.spark.sql.catalyst.util
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.StringUtils._
+import org.apache.spark.sql.internal.SQLConf
 
-class StringUtilsSuite extends SparkFunSuite {
+class StringUtilsSuite extends SparkFunSuite with SQLHelper {
 
   test("escapeLikeRegex") {
     val expectedEscapedStrOne = "(?s)\\Qa\\E\\Qb\\E\\Qd\\E\\Qe\\E\\Qf\\E"
@@ -98,4 +100,32 @@ class StringUtilsSuite extends SparkFunSuite {
     assert(checkLimit("1234567"))
     assert(checkLimit("1234567890"))
   }
+
+  test("SPARK-31916: StringConcat doesn't overflow on many inputs") {
+    val concat = new StringConcat(maxLength = 100)
+    val stringToAppend = "Test internal index of StringConcat does not overflow with many " +
+      "append calls"
+    0.to((Integer.MAX_VALUE / stringToAppend.length) + 1).foreach { _ =>
+      concat.append(stringToAppend)
+    }
+    assert(concat.toString.length === 100)
+  }
+
+  test("SPARK-31916: verify that PlanStringConcat's output shows the actual length of the plan") {
+    withSQLConf(SQLConf.MAX_PLAN_STRING_LENGTH.key -> "0") {
+      val concat = new PlanStringConcat()
+      0.to(3).foreach { i =>
+        concat.append(s"plan fragment $i")
+      }
+      assert(concat.toString === "Truncated plan of 60 characters")
+    }
+
+    withSQLConf(SQLConf.MAX_PLAN_STRING_LENGTH.key -> "60") {
+      val concat = new PlanStringConcat()
+      0.to(2).foreach { i =>
+        concat.append(s"plan fragment $i")
+      }
+      assert(concat.toString === "plan fragment 0plan fragment 1... 15 more characters")
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
new file mode 100644
index 0000000000000..46b216accbadb
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
@@ -0,0 +1,445 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.time.{DateTimeException, Instant, LocalDateTime, LocalTime}
+import java.util.concurrent.TimeUnit
+
+import org.apache.spark.SparkUpgradeException
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
+import org.apache.spark.unsafe.types.UTF8String
+
+class TimestampFormatterSuite extends DatetimeFormatterSuite {
+
+  override def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit = {
+    TimestampFormatter(pattern, UTC, isParsing)
+  }
+
+  override protected def useDateFormatter: Boolean = false
+
+  test("parsing timestamps using time zones") {
+    val localDate = "2018-12-02T10:11:12.001234"
+    val expectedMicros = Map(
+      "UTC" -> 1543745472001234L,
+      PST.getId -> 1543774272001234L,
+      CET.getId -> 1543741872001234L,
+      "Africa/Dakar" -> 1543745472001234L,
+      "America/Los_Angeles" -> 1543774272001234L,
+      "Antarctica/Vostok" -> 1543723872001234L,
+      "Asia/Hong_Kong" -> 1543716672001234L,
+      "Europe/Amsterdam" -> 1543741872001234L)
+    outstandingTimezonesIds.foreach { zoneId =>
+      val formatter = TimestampFormatter(
+        "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
+        getZoneId(zoneId),
+        isParsing = true)
+      val microsSinceEpoch = formatter.parse(localDate)
+      assert(microsSinceEpoch === expectedMicros(zoneId))
+    }
+  }
+
+  test("format timestamps using time zones") {
+    val microsSinceEpoch = 1543745472001234L
+    val expectedTimestamp = Map(
+      "UTC" -> "2018-12-02 10:11:12.001234",
+      PST.getId -> "2018-12-02 02:11:12.001234",
+      CET.getId -> "2018-12-02 11:11:12.001234",
+      "Africa/Dakar" -> "2018-12-02 10:11:12.001234",
+      "America/Los_Angeles" -> "2018-12-02 02:11:12.001234",
+      "Antarctica/Vostok" -> "2018-12-02 16:11:12.001234",
+      "Asia/Hong_Kong" -> "2018-12-02 18:11:12.001234",
+      "Europe/Amsterdam" -> "2018-12-02 11:11:12.001234")
+    outstandingTimezonesIds.foreach { zoneId =>
+      Seq(
+        TimestampFormatter(
+          "yyyy-MM-dd HH:mm:ss.SSSSSS",
+          getZoneId(zoneId),
+          // Test only FAST_DATE_FORMAT because other legacy formats don't support formatting
+          // in microsecond precision.
+          LegacyDateFormats.FAST_DATE_FORMAT,
+          isParsing = false),
+        TimestampFormatter.getFractionFormatter(getZoneId(zoneId))).foreach { formatter =>
+        val timestamp = formatter.format(microsSinceEpoch)
+        assert(timestamp === expectedTimestamp(zoneId))
+        assert(formatter.format(microsToInstant(microsSinceEpoch)) === expectedTimestamp(zoneId))
+        assert(formatter.format(toJavaTimestamp(microsSinceEpoch)) === expectedTimestamp(zoneId))
+      }
+    }
+  }
+
+  test("roundtrip micros -> timestamp -> micros using timezones") {
+    Seq("yyyy-MM-dd'T'HH:mm:ss.SSSSSS", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXXXX").foreach { pattern =>
+      Seq(
+        -58710115316212000L,
+        -18926315945345679L,
+        -9463427405253013L,
+        -244000001L,
+        0L,
+        99628200102030L,
+        1543749753123456L,
+        2177456523456789L,
+        11858049903010203L).foreach { micros =>
+        outstandingZoneIds.foreach { zoneId =>
+          val timestamp = TimestampFormatter(pattern, zoneId, isParsing = false).format(micros)
+          val parsed = TimestampFormatter(
+            pattern, zoneId, isParsing = true).parse(timestamp)
+          assert(micros === parsed)
+        }
+      }
+    }
+  }
+
+  test("roundtrip timestamp -> micros -> timestamp using timezones") {
+    Seq(
+      "0109-07-20T18:38:03.788000",
+      "1370-04-01T10:00:54.654321",
+      "1670-02-11T14:09:54.746987",
+      "1969-12-31T23:55:55.999999",
+      "1970-01-01T00:00:00.000000",
+      "1973-02-27T02:30:00.102030",
+      "2018-12-02T11:22:33.123456",
+      "2039-01-01T01:02:03.456789",
+      "2345-10-07T22:45:03.010203").foreach { timestamp =>
+      outstandingZoneIds.foreach { zoneId =>
+        val pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"
+        val micros = TimestampFormatter(
+          pattern, zoneId, isParsing = true).parse(timestamp)
+        val formatted = TimestampFormatter(pattern, zoneId, isParsing = false).format(micros)
+        assert(timestamp === formatted)
+      }
+    }
+  }
+
+  test("case insensitive parsing of am and pm") {
+    val formatter = TimestampFormatter("yyyy MMM dd hh:mm:ss a", UTC, isParsing = false)
+    val micros = formatter.parse("2009 Mar 20 11:30:01 am")
+    assert(micros === date(2009, 3, 20, 11, 30, 1))
+  }
+
+  test("format fraction of second") {
+    val formatter = TimestampFormatter.getFractionFormatter(UTC)
+    Seq(
+      0 -> "1970-01-01 00:00:00",
+      1 -> "1970-01-01 00:00:00.000001",
+      1000 -> "1970-01-01 00:00:00.001",
+      900000 -> "1970-01-01 00:00:00.9",
+      1000000 -> "1970-01-01 00:00:01").foreach { case (micros, tsStr) =>
+      assert(formatter.format(micros) === tsStr)
+      assert(formatter.format(microsToInstant(micros)) === tsStr)
+      withDefaultTimeZone(UTC) {
+        assert(formatter.format(toJavaTimestamp(micros)) === tsStr)
+      }
+    }
+  }
+
+  test("formatting negative years with default pattern") {
+    val instant = LocalDateTime.of(-99, 1, 1, 0, 0, 0).atZone(UTC).toInstant
+    val micros = instantToMicros(instant)
+    assert(TimestampFormatter(UTC).format(micros) === "-0099-01-01 00:00:00")
+    assert(TimestampFormatter(UTC).format(instant) === "-0099-01-01 00:00:00")
+    withDefaultTimeZone(UTC) { // toJavaTimestamp depends on the default time zone
+      assert(TimestampFormatter("yyyy-MM-dd HH:mm:SS G", UTC, isParsing = false)
+        .format(toJavaTimestamp(micros)) === "0100-01-01 00:00:00 BC")
+    }
+  }
+
+  test("special timestamp values") {
+    testSpecialDatetimeValues { zoneId =>
+      val formatter = TimestampFormatter(zoneId)
+      val tolerance = TimeUnit.SECONDS.toMicros(30)
+
+      assert(formatter.parse("EPOCH") === 0)
+      val now = instantToMicros(Instant.now())
+      formatter.parse("now") should be(now +- tolerance)
+      val localToday = LocalDateTime.now(zoneId)
+        .`with`(LocalTime.MIDNIGHT)
+        .atZone(zoneId)
+      val yesterday = instantToMicros(localToday.minusDays(1).toInstant)
+      formatter.parse("yesterday CET") should be(yesterday +- tolerance)
+      val today = instantToMicros(localToday.toInstant)
+      formatter.parse(" TODAY ") should be(today +- tolerance)
+      val tomorrow = instantToMicros(localToday.plusDays(1).toInstant)
+      formatter.parse("Tomorrow ") should be(tomorrow +- tolerance)
+    }
+  }
+
+  test("parsing timestamp strings with various seconds fractions") {
+    outstandingZoneIds.foreach { zoneId =>
+      def check(pattern: String, input: String, reference: String): Unit = {
+        val formatter = TimestampFormatter(pattern, zoneId, isParsing = true)
+        val expected = stringToTimestamp(UTF8String.fromString(reference), zoneId).get
+        val actual = formatter.parse(input)
+        assert(actual === expected)
+      }
+
+      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSSXXX",
+        "2019-10-14T09:39:07.3220000Z", "2019-10-14T09:39:07.322Z")
+      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
+        "2019-10-14T09:39:07.322000", "2019-10-14T09:39:07.322")
+      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX",
+        "2019-10-14T09:39:07.123456Z", "2019-10-14T09:39:07.123456Z")
+      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX",
+        "2019-10-14T09:39:07.000010Z", "2019-10-14T09:39:07.00001Z")
+      check("yyyy HH:mm:ss.SSSSS", "1970 01:02:03.00004", "1970-01-01 01:02:03.00004")
+      check("yyyy HH:mm:ss.SSSS", "2019 00:00:07.0100", "2019-01-01 00:00:07.0100")
+      check("yyyy-MM-dd'T'HH:mm:ss.SSSX",
+        "2019-10-14T09:39:07.322Z", "2019-10-14T09:39:07.322Z")
+      check("yyyy-MM-dd'T'HH:mm:ss.SS",
+        "2019-10-14T09:39:07.10", "2019-10-14T09:39:07.1")
+      check("yyyy-MM-dd'T'HH:mm:ss.S",
+        "2019-10-14T09:39:07.1", "2019-10-14T09:39:07.1")
+
+      try {
+        TimestampFormatter("yyyy/MM/dd HH_mm_ss.SSSSSS", zoneId, isParsing = true)
+          .parse("2019/11/14 20#25#30.123456")
+        fail("Expected to throw an exception for the invalid input")
+      } catch {
+        case e: java.time.format.DateTimeParseException =>
+          assert(e.getMessage.contains("could not be parsed"))
+      }
+    }
+  }
+
+  test("formatting timestamp strings up to microsecond precision") {
+    outstandingZoneIds.foreach { zoneId =>
+      def check(pattern: String, input: String, expected: String): Unit = {
+        val formatter = TimestampFormatter(pattern, zoneId, isParsing = false)
+        val timestamp = stringToTimestamp(UTF8String.fromString(input), zoneId).get
+        val actual = formatter.format(timestamp)
+        assert(actual === expected)
+      }
+
+      check(
+        "yyyy-MM-dd HH:mm:ss.SSSSSSS", "2019-10-14T09:39:07.123456",
+        "2019-10-14 09:39:07.1234560")
+      check(
+        "yyyy-MM-dd HH:mm:ss.SSSSSS", "1960-01-01T09:39:07.123456",
+        "1960-01-01 09:39:07.123456")
+      check(
+        "yyyy-MM-dd HH:mm:ss.SSSSS", "0001-10-14T09:39:07.1",
+        "0001-10-14 09:39:07.10000")
+      check(
+        "yyyy-MM-dd HH:mm:ss.SSSS", "9999-12-31T23:59:59.999",
+        "9999-12-31 23:59:59.9990")
+      check(
+        "yyyy-MM-dd HH:mm:ss.SSS", "1970-01-01T00:00:00.0101",
+        "1970-01-01 00:00:00.010")
+      check(
+        "yyyy-MM-dd HH:mm:ss.SS", "2019-10-14T09:39:07.09",
+        "2019-10-14 09:39:07.09")
+      check(
+        "yyyy-MM-dd HH:mm:ss.S", "2019-10-14T09:39:07.2",
+        "2019-10-14 09:39:07.2")
+      check(
+        "yyyy-MM-dd HH:mm:ss.S", "2019-10-14T09:39:07",
+        "2019-10-14 09:39:07.0")
+      check(
+        "yyyy-MM-dd HH:mm:ss", "2019-10-14T09:39:07.123456",
+        "2019-10-14 09:39:07")
+    }
+  }
+
+  test("SPARK-30958: parse timestamp with negative year") {
+    val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC, isParsing = true)
+    assert(formatter1.parse("-1234-02-22 02:22:22") === date(-1234, 2, 22, 2, 22, 22))
+
+    def assertParsingError(f: => Unit): Unit = {
+      intercept[Exception](f) match {
+        case e: SparkUpgradeException =>
+          assert(e.getCause.isInstanceOf[DateTimeException])
+        case e =>
+          assert(e.isInstanceOf[DateTimeException])
+      }
+    }
+
+    // "yyyy" with "G" can't parse negative year or year 0000.
+    val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss", UTC, isParsing = true)
+    assertParsingError(formatter2.parse("BC -1234-02-22 02:22:22"))
+    assertParsingError(formatter2.parse("AC 0000-02-22 02:22:22"))
+
+    assert(formatter2.parse("BC 1234-02-22 02:22:22") === date(-1233, 2, 22, 2, 22, 22))
+    assert(formatter2.parse("AD 1234-02-22 02:22:22") === date(1234, 2, 22, 2, 22, 22))
+  }
+
+  test("SPARK-31557: rebasing in legacy formatters/parsers") {
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
+      outstandingZoneIds.foreach { zoneId =>
+        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zoneId.getId) {
+          withDefaultTimeZone(zoneId) {
+            withClue(s"zoneId = ${zoneId.getId}") {
+              val formatters = LegacyDateFormats.values.map { legacyFormat =>
+                TimestampFormatter(
+                  TimestampFormatter.defaultPattern,
+                  zoneId,
+                  TimestampFormatter.defaultLocale,
+                  legacyFormat,
+                  isParsing = false)
+              }.toSeq :+ TimestampFormatter.getFractionFormatter(zoneId)
+              formatters.foreach { formatter =>
+                assert(microsToInstant(formatter.parse("1000-01-01 01:02:03"))
+                  .atZone(zoneId)
+                  .toLocalDateTime === LocalDateTime.of(1000, 1, 1, 1, 2, 3))
+
+                assert(formatter.format(
+                  LocalDateTime.of(1000, 1, 1, 1, 2, 3).atZone(zoneId).toInstant) ===
+                  "1000-01-01 01:02:03")
+                assert(formatter.format(instantToMicros(
+                  LocalDateTime.of(1000, 1, 1, 1, 2, 3)
+                    .atZone(zoneId).toInstant)) === "1000-01-01 01:02:03")
+                assert(formatter.format(java.sql.Timestamp.valueOf("1000-01-01 01:02:03")) ===
+                  "1000-01-01 01:02:03")
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("parsing hour with various patterns") {
+    def createFormatter(pattern: String): TimestampFormatter = {
+      // Use `SIMPLE_DATE_FORMAT`, so that the legacy parser also fails with invalid value range.
+      TimestampFormatter(pattern, UTC, LegacyDateFormats.SIMPLE_DATE_FORMAT, isParsing = true)
+    }
+
+    withClue("HH") {
+      val formatter = createFormatter("yyyy-MM-dd HH")
+
+      val micros1 = formatter.parse("2009-12-12 00")
+      assert(micros1 === date(2009, 12, 12))
+
+      val micros2 = formatter.parse("2009-12-12 15")
+      assert(micros2 === date(2009, 12, 12, 15))
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 24"))
+    }
+
+    withClue("kk") {
+      val formatter = createFormatter("yyyy-MM-dd kk")
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 00"))
+
+      val micros1 = formatter.parse("2009-12-12 15")
+      assert(micros1 === date(2009, 12, 12, 15))
+
+      val micros2 = formatter.parse("2009-12-12 24")
+      assert(micros2 === date(2009, 12, 12))
+    }
+
+    withClue("KK") {
+      val formatter = createFormatter("yyyy-MM-dd KK a")
+
+      val micros1 = formatter.parse("2009-12-12 00 am")
+      assert(micros1 === date(2009, 12, 12))
+
+      // For `KK`, "12:00:00 am" is the same as "00:00:00 pm".
+      val micros2 = formatter.parse("2009-12-12 12 am")
+      assert(micros2 === date(2009, 12, 12, 12))
+
+      val micros3 = formatter.parse("2009-12-12 00 pm")
+      assert(micros3 === date(2009, 12, 12, 12))
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 12 pm"))
+    }
+
+    withClue("hh") {
+      val formatter = createFormatter("yyyy-MM-dd hh a")
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 00 am"))
+
+      val micros1 = formatter.parse("2009-12-12 12 am")
+      assert(micros1 === date(2009, 12, 12))
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 00 pm"))
+
+      val micros2 = formatter.parse("2009-12-12 12 pm")
+      assert(micros2 === date(2009, 12, 12, 12))
+    }
+  }
+
+  test("missing date fields") {
+    val formatter = TimestampFormatter("HH:mm:ss", UTC, isParsing = true)
+    val micros = formatter.parse("11:30:01")
+    assert(micros === date(1970, 1, 1, 11, 30, 1))
+  }
+
+  test("missing year field with invalid date") {
+    // Use `SIMPLE_DATE_FORMAT`, so that the legacy parser also fails with invalid date.
+    val formatter =
+      TimestampFormatter("MM-dd", UTC, LegacyDateFormats.SIMPLE_DATE_FORMAT, isParsing = true)
+    withDefaultTimeZone(UTC)(intercept[DateTimeException](formatter.parse("02-29")))
+  }
+
+  test("missing am/pm field") {
+    Seq("HH", "hh", "KK", "kk").foreach { hour =>
+      val formatter = TimestampFormatter(s"yyyy $hour:mm:ss", UTC, isParsing = true)
+      val micros = formatter.parse("2009 11:30:01")
+      assert(micros === date(2009, 1, 1, 11, 30, 1))
+    }
+  }
+
+  test("missing time fields") {
+    val formatter = TimestampFormatter("yyyy HH", UTC, isParsing = true)
+    val micros = formatter.parse("2009 11")
+    assert(micros === date(2009, 1, 1, 11))
+  }
+
+  test("missing hour field") {
+    val f1 = TimestampFormatter("mm:ss a", UTC, isParsing = true)
+    val t1 = f1.parse("30:01 PM")
+    assert(t1 === date(1970, 1, 1, 12, 30, 1))
+    val t2 = f1.parse("30:01 AM")
+    assert(t2 === date(1970, 1, 1, 0, 30, 1))
+    val f2 = TimestampFormatter("mm:ss", UTC, isParsing = true)
+    val t3 = f2.parse("30:01")
+    assert(t3 === date(1970, 1, 1, 0, 30, 1))
+    val f3 = TimestampFormatter("a", UTC, isParsing = true)
+    val t4 = f3.parse("PM")
+    assert(t4 === date(1970, 1, 1, 12))
+    val t5 = f3.parse("AM")
+    assert(t5 === date(1970))
+  }
+
+  test("check result differences for datetime formatting") {
+    val formatter = TimestampFormatter("DD", UTC, isParsing = false)
+    assert(formatter.format(date(1970, 1, 3)) == "03")
+    assert(formatter.format(date(1970, 4, 9)) == "99")
+
+    if (System.getProperty("java.version").split("\\D+")(0).toInt < 9) {
+      // https://bugs.openjdk.java.net/browse/JDK-8079628
+      intercept[SparkUpgradeException] {
+        formatter.format(date(1970, 4, 10))
+      }
+    } else {
+      assert(formatter.format(date(1970, 4, 10)) == "100")
+    }
+  }
+
+  test("SPARK-32424: avoid silent data change when timestamp overflows") {
+    val formatter = TimestampFormatter("y", UTC, isParsing = true)
+    assert(formatter.parse("294247") === date(294247))
+    assert(formatter.parse("-290307") === date(-290307))
+    val e1 = intercept[ArithmeticException](formatter.parse("294248"))
+    assert(e1.getMessage === "long overflow")
+    val e2 = intercept[ArithmeticException](formatter.parse("-290308"))
+    assert(e2.getMessage === "long overflow")
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
index e7b1c0810a033..6d8ef68473778 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.util
 
 import java.time.{ZoneId, ZoneOffset}
 
+import scala.reflect.runtime.universe.TypeTag
+
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.serializer.{JavaSerializer, KryoSerializer}
 import org.apache.spark.sql.Row
@@ -70,75 +72,55 @@ class UnsafeArraySuite extends SparkFunSuite {
     arrayData
   }
 
+  private def toUnsafeArray[T : TypeTag](array: Array[T]): ArrayData = {
+    val converted = ExpressionEncoder[Array[T]].createSerializer().apply(array).getArray(0)
+    assert(converted.isInstanceOf[T])
+    assert(converted.numElements == array.length)
+    converted
+  }
+
   test("read array") {
-    val unsafeBoolean = ExpressionEncoder[Array[Boolean]].resolveAndBind().
-      toRow(booleanArray).getArray(0)
-    assert(unsafeBoolean.isInstanceOf[UnsafeArrayData])
-    assert(unsafeBoolean.numElements == booleanArray.length)
+    val unsafeBoolean = toUnsafeArray(booleanArray)
     booleanArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeBoolean.getBoolean(i) == e)
     }
 
-    val unsafeShort = ExpressionEncoder[Array[Short]].resolveAndBind().
-      toRow(shortArray).getArray(0)
-    assert(unsafeShort.isInstanceOf[UnsafeArrayData])
-    assert(unsafeShort.numElements == shortArray.length)
+    val unsafeShort = toUnsafeArray(shortArray)
     shortArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeShort.getShort(i) == e)
     }
 
-    val unsafeInt = ExpressionEncoder[Array[Int]].resolveAndBind().
-      toRow(intArray).getArray(0)
-    assert(unsafeInt.isInstanceOf[UnsafeArrayData])
-    assert(unsafeInt.numElements == intArray.length)
+    val unsafeInt = toUnsafeArray(intArray)
     intArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeInt.getInt(i) == e)
     }
 
-    val unsafeLong = ExpressionEncoder[Array[Long]].resolveAndBind().
-      toRow(longArray).getArray(0)
-    assert(unsafeLong.isInstanceOf[UnsafeArrayData])
-    assert(unsafeLong.numElements == longArray.length)
+    val unsafeLong = toUnsafeArray(longArray)
     longArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeLong.getLong(i) == e)
     }
 
-    val unsafeFloat = ExpressionEncoder[Array[Float]].resolveAndBind().
-      toRow(floatArray).getArray(0)
-    assert(unsafeFloat.isInstanceOf[UnsafeArrayData])
-    assert(unsafeFloat.numElements == floatArray.length)
+    val unsafeFloat = toUnsafeArray(floatArray)
     floatArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeFloat.getFloat(i) == e)
     }
 
-    val unsafeDouble = ExpressionEncoder[Array[Double]].resolveAndBind().
-      toRow(doubleArray).getArray(0)
-    assert(unsafeDouble.isInstanceOf[UnsafeArrayData])
-    assert(unsafeDouble.numElements == doubleArray.length)
+    val unsafeDouble = toUnsafeArray(doubleArray)
     doubleArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeDouble.getDouble(i) == e)
     }
 
-    val unsafeString = ExpressionEncoder[Array[String]].resolveAndBind().
-      toRow(stringArray).getArray(0)
-    assert(unsafeString.isInstanceOf[UnsafeArrayData])
-    assert(unsafeString.numElements == stringArray.length)
+    val unsafeString = toUnsafeArray(stringArray)
     stringArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeString.getUTF8String(i).toString().equals(e))
     }
 
-    val unsafeDate = ExpressionEncoder[Array[Int]].resolveAndBind().
-      toRow(dateArray).getArray(0)
-    assert(unsafeDate.isInstanceOf[UnsafeArrayData])
-    assert(unsafeDate.numElements == dateArray.length)
+    val unsafeDate = toUnsafeArray(dateArray)
     dateArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeDate.get(i, DateType).asInstanceOf[Int] == e)
     }
 
-    val unsafeTimestamp = ExpressionEncoder[Array[Long]].resolveAndBind().
-      toRow(timestampArray).getArray(0)
-    assert(unsafeTimestamp.isInstanceOf[UnsafeArrayData])
-    assert(unsafeTimestamp.numElements == timestampArray.length)
+    val unsafeTimestamp = toUnsafeArray(timestampArray)
     timestampArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeTimestamp.get(i, TimestampType).asInstanceOf[Long] == e)
     }
@@ -149,7 +131,7 @@ class UnsafeArraySuite extends SparkFunSuite {
         "array", ArrayType(DecimalType(decimal.precision, decimal.scale)))
       val encoder = RowEncoder(schema).resolveAndBind()
       val externalRow = Row(decimalArray)
-      val ir = encoder.toRow(externalRow)
+      val ir = encoder.createSerializer().apply(externalRow)
 
       val unsafeDecimal = ir.getArray(0)
       assert(unsafeDecimal.isInstanceOf[UnsafeArrayData])
@@ -162,7 +144,7 @@ class UnsafeArraySuite extends SparkFunSuite {
     val schema = new StructType().add("array", ArrayType(CalendarIntervalType))
     val encoder = RowEncoder(schema).resolveAndBind()
     val externalRow = Row(calenderintervalArray)
-    val ir = encoder.toRow(externalRow)
+    val ir = encoder.createSerializer().apply(externalRow)
     val unsafeCalendar = ir.getArray(0)
     assert(unsafeCalendar.isInstanceOf[UnsafeArrayData])
     assert(unsafeCalendar.numElements == calenderintervalArray.length)
@@ -170,10 +152,7 @@ class UnsafeArraySuite extends SparkFunSuite {
       assert(unsafeCalendar.getInterval(i) == e)
     }
 
-    val unsafeMultiDimInt = ExpressionEncoder[Array[Array[Int]]].resolveAndBind().
-      toRow(intMultiDimArray).getArray(0)
-    assert(unsafeMultiDimInt.isInstanceOf[UnsafeArrayData])
-    assert(unsafeMultiDimInt.numElements == intMultiDimArray.length)
+    val unsafeMultiDimInt = toUnsafeArray(intMultiDimArray)
     intMultiDimArray.zipWithIndex.map { case (a, j) =>
       val u = unsafeMultiDimInt.getArray(j)
       assert(u.isInstanceOf[UnsafeArrayData])
@@ -183,10 +162,7 @@ class UnsafeArraySuite extends SparkFunSuite {
       }
     }
 
-    val unsafeMultiDimDouble = ExpressionEncoder[Array[Array[Double]]].resolveAndBind().
-      toRow(doubleMultiDimArray).getArray(0)
-    assert(unsafeDouble.isInstanceOf[UnsafeArrayData])
-    assert(unsafeMultiDimDouble.numElements == doubleMultiDimArray.length)
+    val unsafeMultiDimDouble = toUnsafeArray(doubleMultiDimArray)
     doubleMultiDimArray.zipWithIndex.map { case (a, j) =>
       val u = unsafeMultiDimDouble.getArray(j)
       assert(u.isInstanceOf[UnsafeArrayData])
@@ -216,11 +192,9 @@ class UnsafeArraySuite extends SparkFunSuite {
   }
 
   test("to primitive array") {
-    val intEncoder = ExpressionEncoder[Array[Int]].resolveAndBind()
-    assert(intEncoder.toRow(intArray).getArray(0).toIntArray.sameElements(intArray))
+    assert(toUnsafeArray(intArray).toIntArray().sameElements(intArray))
 
-    val doubleEncoder = ExpressionEncoder[Array[Double]].resolveAndBind()
-    assert(doubleEncoder.toRow(doubleArray).getArray(0).toDoubleArray.sameElements(doubleArray))
+    assert(toUnsafeArray(doubleArray).toDoubleArray().sameElements(doubleArray))
   }
 
   test("unsafe java serialization") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
index c9e4e0aad5704..98b6a3bf682ad 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.connector
 
+import java.time.{Instant, ZoneId}
+import java.time.temporal.ChronoUnit
 import java.util
 
 import scala.collection.JavaConverters._
@@ -25,12 +27,13 @@ import scala.collection.mutable
 import org.scalatest.Assertions._
 
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.connector.catalog._
-import org.apache.spark.sql.connector.expressions.{IdentityTransform, Transform}
+import org.apache.spark.sql.connector.expressions.{BucketTransform, DaysTransform, HoursTransform, IdentityTransform, MonthsTransform, Transform, YearsTransform}
 import org.apache.spark.sql.connector.read._
 import org.apache.spark.sql.connector.write._
 import org.apache.spark.sql.sources.{And, EqualTo, Filter, IsNotNull}
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.types.{DataType, DateType, StructType, TimestampType}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 /**
@@ -46,10 +49,15 @@ class InMemoryTable(
   private val allowUnsupportedTransforms =
     properties.getOrDefault("allow-unsupported-transforms", "false").toBoolean
 
-  partitioning.foreach { t =>
-    if (!t.isInstanceOf[IdentityTransform] && !allowUnsupportedTransforms) {
-      throw new IllegalArgumentException(s"Transform $t must be IdentityTransform")
-    }
+  partitioning.foreach {
+    case _: IdentityTransform =>
+    case _: YearsTransform =>
+    case _: MonthsTransform =>
+    case _: DaysTransform =>
+    case _: HoursTransform =>
+    case _: BucketTransform =>
+    case t if !allowUnsupportedTransforms =>
+      throw new IllegalArgumentException(s"Transform $t is not a supported transform")
   }
 
   // The key `Seq[Any]` is the partition values.
@@ -59,10 +67,72 @@ class InMemoryTable(
 
   def rows: Seq[InternalRow] = dataMap.values.flatMap(_.rows).toSeq
 
-  private val partFieldNames = partitioning.flatMap(_.references).toSeq.flatMap(_.fieldNames)
-  private val partIndexes = partFieldNames.map(schema.fieldIndex)
+  private val partCols: Array[Array[String]] = partitioning.flatMap(_.references).map { ref =>
+    schema.findNestedField(ref.fieldNames(), includeCollections = false) match {
+      case Some(_) => ref.fieldNames()
+      case None => throw new IllegalArgumentException(s"${ref.describe()} does not exist.")
+    }
+  }
+
+  private val UTC = ZoneId.of("UTC")
+  private val EPOCH_LOCAL_DATE = Instant.EPOCH.atZone(UTC).toLocalDate
+
+  private def getKey(row: InternalRow): Seq[Any] = {
+    def extractor(
+        fieldNames: Array[String],
+        schema: StructType,
+        row: InternalRow): (Any, DataType) = {
+      val index = schema.fieldIndex(fieldNames(0))
+      val value = row.toSeq(schema).apply(index)
+      if (fieldNames.length > 1) {
+        (value, schema(index).dataType) match {
+          case (row: InternalRow, nestedSchema: StructType) =>
+            extractor(fieldNames.drop(1), nestedSchema, row)
+          case (_, dataType) =>
+            throw new IllegalArgumentException(s"Unsupported type, ${dataType.simpleString}")
+        }
+      } else {
+        (value, schema(index).dataType)
+      }
+    }
 
-  private def getKey(row: InternalRow): Seq[Any] = partIndexes.map(row.toSeq(schema)(_))
+    partitioning.map {
+      case IdentityTransform(ref) =>
+        extractor(ref.fieldNames, schema, row)._1
+      case YearsTransform(ref) =>
+        extractor(ref.fieldNames, schema, row) match {
+          case (days: Int, DateType) =>
+            ChronoUnit.YEARS.between(EPOCH_LOCAL_DATE, DateTimeUtils.daysToLocalDate(days))
+          case (micros: Long, TimestampType) =>
+            val localDate = DateTimeUtils.microsToInstant(micros).atZone(UTC).toLocalDate
+            ChronoUnit.YEARS.between(EPOCH_LOCAL_DATE, localDate)
+        }
+      case MonthsTransform(ref) =>
+        extractor(ref.fieldNames, schema, row) match {
+          case (days: Int, DateType) =>
+            ChronoUnit.MONTHS.between(EPOCH_LOCAL_DATE, DateTimeUtils.daysToLocalDate(days))
+          case (micros: Long, TimestampType) =>
+            val localDate = DateTimeUtils.microsToInstant(micros).atZone(UTC).toLocalDate
+            ChronoUnit.MONTHS.between(EPOCH_LOCAL_DATE, localDate)
+        }
+      case DaysTransform(ref) =>
+        extractor(ref.fieldNames, schema, row) match {
+          case (days, DateType) =>
+            days
+          case (micros: Long, TimestampType) =>
+            ChronoUnit.DAYS.between(Instant.EPOCH, DateTimeUtils.microsToInstant(micros))
+        }
+      case HoursTransform(ref) =>
+        extractor(ref.fieldNames, schema, row) match {
+          case (micros: Long, TimestampType) =>
+            ChronoUnit.HOURS.between(Instant.EPOCH, DateTimeUtils.microsToInstant(micros))
+        }
+      case BucketTransform(numBuckets, ref) =>
+        val (value, dataType) = extractor(ref.fieldNames, schema, row)
+        val valueHashCode = if (value == null) 0 else value.hashCode
+        ((valueHashCode + 31 * dataType.hashCode()) & Integer.MAX_VALUE) % numBuckets
+    }
+  }
 
   def withData(data: Array[BufferedRows]): InMemoryTable = dataMap.synchronized {
     data.foreach(_.rows.foreach { row =>
@@ -96,6 +166,7 @@ class InMemoryTable(
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+    InMemoryTable.maybeSimulateFailedTableWrite(new CaseInsensitiveStringMap(properties))
     InMemoryTable.maybeSimulateFailedTableWrite(info.options)
 
     new WriteBuilder with SupportsTruncate with SupportsOverwrite with SupportsDynamicOverwrite {
@@ -146,8 +217,10 @@ class InMemoryTable(
   }
 
   private class Overwrite(filters: Array[Filter]) extends TestBatchWrite {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
     override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized {
-      val deleteKeys = InMemoryTable.filtersToKeys(dataMap.keys, partFieldNames, filters)
+      val deleteKeys = InMemoryTable.filtersToKeys(
+        dataMap.keys, partCols.map(_.toSeq.quoted), filters)
       dataMap --= deleteKeys
       withData(messages.map(_.asInstanceOf[BufferedRows]))
     }
@@ -161,7 +234,8 @@ class InMemoryTable(
   }
 
   override def deleteWhere(filters: Array[Filter]): Unit = dataMap.synchronized {
-    dataMap --= InMemoryTable.filtersToKeys(dataMap.keys, partFieldNames, filters)
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+    dataMap --= InMemoryTable.filtersToKeys(dataMap.keys, partCols.map(_.toSeq.quoted), filters)
   }
 }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala
index b2f27e4740cbe..b36ded3d9d2ac 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala
@@ -50,6 +50,7 @@ class LookupCatalogSuite extends SparkFunSuite with LookupCatalog with Inside {
     })
     when(manager.currentCatalog).thenReturn(sessionCatalog)
     when(manager.v2SessionCatalog).thenReturn(sessionCatalog)
+    when(manager.currentNamespace).thenReturn(Array.empty[String])
     manager
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
index b71dc91eed4fa..52f80b1a52e72 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
@@ -423,10 +423,15 @@ class DataTypeSuite extends SparkFunSuite {
   checkCatalogString(MapType(IntegerType, StringType))
   checkCatalogString(MapType(IntegerType, createStruct(40)))
 
-  def checkEqualsStructurally(from: DataType, to: DataType, expected: Boolean): Unit = {
-    val testName = s"equalsStructurally: (from: $from, to: $to)"
+  def checkEqualsStructurally(
+      from: DataType,
+      to: DataType,
+      expected: Boolean,
+      ignoreNullability: Boolean = false): Unit = {
+    val testName = s"equalsStructurally: (from: $from, to: $to, " +
+      s"ignoreNullability: $ignoreNullability)"
     test(testName) {
-      assert(DataType.equalsStructurally(from, to) === expected)
+      assert(DataType.equalsStructurally(from, to, ignoreNullability) === expected)
     }
   }
 
@@ -453,6 +458,105 @@ class DataTypeSuite extends SparkFunSuite {
     new StructType().add("f1", IntegerType).add("f", new StructType().add("f2", StringType, false)),
     new StructType().add("f2", IntegerType).add("g", new StructType().add("f1", StringType)),
     false)
+  checkEqualsStructurally(
+    new StructType().add("f1", IntegerType).add("f", new StructType().add("f2", StringType, false)),
+    new StructType().add("f2", IntegerType).add("g", new StructType().add("f1", StringType)),
+    true,
+    ignoreNullability = true)
+  checkEqualsStructurally(
+    new StructType().add("f1", IntegerType).add("f", new StructType().add("f2", StringType)),
+    new StructType().add("f2", IntegerType, nullable = false)
+      .add("g", new StructType().add("f1", StringType)),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), false),
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    ArrayType(
+      ArrayType(IntegerType, false), true),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), false),
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), false),
+    ArrayType(
+      ArrayType(IntegerType, false), true),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), false),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, false), true),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), false),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, false), true),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, false), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = true)
 
   test("SPARK-25031: MapType should produce current formatted string for complex types") {
     val keyType: DataType = StructType(Seq(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
index c47332f5d9fcb..1a262d646ca10 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
@@ -80,7 +80,7 @@ class StrictDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBa
   test("Check NullType is incompatible with all other types") {
     allNonNullTypes.foreach { t =>
       assertSingleError(NullType, t, "nulls", s"Should not allow writing None to type $t") { err =>
-        assert(err.contains(s"incompatible with $t"))
+        assert(err.contains(s"incompatible with ${t.catalogString}"))
       }
     }
   }
@@ -145,12 +145,12 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase
   test("Conversions between timestamp and long are not allowed") {
     assertSingleError(LongType, TimestampType, "longToTimestamp",
       "Should not allow long to timestamp") { err =>
-      assert(err.contains("Cannot safely cast 'longToTimestamp': LongType to TimestampType"))
+      assert(err.contains("Cannot safely cast 'longToTimestamp': bigint to timestamp"))
     }
 
     assertSingleError(TimestampType, LongType, "timestampToLong",
       "Should not allow timestamp to long") { err =>
-      assert(err.contains("Cannot safely cast 'timestampToLong': TimestampType to LongType"))
+      assert(err.contains("Cannot safely cast 'timestampToLong': timestamp to bigint"))
     }
   }
 
@@ -209,8 +209,8 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
             s"Should not allow writing $w to $r because cast is not safe") { err =>
             assert(err.contains("'t'"), "Should include the field name context")
             assert(err.contains("Cannot safely cast"), "Should identify unsafe cast")
-            assert(err.contains(s"$w"), "Should include write type")
-            assert(err.contains(s"$r"), "Should include read type")
+            assert(err.contains(s"${w.catalogString}"), "Should include write type")
+            assert(err.contains(s"${r.catalogString}"), "Should include read type")
           }
         }
       }
@@ -413,7 +413,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
     assertNumErrors(writeType, readType, "top", "Should catch 14 errors", 14) { errs =>
       assert(errs(0).contains("'top.a.element'"), "Should identify bad type")
       assert(errs(0).contains("Cannot safely cast"))
-      assert(errs(0).contains("StringType to DoubleType"))
+      assert(errs(0).contains("string to double"))
 
       assert(errs(1).contains("'top.a'"), "Should identify bad type")
       assert(errs(1).contains("Cannot write nullable elements to array of non-nulls"))
@@ -430,11 +430,11 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
 
       assert(errs(5).contains("'top.m.key'"), "Should identify bad type")
       assert(errs(5).contains("Cannot safely cast"))
-      assert(errs(5).contains("StringType to LongType"))
+      assert(errs(5).contains("string to bigint"))
 
       assert(errs(6).contains("'top.m.value'"), "Should identify bad type")
       assert(errs(6).contains("Cannot safely cast"))
-      assert(errs(6).contains("BooleanType to FloatType"))
+      assert(errs(6).contains("boolean to float"))
 
       assert(errs(7).contains("'top.m'"), "Should identify bad type")
       assert(errs(7).contains("Cannot write nullable values to map of non-nulls"))
@@ -452,7 +452,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
 
       assert(errs(11).contains("'top.x'"), "Should identify bad type")
       assert(errs(11).contains("Cannot safely cast"))
-      assert(errs(11).contains("StringType to IntegerType"))
+      assert(errs(11).contains("string to int"))
 
       assert(errs(12).contains("'top'"), "Should identify bad type")
       assert(errs(12).contains("expected 'x', found 'y'"), "Should detect name mismatch")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
index 4439a7bb3ae87..0ffb492e702d1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.util
 import org.apache.arrow.vector.types.pojo.ArrowType
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.LA
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.types._
 
@@ -65,7 +66,7 @@ class ArrowUtilsSuite extends SparkFunSuite {
     roundtripWithTz(DateTimeUtils.defaultTimeZone().getID)
     roundtripWithTz("Asia/Tokyo")
     roundtripWithTz("UTC")
-    roundtripWithTz("America/Los_Angeles")
+    roundtripWithTz(LA.getId)
   }
 
   test("array") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
deleted file mode 100644
index d617b1c1d8237..0000000000000
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.util
-
-import java.time.{LocalDate, ZoneOffset}
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util._
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, localDateToDays}
-import org.apache.spark.sql.internal.SQLConf
-
-class DateFormatterSuite extends SparkFunSuite with SQLHelper {
-  test("parsing dates") {
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-        val formatter = DateFormatter(getZoneId(timeZone))
-        val daysSinceEpoch = formatter.parse("2018-12-02")
-        assert(daysSinceEpoch === 17867)
-      }
-    }
-  }
-
-  test("format dates") {
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-        val formatter = DateFormatter(getZoneId(timeZone))
-        val date = formatter.format(17867)
-        assert(date === "2018-12-02")
-      }
-    }
-  }
-
-  test("roundtrip date -> days -> date") {
-    Seq(
-      "0050-01-01",
-      "0953-02-02",
-      "1423-03-08",
-      "1969-12-31",
-      "1972-08-25",
-      "1975-09-26",
-      "2018-12-12",
-      "2038-01-01",
-      "5010-11-17").foreach { date =>
-      DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-          val formatter = DateFormatter(getZoneId(timeZone))
-          val days = formatter.parse(date)
-          val formatted = formatter.format(days)
-          assert(date === formatted)
-        }
-      }
-    }
-  }
-
-  test("roundtrip days -> date -> days") {
-    Seq(
-      -701265,
-      -371419,
-      -199722,
-      -1,
-      0,
-      967,
-      2094,
-      17877,
-      24837,
-      1110657).foreach { days =>
-      DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-          val formatter = DateFormatter(getZoneId(timeZone))
-          val date = formatter.format(days)
-          val parsed = formatter.parse(date)
-          assert(days === parsed)
-        }
-      }
-    }
-  }
-
-  test("parsing date without explicit day") {
-    val formatter = DateFormatter("yyyy MMM", ZoneOffset.UTC)
-    val daysSinceEpoch = formatter.parse("2018 Dec")
-    assert(daysSinceEpoch === LocalDate.of(2018, 12, 1).toEpochDay)
-  }
-
-  test("formatting negative years with default pattern") {
-    val epochDays = LocalDate.of(-99, 1, 1).toEpochDay.toInt
-    assert(DateFormatter(ZoneOffset.UTC).format(epochDays) === "-0099-01-01")
-  }
-
-  test("special date values") {
-    testSpecialDatetimeValues { zoneId =>
-      val formatter = DateFormatter(zoneId)
-
-      assert(formatter.parse("EPOCH") === 0)
-      val today = localDateToDays(LocalDate.now(zoneId))
-      assert(formatter.parse("Yesterday") === today - 1)
-      assert(formatter.parse("now") === today)
-      assert(formatter.parse("today ") === today)
-      assert(formatter.parse("tomorrow UTC") === today + 1)
-    }
-  }
-}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
deleted file mode 100644
index 082849c88669a..0000000000000
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.util
-
-import java.time.{Instant, LocalDateTime, LocalTime, ZoneOffset}
-import java.util.concurrent.TimeUnit
-
-import org.scalatest.Matchers
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils, TimestampFormatter}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.instantToMicros
-import org.apache.spark.unsafe.types.UTF8String
-
-class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
-
-  test("parsing timestamps using time zones") {
-    val localDate = "2018-12-02T10:11:12.001234"
-    val expectedMicros = Map(
-      "UTC" -> 1543745472001234L,
-      "PST" -> 1543774272001234L,
-      "CET" -> 1543741872001234L,
-      "Africa/Dakar" -> 1543745472001234L,
-      "America/Los_Angeles" -> 1543774272001234L,
-      "Antarctica/Vostok" -> 1543723872001234L,
-      "Asia/Hong_Kong" -> 1543716672001234L,
-      "Europe/Amsterdam" -> 1543741872001234L)
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
-      val formatter = TimestampFormatter(
-        "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
-        DateTimeUtils.getZoneId(zoneId))
-      val microsSinceEpoch = formatter.parse(localDate)
-      assert(microsSinceEpoch === expectedMicros(zoneId))
-    }
-  }
-
-  test("format timestamps using time zones") {
-    val microsSinceEpoch = 1543745472001234L
-    val expectedTimestamp = Map(
-      "UTC" -> "2018-12-02T10:11:12.001234",
-      "PST" -> "2018-12-02T02:11:12.001234",
-      "CET" -> "2018-12-02T11:11:12.001234",
-      "Africa/Dakar" -> "2018-12-02T10:11:12.001234",
-      "America/Los_Angeles" -> "2018-12-02T02:11:12.001234",
-      "Antarctica/Vostok" -> "2018-12-02T16:11:12.001234",
-      "Asia/Hong_Kong" -> "2018-12-02T18:11:12.001234",
-      "Europe/Amsterdam" -> "2018-12-02T11:11:12.001234")
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
-      val formatter = TimestampFormatter(
-        "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
-        DateTimeUtils.getZoneId(zoneId))
-      val timestamp = formatter.format(microsSinceEpoch)
-      assert(timestamp === expectedTimestamp(zoneId))
-    }
-  }
-
-  test("roundtrip micros -> timestamp -> micros using timezones") {
-    Seq("yyyy-MM-dd'T'HH:mm:ss.SSSSSS", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXXXX").foreach { pattern =>
-      Seq(
-        -58710115316212000L,
-        -18926315945345679L,
-        -9463427405253013L,
-        -244000001L,
-        0L,
-        99628200102030L,
-        1543749753123456L,
-        2177456523456789L,
-        11858049903010203L).foreach { micros =>
-        DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-          val formatter = TimestampFormatter(pattern, zoneId)
-          val timestamp = formatter.format(micros)
-          val parsed = formatter.parse(timestamp)
-          assert(micros === parsed)
-        }
-      }
-    }
-  }
-
-  test("roundtrip timestamp -> micros -> timestamp using timezones") {
-    Seq(
-      "0109-07-20T18:38:03.788000",
-      "1370-04-01T10:00:54.654321",
-      "1670-02-11T14:09:54.746987",
-      "1969-12-31T23:55:55.999999",
-      "1970-01-01T00:00:00.000000",
-      "1973-02-27T02:30:00.102030",
-      "2018-12-02T11:22:33.123456",
-      "2039-01-01T01:02:03.456789",
-      "2345-10-07T22:45:03.010203").foreach { timestamp =>
-      DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-        val formatter = TimestampFormatter("yyyy-MM-dd'T'HH:mm:ss.SSSSSS", zoneId)
-        val micros = formatter.parse(timestamp)
-        val formatted = formatter.format(micros)
-        assert(timestamp === formatted)
-      }
-    }
-  }
-
-  test(" case insensitive parsing of am and pm") {
-    val formatter = TimestampFormatter("yyyy MMM dd hh:mm:ss a", ZoneOffset.UTC)
-    val micros = formatter.parse("2009 Mar 20 11:30:01 am")
-    assert(micros === TimeUnit.SECONDS.toMicros(
-      LocalDateTime.of(2009, 3, 20, 11, 30, 1).toEpochSecond(ZoneOffset.UTC)))
-  }
-
-  test("format fraction of second") {
-    val formatter = TimestampFormatter.getFractionFormatter(ZoneOffset.UTC)
-    assert(formatter.format(0) === "1970-01-01 00:00:00")
-    assert(formatter.format(1) === "1970-01-01 00:00:00.000001")
-    assert(formatter.format(1000) === "1970-01-01 00:00:00.001")
-    assert(formatter.format(900000) === "1970-01-01 00:00:00.9")
-    assert(formatter.format(1000000) === "1970-01-01 00:00:01")
-  }
-
-  test("formatting negative years with default pattern") {
-    val instant = LocalDateTime.of(-99, 1, 1, 0, 0, 0)
-      .atZone(ZoneOffset.UTC)
-      .toInstant
-    val micros = DateTimeUtils.instantToMicros(instant)
-    assert(TimestampFormatter(ZoneOffset.UTC).format(micros) === "-0099-01-01 00:00:00")
-  }
-
-  test("special timestamp values") {
-    testSpecialDatetimeValues { zoneId =>
-      val formatter = TimestampFormatter(zoneId)
-      val tolerance = TimeUnit.SECONDS.toMicros(30)
-
-      assert(formatter.parse("EPOCH") === 0)
-      val now = instantToMicros(Instant.now())
-      formatter.parse("now") should be(now +- tolerance)
-      val localToday = LocalDateTime.now(zoneId)
-        .`with`(LocalTime.MIDNIGHT)
-        .atZone(zoneId)
-      val yesterday = instantToMicros(localToday.minusDays(1).toInstant)
-      formatter.parse("yesterday CET") should be(yesterday +- tolerance)
-      val today = instantToMicros(localToday.toInstant)
-      formatter.parse(" TODAY ") should be(today +- tolerance)
-      val tomorrow = instantToMicros(localToday.plusDays(1).toInstant)
-      formatter.parse("Tomorrow ") should be(tomorrow +- tolerance)
-    }
-  }
-
-  test("parsing timestamp strings with various seconds fractions") {
-    DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-      def check(pattern: String, input: String, reference: String): Unit = {
-        val formatter = TimestampFormatter(pattern, zoneId)
-        val expected = DateTimeUtils.stringToTimestamp(
-          UTF8String.fromString(reference), zoneId).get
-        val actual = formatter.parse(input)
-        assert(actual === expected)
-      }
-
-      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSSXXX",
-        "2019-10-14T09:39:07.3220000Z", "2019-10-14T09:39:07.322Z")
-      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
-        "2019-10-14T09:39:07.322000", "2019-10-14T09:39:07.322")
-      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX",
-        "2019-10-14T09:39:07.123456Z", "2019-10-14T09:39:07.123456Z")
-      check("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX",
-        "2019-10-14T09:39:07.000010Z", "2019-10-14T09:39:07.00001Z")
-      check("yyyy HH:mm:ss.SSSSS", "1970 01:02:03.00004", "1970-01-01 01:02:03.00004")
-      check("yyyy HH:mm:ss.SSSS", "2019 00:00:07.0100", "2019-01-01 00:00:07.0100")
-      check("yyyy-MM-dd'T'HH:mm:ss.SSSX",
-        "2019-10-14T09:39:07.322Z", "2019-10-14T09:39:07.322Z")
-      check("yyyy-MM-dd'T'HH:mm:ss.SS",
-        "2019-10-14T09:39:07.10", "2019-10-14T09:39:07.1")
-      check("yyyy-MM-dd'T'HH:mm:ss.S",
-        "2019-10-14T09:39:07.1", "2019-10-14T09:39:07.1")
-
-      try {
-        TimestampFormatter("yyyy/MM/dd HH_mm_ss.SSSSSS", zoneId)
-          .parse("2019/11/14 20#25#30.123456")
-        fail("Expected to throw an exception for the invalid input")
-      } catch {
-        case e: java.time.format.DateTimeParseException =>
-          assert(e.getMessage.contains("could not be parsed"))
-      }
-    }
-  }
-
-  test("formatting timestamp strings up to microsecond precision") {
-    DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-      def check(pattern: String, input: String, expected: String): Unit = {
-        val formatter = TimestampFormatter(pattern, zoneId)
-        val timestamp = DateTimeUtils.stringToTimestamp(
-          UTF8String.fromString(input), zoneId).get
-        val actual = formatter.format(timestamp)
-        assert(actual === expected)
-      }
-
-      check(
-        "yyyy-MM-dd HH:mm:ss.SSSSSSS", "2019-10-14T09:39:07.123456",
-        "2019-10-14 09:39:07.1234560")
-      check(
-        "yyyy-MM-dd HH:mm:ss.SSSSSS", "1960-01-01T09:39:07.123456",
-        "1960-01-01 09:39:07.123456")
-      check(
-        "yyyy-MM-dd HH:mm:ss.SSSSS", "0001-10-14T09:39:07.1",
-        "0001-10-14 09:39:07.10000")
-      check(
-        "yyyy-MM-dd HH:mm:ss.SSSS", "9999-12-31T23:59:59.999",
-        "9999-12-31 23:59:59.9990")
-      check(
-        "yyyy-MM-dd HH:mm:ss.SSS", "1970-01-01T00:00:00.0101",
-        "1970-01-01 00:00:00.010")
-      check(
-        "yyyy-MM-dd HH:mm:ss.SS", "2019-10-14T09:39:07.09",
-        "2019-10-14 09:39:07.09")
-      check(
-        "yyyy-MM-dd HH:mm:ss.S", "2019-10-14T09:39:07.2",
-        "2019-10-14 09:39:07.2")
-      check(
-        "yyyy-MM-dd HH:mm:ss.S", "2019-10-14T09:39:07",
-        "2019-10-14 09:39:07.0")
-      check(
-        "yyyy-MM-dd HH:mm:ss", "2019-10-14T09:39:07.123456",
-        "2019-10-14 09:39:07")
-    }
-  }
-}
diff --git a/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt b/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
index d8071e7bbdb35..0e82b632793d2 100644
--- a/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
@@ -2,66 +2,66 @@
 Benchmark to measure CSV read/write performance
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Parsing quoted values:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-One quoted string                                 44297          44515         373          0.0      885948.7       1.0X
+One quoted string                                 46568          46683         198          0.0      931358.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Wide rows with 1000 columns:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 1000 columns                              196720         197783        1560          0.0      196719.8       1.0X
-Select 100 columns                                46691          46861         219          0.0       46691.4       4.2X
-Select one column                                 36811          36922         111          0.0       36811.3       5.3X
-count()                                            8520           8610         106          0.1        8520.5      23.1X
-Select 100 columns, one bad input field           67914          67994         136          0.0       67914.0       2.9X
-Select 100 columns, corrupt record field          77272          77445         214          0.0       77272.0       2.5X
+Select 1000 columns                              129836         130796        1404          0.0      129836.0       1.0X
+Select 100 columns                                40444          40679         261          0.0       40443.5       3.2X
+Select one column                                 33429          33475          73          0.0       33428.6       3.9X
+count()                                            7967           8047          73          0.1        7966.7      16.3X
+Select 100 columns, one bad input field           90639          90832         266          0.0       90638.6       1.4X
+Select 100 columns, corrupt record field         109023         109084          74          0.0      109023.3       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Count a dataset with 10 columns:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns + count()                       25965          26054         103          0.4        2596.5       1.0X
-Select 1 column + count()                         18591          18666          91          0.5        1859.1       1.4X
-count()                                            6102           6119          18          1.6         610.2       4.3X
+Select 10 columns + count()                       20685          20707          35          0.5        2068.5       1.0X
+Select 1 column + count()                         13096          13149          49          0.8        1309.6       1.6X
+count()                                            3994           4001           7          2.5         399.4       5.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     2142           2161          17          4.7         214.2       1.0X
-to_csv(timestamp)                                 14744          14950         182          0.7        1474.4       0.1X
-write timestamps to files                         12078          12202         175          0.8        1207.8       0.2X
-Create a dataset of dates                          2275           2291          18          4.4         227.5       0.9X
-to_csv(date)                                      11407          11464          51          0.9        1140.7       0.2X
-write dates to files                               7638           7702          90          1.3         763.8       0.3X
+Create a dataset of timestamps                     2169           2203          32          4.6         216.9       1.0X
+to_csv(timestamp)                                 14401          14591         168          0.7        1440.1       0.2X
+write timestamps to files                         13209          13276          59          0.8        1320.9       0.2X
+Create a dataset of dates                          2231           2248          17          4.5         223.1       1.0X
+to_csv(date)                                      10406          10473          68          1.0        1040.6       0.2X
+write dates to files                               7970           7976           9          1.3         797.0       0.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2578           2590          10          3.9         257.8       1.0X
-read timestamps from files                        60103          60694         512          0.2        6010.3       0.0X
-infer timestamps from files                      107871         108268         351          0.1       10787.1       0.0X
-read date text from files                          2306           2310           4          4.3         230.6       1.1X
-read date from files                              47415          47657         367          0.2        4741.5       0.1X
-infer date from files                             35261          35447         164          0.3        3526.1       0.1X
-timestamp strings                                  3045           3056          11          3.3         304.5       0.8X
-parse timestamps from Dataset[String]             62221          63173         849          0.2        6222.1       0.0X
-infer timestamps from Dataset[String]            118838         119629         697          0.1       11883.8       0.0X
-date strings                                       3459           3481          19          2.9         345.9       0.7X
-parse dates from Dataset[String]                  51026          51447         503          0.2        5102.6       0.1X
-from_csv(timestamp)                               60738          61818         936          0.2        6073.8       0.0X
-from_csv(date)                                    46012          46278         370          0.2        4601.2       0.1X
+read timestamp text from files                     2387           2391           6          4.2         238.7       1.0X
+read timestamps from files                        53503          53593         124          0.2        5350.3       0.0X
+infer timestamps from files                      107988         108668         647          0.1       10798.8       0.0X
+read date text from files                          2121           2133          12          4.7         212.1       1.1X
+read date from files                              29983          30039          48          0.3        2998.3       0.1X
+infer date from files                             30196          30436         218          0.3        3019.6       0.1X
+timestamp strings                                  3098           3109          10          3.2         309.8       0.8X
+parse timestamps from Dataset[String]             63331          63426          84          0.2        6333.1       0.0X
+infer timestamps from Dataset[String]            124003         124463         490          0.1       12400.3       0.0X
+date strings                                       3423           3429          11          2.9         342.3       0.7X
+parse dates from Dataset[String]                  34235          34314          76          0.3        3423.5       0.1X
+from_csv(timestamp)                               60829          61600         668          0.2        6082.9       0.0X
+from_csv(date)                                    33047          33173         139          0.3        3304.7       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10 on Mac OS X 10.15.2
-Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-w/o filters                                       11889          11945          52          0.0      118893.1       1.0X
-pushdown disabled                                 11790          11860         115          0.0      117902.3       1.0X
-w/ filters                                         1240           1278          33          0.1       12400.8       9.6X
+w/o filters                                       28752          28765          16          0.0      287516.5       1.0X
+pushdown disabled                                 28856          28880          22          0.0      288556.3       1.0X
+w/ filters                                         1714           1731          15          0.1       17137.3      16.8X
 
 
diff --git a/sql/core/benchmarks/CSVBenchmark-results.txt b/sql/core/benchmarks/CSVBenchmark-results.txt
index b3ba69c9eb6b1..a3af46c037bf9 100644
--- a/sql/core/benchmarks/CSVBenchmark-results.txt
+++ b/sql/core/benchmarks/CSVBenchmark-results.txt
@@ -2,66 +2,66 @@
 Benchmark to measure CSV read/write performance
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Parsing quoted values:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-One quoted string                                 51602          51659          59          0.0     1032039.4       1.0X
+One quoted string                                 45457          45731         344          0.0      909136.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Wide rows with 1000 columns:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 1000 columns                              191926         192879        1615          0.0      191925.6       1.0X
-Select 100 columns                                46766          46846          69          0.0       46766.1       4.1X
-Select one column                                 35877          35930          83          0.0       35876.8       5.3X
-count()                                           11186          11262          65          0.1       11186.0      17.2X
-Select 100 columns, one bad input field           59943          60107         232          0.0       59943.0       3.2X
-Select 100 columns, corrupt record field          73062          73406         479          0.0       73062.2       2.6X
+Select 1000 columns                              129646         130527        1412          0.0      129646.3       1.0X
+Select 100 columns                                42444          42551         119          0.0       42444.0       3.1X
+Select one column                                 35415          35428          20          0.0       35414.6       3.7X
+count()                                           11114          11128          16          0.1       11113.6      11.7X
+Select 100 columns, one bad input field           93353          93670         275          0.0       93352.6       1.4X
+Select 100 columns, corrupt record field         113569         113952         373          0.0      113568.8       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Count a dataset with 10 columns:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns + count()                       22389          22447          87          0.4        2238.9       1.0X
-Select 1 column + count()                         14844          14890          43          0.7        1484.4       1.5X
-count()                                            5519           5538          18          1.8         551.9       4.1X
+Select 10 columns + count()                       18498          18589          87          0.5        1849.8       1.0X
+Select 1 column + count()                         11078          11095          27          0.9        1107.8       1.7X
+count()                                            3928           3950          22          2.5         392.8       4.7X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     1949           1977          25          5.1         194.9       1.0X
-to_csv(timestamp)                                 14944          15702         714          0.7        1494.4       0.1X
-write timestamps to files                         12983          12998          14          0.8        1298.3       0.2X
-Create a dataset of dates                          2156           2164           7          4.6         215.6       0.9X
-to_csv(date)                                       9675           9709          41          1.0         967.5       0.2X
-write dates to files                               7880           7897          15          1.3         788.0       0.2X
+Create a dataset of timestamps                     1933           1940          11          5.2         193.3       1.0X
+to_csv(timestamp)                                 18078          18243         255          0.6        1807.8       0.1X
+write timestamps to files                         12668          12786         134          0.8        1266.8       0.2X
+Create a dataset of dates                          2196           2201           5          4.6         219.6       0.9X
+to_csv(date)                                       9583           9597          21          1.0         958.3       0.2X
+write dates to files                               7091           7110          20          1.4         709.1       0.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2235           2245          10          4.5         223.5       1.0X
-read timestamps from files                        54490          54690         283          0.2        5449.0       0.0X
-infer timestamps from files                      104501         104737         236          0.1       10450.1       0.0X
-read date text from files                          2035           2040           6          4.9         203.5       1.1X
-read date from files                              39650          39707          52          0.3        3965.0       0.1X
-infer date from files                             29235          29363         164          0.3        2923.5       0.1X
-timestamp strings                                  3412           3426          18          2.9         341.2       0.7X
-parse timestamps from Dataset[String]             66864          67804         981          0.1        6686.4       0.0X
-infer timestamps from Dataset[String]            118780         119284         837          0.1       11878.0       0.0X
-date strings                                       3730           3734           4          2.7         373.0       0.6X
-parse dates from Dataset[String]                  48728          49071         309          0.2        4872.8       0.0X
-from_csv(timestamp)                               62294          62493         260          0.2        6229.4       0.0X
-from_csv(date)                                    44581          44665         117          0.2        4458.1       0.1X
+read timestamp text from files                     2166           2177          10          4.6         216.6       1.0X
+read timestamps from files                        53212          53402         281          0.2        5321.2       0.0X
+infer timestamps from files                      109788         110372         570          0.1       10978.8       0.0X
+read date text from files                          1921           1929           8          5.2         192.1       1.1X
+read date from files                              25470          25499          25          0.4        2547.0       0.1X
+infer date from files                             27201          27342         134          0.4        2720.1       0.1X
+timestamp strings                                  3638           3653          19          2.7         363.8       0.6X
+parse timestamps from Dataset[String]             61894          62532         555          0.2        6189.4       0.0X
+infer timestamps from Dataset[String]            125171         125430         236          0.1       12517.1       0.0X
+date strings                                       3736           3749          14          2.7         373.6       0.6X
+parse dates from Dataset[String]                  30787          30829          43          0.3        3078.7       0.1X
+from_csv(timestamp)                               60842          61035         209          0.2        6084.2       0.0X
+from_csv(date)                                    30123          30196          95          0.3        3012.3       0.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.2
-Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-w/o filters                                       12557          12634          78          0.0      125572.9       1.0X
-pushdown disabled                                 12449          12509          65          0.0      124486.4       1.0X
-w/ filters                                         1372           1393          18          0.1       13724.8       9.1X
+w/o filters                                       28985          29042          80          0.0      289852.9       1.0X
+pushdown disabled                                 29080          29146          58          0.0      290799.4       1.0X
+w/ filters                                         2072           2084          17          0.0       20722.3      14.0X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 729b112de5e75..70d888227141d 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -1,429 +1,461 @@
+================================================================================================
+datetime +/- interval
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+date + interval(m)                                 1660           1745         120          6.0         166.0       1.0X
+date + interval(m, d)                              1672           1685          19          6.0         167.2       1.0X
+date + interval(m, d, ms)                          6462           6481          27          1.5         646.2       0.3X
+date - interval(m)                                 1456           1480          35          6.9         145.6       1.1X
+date - interval(m, d)                              1501           1509          11          6.7         150.1       1.1X
+date - interval(m, d, ms)                          6457           6466          12          1.5         645.7       0.3X
+timestamp + interval(m)                            2941           2944           4          3.4         294.1       0.6X
+timestamp + interval(m, d)                         3008           3012           6          3.3         300.8       0.6X
+timestamp + interval(m, d, ms)                     3329           3333           6          3.0         332.9       0.5X
+timestamp - interval(m)                            2964           2982          26          3.4         296.4       0.6X
+timestamp - interval(m, d)                         3030           3039          13          3.3         303.0       0.5X
+timestamp - interval(m, d, ms)                     3312           3313           1          3.0         331.2       0.5X
+
+
 ================================================================================================
 Extract components
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    546            572          36         18.3          54.6       1.0X
-cast to timestamp wholestage on                     412            438          16         24.3          41.2       1.3X
+cast to timestamp wholestage off                    333            334           0         30.0          33.3       1.0X
+cast to timestamp wholestage on                     349            368          12         28.6          34.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1240           1295          77          8.1         124.0       1.0X
-year of timestamp wholestage on                    1109           1130          24          9.0         110.9       1.1X
+year of timestamp wholestage off                   1229           1229           1          8.1         122.9       1.0X
+year of timestamp wholestage on                    1218           1223           5          8.2         121.8       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1572           1574           3          6.4         157.2       1.0X
-quarter of timestamp wholestage on                 1386           1405          18          7.2         138.6       1.1X
+quarter of timestamp wholestage off                1593           1594           2          6.3         159.3       1.0X
+quarter of timestamp wholestage on                 1515           1529          14          6.6         151.5       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1194           1196           2          8.4         119.4       1.0X
-month of timestamp wholestage on                   1057           1069          12          9.5         105.7       1.1X
+month of timestamp wholestage off                  1222           1246          34          8.2         122.2       1.0X
+month of timestamp wholestage on                   1207           1232          31          8.3         120.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             2070           2071           2          4.8         207.0       1.0X
-weekofyear of timestamp wholestage on              1549           1555           6          6.5         154.9       1.3X
+weekofyear of timestamp wholestage off             2453           2455           2          4.1         245.3       1.0X
+weekofyear of timestamp wholestage on              2357           2380          22          4.2         235.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1173           1186          18          8.5         117.3       1.0X
-day of timestamp wholestage on                     1056           1076          26          9.5         105.6       1.1X
+day of timestamp wholestage off                    1216           1219           5          8.2         121.6       1.0X
+day of timestamp wholestage on                     1205           1221          25          8.3         120.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1207           1211           6          8.3         120.7       1.0X
-dayofyear of timestamp wholestage on               1097           1108           9          9.1         109.7       1.1X
+dayofyear of timestamp wholestage off              1268           1274           9          7.9         126.8       1.0X
+dayofyear of timestamp wholestage on               1253           1268          10          8.0         125.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1184           1190           8          8.4         118.4       1.0X
-dayofmonth of timestamp wholestage on              1053           1060           9          9.5         105.3       1.1X
+dayofmonth of timestamp wholestage off             1223           1224           1          8.2         122.3       1.0X
+dayofmonth of timestamp wholestage on              1231           1246          14          8.1         123.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1343           1362          27          7.4         134.3       1.0X
-dayofweek of timestamp wholestage on               1228           1239           7          8.1         122.8       1.1X
+dayofweek of timestamp wholestage off              1398           1406          12          7.2         139.8       1.0X
+dayofweek of timestamp wholestage on               1387           1399          15          7.2         138.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1276           1278           3          7.8         127.6       1.0X
-weekday of timestamp wholestage on                 1160           1181          22          8.6         116.0       1.1X
+weekday of timestamp wholestage off                1327           1333           9          7.5         132.7       1.0X
+weekday of timestamp wholestage on                 1329           1333           4          7.5         132.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    854            862          12         11.7          85.4       1.0X
-hour of timestamp wholestage on                     741            748           6         13.5          74.1       1.2X
+hour of timestamp wholestage off                   1005           1016          15          9.9         100.5       1.0X
+hour of timestamp wholestage on                     934            940           4         10.7          93.4       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  853            854           1         11.7          85.3       1.0X
-minute of timestamp wholestage on                   730            737          11         13.7          73.0       1.2X
+minute of timestamp wholestage off                 1003           1009           8         10.0         100.3       1.0X
+minute of timestamp wholestage on                   934            938           7         10.7          93.4       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  726            728           2         13.8          72.6       1.0X
-second of timestamp wholestage on                   614            623           9         16.3          61.4       1.2X
+second of timestamp wholestage off                  997            998           2         10.0          99.7       1.0X
+second of timestamp wholestage on                   925            935           8         10.8          92.5       1.1X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         369            370           2         27.1          36.9       1.0X
-current_date wholestage on                          277            284           8         36.2          27.7       1.3X
+current_date wholestage off                         297            297           0         33.7          29.7       1.0X
+current_date wholestage on                          280            282           2         35.7          28.0       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    410            411           1         24.4          41.0       1.0X
-current_timestamp wholestage on                     283            418         259         35.4          28.3       1.5X
+current_timestamp wholestage off                    307            337          43         32.6          30.7       1.0X
+current_timestamp wholestage on                     260            284          29         38.4          26.0       1.2X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         987            992           7         10.1          98.7       1.0X
-cast to date wholestage on                          891            896           4         11.2          89.1       1.1X
+cast to date wholestage off                        1066           1073          10          9.4         106.6       1.0X
+cast to date wholestage on                          997           1003           6         10.0          99.7       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1179           1179           1          8.5         117.9       1.0X
-last_day wholestage on                             1052           1080          44          9.5         105.2       1.1X
+last_day wholestage off                            1238           1242           6          8.1         123.8       1.0X
+last_day wholestage on                             1259           1272          12          7.9         125.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1073           1081          11          9.3         107.3       1.0X
-next_day wholestage on                              948            955          10         10.5          94.8       1.1X
+next_day wholestage off                            1116           1138          32          9.0         111.6       1.0X
+next_day wholestage on                             1052           1063          11          9.5         105.2       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1006           1009           5          9.9         100.6       1.0X
-date_add wholestage on                              867            870           4         11.5          86.7       1.2X
+date_add wholestage off                            1048           1049           1          9.5         104.8       1.0X
+date_add wholestage on                             1035           1039           3          9.7         103.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             980            988          11         10.2          98.0       1.0X
-date_sub wholestage on                              866            873           9         11.6          86.6       1.1X
+date_sub wholestage off                            1119           1127          11          8.9         111.9       1.0X
+date_sub wholestage on                             1028           1039           7          9.7         102.8       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1329           1332           4          7.5         132.9       1.0X
-add_months wholestage on                           1199           1206           8          8.3         119.9       1.1X
+add_months wholestage off                          1421           1421           0          7.0         142.1       1.0X
+add_months wholestage on                           1423           1434          11          7.0         142.3       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         4724           4736          18          2.1         472.4       1.0X
-format date wholestage on                          4550           4574          26          2.2         455.0       1.0X
+format date wholestage off                         5293           5296           5          1.9         529.3       1.0X
+format date wholestage on                          5143           5157          19          1.9         514.3       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       7171           7183          17          1.4         717.1       1.0X
-from_unixtime wholestage on                        7114           7141          20          1.4         711.4       1.0X
+from_unixtime wholestage off                       7136           7136           1          1.4         713.6       1.0X
+from_unixtime wholestage on                        7049           7068          29          1.4         704.9       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1498           1504           8          6.7         149.8       1.0X
-from_utc_timestamp wholestage on                   1399           1405           5          7.1         139.9       1.1X
+from_utc_timestamp wholestage off                  1325           1329           6          7.5         132.5       1.0X
+from_utc_timestamp wholestage on                   1269           1273           4          7.9         126.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1505           1507           2          6.6         150.5       1.0X
-to_utc_timestamp wholestage on                     1396           1401           5          7.2         139.6       1.1X
+to_utc_timestamp wholestage off                    1684           1691          10          5.9         168.4       1.0X
+to_utc_timestamp wholestage on                     1641           1648           9          6.1         164.1       1.0X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        423            428           7         23.6          42.3       1.0X
-cast interval wholestage on                         300            302           2         33.3          30.0       1.4X
+cast interval wholestage off                        343            346           4         29.1          34.3       1.0X
+cast interval wholestage on                         281            282           1         35.6          28.1       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1626           1630           6          6.1         162.6       1.0X
-datediff wholestage on                             1467           1471           3          6.8         146.7       1.1X
+datediff wholestage off                            1831           1840          13          5.5         183.1       1.0X
+datediff wholestage on                             1759           1769          15          5.7         175.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      1988           1992           5          5.0         198.8       1.0X
-months_between wholestage on                       1812           1834          24          5.5         181.2       1.1X
+months_between wholestage off                      5729           5747          25          1.7         572.9       1.0X
+months_between wholestage on                       5710           5720           9          1.8         571.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2277           2334          80          0.4        2277.1       1.0X
-window wholestage on                              48996          49048          67          0.0       48996.0       0.0X
+window wholestage off                              2183           2189           9          0.5        2182.6       1.0X
+window wholestage on                              46835          46944          88          0.0       46834.8       0.0X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                      867            870           6         11.5          86.7       1.0X
-date_trunc YEAR wholestage on                       815            819           6         12.3          81.5       1.1X
+date_trunc YEAR wholestage off                     2668           2672           5          3.7         266.8       1.0X
+date_trunc YEAR wholestage on                      2719           2731           9          3.7         271.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                      866            875          13         11.5          86.6       1.0X
-date_trunc YYYY wholestage on                       811            813           2         12.3          81.1       1.1X
+date_trunc YYYY wholestage off                     2672           2677           8          3.7         267.2       1.0X
+date_trunc YYYY wholestage on                      2710           2726          12          3.7         271.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                        864            867           4         11.6          86.4       1.0X
-date_trunc YY wholestage on                         812            824          10         12.3          81.2       1.1X
+date_trunc YY wholestage off                       2670           2673           4          3.7         267.0       1.0X
+date_trunc YY wholestage on                        2711           2720           7          3.7         271.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                       881            884           4         11.3          88.1       1.0X
-date_trunc MON wholestage on                        820            826           7         12.2          82.0       1.1X
+date_trunc MON wholestage off                      2674           2674           0          3.7         267.4       1.0X
+date_trunc MON wholestage on                       2667           2677          10          3.7         266.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                     880            881           2         11.4          88.0       1.0X
-date_trunc MONTH wholestage on                      819            822           4         12.2          81.9       1.1X
+date_trunc MONTH wholestage off                    2675           2686          16          3.7         267.5       1.0X
+date_trunc MONTH wholestage on                     2667           2674           6          3.7         266.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                        889            904          21         11.2          88.9       1.0X
-date_trunc MM wholestage on                         818            828           8         12.2          81.8       1.1X
+date_trunc MM wholestage off                       2673           2674           1          3.7         267.3       1.0X
+date_trunc MM wholestage on                        2664           2669           4          3.8         266.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                       590            593           4         16.9          59.0       1.0X
-date_trunc DAY wholestage on                        510            514           4         19.6          51.0       1.2X
+date_trunc DAY wholestage off                      2281           2288          10          4.4         228.1       1.0X
+date_trunc DAY wholestage on                       2302           2312           8          4.3         230.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                        596            604          11         16.8          59.6       1.0X
-date_trunc DD wholestage on                         511            519           9         19.6          51.1       1.2X
+date_trunc DD wholestage off                       2281           2283           3          4.4         228.1       1.0X
+date_trunc DD wholestage on                        2291           2302          11          4.4         229.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                      586            592           9         17.1          58.6       1.0X
-date_trunc HOUR wholestage on                       507            513           5         19.7          50.7       1.2X
+date_trunc HOUR wholestage off                     2331           2332           1          4.3         233.1       1.0X
+date_trunc HOUR wholestage on                      2290           2304          11          4.4         229.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    561            562           1         17.8          56.1       1.0X
-date_trunc MINUTE wholestage on                     480            485           4         20.8          48.0       1.2X
+date_trunc MINUTE wholestage off                    379            385           9         26.4          37.9       1.0X
+date_trunc MINUTE wholestage on                     371            376           5         27.0          37.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    561            561           1         17.8          56.1       1.0X
-date_trunc SECOND wholestage on                     479            480           2         20.9          47.9       1.2X
+date_trunc SECOND wholestage off                    375            376           1         26.7          37.5       1.0X
+date_trunc SECOND wholestage on                     370            376           8         27.0          37.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                      725            727           2         13.8          72.5       1.0X
-date_trunc WEEK wholestage on                       674            684          11         14.8          67.4       1.1X
+date_trunc WEEK wholestage off                     2597           2604          10          3.9         259.7       1.0X
+date_trunc WEEK wholestage on                      2591           2605          13          3.9         259.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  1653           1659          10          6.1         165.3       1.0X
-date_trunc QUARTER wholestage on                   1588           1601          12          6.3         158.8       1.0X
+date_trunc QUARTER wholestage off                  3501           3511          14          2.9         350.1       1.0X
+date_trunc QUARTER wholestage on                   3477           3489           9          2.9         347.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           391            393           2         25.6          39.1       1.0X
-trunc year wholestage on                            312            316           4         32.1          31.2       1.3X
+trunc year wholestage off                           332            334           3         30.1          33.2       1.0X
+trunc year wholestage on                            332            346          17         30.1          33.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           390            394           5         25.6          39.0       1.0X
-trunc yyyy wholestage on                            316            319           4         31.6          31.6       1.2X
+trunc yyyy wholestage off                           331            331           0         30.2          33.1       1.0X
+trunc yyyy wholestage on                            336            339           4         29.8          33.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             387            390           5         25.8          38.7       1.0X
-trunc yy wholestage on                              313            316           3         31.9          31.3       1.2X
+trunc yy wholestage off                             330            342          17         30.3          33.0       1.0X
+trunc yy wholestage on                              333            337           3         30.0          33.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            388            395          11         25.8          38.8       1.0X
-trunc mon wholestage on                             314            316           2         31.8          31.4       1.2X
+trunc mon wholestage off                            334            335           1         30.0          33.4       1.0X
+trunc mon wholestage on                             333            347           9         30.0          33.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          388            390           4         25.8          38.8       1.0X
-trunc month wholestage on                           315            318           2         31.7          31.5       1.2X
+trunc month wholestage off                          332            333           1         30.1          33.2       1.0X
+trunc month wholestage on                           333            340           7         30.0          33.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             384            388           4         26.0          38.4       1.0X
-trunc mm wholestage on                              314            321           9         31.9          31.4       1.2X
+trunc mm wholestage off                             328            336          11         30.5          32.8       1.0X
+trunc mm wholestage on                              333            343          11         30.0          33.3       1.0X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     178            191          18          5.6         178.5       1.0X
-to timestamp str wholestage on                      157            160           2          6.4         156.6       1.1X
+to timestamp str wholestage off                     170            171           1          5.9         170.1       1.0X
+to timestamp str wholestage on                      172            174           2          5.8         171.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1790           1795           7          0.6        1790.0       1.0X
-to_timestamp wholestage on                         1813           1820          10          0.6        1813.1       1.0X
+to_timestamp wholestage off                        1437           1439           3          0.7        1437.0       1.0X
+to_timestamp wholestage on                         1288           1292           5          0.8        1288.1       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1836           1837           1          0.5        1835.8       1.0X
-to_unix_timestamp wholestage on                    1786           1791           3          0.6        1785.8       1.0X
+to_unix_timestamp wholestage off                   1352           1353           2          0.7        1352.0       1.0X
+to_unix_timestamp wholestage on                    1314           1319           5          0.8        1314.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          169            169           1          5.9         168.8       1.0X
-to date str wholestage on                           151            153           2          6.6         151.2       1.1X
+to date str wholestage off                          211            215           6          4.7         210.7       1.0X
+to date str wholestage on                           217            217           1          4.6         216.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             2504           2512          10          0.4        2504.4       1.0X
-to_date wholestage on                              2522           2536          19          0.4        2522.3       1.0X
+to_date wholestage off                             3281           3295          20          0.3        3280.9       1.0X
+to_date wholestage on                              3223           3239          17          0.3        3222.8       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             346            367          35         14.5          69.2       1.0X
-Collect longs                                      2139           2329         289          2.3         427.7       0.2X
-Collect timestamps                                 1883           2086         303          2.7         376.5       0.2X
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+From java.sql.Date                                  446            447           1         11.2          89.1       1.0X
+From java.time.LocalDate                            354            356           1         14.1          70.8       1.3X
+Collect java.sql.Date                              2722           3091         495          1.8         544.4       0.2X
+Collect java.time.LocalDate                        1786           1836          60          2.8         357.2       0.2X
+From java.sql.Timestamp                             275            287          19         18.2          55.0       1.6X
+From java.time.Instant                              325            328           3         15.4          65.0       1.4X
+Collect longs                                      1300           1321          25          3.8         260.0       0.3X
+Collect java.sql.Timestamp                         1450           1557         102          3.4         290.0       0.3X
+Collect java.time.Instant                          1499           1599          87          3.3         299.9       0.3X
+java.sql.Date to Hive string                      17536          18367        1059          0.3        3507.2       0.0X
+java.time.LocalDate to Hive string                12089          12897         725          0.4        2417.8       0.0X
+java.sql.Timestamp to Hive string                 48014          48625         752          0.1        9602.9       0.0X
+java.time.Instant to Hive string                  37346          37445          93          0.1        7469.1       0.0X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 0c30534dd71ca..0795f11a57f28 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -1,429 +1,461 @@
+================================================================================================
+datetime +/- interval
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+date + interval(m)                                 1555           1634         113          6.4         155.5       1.0X
+date + interval(m, d)                              1774           1797          33          5.6         177.4       0.9X
+date + interval(m, d, ms)                          6293           6335          59          1.6         629.3       0.2X
+date - interval(m)                                 1461           1468          10          6.8         146.1       1.1X
+date - interval(m, d)                              1741           1741           0          5.7         174.1       0.9X
+date - interval(m, d, ms)                          6503           6518          21          1.5         650.3       0.2X
+timestamp + interval(m)                            2384           2385           1          4.2         238.4       0.7X
+timestamp + interval(m, d)                         2683           2684           2          3.7         268.3       0.6X
+timestamp + interval(m, d, ms)                     2987           3001          19          3.3         298.7       0.5X
+timestamp - interval(m)                            2391           2395           5          4.2         239.1       0.7X
+timestamp - interval(m, d)                         2674           2684          14          3.7         267.4       0.6X
+timestamp - interval(m, d, ms)                     3005           3007           3          3.3         300.5       0.5X
+
+
 ================================================================================================
 Extract components
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    425            447          30         23.5          42.5       1.0X
-cast to timestamp wholestage on                     368            401          29         27.2          36.8       1.2X
+cast to timestamp wholestage off                    313            320          10         31.9          31.3       1.0X
+cast to timestamp wholestage on                     325            341          18         30.8          32.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1158           1215          80          8.6         115.8       1.0X
-year of timestamp wholestage on                    1158           1179          31          8.6         115.8       1.0X
+year of timestamp wholestage off                   1216           1216           1          8.2         121.6       1.0X
+year of timestamp wholestage on                    1226           1243          13          8.2         122.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1285           1295          15          7.8         128.5       1.0X
-quarter of timestamp wholestage on                 1243           1257          11          8.0         124.3       1.0X
+quarter of timestamp wholestage off                1417           1421           5          7.1         141.7       1.0X
+quarter of timestamp wholestage on                 1358           1365           8          7.4         135.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1076           1082           8          9.3         107.6       1.0X
-month of timestamp wholestage on                   1088           1098           9          9.2         108.8       1.0X
+month of timestamp wholestage off                  1219           1220           1          8.2         121.9       1.0X
+month of timestamp wholestage on                   1222           1227           7          8.2         122.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1649           1659          14          6.1         164.9       1.0X
-weekofyear of timestamp wholestage on              1648           1656           8          6.1         164.8       1.0X
+weekofyear of timestamp wholestage off             1950           1950           0          5.1         195.0       1.0X
+weekofyear of timestamp wholestage on              1890           1899           8          5.3         189.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1083           1084           3          9.2         108.3       1.0X
-day of timestamp wholestage on                     1082           1089          13          9.2         108.2       1.0X
+day of timestamp wholestage off                    1212           1213           2          8.3         121.2       1.0X
+day of timestamp wholestage on                     1216           1227          13          8.2         121.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1102           1103           1          9.1         110.2       1.0X
-dayofyear of timestamp wholestage on               1123           1138          14          8.9         112.3       1.0X
+dayofyear of timestamp wholestage off              1282           1284           3          7.8         128.2       1.0X
+dayofyear of timestamp wholestage on               1269           1274           5          7.9         126.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1068           1073           7          9.4         106.8       1.0X
-dayofmonth of timestamp wholestage on              1082           1095          13          9.2         108.2       1.0X
+dayofmonth of timestamp wholestage off             1214           1219           7          8.2         121.4       1.0X
+dayofmonth of timestamp wholestage on              1216           1224           6          8.2         121.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1265           1294          41          7.9         126.5       1.0X
-dayofweek of timestamp wholestage on               1253           1262          11          8.0         125.3       1.0X
+dayofweek of timestamp wholestage off              1403           1430          39          7.1         140.3       1.0X
+dayofweek of timestamp wholestage on               1378           1386           8          7.3         137.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1189           1191           3          8.4         118.9       1.0X
-weekday of timestamp wholestage on                 1193           1199           6          8.4         119.3       1.0X
+weekday of timestamp wholestage off                1344           1353          13          7.4         134.4       1.0X
+weekday of timestamp wholestage on                 1316           1322           5          7.6         131.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    366            368           3         27.3          36.6       1.0X
-hour of timestamp wholestage on                     360            364           6         27.8          36.0       1.0X
+hour of timestamp wholestage off                    992           1000          10         10.1          99.2       1.0X
+hour of timestamp wholestage on                     960            962           3         10.4          96.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  348            350           2         28.7          34.8       1.0X
-minute of timestamp wholestage on                   355            361           9         28.1          35.5       1.0X
+minute of timestamp wholestage off                  989           1000          16         10.1          98.9       1.0X
+minute of timestamp wholestage on                   965            974          13         10.4          96.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  347            352           7         28.8          34.7       1.0X
-second of timestamp wholestage on                   351            359          10         28.5          35.1       1.0X
+second of timestamp wholestage off                  974            977           5         10.3          97.4       1.0X
+second of timestamp wholestage on                   959            966           8         10.4          95.9       1.0X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         284            287           4         35.2          28.4       1.0X
-current_date wholestage on                          312            318           6         32.1          31.2       0.9X
+current_date wholestage off                         281            282           2         35.6          28.1       1.0X
+current_date wholestage on                          294            300           5         34.0          29.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    291            292           2         34.4          29.1       1.0X
-current_timestamp wholestage on                     297            333          40         33.6          29.7       1.0X
+current_timestamp wholestage off                    282            296          19         35.4          28.2       1.0X
+current_timestamp wholestage on                     304            331          31         32.9          30.4       0.9X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         903            903           1         11.1          90.3       1.0X
-cast to date wholestage on                          897            900           7         11.2          89.7       1.0X
+cast to date wholestage off                        1060           1061           1          9.4         106.0       1.0X
+cast to date wholestage on                         1021           1026          10          9.8         102.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1082           1082           1          9.2         108.2       1.0X
-last_day wholestage on                             1107           1118          16          9.0         110.7       1.0X
+last_day wholestage off                            1278           1280           3          7.8         127.8       1.0X
+last_day wholestage on                             1560           1566           6          6.4         156.0       0.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                             968            974           8         10.3          96.8       1.0X
-next_day wholestage on                              958            959           1         10.4          95.8       1.0X
+next_day wholestage off                            1091           1093           3          9.2         109.1       1.0X
+next_day wholestage on                             1070           1076           9          9.3         107.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                             894            895           1         11.2          89.4       1.0X
-date_add wholestage on                              882            890           9         11.3          88.2       1.0X
+date_add wholestage off                            1041           1047           8          9.6         104.1       1.0X
+date_add wholestage on                             1044           1050           4          9.6         104.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             892            896           6         11.2          89.2       1.0X
-date_sub wholestage on                              881            888           7         11.3          88.1       1.0X
+date_sub wholestage off                            1038           1040           3          9.6         103.8       1.0X
+date_sub wholestage on                             1057           1061           4          9.5         105.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1221           1223           3          8.2         122.1       1.0X
-add_months wholestage on                           1212           1217           5          8.2         121.2       1.0X
+add_months wholestage off                          1401           1401           1          7.1         140.1       1.0X
+add_months wholestage on                           1438           1442           4          7.0         143.8       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         4989           5009          29          2.0         498.9       1.0X
-format date wholestage on                          5037           5055          26          2.0         503.7       1.0X
+format date wholestage off                         5482           5803         454          1.8         548.2       1.0X
+format date wholestage on                          5502           5518           9          1.8         550.2       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       9157           9164          10          1.1         915.7       1.0X
-from_unixtime wholestage on                        9101           9120          16          1.1         910.1       1.0X
+from_unixtime wholestage off                       8538           8553          22          1.2         853.8       1.0X
+from_unixtime wholestage on                        8545           8552           6          1.2         854.5       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                   732            739          10         13.7          73.2       1.0X
-from_utc_timestamp wholestage on                    767            776           8         13.0          76.7       1.0X
+from_utc_timestamp wholestage off                  1094           1099           8          9.1         109.4       1.0X
+from_utc_timestamp wholestage on                   1109           1114           5          9.0         110.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                     802            805           3         12.5          80.2       1.0X
-to_utc_timestamp wholestage on                      776            781           5         12.9          77.6       1.0X
+to_utc_timestamp wholestage off                    1466           1469           4          6.8         146.6       1.0X
+to_utc_timestamp wholestage on                     1401           1408           7          7.1         140.1       1.0X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        328            330           3         30.5          32.8       1.0X
-cast interval wholestage on                         319            326           7         31.3          31.9       1.0X
+cast interval wholestage off                        332            332           0         30.1          33.2       1.0X
+cast interval wholestage on                         315            324          10         31.7          31.5       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1762           1764           3          5.7         176.2       1.0X
-datediff wholestage on                             1495           1502           7          6.7         149.5       1.2X
+datediff wholestage off                            1796           1802           8          5.6         179.6       1.0X
+datediff wholestage on                             1758           1764          10          5.7         175.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      1338           1339           1          7.5         133.8       1.0X
-months_between wholestage on                       1334           1339           5          7.5         133.4       1.0X
+months_between wholestage off                      4833           4836           4          2.1         483.3       1.0X
+months_between wholestage on                       4777           4780           2          2.1         477.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2023           2094         100          0.5        2023.2       1.0X
-window wholestage on                              43505          43551          33          0.0       43504.8       0.0X
+window wholestage off                              1812           1908         136          0.6        1811.7       1.0X
+window wholestage on                              46279          46376          74          0.0       46278.8       0.0X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                      660            661           1         15.1          66.0       1.0X
-date_trunc YEAR wholestage on                       589            599           7         17.0          58.9       1.1X
+date_trunc YEAR wholestage off                     2367           2368           1          4.2         236.7       1.0X
+date_trunc YEAR wholestage on                      2321           2334          22          4.3         232.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                      656            657           1         15.2          65.6       1.0X
-date_trunc YYYY wholestage on                       593            604          16         16.9          59.3       1.1X
+date_trunc YYYY wholestage off                     2330           2334           5          4.3         233.0       1.0X
+date_trunc YYYY wholestage on                      2326           2332           5          4.3         232.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                        666            669           4         15.0          66.6       1.0X
-date_trunc YY wholestage on                         591            603          19         16.9          59.1       1.1X
+date_trunc YY wholestage off                       2334           2335           1          4.3         233.4       1.0X
+date_trunc YY wholestage on                        2315           2324           6          4.3         231.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                       592            592           1         16.9          59.2       1.0X
-date_trunc MON wholestage on                        569            580           8         17.6          56.9       1.0X
+date_trunc MON wholestage off                      2327           2330           4          4.3         232.7       1.0X
+date_trunc MON wholestage on                       2279           2289          12          4.4         227.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                     593            594           2         16.9          59.3       1.0X
-date_trunc MONTH wholestage on                      575            579           4         17.4          57.5       1.0X
+date_trunc MONTH wholestage off                    2330           2332           2          4.3         233.0       1.0X
+date_trunc MONTH wholestage on                     2277           2284           6          4.4         227.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                        589            590           2         17.0          58.9       1.0X
-date_trunc MM wholestage on                         569            575           4         17.6          56.9       1.0X
+date_trunc MM wholestage off                       2328           2329           2          4.3         232.8       1.0X
+date_trunc MM wholestage on                        2279           2284           4          4.4         227.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                       438            442           5         22.8          43.8       1.0X
-date_trunc DAY wholestage on                        346            350           4         28.9          34.6       1.3X
+date_trunc DAY wholestage off                      1974           1984          14          5.1         197.4       1.0X
+date_trunc DAY wholestage on                       1914           1922           7          5.2         191.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                        438            439           2         22.8          43.8       1.0X
-date_trunc DD wholestage on                         347            354           7         28.8          34.7       1.3X
+date_trunc DD wholestage off                       1967           1976          12          5.1         196.7       1.0X
+date_trunc DD wholestage on                        1913           1917           4          5.2         191.3       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                      384            386           2         26.0          38.4       1.0X
-date_trunc HOUR wholestage on                       357            365           6         28.0          35.7       1.1X
+date_trunc HOUR wholestage off                     1970           1970           0          5.1         197.0       1.0X
+date_trunc HOUR wholestage on                      1945           1946           2          5.1         194.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    373            375           3         26.8          37.3       1.0X
-date_trunc MINUTE wholestage on                     327            331           5         30.6          32.7       1.1X
+date_trunc MINUTE wholestage off                    361            361           1         27.7          36.1       1.0X
+date_trunc MINUTE wholestage on                     331            336           4         30.2          33.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    361            363           3         27.7          36.1       1.0X
-date_trunc SECOND wholestage on                     335            341           8         29.9          33.5       1.1X
+date_trunc SECOND wholestage off                    360            361           1         27.8          36.0       1.0X
+date_trunc SECOND wholestage on                     335            348          15         29.8          33.5       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                      515            516           2         19.4          51.5       1.0X
-date_trunc WEEK wholestage on                       455            459           4         22.0          45.5       1.1X
+date_trunc WEEK wholestage off                     2232           2236           6          4.5         223.2       1.0X
+date_trunc WEEK wholestage on                      2225           2232           6          4.5         222.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  1337           1341           6          7.5         133.7       1.0X
-date_trunc QUARTER wholestage on                   1328           1334          10          7.5         132.8       1.0X
+date_trunc QUARTER wholestage off                  3083           3086           4          3.2         308.3       1.0X
+date_trunc QUARTER wholestage on                   3073           3086          16          3.3         307.3       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           318            328          14         31.4          31.8       1.0X
-trunc year wholestage on                            297            308          17         33.6          29.7       1.1X
+trunc year wholestage off                           321            321           0         31.1          32.1       1.0X
+trunc year wholestage on                            299            303           5         33.5          29.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           318            319           2         31.5          31.8       1.0X
-trunc yyyy wholestage on                            296            302          10         33.8          29.6       1.1X
+trunc yyyy wholestage off                           323            327           5         30.9          32.3       1.0X
+trunc yyyy wholestage on                            299            302           3         33.4          29.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             321            345          35         31.2          32.1       1.0X
-trunc yy wholestage on                              297            319          45         33.6          29.7       1.1X
+trunc yy wholestage off                             315            315           1         31.8          31.5       1.0X
+trunc yy wholestage on                              299            304           4         33.4          29.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            318            318           0         31.5          31.8       1.0X
-trunc mon wholestage on                             299            306           6         33.4          29.9       1.1X
+trunc mon wholestage off                            320            321           1         31.2          32.0       1.0X
+trunc mon wholestage on                             299            307          10         33.4          29.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          316            318           3         31.6          31.6       1.0X
-trunc month wholestage on                           296            301           7         33.8          29.6       1.1X
+trunc month wholestage off                          316            317           1         31.6          31.6       1.0X
+trunc month wholestage on                           299            302           5         33.5          29.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             316            321           8         31.7          31.6       1.0X
-trunc mm wholestage on                              295            302           8         33.9          29.5       1.1X
+trunc mm wholestage off                             313            313           1         32.0          31.3       1.0X
+trunc mm wholestage on                              298            302           4         33.5          29.8       1.0X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     219            220           1          4.6         219.3       1.0X
-to timestamp str wholestage on                      212            214           2          4.7         212.3       1.0X
+to timestamp str wholestage off                     217            217           0          4.6         217.3       1.0X
+to timestamp str wholestage on                      209            212           2          4.8         209.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1852           1852           1          0.5        1851.9       1.0X
-to_timestamp wholestage on                         1862           1869           9          0.5        1861.6       1.0X
+to_timestamp wholestage off                        1676           1677           2          0.6        1675.6       1.0X
+to_timestamp wholestage on                         1599           1606           8          0.6        1599.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1839           1842           4          0.5        1839.1       1.0X
-to_unix_timestamp wholestage on                    1861           1866           4          0.5        1861.2       1.0X
+to_unix_timestamp wholestage off                   1582           1589           9          0.6        1582.1       1.0X
+to_unix_timestamp wholestage on                    1634           1637           3          0.6        1633.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          222            228           9          4.5         221.6       1.0X
-to date str wholestage on                           210            211           2          4.8         209.5       1.1X
+to date str wholestage off                          275            282           9          3.6         275.0       1.0X
+to date str wholestage on                           264            265           2          3.8         263.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             2386           2392           8          0.4        2386.3       1.0X
-to_date wholestage on                              2438           2457          18          0.4        2437.7       1.0X
+to_date wholestage off                             3170           3188          25          0.3        3170.1       1.0X
+to_date wholestage on                              3134           3143          10          0.3        3134.3       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
-------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             287            291           7         17.4          57.3       1.0X
-Collect longs                                      1903           2672         694          2.6         380.6       0.2X
-Collect timestamps                                 1544           1644          89          3.2         308.8       0.2X
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+From java.sql.Date                                  407            413           7         12.3          81.5       1.0X
+From java.time.LocalDate                            340            344           5         14.7          68.1       1.2X
+Collect java.sql.Date                              1700           2658        1422          2.9         340.0       0.2X
+Collect java.time.LocalDate                        1473           1494          30          3.4         294.6       0.3X
+From java.sql.Timestamp                             252            266          13         19.8          50.5       1.6X
+From java.time.Instant                              236            243           7         21.1          47.3       1.7X
+Collect longs                                      1280           1337          79          3.9         256.1       0.3X
+Collect java.sql.Timestamp                         1485           1501          15          3.4         297.0       0.3X
+Collect java.time.Instant                          1441           1465          37          3.5         288.1       0.3X
+java.sql.Date to Hive string                      18745          20895        1364          0.3        3749.0       0.0X
+java.time.LocalDate to Hive string                15296          15450         143          0.3        3059.2       0.0X
+java.sql.Timestamp to Hive string                 46421          47210         946          0.1        9284.2       0.0X
+java.time.Instant to Hive string                  34747          35187         382          0.1        6949.4       0.0X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..05896a4d69b47
--- /dev/null
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -0,0 +1,154 @@
+================================================================================================
+Rebasing dates/timestamps in Parquet datasource
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save DATE to parquet:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                  20023          20023           0          5.0         200.2       1.0X
+before 1582, noop                                 10729          10729           0          9.3         107.3       1.9X
+after 1582, rebase EXCEPTION                      31834          31834           0          3.1         318.3       0.6X
+after 1582, rebase LEGACY                         31997          31997           0          3.1         320.0       0.6X
+after 1582, rebase CORRECTED                      31712          31712           0          3.2         317.1       0.6X
+before 1582, rebase LEGACY                        23663          23663           0          4.2         236.6       0.8X
+before 1582, rebase CORRECTED                     22749          22749           0          4.4         227.5       0.9X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load DATE from parquet:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off, rebase EXCEPTION             12984          13262         257          7.7         129.8       1.0X
+after 1582, vec off, rebase LEGACY                13278          13330          50          7.5         132.8       1.0X
+after 1582, vec off, rebase CORRECTED             13202          13255          50          7.6         132.0       1.0X
+after 1582, vec on, rebase EXCEPTION               3823           3853          40         26.2          38.2       3.4X
+after 1582, vec on, rebase LEGACY                  3846           3876          27         26.0          38.5       3.4X
+after 1582, vec on, rebase CORRECTED               3775           3838          62         26.5          37.7       3.4X
+before 1582, vec off, rebase LEGACY               13671          13692          26          7.3         136.7       0.9X
+before 1582, vec off, rebase CORRECTED            13387          13476         106          7.5         133.9       1.0X
+before 1582, vec on, rebase LEGACY                 4477           4484           7         22.3          44.8       2.9X
+before 1582, vec on, rebase CORRECTED              3729           3773          50         26.8          37.3       3.5X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP_INT96 to parquet:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   3020           3020           0         33.1          30.2       1.0X
+before 1900, noop                                  3013           3013           0         33.2          30.1       1.0X
+after 1900, rebase EXCEPTION                      28796          28796           0          3.5         288.0       0.1X
+after 1900, rebase LEGACY                         28869          28869           0          3.5         288.7       0.1X
+after 1900, rebase CORRECTED                      28522          28522           0          3.5         285.2       0.1X
+before 1900, rebase LEGACY                        30594          30594           0          3.3         305.9       0.1X
+before 1900, rebase CORRECTED                     30743          30743           0          3.3         307.4       0.1X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP_INT96 from parquet:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off, rebase EXCEPTION             19325          19468         135          5.2         193.3       1.0X
+after 1900, vec off, rebase LEGACY                19568          19602          30          5.1         195.7       1.0X
+after 1900, vec off, rebase CORRECTED             19532          19538           6          5.1         195.3       1.0X
+after 1900, vec on, rebase EXCEPTION               9884           9990          94         10.1          98.8       2.0X
+after 1900, vec on, rebase LEGACY                  9933           9985          49         10.1          99.3       1.9X
+after 1900, vec on, rebase CORRECTED               9967          10043          76         10.0          99.7       1.9X
+before 1900, vec off, rebase LEGACY               24162          24198          37          4.1         241.6       0.8X
+before 1900, vec off, rebase CORRECTED            24034          24056          20          4.2         240.3       0.8X
+before 1900, vec on, rebase LEGACY                12548          12625          72          8.0         125.5       1.5X
+before 1900, vec on, rebase CORRECTED             12580          12660         115          7.9         125.8       1.5X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP_MICROS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   3159           3159           0         31.7          31.6       1.0X
+before 1900, noop                                  3038           3038           0         32.9          30.4       1.0X
+after 1900, rebase EXCEPTION                      16885          16885           0          5.9         168.8       0.2X
+after 1900, rebase LEGACY                         17171          17171           0          5.8         171.7       0.2X
+after 1900, rebase CORRECTED                      17353          17353           0          5.8         173.5       0.2X
+before 1900, rebase LEGACY                        20579          20579           0          4.9         205.8       0.2X
+before 1900, rebase CORRECTED                     17544          17544           0          5.7         175.4       0.2X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP_MICROS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off, rebase EXCEPTION             16304          16345          58          6.1         163.0       1.0X
+after 1900, vec off, rebase LEGACY                16503          16585          75          6.1         165.0       1.0X
+after 1900, vec off, rebase CORRECTED             16413          16463          44          6.1         164.1       1.0X
+after 1900, vec on, rebase EXCEPTION               5017           5034          29         19.9          50.2       3.2X
+after 1900, vec on, rebase LEGACY                  5060           5094          30         19.8          50.6       3.2X
+after 1900, vec on, rebase CORRECTED               4969           4971           1         20.1          49.7       3.3X
+before 1900, vec off, rebase LEGACY               19767          20001         203          5.1         197.7       0.8X
+before 1900, vec off, rebase CORRECTED            16421          16465          38          6.1         164.2       1.0X
+before 1900, vec on, rebase LEGACY                 8535           8608          64         11.7          85.4       1.9X
+before 1900, vec on, rebase CORRECTED              5044           5077          32         19.8          50.4       3.2X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP_MILLIS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   2995           2995           0         33.4          29.9       1.0X
+before 1900, noop                                  2981           2981           0         33.5          29.8       1.0X
+after 1900, rebase EXCEPTION                      16196          16196           0          6.2         162.0       0.2X
+after 1900, rebase LEGACY                         16550          16550           0          6.0         165.5       0.2X
+after 1900, rebase CORRECTED                      16908          16908           0          5.9         169.1       0.2X
+before 1900, rebase LEGACY                        20087          20087           0          5.0         200.9       0.1X
+before 1900, rebase CORRECTED                     17171          17171           0          5.8         171.7       0.2X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP_MILLIS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off, rebase EXCEPTION             16688          16787          88          6.0         166.9       1.0X
+after 1900, vec off, rebase LEGACY                17383          17462          73          5.8         173.8       1.0X
+after 1900, vec off, rebase CORRECTED             17317          17329          11          5.8         173.2       1.0X
+after 1900, vec on, rebase EXCEPTION               6342           6348           6         15.8          63.4       2.6X
+after 1900, vec on, rebase LEGACY                  6500           6521          18         15.4          65.0       2.6X
+after 1900, vec on, rebase CORRECTED               6164           6172          11         16.2          61.6       2.7X
+before 1900, vec off, rebase LEGACY               20575          20665          81          4.9         205.7       0.8X
+before 1900, vec off, rebase CORRECTED            17239          17290          61          5.8         172.4       1.0X
+before 1900, vec on, rebase LEGACY                 9310           9373          60         10.7          93.1       1.8X
+before 1900, vec on, rebase CORRECTED              6091           6105          16         16.4          60.9       2.7X
+
+
+================================================================================================
+Rebasing dates/timestamps in ORC datasource
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save DATE to ORC:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                  19583          19583           0          5.1         195.8       1.0X
+before 1582, noop                                 10711          10711           0          9.3         107.1       1.8X
+after 1582                                        27864          27864           0          3.6         278.6       0.7X
+before 1582                                       19648          19648           0          5.1         196.5       1.0X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load DATE from ORC:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off                               10383          10560         192          9.6         103.8       1.0X
+after 1582, vec on                                 3844           3864          33         26.0          38.4       2.7X
+before 1582, vec off                              10867          10916          48          9.2         108.7       1.0X
+before 1582, vec on                                4158           4170          12         24.0          41.6       2.5X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP to ORC:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   2989           2989           0         33.5          29.9       1.0X
+before 1900, noop                                  3000           3000           0         33.3          30.0       1.0X
+after 1900                                        19426          19426           0          5.1         194.3       0.2X
+before 1900                                       23282          23282           0          4.3         232.8       0.1X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP from ORC:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off                               12089          12102          15          8.3         120.9       1.0X
+after 1900, vec on                                 5210           5325         100         19.2          52.1       2.3X
+before 1900, vec off                              15320          15373          46          6.5         153.2       0.8X
+before 1900, vec on                                7937           7970          48         12.6          79.4       1.5X
+
+
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
new file mode 100644
index 0000000000000..3e94d6c6fcfa7
--- /dev/null
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -0,0 +1,154 @@
+================================================================================================
+Rebasing dates/timestamps in Parquet datasource
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save DATE to parquet:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                  23300          23300           0          4.3         233.0       1.0X
+before 1582, noop                                 10585          10585           0          9.4         105.9       2.2X
+after 1582, rebase EXCEPTION                      35215          35215           0          2.8         352.1       0.7X
+after 1582, rebase LEGACY                         34927          34927           0          2.9         349.3       0.7X
+after 1582, rebase CORRECTED                      35479          35479           0          2.8         354.8       0.7X
+before 1582, rebase LEGACY                        22767          22767           0          4.4         227.7       1.0X
+before 1582, rebase CORRECTED                     22527          22527           0          4.4         225.3       1.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load DATE from parquet:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off, rebase EXCEPTION             13480          13577          94          7.4         134.8       1.0X
+after 1582, vec off, rebase LEGACY                13466          13586         118          7.4         134.7       1.0X
+after 1582, vec off, rebase CORRECTED             13526          13558          41          7.4         135.3       1.0X
+after 1582, vec on, rebase EXCEPTION               3759           3778          28         26.6          37.6       3.6X
+after 1582, vec on, rebase LEGACY                  3957           4004          57         25.3          39.6       3.4X
+after 1582, vec on, rebase CORRECTED               3739           3755          25         26.7          37.4       3.6X
+before 1582, vec off, rebase LEGACY               13986          14038          67          7.1         139.9       1.0X
+before 1582, vec off, rebase CORRECTED            13453          13491          49          7.4         134.5       1.0X
+before 1582, vec on, rebase LEGACY                 4716           4724          10         21.2          47.2       2.9X
+before 1582, vec on, rebase CORRECTED              3701           3750          50         27.0          37.0       3.6X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP_INT96 to parquet:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   2790           2790           0         35.8          27.9       1.0X
+before 1900, noop                                  2812           2812           0         35.6          28.1       1.0X
+after 1900, rebase EXCEPTION                      24789          24789           0          4.0         247.9       0.1X
+after 1900, rebase LEGACY                         24539          24539           0          4.1         245.4       0.1X
+after 1900, rebase CORRECTED                      24543          24543           0          4.1         245.4       0.1X
+before 1900, rebase LEGACY                        30496          30496           0          3.3         305.0       0.1X
+before 1900, rebase CORRECTED                     30428          30428           0          3.3         304.3       0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP_INT96 from parquet:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off, rebase EXCEPTION             17106          17192          75          5.8         171.1       1.0X
+after 1900, vec off, rebase LEGACY                17273          17337          55          5.8         172.7       1.0X
+after 1900, vec off, rebase CORRECTED             17073          17215         128          5.9         170.7       1.0X
+after 1900, vec on, rebase EXCEPTION               8903           8976         117         11.2          89.0       1.9X
+after 1900, vec on, rebase LEGACY                  8793           8876          84         11.4          87.9       1.9X
+after 1900, vec on, rebase CORRECTED               8820           8878          53         11.3          88.2       1.9X
+before 1900, vec off, rebase LEGACY               20997          21069          82          4.8         210.0       0.8X
+before 1900, vec off, rebase CORRECTED            20874          20946          90          4.8         208.7       0.8X
+before 1900, vec on, rebase LEGACY                12024          12090          58          8.3         120.2       1.4X
+before 1900, vec on, rebase CORRECTED             12020          12069          64          8.3         120.2       1.4X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP_MICROS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   2939           2939           0         34.0          29.4       1.0X
+before 1900, noop                                  2917           2917           0         34.3          29.2       1.0X
+after 1900, rebase EXCEPTION                      15954          15954           0          6.3         159.5       0.2X
+after 1900, rebase LEGACY                         16402          16402           0          6.1         164.0       0.2X
+after 1900, rebase CORRECTED                      16541          16541           0          6.0         165.4       0.2X
+before 1900, rebase LEGACY                        20500          20500           0          4.9         205.0       0.1X
+before 1900, rebase CORRECTED                     16764          16764           0          6.0         167.6       0.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP_MICROS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off, rebase EXCEPTION             15607          15655          81          6.4         156.1       1.0X
+after 1900, vec off, rebase LEGACY                15616          15676          54          6.4         156.2       1.0X
+after 1900, vec off, rebase CORRECTED             15634          15732         108          6.4         156.3       1.0X
+after 1900, vec on, rebase EXCEPTION               5041           5057          16         19.8          50.4       3.1X
+after 1900, vec on, rebase LEGACY                  5516           5539          29         18.1          55.2       2.8X
+after 1900, vec on, rebase CORRECTED               5087           5104          28         19.7          50.9       3.1X
+before 1900, vec off, rebase LEGACY               19262          19338          79          5.2         192.6       0.8X
+before 1900, vec off, rebase CORRECTED            15718          15755          53          6.4         157.2       1.0X
+before 1900, vec on, rebase LEGACY                10147          10240         114          9.9         101.5       1.5X
+before 1900, vec on, rebase CORRECTED              5062           5080          21         19.8          50.6       3.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP_MILLIS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   2915           2915           0         34.3          29.2       1.0X
+before 1900, noop                                  2894           2894           0         34.6          28.9       1.0X
+after 1900, rebase EXCEPTION                      15545          15545           0          6.4         155.4       0.2X
+after 1900, rebase LEGACY                         15840          15840           0          6.3         158.4       0.2X
+after 1900, rebase CORRECTED                      16324          16324           0          6.1         163.2       0.2X
+before 1900, rebase LEGACY                        20359          20359           0          4.9         203.6       0.1X
+before 1900, rebase CORRECTED                     16292          16292           0          6.1         162.9       0.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP_MILLIS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off, rebase EXCEPTION             15857          16015         223          6.3         158.6       1.0X
+after 1900, vec off, rebase LEGACY                16174          16231          63          6.2         161.7       1.0X
+after 1900, vec off, rebase CORRECTED             16353          16400          67          6.1         163.5       1.0X
+after 1900, vec on, rebase EXCEPTION               6449           6459           9         15.5          64.5       2.5X
+after 1900, vec on, rebase LEGACY                  7028           7035           6         14.2          70.3       2.3X
+after 1900, vec on, rebase CORRECTED               6585           6623          37         15.2          65.8       2.4X
+before 1900, vec off, rebase LEGACY               19929          20027          95          5.0         199.3       0.8X
+before 1900, vec off, rebase CORRECTED            16401          16451          49          6.1         164.0       1.0X
+before 1900, vec on, rebase LEGACY                10517          10563          40          9.5         105.2       1.5X
+before 1900, vec on, rebase CORRECTED              6659           6675          26         15.0          66.6       2.4X
+
+
+================================================================================================
+Rebasing dates/timestamps in ORC datasource
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save DATE to ORC:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                  22782          22782           0          4.4         227.8       1.0X
+before 1582, noop                                 10555          10555           0          9.5         105.6       2.2X
+after 1582                                        31497          31497           0          3.2         315.0       0.7X
+before 1582                                       19803          19803           0          5.0         198.0       1.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load DATE from ORC:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off                               10180          10214          44          9.8         101.8       1.0X
+after 1582, vec on                                 3785           3804          24         26.4          37.8       2.7X
+before 1582, vec off                              10537          10582          39          9.5         105.4       1.0X
+before 1582, vec on                                4117           4146          25         24.3          41.2       2.5X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP to ORC:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   2853           2853           0         35.1          28.5       1.0X
+before 1900, noop                                  2999           2999           0         33.3          30.0       1.0X
+after 1900                                        16757          16757           0          6.0         167.6       0.2X
+before 1900                                       21542          21542           0          4.6         215.4       0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP from ORC:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off                               12212          12254          39          8.2         122.1       1.0X
+after 1900, vec on                                 5369           5390          35         18.6          53.7       2.3X
+before 1900, vec off                              15661          15705          73          6.4         156.6       0.8X
+before 1900, vec on                                8720           8744          29         11.5          87.2       1.4X
+
+
diff --git a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
index 0572cc25bf6c2..77f1bdb5eaee0 100644
--- a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
@@ -1,119 +1,104 @@
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   380            456          67         26.3          38.0       1.0X
-MILLENNIUM of timestamp                            1274           1361         138          7.9         127.4       0.3X
-CENTURY of timestamp                               1119           1132          19          8.9         111.9       0.3X
-DECADE of timestamp                                1076           1083           6          9.3         107.6       0.4X
-YEAR of timestamp                                  1066           1098          43          9.4         106.6       0.4X
-ISOYEAR of timestamp                               1190           1194           4          8.4         119.0       0.3X
-QUARTER of timestamp                               1269           1273           4          7.9         126.9       0.3X
-MONTH of timestamp                                 1060           1075          22          9.4         106.0       0.4X
-WEEK of timestamp                                  1560           1565           8          6.4         156.0       0.2X
-DAY of timestamp                                   1039           1046           8          9.6         103.9       0.4X
-DAYOFWEEK of timestamp                             1248           1274          24          8.0         124.8       0.3X
-DOW of timestamp                                   1252           1273          25          8.0         125.2       0.3X
-ISODOW of timestamp                                1195           1204           9          8.4         119.5       0.3X
-DOY of timestamp                                   1081           1086           6          9.3         108.1       0.4X
-HOUR of timestamp                                   778            781           5         12.9          77.8       0.5X
-MINUTE of timestamp                                 779            780           1         12.8          77.9       0.5X
-SECOND of timestamp                                 597            611          20         16.7          59.7       0.6X
-MILLISECONDS of timestamp                           636            642           6         15.7          63.6       0.6X
-MICROSECONDS of timestamp                           498            504           5         20.1          49.8       0.8X
-EPOCH of timestamp                                  946            956           9         10.6          94.6       0.4X
+cast to timestamp                                   333            345          13         30.0          33.3       1.0X
+YEAR of timestamp                                   826            840          16         12.1          82.6       0.4X
+YEAROFWEEK of timestamp                            1161           1168           7          8.6         116.1       0.3X
+QUARTER of timestamp                                961            966           6         10.4          96.1       0.3X
+MONTH of timestamp                                  831            838           6         12.0          83.1       0.4X
+WEEK of timestamp                                  1155           1159           4          8.7         115.5       0.3X
+DAY of timestamp                                    822            828           6         12.2          82.2       0.4X
+DAYOFWEEK of timestamp                              968            972           6         10.3          96.8       0.3X
+DOW of timestamp                                    960            966           9         10.4          96.0       0.3X
+DOW_ISO of timestamp                                961            967           9         10.4          96.1       0.3X
+DAYOFWEEK_ISO of timestamp                          965            968           5         10.4          96.5       0.3X
+DOY of timestamp                                    864            873           9         11.6          86.4       0.4X
+HOUR of timestamp                                   631            647          14         15.9          63.1       0.5X
+MINUTE of timestamp                                 638            651          12         15.7          63.8       0.5X
+SECOND of timestamp                                 758            760           2         13.2          75.8       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   356            362           9         28.1          35.6       1.0X
-MILLENNIUM of timestamp                            1260           1297          41          7.9         126.0       0.3X
-CENTURY of timestamp                               1082           1085           3          9.2         108.2       0.3X
-DECADE of timestamp                                1056           1068          19          9.5         105.6       0.3X
-YEAR of timestamp                                  1045           1053          13          9.6         104.5       0.3X
-ISOYEAR of timestamp                               1300           1316          25          7.7         130.0       0.3X
-QUARTER of timestamp                               1279           1280           2          7.8         127.9       0.3X
-MONTH of timestamp                                 1037           1046          11          9.6         103.7       0.3X
-WEEK of timestamp                                  1539           1557          28          6.5         153.9       0.2X
-DAY of timestamp                                   1032           1038           6          9.7         103.2       0.3X
-DAYOFWEEK of timestamp                             1241           1244           4          8.1         124.1       0.3X
-DOW of timestamp                                   1237           1241           7          8.1         123.7       0.3X
-ISODOW of timestamp                                1155           1158           3          8.7         115.5       0.3X
-DOY of timestamp                                   1075           1080           4          9.3         107.5       0.3X
-HOUR of timestamp                                   766            770           5         13.1          76.6       0.5X
-MINUTE of timestamp                                 764            769           4         13.1          76.4       0.5X
-SECOND of timestamp                                 590            592           2         16.9          59.0       0.6X
-MILLISECONDS of timestamp                           627            636          10         16.0          62.7       0.6X
-MICROSECONDS of timestamp                           493            505          15         20.3          49.3       0.7X
-EPOCH of timestamp                                  962            966           4         10.4          96.2       0.4X
+cast to timestamp                                   301            307           8         33.2          30.1       1.0X
+YEAR of timestamp                                   797            805           7         12.6          79.7       0.4X
+YEAROFWEEK of timestamp                            1147           1156           8          8.7         114.7       0.3X
+QUARTER of timestamp                                965            974           9         10.4          96.5       0.3X
+MONTH of timestamp                                  801            815          17         12.5          80.1       0.4X
+WEEK of timestamp                                  1133           1157          21          8.8         113.3       0.3X
+DAY of timestamp                                    825            828           4         12.1          82.5       0.4X
+DAYOFWEEK of timestamp                              929            937           8         10.8          92.9       0.3X
+DOW of timestamp                                    933            940          11         10.7          93.3       0.3X
+DOW_ISO of timestamp                                944            954          11         10.6          94.4       0.3X
+DAYOFWEEK_ISO of timestamp                          948            965          21         10.5          94.8       0.3X
+DOY of timestamp                                    894            898           6         11.2          89.4       0.3X
+HOUR of timestamp                                   678            683           6         14.7          67.8       0.4X
+MINUTE of timestamp                                 669            670           2         15.0          66.9       0.5X
+SECOND of timestamp                                 762            771          12         13.1          76.2       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        886            905          17         11.3          88.6       1.0X
-MILLENNIUM of date                                 1253           1261           7          8.0         125.3       0.7X
-CENTURY of date                                    1068           1079          10          9.4         106.8       0.8X
-DECADE of date                                     1040           1068          25          9.6         104.0       0.9X
-YEAR of date                                       1032           1043          11          9.7         103.2       0.9X
-ISOYEAR of date                                    1304           1313          12          7.7         130.4       0.7X
-QUARTER of date                                    1284           1301          16          7.8         128.4       0.7X
-MONTH of date                                      1033           1036           4          9.7         103.3       0.9X
-WEEK of date                                       1535           1545           8          6.5         153.5       0.6X
-DAY of date                                        1023           1033          11          9.8         102.3       0.9X
-DAYOFWEEK of date                                  1230           1236           6          8.1         123.0       0.7X
-DOW of date                                        1238           1247           9          8.1         123.8       0.7X
-ISODOW of date                                     1159           1169          17          8.6         115.9       0.8X
-DOY of date                                        1082           1084           3          9.2         108.2       0.8X
-HOUR of date                                       1879           1891          11          5.3         187.9       0.5X
-MINUTE of date                                     1881           1905          21          5.3         188.1       0.5X
-SECOND of date                                     1718           1724           5          5.8         171.8       0.5X
-MILLISECONDS of date                               1733           1737           6          5.8         173.3       0.5X
-MICROSECONDS of date                               1629           1644          23          6.1         162.9       0.5X
-EPOCH of date                                      2085           2090           5          4.8         208.5       0.4X
+cast to date                                        737            751          23         13.6          73.7       1.0X
+YEAR of date                                        804            811          11         12.4          80.4       0.9X
+YEAROFWEEK of date                                 1151           1155           4          8.7         115.1       0.6X
+QUARTER of date                                     964            987          26         10.4          96.4       0.8X
+MONTH of date                                       831            835           6         12.0          83.1       0.9X
+WEEK of date                                       1162           1163           2          8.6         116.2       0.6X
+DAY of date                                         826            833           7         12.1          82.6       0.9X
+DAYOFWEEK of date                                   984           1003          26         10.2          98.4       0.7X
+DOW of date                                         995            997           2         10.0          99.5       0.7X
+DOW_ISO of date                                     985            992           8         10.2          98.5       0.7X
+DAYOFWEEK_ISO of date                               985           1001          19         10.1          98.5       0.7X
+DOY of date                                         884            890           8         11.3          88.4       0.8X
+HOUR of date                                       1630           1640          13          6.1         163.0       0.5X
+MINUTE of date                                     1631           1636           6          6.1         163.1       0.5X
+SECOND of date                                     1737           1739           2          5.8         173.7       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        888            891           6         11.3          88.8       1.0X
-MILLENNIUM of date                                 1250           1260          17          8.0         125.0       0.7X
-CENTURY of date                                    1070           1076           6          9.3         107.0       0.8X
-DECADE of date                                     1036           1041           5          9.6         103.6       0.9X
-YEAR of date                                       1037           1038           1          9.6         103.7       0.9X
-ISOYEAR of date                                    1300           1307           9          7.7         130.0       0.7X
-QUARTER of date                                    1267           1277           9          7.9         126.7       0.7X
-MONTH of date                                      1034           1037           4          9.7         103.4       0.9X
-WEEK of date                                       1543           1554          10          6.5         154.3       0.6X
-DAY of date                                        1022           1030          12          9.8         102.2       0.9X
-DAYOFWEEK of date                                  1230           1232           4          8.1         123.0       0.7X
-DOW of date                                        1227           1242          15          8.1         122.7       0.7X
-ISODOW of date                                     1157           1173          20          8.6         115.7       0.8X
-DOY of date                                        1073           1083          18          9.3         107.3       0.8X
-HOUR of date                                       1873           1878           7          5.3         187.3       0.5X
-MINUTE of date                                     1861           1876          14          5.4         186.1       0.5X
-SECOND of date                                     1717           1724           6          5.8         171.7       0.5X
-MILLISECONDS of date                               1729           1736           7          5.8         172.9       0.5X
-MICROSECONDS of date                               1622           1627           5          6.2         162.2       0.5X
-EPOCH of date                                      2066           2079          19          4.8         206.6       0.4X
+cast to date                                        746            753           8         13.4          74.6       1.0X
+YEAR of date                                        841            843           1         11.9          84.1       0.9X
+YEAROFWEEK of date                                 1169           1191          20          8.6         116.9       0.6X
+QUARTER of date                                    1008           1012           4          9.9         100.8       0.7X
+MONTH of date                                       838            844           6         11.9          83.8       0.9X
+WEEK of date                                       1179           1182           3          8.5         117.9       0.6X
+DAY of date                                         849            850           1         11.8          84.9       0.9X
+DAYOFWEEK of date                                   984            988           4         10.2          98.4       0.8X
+DOW of date                                         986            994           7         10.1          98.6       0.8X
+DOW_ISO of date                                     974            992          17         10.3          97.4       0.8X
+DAYOFWEEK_ISO of date                               979            994          14         10.2          97.9       0.8X
+DOY of date                                         874            878           3         11.4          87.4       0.9X
+HOUR of date                                       1644           1649           6          6.1         164.4       0.5X
+MINUTE of date                                     1610           1626          15          6.2         161.0       0.5X
+SECOND of date                                     1768           1789          30          5.7         176.8       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+Invoke extract for interval:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+cast to interval                                    989           1003          13         10.1          98.9       1.0X
+YEAR of interval                                    980            999          16         10.2          98.0       1.0X
+MONTH of interval                                   988           1000          15         10.1          98.8       1.0X
+DAY of interval                                     970            983          11         10.3          97.0       1.0X
+HOUR of interval                                    989           1000          13         10.1          98.9       1.0X
+MINUTE of interval                                 1019           1034          13          9.8         101.9       1.0X
+SECOND of interval                                 1084           1095          10          9.2         108.4       0.9X
+
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                   1273           1279           6          7.9         127.3       1.0X
-MILLENNIUM of interval                             1323           1352          36          7.6         132.3       1.0X
-CENTURY of interval                                1353           1356           4          7.4         135.3       0.9X
-DECADE of interval                                 1326           1339          11          7.5         132.6       1.0X
-YEAR of interval                                   1341           1345           3          7.5         134.1       0.9X
-QUARTER of interval                                1368           1372           4          7.3         136.8       0.9X
-MONTH of interval                                  1320           1326           6          7.6         132.0       1.0X
-DAY of interval                                    1306           1310           4          7.7         130.6       1.0X
-HOUR of interval                                   1341           1347           8          7.5         134.1       0.9X
-MINUTE of interval                                 1337           1349          11          7.5         133.7       1.0X
-SECOND of interval                                 1450           1451           1          6.9         145.0       0.9X
-MILLISECONDS of interval                           1476           1490          23          6.8         147.6       0.9X
-MICROSECONDS of interval                           1316           1331          25          7.6         131.6       1.0X
-EPOCH of interval                                  1461           1462           1          6.8         146.1       0.9X
+cast to interval                                    955            969          15         10.5          95.5       1.0X
+YEAR of interval                                   1004           1007           3         10.0         100.4       1.0X
+MONTH of interval                                   989            990           2         10.1          98.9       1.0X
+DAY of interval                                    1006           1011           6          9.9         100.6       0.9X
+HOUR of interval                                    986            997          10         10.1          98.6       1.0X
+MINUTE of interval                                 1004           1014          15         10.0         100.4       1.0X
+SECOND of interval                                 1083           1088           6          9.2         108.3       0.9X
 
diff --git a/sql/core/benchmarks/ExtractBenchmark-results.txt b/sql/core/benchmarks/ExtractBenchmark-results.txt
index 317f107bd746c..11144a0547e43 100644
--- a/sql/core/benchmarks/ExtractBenchmark-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-results.txt
@@ -1,119 +1,104 @@
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   409            457          45         24.5          40.9       1.0X
-MILLENNIUM of timestamp                            1240           1295          57          8.1         124.0       0.3X
-CENTURY of timestamp                               1186           1240          49          8.4         118.6       0.3X
-DECADE of timestamp                                1083           1104          20          9.2         108.3       0.4X
-YEAR of timestamp                                  1061           1073          15          9.4         106.1       0.4X
-ISOYEAR of timestamp                               1198           1213          25          8.3         119.8       0.3X
-QUARTER of timestamp                               1304           1322          23          7.7         130.4       0.3X
-MONTH of timestamp                                 1052           1067          19          9.5         105.2       0.4X
-WEEK of timestamp                                  1534           1558          25          6.5         153.4       0.3X
-DAY of timestamp                                   1038           1057          26          9.6         103.8       0.4X
-DAYOFWEEK of timestamp                             1226           1239          22          8.2         122.6       0.3X
-DOW of timestamp                                   1212           1224          13          8.3         121.2       0.3X
-ISODOW of timestamp                                1148           1165          24          8.7         114.8       0.4X
-DOY of timestamp                                   1066           1075          14          9.4         106.6       0.4X
-HOUR of timestamp                                   358            362           6         27.9          35.8       1.1X
-MINUTE of timestamp                                 364            369           4         27.4          36.4       1.1X
-SECOND of timestamp                                 453            471          26         22.1          45.3       0.9X
-MILLISECONDS of timestamp                           497            500           2         20.1          49.7       0.8X
-MICROSECONDS of timestamp                           360            363           4         27.8          36.0       1.1X
-EPOCH of timestamp                                 1100           1104           5          9.1         110.0       0.4X
+cast to timestamp                                   292            310          16         34.3          29.2       1.0X
+YEAR of timestamp                                   847            875          26         11.8          84.7       0.3X
+YEAROFWEEK of timestamp                             964            981          24         10.4          96.4       0.3X
+QUARTER of timestamp                               1217           1219           2          8.2         121.7       0.2X
+MONTH of timestamp                                  835            844          10         12.0          83.5       0.3X
+WEEK of timestamp                                  1173           1183          15          8.5         117.3       0.2X
+DAY of timestamp                                    851            878          25         11.7          85.1       0.3X
+DAYOFWEEK of timestamp                              946            970          22         10.6          94.6       0.3X
+DOW of timestamp                                    935            959          21         10.7          93.5       0.3X
+DOW_ISO of timestamp                                947            961          13         10.6          94.7       0.3X
+DAYOFWEEK_ISO of timestamp                          965            992          26         10.4          96.5       0.3X
+DOY of timestamp                                    886            904          26         11.3          88.6       0.3X
+HOUR of timestamp                                   697            700           4         14.3          69.7       0.4X
+MINUTE of timestamp                                 654            665          10         15.3          65.4       0.4X
+SECOND of timestamp                                 770            778           8         13.0          77.0       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   315            317           2         31.8          31.5       1.0X
-MILLENNIUM of timestamp                            1155           1162           8          8.7         115.5       0.3X
-CENTURY of timestamp                               1146           1152           5          8.7         114.6       0.3X
-DECADE of timestamp                                1031           1043          11          9.7         103.1       0.3X
-YEAR of timestamp                                  1033           1041          10          9.7         103.3       0.3X
-ISOYEAR of timestamp                               1274           1278           5          7.8         127.4       0.2X
-QUARTER of timestamp                               1326           1346          30          7.5         132.6       0.2X
-MONTH of timestamp                                 1027           1031           7          9.7         102.7       0.3X
-WEEK of timestamp                                  1529           1535           6          6.5         152.9       0.2X
-DAY of timestamp                                   1024           1031           9          9.8         102.4       0.3X
-DAYOFWEEK of timestamp                             1197           1201           5          8.4         119.7       0.3X
-DOW of timestamp                                   1201           1218          15          8.3         120.1       0.3X
-ISODOW of timestamp                                1143           1149           6          8.8         114.3       0.3X
-DOY of timestamp                                   1074           1085           9          9.3         107.4       0.3X
-HOUR of timestamp                                   354            354           0         28.3          35.4       0.9X
-MINUTE of timestamp                                 358            361           5         27.9          35.8       0.9X
-SECOND of timestamp                                 445            456          17         22.5          44.5       0.7X
-MILLISECONDS of timestamp                           504            514          12         19.8          50.4       0.6X
-MICROSECONDS of timestamp                           361            370           8         27.7          36.1       0.9X
-EPOCH of timestamp                                 1111           1117           9          9.0         111.1       0.3X
+cast to timestamp                                   233            243           9         43.0          23.3       1.0X
+YEAR of timestamp                                   810            826          15         12.3          81.0       0.3X
+YEAROFWEEK of timestamp                             996           1019          21         10.0          99.6       0.2X
+QUARTER of timestamp                               1037           1049          11          9.6         103.7       0.2X
+MONTH of timestamp                                  822            852          30         12.2          82.2       0.3X
+WEEK of timestamp                                  1179           1220          35          8.5         117.9       0.2X
+DAY of timestamp                                    822            825           3         12.2          82.2       0.3X
+DAYOFWEEK of timestamp                              937            941           3         10.7          93.7       0.2X
+DOW of timestamp                                    931            970          34         10.7          93.1       0.2X
+DOW_ISO of timestamp                                927            948          22         10.8          92.7       0.3X
+DAYOFWEEK_ISO of timestamp                          896            918          20         11.2          89.6       0.3X
+DOY of timestamp                                    863            891          25         11.6          86.3       0.3X
+HOUR of timestamp                                   639            645           6         15.7          63.9       0.4X
+MINUTE of timestamp                                 639            647          12         15.7          63.9       0.4X
+SECOND of timestamp                                 785            796          11         12.7          78.5       0.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        849            851           3         11.8          84.9       1.0X
-MILLENNIUM of date                                 1129           1139          11          8.9         112.9       0.8X
-CENTURY of date                                    1136           1143           7          8.8         113.6       0.7X
-DECADE of date                                     1039           1043           5          9.6         103.9       0.8X
-YEAR of date                                       1030           1037          10          9.7         103.0       0.8X
-ISOYEAR of date                                    1269           1278           9          7.9         126.9       0.7X
-QUARTER of date                                    1323           1330           6          7.6         132.3       0.6X
-MONTH of date                                      1021           1023           2          9.8         102.1       0.8X
-WEEK of date                                       1541           1549           8          6.5         154.1       0.6X
-DAY of date                                        1021           1033          13          9.8         102.1       0.8X
-DAYOFWEEK of date                                  1196           1209          11          8.4         119.6       0.7X
-DOW of date                                        1214           1229          13          8.2         121.4       0.7X
-ISODOW of date                                     1148           1153           7          8.7         114.8       0.7X
-DOY of date                                        1073           1079           5          9.3         107.3       0.8X
-HOUR of date                                       1311           1314           4          7.6         131.1       0.6X
-MINUTE of date                                     1311           1311           1          7.6         131.1       0.6X
-SECOND of date                                     1420           1434          13          7.0         142.0       0.6X
-MILLISECONDS of date                               1426           1442          14          7.0         142.6       0.6X
-MICROSECONDS of date                               1312           1318           6          7.6         131.2       0.6X
-EPOCH of date                                      2034           2050          16          4.9         203.4       0.4X
+cast to date                                        690            693           3         14.5          69.0       1.0X
+YEAR of date                                        822            841          17         12.2          82.2       0.8X
+YEAROFWEEK of date                                  967            974           7         10.3          96.7       0.7X
+QUARTER of date                                    1034           1044          11          9.7         103.4       0.7X
+MONTH of date                                       831            836           5         12.0          83.1       0.8X
+WEEK of date                                       1152           1177          34          8.7         115.2       0.6X
+DAY of date                                         836            873          34         12.0          83.6       0.8X
+DAYOFWEEK of date                                   960            992          30         10.4          96.0       0.7X
+DOW of date                                        1011           1016           4          9.9         101.1       0.7X
+DOW_ISO of date                                     969            984          16         10.3          96.9       0.7X
+DAYOFWEEK_ISO of date                               967            986          19         10.3          96.7       0.7X
+DOY of date                                         901            953          47         11.1          90.1       0.8X
+HOUR of date                                       1581           1586           5          6.3         158.1       0.4X
+MINUTE of date                                     1570           1584          13          6.4         157.0       0.4X
+SECOND of date                                     1713           1740          27          5.8         171.3       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        852            879          42         11.7          85.2       1.0X
-MILLENNIUM of date                                 1131           1136           7          8.8         113.1       0.8X
-CENTURY of date                                    1138           1145           6          8.8         113.8       0.7X
-DECADE of date                                     1030           1043          13          9.7         103.0       0.8X
-YEAR of date                                       1022           1028           8          9.8         102.2       0.8X
-ISOYEAR of date                                    1260           1265           6          7.9         126.0       0.7X
-QUARTER of date                                    1326           1330           7          7.5         132.6       0.6X
-MONTH of date                                      1014           1034          26          9.9         101.4       0.8X
-WEEK of date                                       1523           1526           5          6.6         152.3       0.6X
-DAY of date                                        1022           1023           2          9.8         102.2       0.8X
-DAYOFWEEK of date                                  1197           1203           9          8.4         119.7       0.7X
-DOW of date                                        1188           1198          16          8.4         118.8       0.7X
-ISODOW of date                                     1143           1153           9          8.8         114.3       0.7X
-DOY of date                                        1052           1058           7          9.5         105.2       0.8X
-HOUR of date                                       1309           1311           4          7.6         130.9       0.7X
-MINUTE of date                                     1302           1305           6          7.7         130.2       0.7X
-SECOND of date                                     1414           1432          16          7.1         141.4       0.6X
-MILLISECONDS of date                               1441           1450          11          6.9         144.1       0.6X
-MICROSECONDS of date                               1292           1301           8          7.7         129.2       0.7X
-EPOCH of date                                      2030           2036           8          4.9         203.0       0.4X
+cast to date                                        756            762           6         13.2          75.6       1.0X
+YEAR of date                                        843            857          13         11.9          84.3       0.9X
+YEAROFWEEK of date                                 1055           1065          16          9.5         105.5       0.7X
+QUARTER of date                                    1066           1073           6          9.4         106.6       0.7X
+MONTH of date                                       856            890          44         11.7          85.6       0.9X
+WEEK of date                                       1155           1204          59          8.7         115.5       0.7X
+DAY of date                                         749            762          12         13.3          74.9       1.0X
+DAYOFWEEK of date                                   850            865          15         11.8          85.0       0.9X
+DOW of date                                         878            893          16         11.4          87.8       0.9X
+DOW_ISO of date                                     865            869           5         11.6          86.5       0.9X
+DAYOFWEEK_ISO of date                               914            967          76         10.9          91.4       0.8X
+DOY of date                                         789            792           4         12.7          78.9       1.0X
+HOUR of date                                       1558           1659         168          6.4         155.8       0.5X
+MINUTE of date                                     1581           1673          80          6.3         158.1       0.5X
+SECOND of date                                     1646           1881         319          6.1         164.6       0.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+Invoke extract for interval:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+cast to interval                                    925            941          20         10.8          92.5       1.0X
+YEAR of interval                                    903            919          14         11.1          90.3       1.0X
+MONTH of interval                                   944            958          17         10.6          94.4       1.0X
+DAY of interval                                     917            925           7         10.9          91.7       1.0X
+HOUR of interval                                    925            940          17         10.8          92.5       1.0X
+MINUTE of interval                                  951            962          12         10.5          95.1       1.0X
+SECOND of interval                                 1017           1036          19          9.8         101.7       0.9X
+
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                   1249           1254           6          8.0         124.9       1.0X
-MILLENNIUM of interval                             1310           1316           9          7.6         131.0       1.0X
-CENTURY of interval                                1304           1315          10          7.7         130.4       1.0X
-DECADE of interval                                 1306           1313           7          7.7         130.6       1.0X
-YEAR of interval                                   1304           1313          11          7.7         130.4       1.0X
-QUARTER of interval                                1310           1317           7          7.6         131.0       1.0X
-MONTH of interval                                  1311           1319          12          7.6         131.1       1.0X
-DAY of interval                                    1295           1304          13          7.7         129.5       1.0X
-HOUR of interval                                   1301           1306           8          7.7         130.1       1.0X
-MINUTE of interval                                 1316           1319           3          7.6         131.6       0.9X
-SECOND of interval                                 1437           1440           3          7.0         143.7       0.9X
-MILLISECONDS of interval                           1435           1449          16          7.0         143.5       0.9X
-MICROSECONDS of interval                           1304           1314           9          7.7         130.4       1.0X
-EPOCH of interval                                  1440           1453          19          6.9         144.0       0.9X
+cast to interval                                    913            920           8         11.0          91.3       1.0X
+YEAR of interval                                    930            935           4         10.8          93.0       1.0X
+MONTH of interval                                   930            943          14         10.7          93.0       1.0X
+DAY of interval                                     933            946          12         10.7          93.3       1.0X
+HOUR of interval                                    951            953           3         10.5          95.1       1.0X
+MINUTE of interval                                  923            958          30         10.8          92.3       1.0X
+SECOND of interval                                  993            995           1         10.1          99.3       0.9X
 
diff --git a/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt b/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt
index 7d23e5467baed..8958d7c53413f 100644
--- a/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt
@@ -1,29 +1,29 @@
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast strings to intervals:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-prepare string w/ interval                          756            831          78          1.3         756.4       1.0X
-prepare string w/o interval                         622            637          15          1.6         622.4       1.2X
-1 units w/ interval                                 506            532          22          2.0         506.2       1.5X
-1 units w/o interval                                472            477           9          2.1         471.6       1.6X
-2 units w/ interval                                 699            710          11          1.4         698.5       1.1X
-2 units w/o interval                                670            674           6          1.5         669.9       1.1X
-3 units w/ interval                                1431           1437           7          0.7        1431.3       0.5X
-3 units w/o interval                               1418           1429          11          0.7        1418.2       0.5X
-4 units w/ interval                                1687           1692           8          0.6        1686.8       0.4X
-4 units w/o interval                               1679           1688           7          0.6        1679.4       0.5X
-5 units w/ interval                                1862           1864           3          0.5        1861.9       0.4X
-5 units w/o interval                               1847           1864          15          0.5        1846.9       0.4X
-6 units w/ interval                                2067           2081          12          0.5        2066.9       0.4X
-6 units w/o interval                               2070           2071           2          0.5        2069.6       0.4X
-7 units w/ interval                                2458           2468          13          0.4        2457.7       0.3X
-7 units w/o interval                               2450           2453           3          0.4        2450.1       0.3X
-8 units w/ interval                                2833           2838           8          0.4        2832.6       0.3X
-8 units w/o interval                               2830           2839           8          0.4        2829.8       0.3X
-9 units w/ interval                                2873           2880           6          0.3        2873.4       0.3X
-9 units w/o interval                               2860           2863           3          0.3        2860.1       0.3X
-10 units w/ interval                               3252           3257           5          0.3        3252.2       0.2X
-10 units w/o interval                              3212           3220           8          0.3        3211.6       0.2X
-11 units w/ interval                               3369           3376           6          0.3        3368.5       0.2X
-11 units w/o interval                              3384           3395          15          0.3        3384.2       0.2X
+prepare string w/ interval                          448            469          20          2.2         447.6       1.0X
+prepare string w/o interval                         405            409           4          2.5         404.6       1.1X
+1 units w/ interval                                 321            328           6          3.1         321.4       1.4X
+1 units w/o interval                                303            307           4          3.3         303.1       1.5X
+2 units w/ interval                                 445            458          12          2.2         444.6       1.0X
+2 units w/o interval                                416            424          10          2.4         416.2       1.1X
+3 units w/ interval                                1006           1012           8          1.0        1006.4       0.4X
+3 units w/o interval                               1240           1249           8          0.8        1239.6       0.4X
+4 units w/ interval                                1295           1418         106          0.8        1295.4       0.3X
+4 units w/o interval                               1172           1188          15          0.9        1171.6       0.4X
+5 units w/ interval                                1326           1335          11          0.8        1325.6       0.3X
+5 units w/o interval                               1309           1336          44          0.8        1308.7       0.3X
+6 units w/ interval                                1441           1464          29          0.7        1441.0       0.3X
+6 units w/o interval                               1350           1369          17          0.7        1350.1       0.3X
+7 units w/ interval                                1606           1669          99          0.6        1605.6       0.3X
+7 units w/o interval                               1546           1557          12          0.6        1546.3       0.3X
+8 units w/ interval                                1771           1875         120          0.6        1770.6       0.3X
+8 units w/o interval                               1775           1789          13          0.6        1775.2       0.3X
+9 units w/ interval                                2126           2757         849          0.5        2126.4       0.2X
+9 units w/o interval                               2053           2070          21          0.5        2053.3       0.2X
+10 units w/ interval                               2209           2243          30          0.5        2209.1       0.2X
+10 units w/o interval                              2400           2702         365          0.4        2400.2       0.2X
+11 units w/ interval                               2616           2699          72          0.4        2616.5       0.2X
+11 units w/o interval                              3218           3380         195          0.3        3218.4       0.1X
 
diff --git a/sql/core/benchmarks/IntervalBenchmark-results.txt b/sql/core/benchmarks/IntervalBenchmark-results.txt
index da2f0e66dacac..48af333b78ba4 100644
--- a/sql/core/benchmarks/IntervalBenchmark-results.txt
+++ b/sql/core/benchmarks/IntervalBenchmark-results.txt
@@ -1,29 +1,29 @@
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast strings to intervals:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-prepare string w/ interval                          648            721          94          1.5         648.3       1.0X
-prepare string w/o interval                         562            596          49          1.8         562.3       1.2X
-1 units w/ interval                                 568            590          21          1.8         568.5       1.1X
-1 units w/o interval                                522            538          20          1.9         521.7       1.2X
-2 units w/ interval                                 751            754           3          1.3         751.5       0.9X
-2 units w/o interval                                716            723           6          1.4         716.1       0.9X
-3 units w/ interval                                1402           1411          11          0.7        1401.6       0.5X
-3 units w/o interval                               1381           1387           5          0.7        1381.2       0.5X
-4 units w/ interval                                1591           1595           6          0.6        1591.2       0.4X
-4 units w/o interval                               1582           1585           3          0.6        1582.3       0.4X
-5 units w/ interval                                1747           1749           2          0.6        1747.3       0.4X
-5 units w/o interval                               1738           1746          10          0.6        1737.7       0.4X
-6 units w/ interval                                1929           1931           3          0.5        1929.1       0.3X
-6 units w/o interval                               1919           1922           2          0.5        1919.0       0.3X
-7 units w/ interval                                2345           2354           8          0.4        2345.0       0.3X
-7 units w/o interval                               2334           2336           2          0.4        2334.1       0.3X
-8 units w/ interval                                2533           2546          16          0.4        2533.0       0.3X
-8 units w/o interval                               2519           2521           1          0.4        2519.4       0.3X
-9 units w/ interval                                2885           2889           5          0.3        2884.5       0.2X
-9 units w/o interval                               2804           2813          12          0.4        2803.9       0.2X
-10 units w/ interval                               3041           3060          16          0.3        3041.3       0.2X
-10 units w/o interval                              3031           3043          15          0.3        3031.2       0.2X
-11 units w/ interval                               3270           3280           9          0.3        3269.9       0.2X
-11 units w/o interval                              3273           3280           7          0.3        3272.6       0.2X
+prepare string w/ interval                          389            410          21          2.6         388.7       1.0X
+prepare string w/o interval                         340            360          18          2.9         340.5       1.1X
+1 units w/ interval                                 378            389          16          2.6         377.8       1.0X
+1 units w/o interval                                346            350           5          2.9         346.2       1.1X
+2 units w/ interval                                 444            457          11          2.3         444.2       0.9X
+2 units w/o interval                                455            464          12          2.2         455.1       0.9X
+3 units w/ interval                                 942            964          20          1.1         941.5       0.4X
+3 units w/o interval                                927           1020          93          1.1         927.3       0.4X
+4 units w/ interval                                1114           1127          17          0.9        1113.9       0.3X
+4 units w/o interval                               1100           1105           4          0.9        1100.3       0.4X
+5 units w/ interval                                1180           1244          57          0.8        1180.1       0.3X
+5 units w/o interval                               1135           1141           6          0.9        1135.2       0.3X
+6 units w/ interval                                1284           1316          48          0.8        1284.0       0.3X
+6 units w/o interval                               1276           1357         122          0.8        1276.1       0.3X
+7 units w/ interval                                1609           1636          32          0.6        1609.1       0.2X
+7 units w/o interval                               1551           1578          36          0.6        1550.9       0.3X
+8 units w/ interval                                1787           1874         129          0.6        1787.1       0.2X
+8 units w/o interval                               1751           1767          15          0.6        1750.6       0.2X
+9 units w/ interval                                1960           2065         141          0.5        1959.7       0.2X
+9 units w/o interval                               1885           1908          39          0.5        1885.1       0.2X
+10 units w/ interval                               2178           2185          11          0.5        2177.9       0.2X
+10 units w/o interval                              2150           2255         164          0.5        2150.1       0.2X
+11 units w/ interval                               2457           2542         139          0.4        2456.7       0.2X
+11 units w/o interval                              2557           2770         188          0.4        2556.7       0.2X
 
diff --git a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
index 920e0a7723e70..2d506f03d9f7e 100644
--- a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
@@ -3,110 +3,110 @@ Benchmark for performance of JSON parsing
 ================================================================================================
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       84774          84927         264          1.2         847.7       1.0X
-UTF-8 is set                                     119081         120155        1773          0.8        1190.8       0.7X
+No encoding                                       73307          73400         141          1.4         733.1       1.0X
+UTF-8 is set                                     143834         143925         152          0.7        1438.3       0.5X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       49293          49356          70          2.0         492.9       1.0X
-UTF-8 is set                                      80183          80211          25          1.2         801.8       0.6X
+No encoding                                       50894          51065         292          2.0         508.9       1.0X
+UTF-8 is set                                      98462          99455        1173          1.0         984.6       0.5X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       61070          61476         536          0.2        6107.0       1.0X
-UTF-8 is set                                     109765         109881         102          0.1       10976.5       0.6X
+No encoding                                       64011          64969        1001          0.2        6401.1       1.0X
+UTF-8 is set                                     102757         102984         311          0.1       10275.7       0.6X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                      176999         178163        1008          0.0      353997.9       1.0X
-UTF-8 is set                                     201209         201641         614          0.0      402419.0       0.9X
+No encoding                                      132559         133561        1010          0.0      265117.3       1.0X
+UTF-8 is set                                     151458         152129         611          0.0      302915.4       0.9X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                 18768          20587         496          0.5        1876.8       1.0X
-Select 1 column                                   22642          22644           3          0.4        2264.2       0.8X
+Select 10 columns                                 21148          21202          87          0.5        2114.8       1.0X
+Select 1 column                                   24701          24724          21          0.4        2470.1       0.9X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      7697           7738          55          1.3         769.7       1.0X
-Short column with UTF-8                           14051          14189         176          0.7        1405.1       0.5X
-Wide column without encoding                     108999         110075        1085          0.1       10899.9       0.1X
-Wide column with UTF-8                           157433         157779         308          0.1       15743.3       0.0X
+Short column without encoding                      6945           6998          59          1.4         694.5       1.0X
+Short column with UTF-8                           11510          11569          51          0.9        1151.0       0.6X
+Wide column without encoding                      95004          95795         790          0.1        9500.4       0.1X
+Wide column with UTF-8                           149223         149409         276          0.1       14922.3       0.0X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                           644            647           4         15.5          64.4       1.0X
-from_json                                         25859          25872          12          0.4        2585.9       0.0X
-json_tuple                                        31679          31761          71          0.3        3167.9       0.0X
-get_json_object                                   24772          25220         389          0.4        2477.2       0.0X
+Text read                                           649            652           3         15.4          64.9       1.0X
+from_json                                         22284          22393          99          0.4        2228.4       0.0X
+json_tuple                                        32310          32824         484          0.3        3231.0       0.0X
+get_json_object                                   22111          22751         568          0.5        2211.1       0.0X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          3135           3165          52         15.9          62.7       1.0X
-schema inferring                                  29383          29389          10          1.7         587.7       0.1X
-parsing                                           32623          35183         NaN          1.5         652.5       0.1X
+Text read                                          2894           2903           8         17.3          57.9       1.0X
+schema inferring                                  26724          26785          62          1.9         534.5       0.1X
+parsing                                           37502          37632         131          1.3         750.0       0.1X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                         11874          11948          82          4.2         237.5       1.0X
-Schema inferring                                  42382          42398          23          1.2         847.6       0.3X
-Parsing without charset                           36410          36442          54          1.4         728.2       0.3X
-Parsing with UTF-8                                62412          62463          48          0.8        1248.2       0.2X
+Text read                                         10994          11010          16          4.5         219.9       1.0X
+Schema inferring                                  45654          45677          37          1.1         913.1       0.2X
+Parsing without charset                           34476          34559          73          1.5         689.5       0.3X
+Parsing with UTF-8                                56987          57002          13          0.9        1139.7       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     2191           2209          20          4.6         219.1       1.0X
-to_json(timestamp)                                18670          19042         565          0.5        1867.0       0.1X
-write timestamps to files                         11836          13156         NaN          0.8        1183.6       0.2X
-Create a dataset of dates                          2321           2351          33          4.3         232.1       0.9X
-to_json(date)                                     12703          12726          24          0.8        1270.3       0.2X
-write dates to files                               8230           8303          76          1.2         823.0       0.3X
+Create a dataset of timestamps                     2150           2188          35          4.7         215.0       1.0X
+to_json(timestamp)                                17874          18080         294          0.6        1787.4       0.1X
+write timestamps to files                         12518          12538          34          0.8        1251.8       0.2X
+Create a dataset of dates                          2298           2310          18          4.4         229.8       0.9X
+to_json(date)                                     11673          11703          27          0.9        1167.3       0.2X
+write dates to files                               7121           7135          12          1.4         712.1       0.3X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2780           2795          13          3.6         278.0       1.0X
-read timestamps from files                        37158          37305         137          0.3        3715.8       0.1X
-infer timestamps from files                       73666          73838         149          0.1        7366.6       0.0X
-read date text from files                          2597           2609          10          3.9         259.7       1.1X
-read date from files                              24439          24501          56          0.4        2443.9       0.1X
-timestamp strings                                  3052           3064          12          3.3         305.2       0.9X
-parse timestamps from Dataset[String]             43611          43665          52          0.2        4361.1       0.1X
-infer timestamps from Dataset[String]             83745          84153         376          0.1        8374.5       0.0X
-date strings                                       4068           4076          10          2.5         406.8       0.7X
-parse dates from Dataset[String]                  34700          34807         118          0.3        3470.0       0.1X
-from_json(timestamp)                              64074          64124          53          0.2        6407.4       0.0X
-from_json(date)                                   52520          52617         101          0.2        5252.0       0.1X
+read timestamp text from files                     2616           2641          34          3.8         261.6       1.0X
+read timestamps from files                        37481          37517          58          0.3        3748.1       0.1X
+infer timestamps from files                       84774          84964         201          0.1        8477.4       0.0X
+read date text from files                          2362           2365           3          4.2         236.2       1.1X
+read date from files                              16583          16612          29          0.6        1658.3       0.2X
+timestamp strings                                  3927           3963          40          2.5         392.7       0.7X
+parse timestamps from Dataset[String]             52827          53004         243          0.2        5282.7       0.0X
+infer timestamps from Dataset[String]            101108         101644         769          0.1       10110.8       0.0X
+date strings                                       4886           4906          26          2.0         488.6       0.5X
+parse dates from Dataset[String]                  27623          27694          62          0.4        2762.3       0.1X
+from_json(timestamp)                              71764          71887         124          0.1        7176.4       0.0X
+from_json(date)                                   46200          46314          99          0.2        4620.0       0.1X
 
 
diff --git a/sql/core/benchmarks/JsonBenchmark-results.txt b/sql/core/benchmarks/JsonBenchmark-results.txt
index e435f573a5d92..c22118f91b3fc 100644
--- a/sql/core/benchmarks/JsonBenchmark-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-results.txt
@@ -3,110 +3,110 @@ Benchmark for performance of JSON parsing
 ================================================================================================
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       61888          61918          27          1.6         618.9       1.0X
-UTF-8 is set                                     109057         113663         NaN          0.9        1090.6       0.6X
+No encoding                                       63839          64000         263          1.6         638.4       1.0X
+UTF-8 is set                                     124633         124945         429          0.8        1246.3       0.5X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       44517          44535          29          2.2         445.2       1.0X
-UTF-8 is set                                      75722          75840         111          1.3         757.2       0.6X
+No encoding                                       51720          51901         157          1.9         517.2       1.0X
+UTF-8 is set                                      91161          91190          25          1.1         911.6       0.6X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       63677          64090         633          0.2        6367.7       1.0X
-UTF-8 is set                                      99424          99615         185          0.1        9942.4       0.6X
+No encoding                                       58486          59038         714          0.2        5848.6       1.0X
+UTF-8 is set                                     103045         103350         358          0.1       10304.5       0.6X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                      174052         174251         174          0.0      348104.1       1.0X
-UTF-8 is set                                     189000         189098         113          0.0      378000.9       0.9X
+No encoding                                      134909         135024         105          0.0      269818.6       1.0X
+UTF-8 is set                                     154418         154593         155          0.0      308836.7       0.9X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                 18387          18473         142          0.5        1838.7       1.0X
-Select 1 column                                   25560          25571          13          0.4        2556.0       0.7X
+Select 10 columns                                 19538          19620          70          0.5        1953.8       1.0X
+Select 1 column                                   26142          26159          15          0.4        2614.2       0.7X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      9323           9384          58          1.1         932.3       1.0X
-Short column with UTF-8                           14016          14058          55          0.7        1401.6       0.7X
-Wide column without encoding                     133258         133532         382          0.1       13325.8       0.1X
-Wide column with UTF-8                           181212         181283          61          0.1       18121.2       0.1X
+Short column without encoding                      8103           8162          53          1.2         810.3       1.0X
+Short column with UTF-8                           13104          13150          58          0.8        1310.4       0.6X
+Wide column without encoding                     135280         135593         375          0.1       13528.0       0.1X
+Wide column with UTF-8                           175189         175483         278          0.1       17518.9       0.0X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          1168           1174           5          8.6         116.8       1.0X
-from_json                                         22604          23571         883          0.4        2260.4       0.1X
-json_tuple                                        29979          30053          91          0.3        2997.9       0.0X
-get_json_object                                   21987          22263         241          0.5        2198.7       0.1X
+Text read                                          1225           1234           8          8.2         122.5       1.0X
+from_json                                         22482          22552          95          0.4        2248.2       0.1X
+json_tuple                                        30203          30338         146          0.3        3020.3       0.0X
+get_json_object                                   22219          22245          26          0.5        2221.9       0.1X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          5831           5842          14          8.6         116.6       1.0X
-schema inferring                                  31372          31456          73          1.6         627.4       0.2X
-parsing                                           35911          36191         254          1.4         718.2       0.2X
+Text read                                          5897           5904          10          8.5         117.9       1.0X
+schema inferring                                  30282          30340          50          1.7         605.6       0.2X
+parsing                                           33304          33577         289          1.5         666.1       0.2X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                         10249          10314          77          4.9         205.0       1.0X
-Schema inferring                                  35403          35436          40          1.4         708.1       0.3X
-Parsing without charset                           32875          32879           4          1.5         657.5       0.3X
-Parsing with UTF-8                                53444          53519         100          0.9        1068.9       0.2X
+Text read                                          9710           9757          80          5.1         194.2       1.0X
+Schema inferring                                  35929          35939           9          1.4         718.6       0.3X
+Parsing without charset                           39175          39227          87          1.3         783.5       0.2X
+Parsing with UTF-8                                59188          59294         109          0.8        1183.8       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     1909           1924          17          5.2         190.9       1.0X
-to_json(timestamp)                                18956          19122         208          0.5        1895.6       0.1X
-write timestamps to files                         13446          13472          43          0.7        1344.6       0.1X
-Create a dataset of dates                          2180           2200          28          4.6         218.0       0.9X
-to_json(date)                                     12780          12899         109          0.8        1278.0       0.1X
-write dates to files                               7835           7865          29          1.3         783.5       0.2X
+Create a dataset of timestamps                     1967           1977           9          5.1         196.7       1.0X
+to_json(timestamp)                                17086          17304         371          0.6        1708.6       0.1X
+write timestamps to files                         12691          12716          28          0.8        1269.1       0.2X
+Create a dataset of dates                          2192           2217          39          4.6         219.2       0.9X
+to_json(date)                                     10541          10656         137          0.9        1054.1       0.2X
+write dates to files                               7259           7311          46          1.4         725.9       0.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2467           2477           9          4.1         246.7       1.0X
-read timestamps from files                        40186          40342         135          0.2        4018.6       0.1X
-infer timestamps from files                       82005          82079          71          0.1        8200.5       0.0X
-read date text from files                          2243           2264          22          4.5         224.3       1.1X
-read date from files                              24852          24863          19          0.4        2485.2       0.1X
-timestamp strings                                  3836           3854          16          2.6         383.6       0.6X
-parse timestamps from Dataset[String]             51521          51697         242          0.2        5152.1       0.0X
-infer timestamps from Dataset[String]             97300          97398         133          0.1        9730.0       0.0X
-date strings                                       4488           4491           5          2.2         448.8       0.5X
-parse dates from Dataset[String]                  37918          37976          68          0.3        3791.8       0.1X
-from_json(timestamp)                              69611          69632          36          0.1        6961.1       0.0X
-from_json(date)                                   56598          56974         347          0.2        5659.8       0.0X
+read timestamp text from files                     2318           2326          13          4.3         231.8       1.0X
+read timestamps from files                        43345          43627         258          0.2        4334.5       0.1X
+infer timestamps from files                       89570          89621          59          0.1        8957.0       0.0X
+read date text from files                          2099           2107           9          4.8         209.9       1.1X
+read date from files                              18000          18065          98          0.6        1800.0       0.1X
+timestamp strings                                  3937           3956          32          2.5         393.7       0.6X
+parse timestamps from Dataset[String]             56001          56429         539          0.2        5600.1       0.0X
+infer timestamps from Dataset[String]            109410         109963         559          0.1       10941.0       0.0X
+date strings                                       4530           4540           9          2.2         453.0       0.5X
+parse dates from Dataset[String]                  29723          29767          72          0.3        2972.3       0.1X
+from_json(timestamp)                              74106          74619         728          0.1        7410.6       0.0X
+from_json(date)                                   46599          46632          32          0.2        4659.9       0.0X
 
 
diff --git a/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..c33f8a372d82b
--- /dev/null
+++ b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt
@@ -0,0 +1,21 @@
+OpenJDK 64-Bit Server VM 11.0.2+9 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip all row groups:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 34214          35752         NaN          3.1         326.3       1.0X
+With nested predicate Pushdown                       86            102          11       1216.2           0.8     396.8X
+
+OpenJDK 64-Bit Server VM 11.0.2+9 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip some row groups:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 34211          35162         843          3.1         326.3       1.0X
+With nested predicate Pushdown                     3470           3514          36         30.2          33.1       9.9X
+
+OpenJDK 64-Bit Server VM 11.0.2+9 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip no row groups:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 37533          37919         329          2.8         357.9       1.0X
+With nested predicate Pushdown                    37876          39132         536          2.8         361.2       1.0X
+
diff --git a/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt
new file mode 100644
index 0000000000000..35dd4f0a5e9cb
--- /dev/null
+++ b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt
@@ -0,0 +1,21 @@
+OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip all row groups:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 30687          31552         NaN          3.4         292.7       1.0X
+With nested predicate Pushdown                      105            150          61        999.3           1.0     292.5X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip some row groups:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 30505          31828         NaN          3.4         290.9       1.0X
+With nested predicate Pushdown                     3156           3215          77         33.2          30.1       9.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip no row groups:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 34475          35302         NaN          3.0         328.8       1.0X
+With nested predicate Pushdown                    34003          34596         567          3.1         324.3       1.0X
+
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 0e664eca6a820..1a1673fdfd212 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
@@ -150,6 +150,16 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.seleniumhq.selenium</groupId>
+      <artifactId>selenium-java</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.seleniumhq.selenium</groupId>
+      <artifactId>selenium-htmlunit-driver</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
diff --git a/sql/core/src/main/java/org/apache/parquet/filter2/predicate/SparkFilterApi.java b/sql/core/src/main/java/org/apache/parquet/filter2/predicate/SparkFilterApi.java
new file mode 100644
index 0000000000000..884042c824046
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/parquet/filter2/predicate/SparkFilterApi.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.filter2.predicate;
+
+import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.filter2.predicate.Operators.BinaryColumn;
+import org.apache.parquet.filter2.predicate.Operators.BooleanColumn;
+import org.apache.parquet.filter2.predicate.Operators.DoubleColumn;
+import org.apache.parquet.filter2.predicate.Operators.FloatColumn;
+import org.apache.parquet.filter2.predicate.Operators.IntColumn;
+import org.apache.parquet.filter2.predicate.Operators.LongColumn;
+
+/**
+ * TODO (PARQUET-1809): This is a temporary workaround; it is intended to be moved to Parquet.
+ */
+public final class SparkFilterApi {
+  public static IntColumn intColumn(String[] path) {
+    return new IntColumn(ColumnPath.get(path));
+  }
+
+  public static LongColumn longColumn(String[] path) {
+    return new LongColumn(ColumnPath.get(path));
+  }
+
+  public static FloatColumn floatColumn(String[] path) {
+    return new FloatColumn(ColumnPath.get(path));
+  }
+
+  public static DoubleColumn doubleColumn(String[] path) {
+    return new DoubleColumn(ColumnPath.get(path));
+  }
+
+  public static BooleanColumn booleanColumn(String[] path) {
+    return new BooleanColumn(ColumnPath.get(path));
+  }
+
+  public static BinaryColumn binaryColumn(String[] path) {
+    return new BinaryColumn(ColumnPath.get(path));
+  }
+}
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
index acd54fe25d62d..42ceebccdd1ee 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
@@ -35,6 +35,7 @@
 import org.apache.spark.storage.BlockManager;
 import org.apache.spark.unsafe.KVIterator;
 import org.apache.spark.unsafe.Platform;
+import org.apache.spark.unsafe.UnsafeAlignedOffset;
 import org.apache.spark.unsafe.array.LongArray;
 import org.apache.spark.unsafe.map.BytesToBytesMap;
 import org.apache.spark.unsafe.memory.MemoryBlock;
@@ -141,9 +142,10 @@ public UnsafeKVExternalSorter(
 
         // Get encoded memory address
         // baseObject + baseOffset point to the beginning of the key data in the map, but that
-        // the KV-pair's length data is stored in the word immediately before that address
+        // the KV-pair's length data is stored at 2 * uaoSize bytes immediately before that address
         MemoryBlock page = loc.getMemoryPage();
-        long address = taskMemoryManager.encodePageNumberAndOffset(page, baseOffset - 8);
+        long address = taskMemoryManager.encodePageNumberAndOffset(page,
+            baseOffset - 2 * UnsafeAlignedOffset.getUaoSize());
 
         // Compute prefix
         row.pointTo(baseObject, baseOffset, loc.getKeyLength());
@@ -163,7 +165,8 @@ public UnsafeKVExternalSorter(
         (int) (long) SparkEnv.get().conf().get(package$.MODULE$.SHUFFLE_SORT_INIT_BUFFER_SIZE()),
         pageSizeBytes,
         numElementsForSpillThreshold,
-        inMemSorter);
+        inMemSorter,
+        map.getTotalMemoryConsumption());
 
       // reset the map, so we can re-use it to insert new records. the inMemSorter will not used
       // anymore, so the underline array could be used by map again.
@@ -262,10 +265,11 @@ public int compare(
         Object baseObj2,
         long baseOff2,
         int baseLen2) {
+      int uaoSize = UnsafeAlignedOffset.getUaoSize();
       // Note that since ordering doesn't need the total length of the record, we just pass 0
       // into the row.
-      row1.pointTo(baseObj1, baseOff1 + 4, 0);
-      row2.pointTo(baseObj2, baseOff2 + 4, 0);
+      row1.pointTo(baseObj1, baseOff1 + uaoSize, 0);
+      row2.pointTo(baseObj2, baseOff2 + uaoSize, 0);
       return ordering.compare(row1, row2);
     }
   }
@@ -289,11 +293,12 @@ public boolean next() throws IOException {
           long recordOffset = underlying.getBaseOffset();
           int recordLen = underlying.getRecordLength();
 
-          // Note that recordLen = keyLen + valueLen + 4 bytes (for the keyLen itself)
+          // Note that recordLen = keyLen + valueLen + uaoSize (for the keyLen itself)
+          int uaoSize = UnsafeAlignedOffset.getUaoSize();
           int keyLen = Platform.getInt(baseObj, recordOffset);
-          int valueLen = recordLen - keyLen - 4;
-          key.pointTo(baseObj, recordOffset + 4, keyLen);
-          value.pointTo(baseObj, recordOffset + 4 + keyLen, valueLen);
+          int valueLen = recordLen - keyLen - uaoSize;
+          key.pointTo(baseObj, recordOffset + uaoSize, keyLen);
+          value.pointTo(baseObj, recordOffset + uaoSize + keyLen, valueLen);
 
           return true;
         } else {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index ba26b57567e64..f264281c02f45 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -18,8 +18,9 @@
 package org.apache.spark.sql.execution.datasources.parquet;
 
 import java.io.IOException;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.util.Arrays;
-import java.util.TimeZone;
 
 import org.apache.parquet.bytes.ByteBufferInputStream;
 import org.apache.parquet.bytes.BytesInput;
@@ -34,6 +35,8 @@
 import org.apache.parquet.schema.PrimitiveType;
 
 import org.apache.spark.sql.catalyst.util.DateTimeUtils;
+import org.apache.spark.sql.catalyst.util.RebaseDateTime;
+import org.apache.spark.sql.execution.datasources.DataSourceUtils;
 import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 import org.apache.spark.sql.types.DataTypes;
@@ -98,14 +101,16 @@ public class VectorizedColumnReader {
   private final ColumnDescriptor descriptor;
   private final OriginalType originalType;
   // The timezone conversion to apply to int96 timestamps. Null if no conversion.
-  private final TimeZone convertTz;
-  private static final TimeZone UTC = DateTimeUtils.TimeZoneUTC();
+  private final ZoneId convertTz;
+  private static final ZoneId UTC = ZoneOffset.UTC;
+  private final String datetimeRebaseMode;
 
   public VectorizedColumnReader(
       ColumnDescriptor descriptor,
       OriginalType originalType,
       PageReader pageReader,
-      TimeZone convertTz) throws IOException {
+      ZoneId convertTz,
+      String datetimeRebaseMode) throws IOException {
     this.descriptor = descriptor;
     this.pageReader = pageReader;
     this.convertTz = convertTz;
@@ -128,6 +133,9 @@ public VectorizedColumnReader(
     if (totalValueCount == 0) {
       throw new IOException("totalValueCount == 0");
     }
+    assert "LEGACY".equals(datetimeRebaseMode) || "EXCEPTION".equals(datetimeRebaseMode) ||
+      "CORRECTED".equals(datetimeRebaseMode);
+    this.datetimeRebaseMode = datetimeRebaseMode;
   }
 
   /**
@@ -147,6 +155,52 @@ private boolean next() throws IOException {
     return definitionLevelColumn.nextInt() == maxDefLevel;
   }
 
+  private boolean isLazyDecodingSupported(PrimitiveType.PrimitiveTypeName typeName) {
+    boolean isSupported = false;
+    switch (typeName) {
+      case INT32:
+        isSupported = originalType != OriginalType.DATE || "CORRECTED".equals(datetimeRebaseMode);
+        break;
+      case INT64:
+        if (originalType == OriginalType.TIMESTAMP_MICROS) {
+          isSupported = "CORRECTED".equals(datetimeRebaseMode);
+        } else {
+          isSupported = originalType != OriginalType.TIMESTAMP_MILLIS;
+        }
+        break;
+      case FLOAT:
+      case DOUBLE:
+      case BINARY:
+        isSupported = true;
+        break;
+    }
+    return isSupported;
+  }
+
+  static int rebaseDays(int julianDays, final boolean failIfRebase) {
+    if (failIfRebase) {
+      if (julianDays < RebaseDateTime.lastSwitchJulianDay()) {
+        throw DataSourceUtils.newRebaseExceptionInRead("Parquet");
+      } else {
+        return julianDays;
+      }
+    } else {
+      return RebaseDateTime.rebaseJulianToGregorianDays(julianDays);
+    }
+  }
+
+  static long rebaseMicros(long julianMicros, final boolean failIfRebase) {
+    if (failIfRebase) {
+      if (julianMicros < RebaseDateTime.lastSwitchJulianTs()) {
+        throw DataSourceUtils.newRebaseExceptionInRead("Parquet");
+      } else {
+        return julianMicros;
+      }
+    } else {
+      return RebaseDateTime.rebaseJulianToGregorianMicros(julianMicros);
+    }
+  }
+
   /**
    * Reads `total` values from this columnReader into column.
    */
@@ -176,13 +230,7 @@ void readBatch(int total, WritableColumnVector column) throws IOException {
 
         // TIMESTAMP_MILLIS encoded as INT64 can't be lazily decoded as we need to post process
         // the values to add microseconds precision.
-        if (column.hasDictionary() || (rowId == 0 &&
-            (typeName == PrimitiveType.PrimitiveTypeName.INT32 ||
-            (typeName == PrimitiveType.PrimitiveTypeName.INT64 &&
-              originalType != OriginalType.TIMESTAMP_MILLIS) ||
-            typeName == PrimitiveType.PrimitiveTypeName.FLOAT ||
-            typeName == PrimitiveType.PrimitiveTypeName.DOUBLE ||
-            typeName == PrimitiveType.PrimitiveTypeName.BINARY))) {
+        if (column.hasDictionary() || (rowId == 0 && isLazyDecodingSupported(typeName))) {
           // Column vector supports lazy decoding of dictionary values so just set the dictionary.
           // We can't do this if rowId != 0 AND the column doesn't have a dictionary (i.e. some
           // non-dictionary encoded values have already been added).
@@ -261,7 +309,8 @@ private void decodeDictionaryIds(
     switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
       case INT32:
         if (column.dataType() == DataTypes.IntegerType ||
-            DecimalType.is32BitDecimalType(column.dataType())) {
+            DecimalType.is32BitDecimalType(column.dataType()) ||
+            (column.dataType() == DataTypes.DateType && "CORRECTED".equals(datetimeRebaseMode))) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               column.putInt(i, dictionary.decodeToInt(dictionaryIds.getDictId(i)));
@@ -279,6 +328,14 @@ private void decodeDictionaryIds(
               column.putShort(i, (short) dictionary.decodeToInt(dictionaryIds.getDictId(i)));
             }
           }
+        } else if (column.dataType() == DataTypes.DateType) {
+          final boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
+          for (int i = rowId; i < rowId + num; ++i) {
+            if (!column.isNullAt(i)) {
+              int julianDays = dictionary.decodeToInt(dictionaryIds.getDictId(i));
+              column.putInt(i, rebaseDays(julianDays, failIfRebase));
+            }
+          }
         } else {
           throw constructConvertNotSupportedException(descriptor, column);
         }
@@ -287,17 +344,37 @@ private void decodeDictionaryIds(
       case INT64:
         if (column.dataType() == DataTypes.LongType ||
             DecimalType.is64BitDecimalType(column.dataType()) ||
-            originalType == OriginalType.TIMESTAMP_MICROS) {
+            (originalType == OriginalType.TIMESTAMP_MICROS &&
+              "CORRECTED".equals(datetimeRebaseMode))) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               column.putLong(i, dictionary.decodeToLong(dictionaryIds.getDictId(i)));
             }
           }
         } else if (originalType == OriginalType.TIMESTAMP_MILLIS) {
+          if ("CORRECTED".equals(datetimeRebaseMode)) {
+            for (int i = rowId; i < rowId + num; ++i) {
+              if (!column.isNullAt(i)) {
+                long gregorianMillis = dictionary.decodeToLong(dictionaryIds.getDictId(i));
+                column.putLong(i, DateTimeUtils.fromMillis(gregorianMillis));
+              }
+            }
+          } else {
+            final boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
+            for (int i = rowId; i < rowId + num; ++i) {
+              if (!column.isNullAt(i)) {
+                long julianMillis = dictionary.decodeToLong(dictionaryIds.getDictId(i));
+                long julianMicros = DateTimeUtils.fromMillis(julianMillis);
+                column.putLong(i, rebaseMicros(julianMicros, failIfRebase));
+              }
+            }
+          }
+        } else if (originalType == OriginalType.TIMESTAMP_MICROS) {
+          final boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
-              column.putLong(i,
-                DateTimeUtils.fromMillis(dictionary.decodeToLong(dictionaryIds.getDictId(i))));
+              long julianMicros = dictionary.decodeToLong(dictionaryIds.getDictId(i));
+              column.putLong(i, rebaseMicros(julianMicros, failIfRebase));
             }
           }
         } else {
@@ -406,7 +483,7 @@ private void readBooleanBatch(int rowId, int num, WritableColumnVector column)
   private void readIntBatch(int rowId, int num, WritableColumnVector column) throws IOException {
     // This is where we implement support for the valid type conversions.
     // TODO: implement remaining type conversions
-    if (column.dataType() == DataTypes.IntegerType || column.dataType() == DataTypes.DateType ||
+    if (column.dataType() == DataTypes.IntegerType ||
         DecimalType.is32BitDecimalType(column.dataType())) {
       defColumn.readIntegers(
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
@@ -416,6 +493,15 @@ private void readIntBatch(int rowId, int num, WritableColumnVector column) throw
     } else if (column.dataType() == DataTypes.ShortType) {
       defColumn.readShorts(
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+    } else if (column.dataType() == DataTypes.DateType ) {
+      if ("CORRECTED".equals(datetimeRebaseMode)) {
+        defColumn.readIntegers(
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+      } else {
+        boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
+        defColumn.readIntegersWithRebase(
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn, failIfRebase);
+      }
     } else {
       throw constructConvertNotSupportedException(descriptor, column);
     }
@@ -424,16 +510,36 @@ private void readIntBatch(int rowId, int num, WritableColumnVector column) throw
   private void readLongBatch(int rowId, int num, WritableColumnVector column) throws IOException {
     // This is where we implement support for the valid type conversions.
     if (column.dataType() == DataTypes.LongType ||
-        DecimalType.is64BitDecimalType(column.dataType()) ||
-        originalType == OriginalType.TIMESTAMP_MICROS) {
+        DecimalType.is64BitDecimalType(column.dataType())) {
       defColumn.readLongs(
         num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+    } else if (originalType == OriginalType.TIMESTAMP_MICROS) {
+      if ("CORRECTED".equals(datetimeRebaseMode)) {
+        defColumn.readLongs(
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+      } else {
+        boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
+        defColumn.readLongsWithRebase(
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn, failIfRebase);
+      }
     } else if (originalType == OriginalType.TIMESTAMP_MILLIS) {
-      for (int i = 0; i < num; i++) {
-        if (defColumn.readInteger() == maxDefLevel) {
-          column.putLong(rowId + i, DateTimeUtils.fromMillis(dataColumn.readLong()));
-        } else {
-          column.putNull(rowId + i);
+      if ("CORRECTED".equals(datetimeRebaseMode)) {
+        for (int i = 0; i < num; i++) {
+          if (defColumn.readInteger() == maxDefLevel) {
+            column.putLong(rowId + i, DateTimeUtils.fromMillis(dataColumn.readLong()));
+          } else {
+            column.putNull(rowId + i);
+          }
+        }
+      } else {
+        final boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
+        for (int i = 0; i < num; i++) {
+          if (defColumn.readInteger() == maxDefLevel) {
+            long julianMicros = DateTimeUtils.fromMillis(dataColumn.readLong());
+            column.putLong(rowId + i, rebaseMicros(julianMicros, failIfRebase));
+          } else {
+            column.putNull(rowId + i);
+          }
         }
       }
     } else {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
index f02861355c404..b40cc154d76fe 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
@@ -18,9 +18,9 @@
 package org.apache.spark.sql.execution.datasources.parquet;
 
 import java.io.IOException;
+import java.time.ZoneId;
 import java.util.Arrays;
 import java.util.List;
-import java.util.TimeZone;
 
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -86,7 +86,12 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
    * The timezone that timestamp INT96 values should be converted to. Null if no conversion. Here to
    * workaround incompatibilities between different engines when writing timestamp values.
    */
-  private TimeZone convertTz = null;
+  private final ZoneId convertTz;
+
+  /**
+   * The mode of rebasing date/timestamp from Julian to Proleptic Gregorian calendar.
+   */
+  private final String datetimeRebaseMode;
 
   /**
    * columnBatch object that is used for batch decoding. This is created on first use and triggers
@@ -116,12 +121,19 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
    */
   private final MemoryMode MEMORY_MODE;
 
-  public VectorizedParquetRecordReader(TimeZone convertTz, boolean useOffHeap, int capacity) {
+  public VectorizedParquetRecordReader(
+    ZoneId convertTz, String datetimeRebaseMode, boolean useOffHeap, int capacity) {
     this.convertTz = convertTz;
+    this.datetimeRebaseMode = datetimeRebaseMode;
     MEMORY_MODE = useOffHeap ? MemoryMode.OFF_HEAP : MemoryMode.ON_HEAP;
     this.capacity = capacity;
   }
 
+  // For test only.
+  public VectorizedParquetRecordReader(boolean useOffHeap, int capacity) {
+    this(null, "CORRECTED", useOffHeap, capacity);
+  }
+
   /**
    * Implementation of RecordReader API.
    */
@@ -309,7 +321,7 @@ private void checkEndOfRowGroup() throws IOException {
     for (int i = 0; i < columns.size(); ++i) {
       if (missingColumns[i]) continue;
       columnReaders[i] = new VectorizedColumnReader(columns.get(i), types.get(i).getOriginalType(),
-        pages.getPageReader(columns.get(i)), convertTz);
+        pages.getPageReader(columns.get(i)), convertTz, datetimeRebaseMode);
     }
     totalCountLoadedSoFar += pages.getRowCount();
   }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
index c62dc3d86386e..994779b618829 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
@@ -21,11 +21,13 @@
 import java.nio.ByteOrder;
 
 import org.apache.parquet.bytes.ByteBufferInputStream;
-import org.apache.parquet.io.ParquetDecodingException;
-import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
-
 import org.apache.parquet.column.values.ValuesReader;
 import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.ParquetDecodingException;
+
+import org.apache.spark.sql.catalyst.util.RebaseDateTime;
+import org.apache.spark.sql.execution.datasources.DataSourceUtils;
+import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 
 /**
  * An implementation of the Parquet PLAIN decoder that supports the vectorized interface.
@@ -81,6 +83,38 @@ public final void readIntegers(int total, WritableColumnVector c, int rowId) {
     }
   }
 
+  // A fork of `readIntegers` to rebase the date values. For performance reasons, this method
+  // iterates the values twice: check if we need to rebase first, then go to the optimized branch
+  // if rebase is not needed.
+  @Override
+  public final void readIntegersWithRebase(
+      int total, WritableColumnVector c, int rowId, boolean failIfRebase) {
+    int requiredBytes = total * 4;
+    ByteBuffer buffer = getBuffer(requiredBytes);
+    boolean rebase = false;
+    for (int i = 0; i < total; i += 1) {
+      rebase |= buffer.getInt(buffer.position() + i * 4) < RebaseDateTime.lastSwitchJulianDay();
+    }
+    if (rebase) {
+      if (failIfRebase) {
+        throw DataSourceUtils.newRebaseExceptionInRead("Parquet");
+      } else {
+        for (int i = 0; i < total; i += 1) {
+          c.putInt(rowId + i, RebaseDateTime.rebaseJulianToGregorianDays(buffer.getInt()));
+        }
+      }
+    } else {
+      if (buffer.hasArray()) {
+        int offset = buffer.arrayOffset() + buffer.position();
+        c.putIntsLittleEndian(rowId, total, buffer.array(), offset);
+      } else {
+        for (int i = 0; i < total; i += 1) {
+          c.putInt(rowId + i, buffer.getInt());
+        }
+      }
+    }
+  }
+
   @Override
   public final void readLongs(int total, WritableColumnVector c, int rowId) {
     int requiredBytes = total * 8;
@@ -96,6 +130,38 @@ public final void readLongs(int total, WritableColumnVector c, int rowId) {
     }
   }
 
+  // A fork of `readLongs` to rebase the timestamp values. For performance reasons, this method
+  // iterates the values twice: check if we need to rebase first, then go to the optimized branch
+  // if rebase is not needed.
+  @Override
+  public final void readLongsWithRebase(
+      int total, WritableColumnVector c, int rowId, boolean failIfRebase) {
+    int requiredBytes = total * 8;
+    ByteBuffer buffer = getBuffer(requiredBytes);
+    boolean rebase = false;
+    for (int i = 0; i < total; i += 1) {
+      rebase |= buffer.getLong(buffer.position() + i * 8) < RebaseDateTime.lastSwitchJulianTs();
+    }
+    if (rebase) {
+      if (failIfRebase) {
+        throw DataSourceUtils.newRebaseExceptionInRead("Parquet");
+      } else {
+        for (int i = 0; i < total; i += 1) {
+          c.putLong(rowId + i, RebaseDateTime.rebaseJulianToGregorianMicros(buffer.getLong()));
+        }
+      }
+    } else {
+      if (buffer.hasArray()) {
+        int offset = buffer.arrayOffset() + buffer.position();
+        c.putLongsLittleEndian(rowId, total, buffer.array(), offset);
+      } else {
+        for (int i = 0; i < total; i += 1) {
+          c.putLong(rowId + i, buffer.getLong());
+        }
+      }
+    }
+  }
+
   @Override
   public final void readFloats(int total, WritableColumnVector c, int rowId) {
     int requiredBytes = total * 4;
@@ -103,7 +169,7 @@ public final void readFloats(int total, WritableColumnVector c, int rowId) {
 
     if (buffer.hasArray()) {
       int offset = buffer.arrayOffset() + buffer.position();
-      c.putFloats(rowId, total, buffer.array(), offset);
+      c.putFloatsLittleEndian(rowId, total, buffer.array(), offset);
     } else {
       for (int i = 0; i < total; i += 1) {
         c.putFloat(rowId + i, buffer.getFloat());
@@ -118,7 +184,7 @@ public final void readDoubles(int total, WritableColumnVector c, int rowId) {
 
     if (buffer.hasArray()) {
       int offset = buffer.arrayOffset() + buffer.position();
-      c.putDoubles(rowId, total, buffer.array(), offset);
+      c.putDoublesLittleEndian(rowId, total, buffer.array(), offset);
     } else {
       for (int i = 0; i < total; i += 1) {
         c.putDouble(rowId + i, buffer.getDouble());
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
index fe3d31ae8e746..03595068f0d31 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
@@ -17,6 +17,9 @@
 
 package org.apache.spark.sql.execution.datasources.parquet;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
 import org.apache.parquet.Preconditions;
 import org.apache.parquet.bytes.ByteBufferInputStream;
 import org.apache.parquet.bytes.BytesUtils;
@@ -28,9 +31,6 @@
 
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
 /**
  * A values reader for Parquet's run-length encoded data. This is based off of the version in
  * parquet-mr with these changes:
@@ -46,7 +46,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
   // Current decoding mode. The encoded data contains groups of either run length encoded data
   // (RLE) or bit packed data. Each group contains a header that indicates which group it is and
   // the number of values in the group.
-  // More details here: https://github.com/Parquet/parquet-format/blob/master/Encodings.md
+  // More details here: https://github.com/apache/parquet-format/blob/master/Encodings.md
   private enum MODE {
     RLE,
     PACKED
@@ -203,6 +203,44 @@ public void readIntegers(
     }
   }
 
+  // A fork of `readIntegers`, which rebases the date int value (days) before filling
+  // the Spark column vector.
+  public void readIntegersWithRebase(
+      int total,
+      WritableColumnVector c,
+      int rowId,
+      int level,
+      VectorizedValuesReader data,
+      final boolean failIfRebase) throws IOException {
+    int left = total;
+    while (left > 0) {
+      if (this.currentCount == 0) this.readNextGroup();
+      int n = Math.min(left, this.currentCount);
+      switch (mode) {
+        case RLE:
+          if (currentValue == level) {
+            data.readIntegersWithRebase(n, c, rowId, failIfRebase);
+          } else {
+            c.putNulls(rowId, n);
+          }
+          break;
+        case PACKED:
+          for (int i = 0; i < n; ++i) {
+            if (currentBuffer[currentBufferIdx++] == level) {
+              int julianDays = data.readInteger();
+              c.putInt(rowId + i, VectorizedColumnReader.rebaseDays(julianDays, failIfRebase));
+            } else {
+              c.putNull(rowId + i);
+            }
+          }
+          break;
+      }
+      rowId += n;
+      left -= n;
+      currentCount -= n;
+    }
+  }
+
   // TODO: can this code duplication be removed without a perf penalty?
   public void readBooleans(
       int total,
@@ -342,6 +380,44 @@ public void readLongs(
     }
   }
 
+  // A fork of `readLongs`, which rebases the timestamp long value (microseconds) before filling
+  // the Spark column vector.
+  public void readLongsWithRebase(
+      int total,
+      WritableColumnVector c,
+      int rowId,
+      int level,
+      VectorizedValuesReader data,
+      final boolean failIfRebase) throws IOException {
+    int left = total;
+    while (left > 0) {
+      if (this.currentCount == 0) this.readNextGroup();
+      int n = Math.min(left, this.currentCount);
+      switch (mode) {
+        case RLE:
+          if (currentValue == level) {
+            data.readLongsWithRebase(n, c, rowId, failIfRebase);
+          } else {
+            c.putNulls(rowId, n);
+          }
+          break;
+        case PACKED:
+          for (int i = 0; i < n; ++i) {
+            if (currentBuffer[currentBufferIdx++] == level) {
+              long julianMicros = data.readLong();
+              c.putLong(rowId + i, VectorizedColumnReader.rebaseMicros(julianMicros, failIfRebase));
+            } else {
+              c.putNull(rowId + i);
+            }
+          }
+          break;
+      }
+      rowId += n;
+      left -= n;
+      currentCount -= n;
+    }
+  }
+
   public void readFloats(
       int total,
       WritableColumnVector c,
@@ -508,6 +584,12 @@ public void readIntegers(int total, WritableColumnVector c, int rowId) {
     }
   }
 
+  @Override
+  public void readIntegersWithRebase(
+      int total, WritableColumnVector c, int rowId, boolean failIfRebase) {
+    throw new UnsupportedOperationException("only readInts is valid.");
+  }
+
   @Override
   public byte readByte() {
     throw new UnsupportedOperationException("only readInts is valid.");
@@ -523,6 +605,12 @@ public void readLongs(int total, WritableColumnVector c, int rowId) {
     throw new UnsupportedOperationException("only readInts is valid.");
   }
 
+  @Override
+  public void readLongsWithRebase(
+      int total, WritableColumnVector c, int rowId, boolean failIfRebase) {
+    throw new UnsupportedOperationException("only readInts is valid.");
+  }
+
   @Override
   public void readBinary(int total, WritableColumnVector c, int rowId) {
     throw new UnsupportedOperationException("only readInts is valid.");
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
index 57d92ae27ece8..35db8f235ed60 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
@@ -40,7 +40,9 @@ public interface VectorizedValuesReader {
   void readBooleans(int total, WritableColumnVector c, int rowId);
   void readBytes(int total, WritableColumnVector c, int rowId);
   void readIntegers(int total, WritableColumnVector c, int rowId);
+  void readIntegersWithRebase(int total, WritableColumnVector c, int rowId, boolean failIfRebase);
   void readLongs(int total, WritableColumnVector c, int rowId);
+  void readLongsWithRebase(int total, WritableColumnVector c, int rowId, boolean failIfRebase);
   void readFloats(int total, WritableColumnVector c, int rowId);
   void readDoubles(int total, WritableColumnVector c, int rowId);
   void readBinary(int total, WritableColumnVector c, int rowId);
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
index bce6aa28c42a1..25aabcd086289 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
@@ -54,6 +54,8 @@ public static void populate(WritableColumnVector col, InternalRow row, int field
     } else {
       if (t == DataTypes.BooleanType) {
         col.putBooleans(0, capacity, row.getBoolean(fieldIdx));
+      } else if (t == DataTypes.BinaryType) {
+        col.putByteArray(0, row.getBinary(fieldIdx));
       } else if (t == DataTypes.ByteType) {
         col.putBytes(0, capacity, row.getByte(fieldIdx));
       } else if (t == DataTypes.ShortType) {
@@ -94,6 +96,9 @@ public static void populate(WritableColumnVector col, InternalRow row, int field
         col.putInts(0, capacity, row.getInt(fieldIdx));
       } else if (t instanceof TimestampType) {
         col.putLongs(0, capacity, row.getLong(fieldIdx));
+      } else {
+        throw new RuntimeException(String.format("DataType %s is not supported" +
+            " in column vectorized reader.", t.sql()));
       }
     }
   }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
index 3b919c7d471f4..982a26c0ef775 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
@@ -425,6 +425,19 @@ public void putFloats(int rowId, int count, byte[] src, int srcIndex) {
     }
   }
 
+  @Override
+  public void putFloatsLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
+    if (!bigEndianPlatform) {
+      putFloats(rowId, count, src, srcIndex);
+    } else {
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+      long offset = data + 4L * rowId;
+      for (int i = 0; i < count; ++i, offset += 4) {
+        Platform.putFloat(null, offset, bb.getFloat(srcIndex + (4 * i)));
+      }
+    }
+  }
+
   @Override
   public float getFloat(int rowId) {
     if (dictionary == null) {
@@ -480,6 +493,19 @@ public void putDoubles(int rowId, int count, byte[] src, int srcIndex) {
     }
   }
 
+  @Override
+  public void putDoublesLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
+    if (!bigEndianPlatform) {
+      putDoubles(rowId, count, src, srcIndex);
+    } else {
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+      long offset = data + 8L * rowId;
+      for (int i = 0; i < count; ++i, offset += 8) {
+        Platform.putDouble(null, offset, bb.getDouble(srcIndex + (8 * i)));
+      }
+    }
+  }
+
   @Override
   public double getDouble(int rowId) {
     if (dictionary == null) {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
index 1bf3126664177..625b78be4de94 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
@@ -403,6 +403,18 @@ public void putFloats(int rowId, int count, byte[] src, int srcIndex) {
     }
   }
 
+  @Override
+  public void putFloatsLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
+    if (!bigEndianPlatform) {
+      putFloats(rowId, count, src, srcIndex);
+    } else {
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+      for (int i = 0; i < count; ++i) {
+        floatData[i + rowId] = bb.getFloat(srcIndex + (4 * i));
+      }
+    }
+  }
+
   @Override
   public float getFloat(int rowId) {
     if (dictionary == null) {
@@ -452,6 +464,18 @@ public void putDoubles(int rowId, int count, byte[] src, int srcIndex) {
     }
   }
 
+  @Override
+  public void putDoublesLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
+    if (!bigEndianPlatform) {
+      putDoubles(rowId, count, src, srcIndex);
+    } else {
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+      for (int i = 0; i < count; ++i) {
+        doubleData[i + rowId] = bb.getDouble(srcIndex + (8 * i));
+      }
+    }
+  }
+
   @Override
   public double getDouble(int rowId) {
     if (dictionary == null) {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
index f04b0707ff7b5..8c0f1e1257503 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
@@ -312,6 +312,12 @@ public WritableColumnVector reserveDictionaryIds(int capacity) {
    */
   public abstract void putFloats(int rowId, int count, byte[] src, int srcIndex);
 
+  /**
+   * Sets values from [src[srcIndex], src[srcIndex + count * 4]) to [rowId, rowId + count)
+   * The data in src must be ieee formatted floats in little endian.
+   */
+  public abstract void putFloatsLittleEndian(int rowId, int count, byte[] src, int srcIndex);
+
   /**
    * Sets `value` to the value at rowId.
    */
@@ -333,6 +339,12 @@ public WritableColumnVector reserveDictionaryIds(int capacity) {
    */
   public abstract void putDoubles(int rowId, int count, byte[] src, int srcIndex);
 
+  /**
+   * Sets values from [src[srcIndex], src[srcIndex + count * 8]) to [rowId, rowId + count)
+   * The data in src must be ieee formatted doubles in little endian.
+   */
+  public abstract void putDoublesLittleEndian(int rowId, int count, byte[] src, int srcIndex);
+
   /**
    * Puts a byte array that already exists in this column.
    */
diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
index eff0142dc523f..aea5e2e93c661 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
@@ -18,6 +18,7 @@
 #plan-viz-graph .label {
   font-weight: normal;
   text-shadow: none;
+  color: #333;
 }
 
 #plan-viz-graph svg g.cluster rect {
@@ -32,14 +33,13 @@
   stroke-width: 1px;
 }
 
-/* This declaration is needed to define the width of rectangles */
-#plan-viz-graph svg text :first-child {
+/* Highlight the SparkPlan node name */
+#plan-viz-graph svg text :first-child:not(.stageId-and-taskId-metrics) {
   font-weight: bold;
 }
 
-/* Highlight the SparkPlan node name */
-#plan-viz-graph svg text .operator-name {
-  font-weight: bold;
+#plan-viz-graph svg text {
+  fill: #333;
 }
 
 #plan-viz-graph svg path {
diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
index e6ce641a841b3..724cec10d230e 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
@@ -34,7 +34,6 @@ function renderPlanViz() {
   preprocessGraphLayout(g);
   var renderer = new dagreD3.render();
   renderer(graph, g);
-  adjustPositionOfOperationName();
 
   // Round corners on rectangles
   svg
@@ -48,6 +47,7 @@ function renderPlanViz() {
   }
 
   resizeSvg(svg);
+  postprocessForAdditionalMetrics();
 }
 
 /* -------------------- *
@@ -61,7 +61,7 @@ function planVizContainer() { return d3.select("#plan-viz-graph"); }
  * node, it will display the details of this SparkPlan node in the right.
  */
 function setupTooltipForSparkPlanNode(nodeId) {
-  var nodeTooltip = d3.select("#plan-meta-data-" + nodeId).text()
+  var nodeTooltip = d3.select("#plan-meta-data-" + nodeId).text();
   d3.select("svg g .node_" + nodeId)
     .on('mouseover', function(d) {
       var domNode = d3.select(this).node();
@@ -76,17 +76,42 @@ function setupTooltipForSparkPlanNode(nodeId) {
     })
 }
 
+// labelSeparator should be a non-graphical character in order not to affect the width of boxes.
+var labelSeparator = "\x01";
+var stageAndTaskMetricsPattern = "^(.*)(\\(stage.*task[^)]*\\))(.*)$";
+
 /*
  * Helper function to pre-process the graph layout.
  * This step is necessary for certain styles that affect the positioning
  * and sizes of graph elements, e.g. padding, font style, shape.
  */
 function preprocessGraphLayout(g) {
-  g.graph().ranksep = "90";
+  g.graph().ranksep = "70";
   var nodes = g.nodes();
   for (var i = 0; i < nodes.length; i++) {
-      var node = g.node(nodes[i]);
-      node.padding = "5";
+    var node = g.node(nodes[i]);
+    node.padding = "5";
+
+    var firstSearator;
+    var secondSeparator;
+    var splitter;
+    if (node.isCluster) {
+      firstSearator = secondSeparator = labelSeparator;
+      splitter = "\\n";
+    } else {
+      firstSearator = "<span class='stageId-and-taskId-metrics'>";
+      secondSeparator = "</span>";
+      splitter = "<br>";
+    }
+
+    node.label.split(splitter).forEach(function(text, i) {
+      var newTexts = text.match(stageAndTaskMetricsPattern);
+      if (newTexts) {
+        node.label = node.label.replace(
+            newTexts[0],
+            newTexts[1] + firstSearator + newTexts[2] + secondSeparator + newTexts[3]);
+      }
+    });
   }
   // Curve the edges
   var edges = g.edges();
@@ -102,10 +127,8 @@ function preprocessGraphLayout(g) {
  */
 function resizeSvg(svg) {
   var allClusters = svg.selectAll("g rect")[0];
-  console.log(allClusters);
   var startX = -PlanVizConstants.svgMarginX +
     toFloat(d3.min(allClusters, function(e) {
-      console.log(e);
       return getAbsolutePosition(d3.select(e)).x;
     }));
   var startY = -PlanVizConstants.svgMarginY +
@@ -129,34 +152,6 @@ function resizeSvg(svg) {
      .attr("height", height);
 }
 
-
-/* Helper function to adjust the position of operation name and mark as a operation-name class. */
-function adjustPositionOfOperationName() {
-  $("#plan-viz-graph svg text")
-      .each(function() {
-        var tspans = $(this).find("tspan");
-
-        if (tspans[0].textContent.trim() !== "") {
-          var isOperationNameOnly =
-              $(tspans).filter(function(i, n) {
-                return i !== 0 && n.textContent.trim() !== "";
-              }).length === 0;
-
-          if (isOperationNameOnly) {
-            // If the only text in a node is operation name,
-            // vertically centering the position of operation name.
-            var operationName = tspans[0].textContent;
-            var half = parseInt(tspans.length / 2);
-            tspans[0].textContent = tspans[half].textContent;
-            tspans[half].textContent = operationName;
-            $(tspans[half]).addClass("operator-name");
-          } else {
-            tspans.first().addClass("operator-name");
-          }
-        }
-      });
-}
-
 /* Helper function to convert attributes to numeric values. */
 function toFloat(f) {
   if (f) {
@@ -186,9 +181,53 @@ function getAbsolutePosition(d3selection) {
     // Climb upwards to find how our parents are translated
     obj = d3.select(obj.node().parentNode);
     // Stop when we've reached the graph container itself
-    if (obj.node() == planVizContainer().node()) {
+    if (obj.node() === planVizContainer().node()) {
       break;
     }
   }
   return { x: _x, y: _y };
 }
+
+/*
+ * Helper function for postprocess for additional metrics.
+ */
+function postprocessForAdditionalMetrics() {
+  // With dagre-d3, we can choose normal text (default) or HTML as a label type.
+  // HTML label for node works well but not for cluster so we need to choose the default label type
+  // and manipulate DOM.
+  $("g.cluster text tspan")
+    .each(function() {
+      var originalText = $(this).text();
+        if (originalText.indexOf(labelSeparator) > 0) {
+          var newTexts = originalText.split(labelSeparator);
+          var thisD3Node = d3.selectAll($(this));
+          thisD3Node.text(newTexts[0]);
+          thisD3Node.append("tspan").attr("class", "stageId-and-taskId-metrics").text(newTexts[1]);
+          $(this).append(newTexts[2]);
+        } else {
+          return originalText;
+        }
+  });
+
+  var checkboxNode = $("#stageId-and-taskId-checkbox");
+  checkboxNode.click(function() {
+      onClickAdditionalMetricsCheckbox($(this));
+  });
+  var isChecked = window.localStorage.getItem("stageId-and-taskId-checked") === "true";
+  checkboxNode.prop("checked", isChecked);
+  onClickAdditionalMetricsCheckbox(checkboxNode);
+}
+
+/*
+ * Helper function which defines the action on click the checkbox.
+ */
+function onClickAdditionalMetricsCheckbox(checkboxNode) {
+  var additionalMetrics = $(".stageId-and-taskId-metrics");
+  var isChecked = checkboxNode.prop("checked");
+  if (isChecked) {
+    additionalMetrics.show();
+  } else {
+    additionalMetrics.hide();
+  }
+  window.localStorage.setItem("stageId-and-taskId-checked", isChecked);
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 8bd5835fd931b..e6f7b1d723af6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -31,7 +31,6 @@ import org.apache.spark.sql.catalyst.util.toPrettySQL
 import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
 import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 private[sql] object Column {
@@ -319,6 +318,24 @@ class Column(val expr: Expression) extends Logging {
     */
   def =!= (other: Any): Column = withExpr{ Not(EqualTo(expr, lit(other).expr)) }
 
+  /**
+   * Inequality test.
+   * {{{
+   *   // Scala:
+   *   df.select( df("colA") !== df("colB") )
+   *   df.select( !(df("colA") === df("colB")) )
+   *
+   *   // Java:
+   *   import static org.apache.spark.sql.functions.*;
+   *   df.filter( col("colA").notEqual(col("colB")) );
+   * }}}
+   *
+   * @group expr_ops
+   * @since 1.3.0
+    */
+  @deprecated("!== does not have the same precedence as ===, use =!= instead", "2.0.0")
+  def !== (other: Any): Column = this =!= other
+
   /**
    * Inequality test.
    * {{{
@@ -809,14 +826,7 @@ class Column(val expr: Expression) extends Logging {
    * @group expr_ops
    * @since 2.4.0
    */
-  def isInCollection(values: scala.collection.Iterable[_]): Column = withExpr {
-    val hSet = values.toSet[Any]
-    if (hSet.size > SQLConf.get.optimizerInSetConversionThreshold) {
-      InSet(expr, hSet)
-    } else {
-      In(expr, values.toSeq.map(lit(_).expr))
-    }
-  }
+  def isInCollection(values: scala.collection.Iterable[_]): Column = isin(values.toSeq: _*)
 
   /**
    * A boolean expression that is evaluated to true if the value of this expression is contained
@@ -955,7 +965,8 @@ class Column(val expr: Expression) extends Logging {
    * }}}
    *
    * If the current column has metadata associated with it, this metadata will be propagated
-   * to the new column.  If this not desired, use `as` with explicitly empty metadata.
+   * to the new column. If this not desired, use the API `as(alias: String, metadata: Metadata)`
+   * with explicit metadata.
    *
    * @group expr_ops
    * @since 1.3.0
@@ -994,7 +1005,8 @@ class Column(val expr: Expression) extends Logging {
    * }}}
    *
    * If the current column has metadata associated with it, this metadata will be propagated
-   * to the new column.  If this not desired, use `as` with explicitly empty metadata.
+   * to the new column. If this not desired, use the API `as(alias: String, metadata: Metadata)`
+   * with explicit metadata.
    *
    * @group expr_ops
    * @since 1.3.0
@@ -1023,16 +1035,14 @@ class Column(val expr: Expression) extends Logging {
    * }}}
    *
    * If the current column has metadata associated with it, this metadata will be propagated
-   * to the new column.  If this not desired, use `as` with explicitly empty metadata.
+   * to the new column. If this not desired, use the API `as(alias: String, metadata: Metadata)`
+   * with explicit metadata.
    *
    * @group expr_ops
    * @since 2.0.0
    */
   def name(alias: String): Column = withExpr {
-    normalizedExpr() match {
-      case ne: NamedExpression => Alias(expr, alias)(explicitMetadata = Some(ne.metadata))
-      case other => Alias(other, alias)()
-    }
+    Alias(normalizedExpr(), alias)()
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
index 953db806258a6..bbf0ac1dd85e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
@@ -89,7 +89,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
    * @since 1.3.1
    */
   def drop(how: String, cols: Seq[String]): DataFrame = {
-    drop0(how, toAttributes(cols))
+    drop0(how, cols.map(df.resolve(_)))
   }
 
   /**
@@ -115,7 +115,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
    * @since 1.3.1
    */
   def drop(minNonNulls: Int, cols: Seq[String]): DataFrame = {
-    drop0(minNonNulls, toAttributes(cols))
+    drop0(minNonNulls, cols.map(df.resolve(_)))
   }
 
   /**
@@ -480,7 +480,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
     df.queryExecution.analyzed.output
   }
 
-  private def drop0(how: String, cols: Seq[Attribute]): DataFrame = {
+  private def drop0(how: String, cols: Seq[NamedExpression]): DataFrame = {
     how.toLowerCase(Locale.ROOT) match {
       case "any" => drop0(cols.size, cols)
       case "all" => drop0(1, cols)
@@ -488,12 +488,10 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
     }
   }
 
-  private def drop0(minNonNulls: Int, cols: Seq[Attribute]): DataFrame = {
+  private def drop0(minNonNulls: Int, cols: Seq[NamedExpression]): DataFrame = {
     // Filtering condition:
     // only keep the row if it has at least `minNonNulls` non-null and non-NaN values.
-    val predicate = AtLeastNNonNulls(
-      minNonNulls,
-      outputAttributes.filter{ col => cols.exists(_.semanticEquals(col)) })
+    val predicate = AtLeastNNonNulls(minNonNulls, cols)
     df.filter(Column(predicate))
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 6cce7203127f5..e8e31059aec37 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -31,7 +31,7 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.csv.{CSVHeaderChecker, CSVOptions, UnivocityParser}
 import org.apache.spark.sql.catalyst.expressions.ExprUtils
 import org.apache.spark.sql.catalyst.json.{CreateJacksonParser, JacksonParser, JSONOptions}
-import org.apache.spark.sql.catalyst.util.FailureSafeParser
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, FailureSafeParser}
 import org.apache.spark.sql.connector.catalog.{CatalogV2Util, SupportsCatalogOptions, SupportsRead}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.execution.command.DDLUtils
@@ -94,10 +94,24 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -110,6 +124,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Boolean): DataFrameReader = option(key, value.toString)
@@ -117,6 +134,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Long): DataFrameReader = option(key, value.toString)
@@ -124,6 +144,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Double): DataFrameReader = option(key, value.toString)
@@ -131,10 +154,24 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * (Scala-specific) Adds input options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -147,10 +184,24 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds input options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -205,7 +256,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
         Some("paths" -> objectMapper.writeValueAsString(paths.toArray))
       }
 
-      val finalOptions = sessionOptions ++ extraOptions.toMap ++ pathsOption
+      val finalOptions = sessionOptions ++ extraOptions.originalMap ++ pathsOption
       val dsOptions = new CaseInsensitiveStringMap(finalOptions.asJava)
       val (table, catalog, ident) = provider match {
         case _: SupportsCatalogOptions if userSpecifiedSchema.nonEmpty =>
@@ -243,7 +294,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
         paths = paths,
         userSpecifiedSchema = userSpecifiedSchema,
         className = source,
-        options = extraOptions.toMap).resolveRelation())
+        options = extraOptions.originalMap).resolveRelation())
   }
 
   /**
@@ -257,7 +308,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
     // properties should override settings in extraOptions.
     this.extraOptions ++= properties.asScala
     // explicit url and dbtable should override all
-    this.extraOptions += (JDBCOptions.JDBC_URL -> url, JDBCOptions.JDBC_TABLE_NAME -> table)
+    this.extraOptions ++= Seq(JDBCOptions.JDBC_URL -> url, JDBCOptions.JDBC_TABLE_NAME -> table)
     format("jdbc").load()
   }
 
@@ -328,7 +379,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
       connectionProperties: Properties): DataFrame = {
     assertNoSpecifiedSchema("jdbc")
     // connectionProperties should override settings in extraOptions.
-    val params = extraOptions.toMap ++ connectionProperties.asScala.toMap
+    val params = extraOptions ++ connectionProperties.asScala
     val options = new JDBCOptions(url, table, params)
     val parts: Array[Partition] = predicates.zipWithIndex.map { case (part, i) =>
       JDBCPartition(part, i) : Partition
@@ -390,12 +441,16 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * <li>`columnNameOfCorruptRecord` (default is the value specified in
    * `spark.sql.columnNameOfCorruptRecord`): allows renaming the new field having malformed string
    * created by `PERMISSIVE` mode. This overrides `spark.sql.columnNameOfCorruptRecord`.</li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`multiLine` (default `false`): parse one record, which may span multiple lines,
    * per file</li>
    * <li>`encoding` (by default it is not set): allows to forcibly set one of standard basic
@@ -511,6 +566,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * If the enforceSchema is set to `false`, only the CSV header in the first line is checked
    * to conform specified or inferred schema.
    *
+   * @note if `header` option is set to `true` when calling this API, all lines same with
+   * the header will be removed if exists.
+   *
    * @param csvDataset input Dataset with one CSV row per record
    * @since 2.2.0
    */
@@ -615,12 +673,16 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * value.</li>
    * <li>`negativeInf` (default `-Inf`): sets the string representation of a negative infinity
    * value.</li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`maxColumns` (default `20480`): defines a hard limit of how many columns
    * a record can have.</li>
    * <li>`maxCharsPerColumn` (default `-1`): defines the maximum number of characters allowed
@@ -838,6 +900,6 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
 
   private var userSpecifiedSchema: Option[StructType] = None
 
-  private val extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 4557219abeb15..d16404f3ef6fa 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -26,18 +26,18 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, NoSuchTableException, UnresolvedRelation}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelect, InsertIntoStatement, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelect}
+import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelect, CreateTableAsSelectStatement, InsertIntoStatement, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelectStatement}
 import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
-import org.apache.spark.sql.connector.catalog.{CatalogPlugin, CatalogV2Implicits, CatalogV2Util, Identifier, SupportsCatalogOptions, SupportsWrite, Table, TableCatalog, TableProvider, V1Table}
+import org.apache.spark.sql.connector.catalog.{CatalogPlugin, CatalogV2Implicits, CatalogV2Util, Identifier, SupportsCatalogOptions, Table, TableCatalog, TableProvider, V1Table}
 import org.apache.spark.sql.connector.catalog.TableCapability._
-import org.apache.spark.sql.connector.expressions.{BucketTransform, FieldReference, IdentityTransform, LiteralValue, Transform}
+import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform}
 import org.apache.spark.sql.execution.SQLExecution
 import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, DataSourceUtils, LogicalRelation}
 import org.apache.spark.sql.execution.datasources.v2._
 import org.apache.spark.sql.internal.SQLConf.PartitionOverwriteMode
 import org.apache.spark.sql.sources.BaseRelation
-import org.apache.spark.sql.types.{IntegerType, StructType}
+import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 /**
@@ -60,8 +60,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * <li>`SaveMode.ErrorIfExists`: throw an exception at runtime.</li>
    * </ul>
    * <p>
-   * When writing to data source v1, the default option is `ErrorIfExists`. When writing to data
-   * source v2, the default option is `Append`.
+   * The default option is `ErrorIfExists`.
    *
    * @since 1.4.0
    */
@@ -105,10 +104,24 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -121,6 +134,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Boolean): DataFrameWriter[T] = option(key, value.toString)
@@ -128,6 +144,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Long): DataFrameWriter[T] = option(key, value.toString)
@@ -135,6 +154,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Double): DataFrameWriter[T] = option(key, value.toString)
@@ -142,10 +164,24 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * (Scala-specific) Adds output options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -158,10 +194,24 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds output options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -198,7 +248,8 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
 
   /**
    * Buckets the output by the given columns. If specified, the output is laid out on the file
-   * system similar to Hive's bucketing scheme.
+   * system similar to Hive's bucketing scheme, but with a different bucket hash function
+   * and is not compatible with Hive's bucketing.
    *
    * This is applicable for all file-based data sources (e.g. Parquet, JSON) starting with Spark
    * 2.1.0.
@@ -254,7 +305,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
       val provider = maybeV2Provider.get
       val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
         provider, df.sparkSession.sessionState.conf)
-      val options = sessionOptions ++ extraOptions
+      val options = sessionOptions.filterKeys(!extraOptions.contains(_)) ++ extraOptions.toMap
       val dsOptions = new CaseInsensitiveStringMap(options.asJava)
 
       def getTable: Table = {
@@ -540,12 +591,12 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
     val canUseV2 = lookupV2Provider().isDefined
 
     session.sessionState.sqlParser.parseMultipartIdentifier(tableName) match {
-      case NonSessionCatalogAndIdentifier(catalog, ident) =>
-        saveAsTable(catalog.asTableCatalog, ident)
+      case nameParts @ NonSessionCatalogAndIdentifier(catalog, ident) =>
+        saveAsTable(catalog.asTableCatalog, ident, nameParts)
 
-      case SessionCatalogAndIdentifier(catalog, ident)
+      case nameParts @ SessionCatalogAndIdentifier(catalog, ident)
           if canUseV2 && ident.namespace().length <= 1 =>
-        saveAsTable(catalog.asTableCatalog, ident)
+        saveAsTable(catalog.asTableCatalog, ident, nameParts)
 
       case AsTableIdentifier(tableIdentifier) =>
         saveAsTable(tableIdentifier)
@@ -557,16 +608,12 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   }
 
 
-  private def saveAsTable(catalog: TableCatalog, ident: Identifier): Unit = {
+  private def saveAsTable(
+      catalog: TableCatalog, ident: Identifier, nameParts: Seq[String]): Unit = {
     val tableOpt = try Option(catalog.loadTable(ident)) catch {
       case _: NoSuchTableException => None
     }
 
-    def getLocationIfExists: Option[(String, String)] = {
-      val opts = CaseInsensitiveMap(extraOptions.toMap)
-      opts.get("path").map(TableCatalog.PROP_LOCATION -> _)
-    }
-
     val command = (mode, tableOpt) match {
       case (_, Some(_: V1Table)) =>
         return saveAsTable(TableIdentifier(ident.name(), ident.namespace().headOption))
@@ -577,12 +624,16 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
         AppendData.byName(v2Relation, df.logicalPlan, extraOptions.toMap)
 
       case (SaveMode.Overwrite, _) =>
-        ReplaceTableAsSelect(
-          catalog,
-          ident,
-          partitioningAsV2,
+        ReplaceTableAsSelectStatement(
+          nameParts,
           df.queryExecution.analyzed,
-          Map(TableCatalog.PROP_PROVIDER -> source) ++ getLocationIfExists,
+          partitioningAsV2,
+          None,
+          Map.empty,
+          Some(source),
+          Map.empty,
+          extraOptions.get("path"),
+          extraOptions.get(TableCatalog.PROP_COMMENT),
           extraOptions.toMap,
           orCreate = true)      // Create the table if it doesn't exist
 
@@ -590,14 +641,18 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
         // We have a potential race condition here in AppendMode, if the table suddenly gets
         // created between our existence check and physical execution, but this can't be helped
         // in any case.
-        CreateTableAsSelect(
-          catalog,
-          ident,
-          partitioningAsV2,
+        CreateTableAsSelectStatement(
+          nameParts,
           df.queryExecution.analyzed,
-          Map(TableCatalog.PROP_PROVIDER -> source) ++ getLocationIfExists,
+          partitioningAsV2,
+          None,
+          Map.empty,
+          Some(source),
+          Map.empty,
+          extraOptions.get("path"),
+          extraOptions.get(TableCatalog.PROP_COMMENT),
           extraOptions.toMap,
-          ignoreIfExists = other == SaveMode.Ignore)
+          ifNotExists = other == SaveMode.Ignore)
     }
 
     runCommand(df.sparkSession, "saveAsTable") {
@@ -731,7 +786,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
     // connectionProperties should override settings in extraOptions.
     this.extraOptions ++= connectionProperties.asScala
     // explicit url and dbtable should override all
-    this.extraOptions += ("url" -> url, "dbtable" -> table)
+    this.extraOptions ++= Seq("url" -> url, "dbtable" -> table)
     format("jdbc").save()
   }
 
@@ -748,12 +803,16 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * <li>`compression` (default `null`): compression codec to use when saving to file. This can be
    * one of the known case-insensitive shorten names (`none`, `bzip2`, `gzip`, `lz4`,
    * `snappy` and `deflate`). </li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`encoding` (by default it is not set): specifies encoding (charset) of saved json
    * files. If it is not set, the UTF-8 charset will be used. </li>
    * <li>`lineSep` (default `\n`): defines the line separator that should be used for writing.</li>
@@ -869,12 +928,16 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * <li>`compression` (default `null`): compression codec to use when saving to file. This can be
    * one of the known case-insensitive shorten names (`none`, `bzip2`, `gzip`, `lz4`,
    * `snappy` and `deflate`). </li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`ignoreLeadingWhiteSpace` (default `true`): a flag indicating whether or not leading
    * whitespaces from values being written should be skipped.</li>
    * <li>`ignoreTrailingWhiteSpace` (default `true`): a flag indicating defines whether or not
@@ -896,7 +959,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   private def runCommand(session: SparkSession, name: String)(command: LogicalPlan): Unit = {
     val qe = session.sessionState.executePlan(command)
     // call `QueryExecution.toRDD` to trigger the execution of commands.
-    SQLExecution.withNewExecutionId(session, qe, Some(name))(qe.toRdd)
+    SQLExecution.withNewExecutionId(qe, Some(name))(qe.toRdd)
   }
 
   private def lookupV2Provider(): Option[TableProvider] = {
@@ -915,7 +978,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
 
   private var mode: SaveMode = SaveMode.ErrorIfExists
 
-  private val extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 
   private var partitioningColumns: Option[Seq[String]] = None
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
index f5dd7613d4103..87f35410172d6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
@@ -23,8 +23,7 @@ import scala.collection.mutable
 import org.apache.spark.annotation.Experimental
 import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NoSuchTableException, TableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Bucket, Days, Hours, Literal, Months, Years}
-import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelect, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelect}
-import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelectStatement, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelectStatement}
 import org.apache.spark.sql.connector.expressions.{LogicalExpressions, NamedReference, Transform}
 import org.apache.spark.sql.execution.SQLExecution
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
@@ -47,8 +46,6 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
 
   private val sparkSession = ds.sparkSession
 
-  private val catalogManager = sparkSession.sessionState.analyzer.catalogManager
-
   private val tableName = sparkSession.sessionState.sqlParser.parseMultipartIdentifier(table)
 
   private val (catalog, identifier) = {
@@ -120,19 +117,19 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
   }
 
   override def create(): Unit = {
-    // create and replace could alternatively create ParsedPlan statements, like
-    // `CreateTableFromDataFrameStatement(UnresolvedRelation(tableName), ...)`, to keep the catalog
-    // resolution logic in the analyzer.
     runCommand("create") {
-      CreateTableAsSelect(
-        catalog,
-        identifier,
-        partitioning.getOrElse(Seq.empty),
+      CreateTableAsSelectStatement(
+        tableName,
         logicalPlan,
-        properties = provider.map(p => properties + (TableCatalog.PROP_PROVIDER -> p))
-          .getOrElse(properties).toMap,
-        writeOptions = options.toMap,
-        ignoreIfExists = false)
+        partitioning.getOrElse(Seq.empty),
+        None,
+        properties.toMap,
+        provider,
+        Map.empty,
+        None,
+        None,
+        options.toMap,
+        ifNotExists = false)
     }
   }
 
@@ -226,18 +223,22 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
   private def runCommand(name: String)(command: LogicalPlan): Unit = {
     val qe = sparkSession.sessionState.executePlan(command)
     // call `QueryExecution.toRDD` to trigger the execution of commands.
-    SQLExecution.withNewExecutionId(sparkSession, qe, Some(name))(qe.toRdd)
+    SQLExecution.withNewExecutionId(qe, Some(name))(qe.toRdd)
   }
 
   private def internalReplace(orCreate: Boolean): Unit = {
     runCommand("replace") {
-      ReplaceTableAsSelect(
-        catalog,
-        identifier,
-        partitioning.getOrElse(Seq.empty),
+      ReplaceTableAsSelectStatement(
+        tableName,
         logicalPlan,
-        properties = provider.map(p => properties + ("provider" -> p)).getOrElse(properties).toMap,
-        writeOptions = options.toMap,
+        partitioning.getOrElse(Seq.empty),
+        None,
+        properties.toMap,
+        provider,
+        Map.empty,
+        None,
+        None,
+        options.toMap,
         orCreate = orCreate)
     }
   }
@@ -246,6 +247,7 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
 /**
  * Configuration methods common to create/replace operations and insert/overwrite operations.
  * @tparam R builder type to return
+ * @since 3.0.0
  */
 trait WriteConfigMethods[R] {
   /**
@@ -293,6 +295,8 @@ trait WriteConfigMethods[R] {
 
 /**
  * Trait to restrict calls to create and replace operations.
+ *
+ * @since 3.0.0
  */
 trait CreateTableWriter[T] extends WriteConfigMethods[CreateTableWriter[T]] {
   /**
@@ -320,7 +324,7 @@ trait CreateTableWriter[T] extends WriteConfigMethods[CreateTableWriter[T]] {
    * [[org.apache.spark.sql.catalyst.analysis.CannotReplaceMissingTableException]].
    *
    * @throws org.apache.spark.sql.catalyst.analysis.CannotReplaceMissingTableException
-   *         If the table already exists
+   *         If the table does not exist
    */
   @throws(classOf[CannotReplaceMissingTableException])
   def replace(): Unit
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index a1c33f92d17b4..3b6fd2fcc177c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -22,6 +22,7 @@ import java.io.{ByteArrayOutputStream, CharArrayWriter, DataOutputStream}
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 import scala.language.implicitConversions
+import scala.reflect.runtime.universe.TypeTag
 import scala.util.control.NonFatal
 
 import org.apache.commons.lang3.StringUtils
@@ -34,6 +35,7 @@ import org.apache.spark.api.python.{PythonRDD, SerDeUtil}
 import org.apache.spark.api.r.RRDD
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, ScalaReflection}
 import org.apache.spark.sql.catalyst.QueryPlanningTracker
 import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.catalyst.catalog.HiveTableRelation
@@ -48,6 +50,7 @@ import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, PartitioningC
 import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.catalyst.util.IntervalUtils
 import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
 import org.apache.spark.sql.execution.arrow.{ArrowBatchStreamWriter, ArrowConverters}
 import org.apache.spark.sql.execution.command._
 import org.apache.spark.sql.execution.datasources.LogicalRelation
@@ -82,18 +85,19 @@ private[sql] object Dataset {
     dataset
   }
 
-  def ofRows(sparkSession: SparkSession, logicalPlan: LogicalPlan): DataFrame = {
-    val qe = sparkSession.sessionState.executePlan(logicalPlan)
-    qe.assertAnalyzed()
-    new Dataset[Row](sparkSession, qe, RowEncoder(qe.analyzed.schema))
+  def ofRows(sparkSession: SparkSession, logicalPlan: LogicalPlan): DataFrame =
+    sparkSession.withActive {
+      val qe = sparkSession.sessionState.executePlan(logicalPlan)
+      qe.assertAnalyzed()
+      new Dataset[Row](qe, RowEncoder(qe.analyzed.schema))
   }
 
   /** A variant of ofRows that allows passing in a tracker so we can track query parsing time. */
   def ofRows(sparkSession: SparkSession, logicalPlan: LogicalPlan, tracker: QueryPlanningTracker)
-    : DataFrame = {
+    : DataFrame = sparkSession.withActive {
     val qe = new QueryExecution(sparkSession, logicalPlan, tracker)
     qe.assertAnalyzed()
-    new Dataset[Row](sparkSession, qe, RowEncoder(qe.analyzed.schema))
+    new Dataset[Row](qe, RowEncoder(qe.analyzed.schema))
   }
 }
 
@@ -185,13 +189,12 @@ private[sql] object Dataset {
  */
 @Stable
 class Dataset[T] private[sql](
-    @transient private val _sparkSession: SparkSession,
     @DeveloperApi @Unstable @transient val queryExecution: QueryExecution,
     @DeveloperApi @Unstable @transient val encoder: Encoder[T])
   extends Serializable {
 
   @transient lazy val sparkSession: SparkSession = {
-    if (_sparkSession == null) {
+    if (queryExecution == null || queryExecution.sparkSession == null) {
       throw new SparkException(
       "Dataset transformations and actions can only be invoked by the driver, not inside of" +
         " other Dataset transformations; for example, dataset1.map(x => dataset2.values.count()" +
@@ -199,7 +202,7 @@ class Dataset[T] private[sql](
         "performed inside of the dataset1.map transformation. For more information," +
         " see SPARK-28702.")
     }
-    _sparkSession
+    queryExecution.sparkSession
   }
 
   // A globally unique id of this Dataset.
@@ -211,7 +214,7 @@ class Dataset[T] private[sql](
   // you wrap it with `withNewExecutionId` if this actions doesn't call other action.
 
   def this(sparkSession: SparkSession, logicalPlan: LogicalPlan, encoder: Encoder[T]) = {
-    this(sparkSession, sparkSession.sessionState.executePlan(logicalPlan), encoder)
+    this(sparkSession.sessionState.executePlan(logicalPlan), encoder)
   }
 
   def this(sqlContext: SQLContext, logicalPlan: LogicalPlan, encoder: Encoder[T]) = {
@@ -445,7 +448,7 @@ class Dataset[T] private[sql](
    */
   // This is declared with parentheses to prevent the Scala compiler from treating
   // `ds.toDF("1")` as invoking this toDF and then apply on the returned DataFrame.
-  def toDF(): DataFrame = new Dataset[Row](sparkSession, queryExecution, RowEncoder(schema))
+  def toDF(): DataFrame = new Dataset[Row](queryExecution, RowEncoder(schema))
 
   /**
    * Returns a new Dataset where each record has been mapped on to the specified type. The
@@ -503,7 +506,9 @@ class Dataset[T] private[sql](
    * @group basic
    * @since 1.6.0
    */
-  def schema: StructType = queryExecution.analyzed.schema
+  def schema: StructType = sparkSession.withActive {
+    queryExecution.analyzed.schema
+  }
 
   /**
    * Prints the schema to the console in a nice tree format.
@@ -539,7 +544,7 @@ class Dataset[T] private[sql](
    * @group basic
    * @since 3.0.0
    */
-  def explain(mode: String): Unit = {
+  def explain(mode: String): Unit = sparkSession.withActive {
     // Because temporary views are resolved during analysis when we create a Dataset, and
     // `ExplainCommand` analyzes input query plan and resolves temporary views again. Using
     // `ExplainCommand` here will probably output different query plans, compared to the results
@@ -1430,7 +1435,25 @@ class Dataset[T] private[sql](
    */
   @scala.annotation.varargs
   def select(cols: Column*): DataFrame = withPlan {
-    Project(cols.map(_.named), logicalPlan)
+    val untypedCols = cols.map {
+      case typedCol: TypedColumn[_, _] =>
+        // Checks if a `TypedColumn` has been inserted with
+        // specific input type and schema by `withInputType`.
+        val needInputType = typedCol.expr.find {
+          case ta: TypedAggregateExpression if ta.inputDeserializer.isEmpty => true
+          case _ => false
+        }.isDefined
+
+        if (!needInputType) {
+          typedCol
+        } else {
+          throw new AnalysisException(s"Typed column $typedCol that needs input type and schema " +
+            "cannot be passed in untyped `select` API. Use the typed `Dataset.select` API instead.")
+        }
+
+      case other => other
+    }
+    Project(untypedCols.map(_.named), logicalPlan)
   }
 
   /**
@@ -1502,7 +1525,7 @@ class Dataset[T] private[sql](
     val namedColumns =
       columns.map(_.withInputType(exprEnc, logicalPlan.output).named)
     val execution = new QueryExecution(sparkSession, Project(namedColumns, logicalPlan))
-    new Dataset(sparkSession, execution, ExpressionEncoder.tuple(encoders))
+    new Dataset(execution, ExpressionEncoder.tuple(encoders))
   }
 
   /**
@@ -2244,6 +2267,90 @@ class Dataset[T] private[sql](
     randomSplit(weights.toArray, seed)
   }
 
+  /**
+   * (Scala-specific) Returns a new Dataset where each row has been expanded to zero or more
+   * rows by the provided function. This is similar to a `LATERAL VIEW` in HiveQL. The columns of
+   * the input row are implicitly joined with each row that is output by the function.
+   *
+   * Given that this is deprecated, as an alternative, you can explode columns either using
+   * `functions.explode()` or `flatMap()`. The following example uses these alternatives to count
+   * the number of books that contain a given word:
+   *
+   * {{{
+   *   case class Book(title: String, words: String)
+   *   val ds: Dataset[Book]
+   *
+   *   val allWords = ds.select('title, explode(split('words, " ")).as("word"))
+   *
+   *   val bookCountPerWord = allWords.groupBy("word").agg(countDistinct("title"))
+   * }}}
+   *
+   * Using `flatMap()` this can similarly be exploded as:
+   *
+   * {{{
+   *   ds.flatMap(_.words.split(" "))
+   * }}}
+   *
+   * @group untypedrel
+   * @since 2.0.0
+   */
+  @deprecated("use flatMap() or select() with functions.explode() instead", "2.0.0")
+  def explode[A <: Product : TypeTag](input: Column*)(f: Row => TraversableOnce[A]): DataFrame = {
+    val elementSchema = ScalaReflection.schemaFor[A].dataType.asInstanceOf[StructType]
+
+    val convert = CatalystTypeConverters.createToCatalystConverter(elementSchema)
+
+    val rowFunction =
+      f.andThen(_.map(convert(_).asInstanceOf[InternalRow]))
+    val generator = UserDefinedGenerator(elementSchema, rowFunction, input.map(_.expr))
+
+    withPlan {
+      Generate(generator, unrequiredChildIndex = Nil, outer = false,
+        qualifier = None, generatorOutput = Nil, logicalPlan)
+    }
+  }
+
+  /**
+   * (Scala-specific) Returns a new Dataset where a single column has been expanded to zero
+   * or more rows by the provided function. This is similar to a `LATERAL VIEW` in HiveQL. All
+   * columns of the input row are implicitly joined with each value that is output by the function.
+   *
+   * Given that this is deprecated, as an alternative, you can explode columns either using
+   * `functions.explode()`:
+   *
+   * {{{
+   *   ds.select(explode(split('words, " ")).as("word"))
+   * }}}
+   *
+   * or `flatMap()`:
+   *
+   * {{{
+   *   ds.flatMap(_.words.split(" "))
+   * }}}
+   *
+   * @group untypedrel
+   * @since 2.0.0
+   */
+  @deprecated("use flatMap() or select() with functions.explode() instead", "2.0.0")
+  def explode[A, B : TypeTag](inputColumn: String, outputColumn: String)(f: A => TraversableOnce[B])
+    : DataFrame = {
+    val dataType = ScalaReflection.schemaFor[B].dataType
+    val attributes = AttributeReference(outputColumn, dataType)() :: Nil
+    // TODO handle the metadata?
+    val elementSchema = attributes.toStructType
+
+    def rowFunction(row: Row): TraversableOnce[InternalRow] = {
+      val convert = CatalystTypeConverters.createToCatalystConverter(dataType)
+      f(row(0).asInstanceOf[A]).map(o => InternalRow(convert(o)))
+    }
+    val generator = UserDefinedGenerator(elementSchema, rowFunction, apply(inputColumn).expr :: Nil)
+
+    withPlan {
+      Generate(generator, unrequiredChildIndex = Nil, outer = false,
+        qualifier = None, generatorOutput = Nil, logicalPlan)
+    }
+  }
+
   /**
    * Returns a new Dataset by adding a column or replacing the existing column that has
    * the same name.
@@ -2434,6 +2541,8 @@ class Dataset[T] private[sql](
   def dropDuplicates(colNames: Seq[String]): Dataset[T] = withTypedPlan {
     val resolver = sparkSession.sessionState.analyzer.resolver
     val allColumns = queryExecution.analyzed.output
+    // SPARK-31990: We must keep `toSet.toSeq` here because of the backward compatibility issue
+    // (the Streaming's state store depends on the `groupCols` order).
     val groupCols = colNames.toSet.toSeq.flatMap { (colName: String) =>
       // It is possibly there are more than one columns with the same name,
       // so we call filter instead of find.
@@ -2858,9 +2967,8 @@ class Dataset[T] private[sql](
    */
   def toLocalIterator(): java.util.Iterator[T] = {
     withAction("toLocalIterator", queryExecution) { plan =>
-      // `ExpressionEncoder` is not thread-safe, here we create a new encoder.
-      val enc = resolvedEnc.copy()
-      plan.executeToIterator().map(enc.fromRow).asJava
+      val fromRow = resolvedEnc.createDeserializer()
+      plan.executeToIterator().map(fromRow).asJava
     }
   }
 
@@ -3108,6 +3216,18 @@ class Dataset[T] private[sql](
    */
   def javaRDD: JavaRDD[T] = toJavaRDD
 
+  /**
+   * Registers this Dataset as a temporary table using the given name. The lifetime of this
+   * temporary table is tied to the [[SparkSession]] that was used to create this Dataset.
+   *
+   * @group basic
+   * @since 1.6.0
+   */
+  @deprecated("Use createOrReplaceTempView(viewName) instead.", "2.0.0")
+  def registerTempTable(tableName: String): Unit = {
+    createOrReplaceTempView(tableName)
+  }
+
   /**
    * Creates a local temporary view using the given name. The lifetime of this
    * temporary view is tied to the [[SparkSession]] that was used to create this Dataset.
@@ -3268,9 +3388,10 @@ class Dataset[T] private[sql](
         new JSONOptions(Map.empty[String, String], sessionLocalTimeZone))
 
       new Iterator[String] {
+        private val toRow = exprEnc.createSerializer()
         override def hasNext: Boolean = iter.hasNext
         override def next(): String = {
-          gen.write(exprEnc.toRow(iter.next()))
+          gen.write(toRow(iter.next()))
           gen.flush()
 
           val json = writer.toString
@@ -3362,8 +3483,8 @@ class Dataset[T] private[sql](
   private[sql] def collectAsArrowToR(): Array[Any] = {
     val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
 
-    withAction("collectAsArrowToR", queryExecution) { plan =>
-      RRDD.serveToStream("serve-Arrow") { outputStream =>
+    RRDD.serveToStream("serve-Arrow") { outputStream =>
+      withAction("collectAsArrowToR", queryExecution) { plan =>
         val buffer = new ByteArrayOutputStream()
         val out = new DataOutputStream(outputStream)
         val batchWriter = new ArrowBatchStreamWriter(schema, buffer, timeZoneId)
@@ -3371,7 +3492,7 @@ class Dataset[T] private[sql](
         val numPartitions = arrowBatchRdd.partitions.length
 
         // Store collection results for worst case of 1 to N-1 partitions
-        val results = new Array[Array[Array[Byte]]](numPartitions - 1)
+        val results = new Array[Array[Array[Byte]]](Math.max(0, numPartitions - 1))
         var lastIndex = -1  // index of last partition written
 
         // Handler to eagerly write partitions to Python in order
@@ -3416,8 +3537,8 @@ class Dataset[T] private[sql](
   private[sql] def collectAsArrowToPython: Array[Any] = {
     val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
 
-    withAction("collectAsArrowToPython", queryExecution) { plan =>
-      PythonRDD.serveToStream("serve-Arrow") { outputStream =>
+    PythonRDD.serveToStream("serve-Arrow") { outputStream =>
+      withAction("collectAsArrowToPython", queryExecution) { plan =>
         val out = new DataOutputStream(outputStream)
         val batchWriter = new ArrowBatchStreamWriter(schema, out, timeZoneId)
 
@@ -3472,7 +3593,7 @@ class Dataset[T] private[sql](
    * an execution.
    */
   private def withNewExecutionId[U](body: => U): U = {
-    SQLExecution.withNewExecutionId(sparkSession, queryExecution)(body)
+    SQLExecution.withNewExecutionId(queryExecution)(body)
   }
 
   /**
@@ -3481,7 +3602,7 @@ class Dataset[T] private[sql](
    * reset.
    */
   private def withNewRDDExecutionId[U](body: => U): U = {
-    SQLExecution.withNewExecutionId(sparkSession, rddQueryExecution) {
+    SQLExecution.withNewExecutionId(rddQueryExecution) {
       rddQueryExecution.executedPlan.resetMetrics()
       body
     }
@@ -3492,7 +3613,7 @@ class Dataset[T] private[sql](
    * user-registered callback functions.
    */
   private def withAction[U](name: String, qe: QueryExecution)(action: SparkPlan => U) = {
-    SQLExecution.withNewExecutionId(sparkSession, qe, Some(name)) {
+    SQLExecution.withNewExecutionId(qe, Some(name)) {
       qe.executedPlan.resetMetrics()
       action(qe.executedPlan)
     }
@@ -3502,9 +3623,8 @@ class Dataset[T] private[sql](
    * Collect all elements from a spark plan.
    */
   private def collectFromPlan(plan: SparkPlan): Array[T] = {
-    // `ExpressionEncoder` is not thread-safe, here we create a new encoder.
-    val enc = resolvedEnc.copy()
-    plan.executeCollect().map(enc.fromRow)
+    val fromRow = resolvedEnc.createDeserializer()
+    plan.executeCollect().map(fromRow)
   }
 
   private def sortInternal(global: Boolean, sortExprs: Seq[Column]): Dataset[T] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
index 89cc9735e4f6a..76ee297dfca79 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
@@ -449,10 +449,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
     val aggregate = Aggregate(groupingAttributes, keyColumn +: namedColumns, logicalPlan)
     val execution = new QueryExecution(sparkSession, aggregate)
 
-    new Dataset(
-      sparkSession,
-      execution,
-      ExpressionEncoder.tuple(kExprEnc +: encoders))
+    new Dataset(execution, ExpressionEncoder.tuple(kExprEnc +: encoders))
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
index b1ba7d4538732..c37d8eaa294bf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
@@ -546,9 +546,10 @@ class RelationalGroupedDataset protected[sql](
       case ne: NamedExpression => ne
       case other => Alias(other, other.toString)()
     }
-    val groupingAttributes = groupingNamedExpressions.map(_.toAttribute)
     val child = df.logicalPlan
-    val project = Project(groupingNamedExpressions ++ child.output, child)
+    val project = df.sparkSession.sessionState.executePlan(
+      Project(groupingNamedExpressions ++ child.output, child)).analyzed
+    val groupingAttributes = project.output.take(groupingNamedExpressions.length)
     val output = expr.dataType.asInstanceOf[StructType].toAttributes
     val plan = FlatMapGroupsInPandas(groupingAttributes, expr, output, project)
 
@@ -583,14 +584,16 @@ class RelationalGroupedDataset protected[sql](
       case other => Alias(other, other.toString)()
     }
 
-    val leftAttributes = leftGroupingNamedExpressions.map(_.toAttribute)
-    val rightAttributes = rightGroupingNamedExpressions.map(_.toAttribute)
-
     val leftChild = df.logicalPlan
     val rightChild = r.df.logicalPlan
 
-    val left = Project(leftGroupingNamedExpressions ++ leftChild.output, leftChild)
-    val right = Project(rightGroupingNamedExpressions ++ rightChild.output, rightChild)
+    val left = df.sparkSession.sessionState.executePlan(
+      Project(leftGroupingNamedExpressions ++ leftChild.output, leftChild)).analyzed
+    val right = r.df.sparkSession.sessionState.executePlan(
+      Project(rightGroupingNamedExpressions ++ rightChild.output, rightChild)).analyzed
+
+    val leftAttributes = left.output.take(leftGroupingNamedExpressions.length)
+    val rightAttributes = right.output.take(rightGroupingNamedExpressions.length)
 
     val output = expr.dataType.asInstanceOf[StructType].toAttributes
     val plan = FlatMapCoGroupsInPandas(leftAttributes, rightAttributes, expr, output, left, right)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
index e1b44b5918143..e9bc25d489718 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
@@ -31,7 +31,7 @@ import org.apache.spark.sql.internal.SQLConf.{DeprecatedConfig, RemovedConfig}
  * @since 2.0.0
  */
 @Stable
-class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging {
+class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) {
 
   /**
    * Sets the given Spark runtime configuration property.
@@ -40,8 +40,6 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging
    */
   def set(key: String, value: String): Unit = {
     requireNonStaticConf(key)
-    requireDefaultValueOfRemovedConf(key, value)
-    logDeprecationWarning(key)
     sqlConf.setConfString(key, value)
   }
 
@@ -130,7 +128,6 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging
    */
   def unset(key: String): Unit = {
     requireNonStaticConf(key)
-    logDeprecationWarning(key)
     sqlConf.unsetConf(key)
   }
 
@@ -161,26 +158,4 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging
       throw new AnalysisException(s"Cannot modify the value of a Spark config: $key")
     }
   }
-
-  private def requireDefaultValueOfRemovedConf(key: String, value: String): Unit = {
-    SQLConf.removedSQLConfigs.get(key).foreach {
-      case RemovedConfig(configName, version, defaultValue, comment) =>
-        if (value != defaultValue) {
-          throw new AnalysisException(
-            s"The SQL config '$configName' was removed in the version $version. $comment")
-        }
-    }
-  }
-
-  /**
-   * Logs a warning message if the given config key is deprecated.
-   */
-  private def logDeprecationWarning(key: String): Unit = {
-    SQLConf.deprecatedSQLConfigs.get(key).foreach {
-      case DeprecatedConfig(configName, version, comment) =>
-        logWarning(
-          s"The SQL config '$configName' has been deprecated in Spark v$version " +
-          s"and may be removed in the future. $comment")
-    }
-  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 2054874e5e07b..dd237962110ef 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -24,7 +24,7 @@ import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.annotation.{DeveloperApi, Experimental, Stable, Unstable}
-import org.apache.spark.api.java.JavaRDD
+import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.ConfigEntry
 import org.apache.spark.rdd.RDD
@@ -64,6 +64,15 @@ class SQLContext private[sql](val sparkSession: SparkSession)
 
   // Note: Since Spark 2.0 this class has become a wrapper of SparkSession, where the
   // real functionality resides. This class remains mainly for backward compatibility.
+
+  @deprecated("Use SparkSession.builder instead", "2.0.0")
+  def this(sc: SparkContext) = {
+    this(SparkSession.builder().sparkContext(sc).getOrCreate())
+  }
+
+  @deprecated("Use SparkSession.builder instead", "2.0.0")
+  def this(sparkContext: JavaSparkContext) = this(sparkContext.sc)
+
   // TODO: move this logic into SparkSession
 
   private[sql] def sessionState: SessionState = sparkSession.sessionState
@@ -479,6 +488,97 @@ class SQLContext private[sql](val sparkSession: SparkSession)
   def readStream: DataStreamReader = sparkSession.readStream
 
 
+  /**
+   * Creates an external table from the given path and returns the corresponding DataFrame.
+   * It will use the default data source configured by spark.sql.sources.default.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(tableName: String, path: String): DataFrame = {
+    sparkSession.catalog.createTable(tableName, path)
+  }
+
+  /**
+   * Creates an external table from the given path based on a data source
+   * and returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      path: String,
+      source: String): DataFrame = {
+    sparkSession.catalog.createTable(tableName, path, source)
+  }
+
+  /**
+   * Creates an external table from the given path based on a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      options: java.util.Map[String, String]): DataFrame = {
+    sparkSession.catalog.createTable(tableName, source, options)
+  }
+
+  /**
+   * (Scala-specific)
+   * Creates an external table from the given path based on a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      options: Map[String, String]): DataFrame = {
+    sparkSession.catalog.createTable(tableName, source, options)
+  }
+
+  /**
+   * Create an external table from the given path based on a data source, a schema and
+   * a set of options. Then, returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      options: java.util.Map[String, String]): DataFrame = {
+    sparkSession.catalog.createTable(tableName, source, schema, options)
+  }
+
+  /**
+   * (Scala-specific)
+   * Create an external table from the given path based on a data source, a schema and
+   * a set of options. Then, returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      options: Map[String, String]): DataFrame = {
+    sparkSession.catalog.createTable(tableName, source, schema, options)
+  }
+
   /**
    * Registers the given `DataFrame` as a temporary table in the catalog. Temporary tables exist
    * only during the lifetime of this instance of SQLContext.
@@ -541,8 +641,8 @@ class SQLContext private[sql](val sparkSession: SparkSession)
   }
 
   /**
-   * Executes a SQL query using Spark, returning the result as a `DataFrame`. The dialect that is
-   * used for SQL parsing can be configured with 'spark.sql.dialect'.
+   * Executes a SQL query using Spark, returning the result as a `DataFrame`.
+   * This API eagerly runs DDL/DML commands, but not for SELECT queries.
    *
    * @group basic
    * @since 1.3.0
@@ -561,7 +661,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
 
   /**
    * Returns a `DataFrame` containing names of existing tables in the current database.
-   * The returned DataFrame has two columns, tableName and isTemporary (a Boolean
+   * The returned DataFrame has three columns, database, tableName and isTemporary (a Boolean
    * indicating if a table is a temporary one or not).
    *
    * @group ddl_ops
@@ -573,7 +673,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
 
   /**
    * Returns a `DataFrame` containing names of existing tables in the given database.
-   * The returned DataFrame has two columns, tableName and isTemporary (a Boolean
+   * The returned DataFrame has three columns, database, tableName and isTemporary (a Boolean
    * indicating if a table is a temporary one or not).
    *
    * @group ddl_ops
@@ -611,6 +711,289 @@ class SQLContext private[sql](val sparkSession: SparkSession)
     sessionState.catalog.listTables(databaseName).map(_.table).toArray
   }
 
+  ////////////////////////////////////////////////////////////////////////////
+  ////////////////////////////////////////////////////////////////////////////
+  // Deprecated methods
+  ////////////////////////////////////////////////////////////////////////////
+  ////////////////////////////////////////////////////////////////////////////
+
+  /**
+   * @deprecated As of 1.3.0, replaced by `createDataFrame()`.
+   */
+  @deprecated("Use createDataFrame instead.", "1.3.0")
+  def applySchema(rowRDD: RDD[Row], schema: StructType): DataFrame = {
+    createDataFrame(rowRDD, schema)
+  }
+
+  /**
+   * @deprecated As of 1.3.0, replaced by `createDataFrame()`.
+   */
+  @deprecated("Use createDataFrame instead.", "1.3.0")
+  def applySchema(rowRDD: JavaRDD[Row], schema: StructType): DataFrame = {
+    createDataFrame(rowRDD, schema)
+  }
+
+  /**
+   * @deprecated As of 1.3.0, replaced by `createDataFrame()`.
+   */
+  @deprecated("Use createDataFrame instead.", "1.3.0")
+  def applySchema(rdd: RDD[_], beanClass: Class[_]): DataFrame = {
+    createDataFrame(rdd, beanClass)
+  }
+
+  /**
+   * @deprecated As of 1.3.0, replaced by `createDataFrame()`.
+   */
+  @deprecated("Use createDataFrame instead.", "1.3.0")
+  def applySchema(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame = {
+    createDataFrame(rdd, beanClass)
+  }
+
+  /**
+   * Loads a Parquet file, returning the result as a `DataFrame`. This function returns an empty
+   * `DataFrame` if no paths are passed in.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().parquet()`.
+   */
+  @deprecated("Use read.parquet() instead.", "1.4.0")
+  @scala.annotation.varargs
+  def parquetFile(paths: String*): DataFrame = {
+    if (paths.isEmpty) {
+      emptyDataFrame
+    } else {
+      read.parquet(paths : _*)
+    }
+  }
+
+  /**
+   * Loads a JSON file (one object per line), returning the result as a `DataFrame`.
+   * It goes through the entire dataset once to determine the schema.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonFile(path: String): DataFrame = {
+    read.json(path)
+  }
+
+  /**
+   * Loads a JSON file (one object per line) and applies the given schema,
+   * returning the result as a `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonFile(path: String, schema: StructType): DataFrame = {
+    read.schema(schema).json(path)
+  }
+
+  /**
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonFile(path: String, samplingRatio: Double): DataFrame = {
+    read.option("samplingRatio", samplingRatio.toString).json(path)
+  }
+
+  /**
+   * Loads an RDD[String] storing JSON objects (one object per record), returning the result as a
+   * `DataFrame`.
+   * It goes through the entire dataset once to determine the schema.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: RDD[String]): DataFrame = read.json(json)
+
+  /**
+   * Loads an RDD[String] storing JSON objects (one object per record), returning the result as a
+   * `DataFrame`.
+   * It goes through the entire dataset once to determine the schema.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: JavaRDD[String]): DataFrame = read.json(json)
+
+  /**
+   * Loads an RDD[String] storing JSON objects (one object per record) and applies the given schema,
+   * returning the result as a `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: RDD[String], schema: StructType): DataFrame = {
+    read.schema(schema).json(json)
+  }
+
+  /**
+   * Loads an JavaRDD[String] storing JSON objects (one object per record) and applies the given
+   * schema, returning the result as a `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: JavaRDD[String], schema: StructType): DataFrame = {
+    read.schema(schema).json(json)
+  }
+
+  /**
+   * Loads an RDD[String] storing JSON objects (one object per record) inferring the
+   * schema, returning the result as a `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: RDD[String], samplingRatio: Double): DataFrame = {
+    read.option("samplingRatio", samplingRatio.toString).json(json)
+  }
+
+  /**
+   * Loads a JavaRDD[String] storing JSON objects (one object per record) inferring the
+   * schema, returning the result as a `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: JavaRDD[String], samplingRatio: Double): DataFrame = {
+    read.option("samplingRatio", samplingRatio.toString).json(json)
+  }
+
+  /**
+   * Returns the dataset stored at path as a DataFrame,
+   * using the default data source configured by spark.sql.sources.default.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by `read().load(path)`.
+   */
+  @deprecated("Use read.load(path) instead.", "1.4.0")
+  def load(path: String): DataFrame = {
+    read.load(path)
+  }
+
+  /**
+   * Returns the dataset stored at path as a DataFrame, using the given data source.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by `read().format(source).load(path)`.
+   */
+  @deprecated("Use read.format(source).load(path) instead.", "1.4.0")
+  def load(path: String, source: String): DataFrame = {
+    read.format(source).load(path)
+  }
+
+  /**
+   * (Java-specific) Returns the dataset specified by the given data source and
+   * a set of options as a DataFrame.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by `read().format(source).options(options).load()`.
+   */
+  @deprecated("Use read.format(source).options(options).load() instead.", "1.4.0")
+  def load(source: String, options: java.util.Map[String, String]): DataFrame = {
+    read.options(options).format(source).load()
+  }
+
+  /**
+   * (Scala-specific) Returns the dataset specified by the given data source and
+   * a set of options as a DataFrame.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by `read().format(source).options(options).load()`.
+   */
+  @deprecated("Use read.format(source).options(options).load() instead.", "1.4.0")
+  def load(source: String, options: Map[String, String]): DataFrame = {
+    read.options(options).format(source).load()
+  }
+
+  /**
+   * (Java-specific) Returns the dataset specified by the given data source and
+   * a set of options as a DataFrame, using the given schema as the schema of the DataFrame.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by
+   *            `read().format(source).schema(schema).options(options).load()`.
+   */
+  @deprecated("Use read.format(source).schema(schema).options(options).load() instead.", "1.4.0")
+  def load(
+      source: String,
+      schema: StructType,
+      options: java.util.Map[String, String]): DataFrame = {
+    read.format(source).schema(schema).options(options).load()
+  }
+
+  /**
+   * (Scala-specific) Returns the dataset specified by the given data source and
+   * a set of options as a DataFrame, using the given schema as the schema of the DataFrame.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by
+   *            `read().format(source).schema(schema).options(options).load()`.
+   */
+  @deprecated("Use read.format(source).schema(schema).options(options).load() instead.", "1.4.0")
+  def load(source: String, schema: StructType, options: Map[String, String]): DataFrame = {
+    read.format(source).schema(schema).options(options).load()
+  }
+
+  /**
+   * Construct a `DataFrame` representing the database table accessible via JDBC URL
+   * url named table.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().jdbc()`.
+   */
+  @deprecated("Use read.jdbc() instead.", "1.4.0")
+  def jdbc(url: String, table: String): DataFrame = {
+    read.jdbc(url, table, new Properties)
+  }
+
+  /**
+   * Construct a `DataFrame` representing the database table accessible via JDBC URL
+   * url named table.  Partitions of the table will be retrieved in parallel based on the parameters
+   * passed to this function.
+   *
+   * @param columnName the name of a column of integral type that will be used for partitioning.
+   * @param lowerBound the minimum value of `columnName` used to decide partition stride
+   * @param upperBound the maximum value of `columnName` used to decide partition stride
+   * @param numPartitions the number of partitions.  the range `minValue`-`maxValue` will be split
+   *                      evenly into this many partitions
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().jdbc()`.
+   */
+  @deprecated("Use read.jdbc() instead.", "1.4.0")
+  def jdbc(
+      url: String,
+      table: String,
+      columnName: String,
+      lowerBound: Long,
+      upperBound: Long,
+      numPartitions: Int): DataFrame = {
+    read.jdbc(url, table, columnName, lowerBound, upperBound, numPartitions, new Properties)
+  }
+
+  /**
+   * Construct a `DataFrame` representing the database table accessible via JDBC URL
+   * url named table. The theParts parameter gives a list expressions
+   * suitable for inclusion in WHERE clauses; each one defines one partition
+   * of the `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().jdbc()`.
+   */
+  @deprecated("Use read.jdbc() instead.", "1.4.0")
+  def jdbc(url: String, table: String, theParts: Array[String]): DataFrame = {
+    read.jdbc(url, table, theParts, new Properties)
+  }
 }
 
 /**
@@ -623,6 +1006,45 @@ class SQLContext private[sql](val sparkSession: SparkSession)
  */
 object SQLContext {
 
+  /**
+   * Get the singleton SQLContext if it exists or create a new one using the given SparkContext.
+   *
+   * This function can be used to create a singleton SQLContext object that can be shared across
+   * the JVM.
+   *
+   * If there is an active SQLContext for current thread, it will be returned instead of the global
+   * one.
+   *
+   * @since 1.5.0
+   */
+  @deprecated("Use SparkSession.builder instead", "2.0.0")
+  def getOrCreate(sparkContext: SparkContext): SQLContext = {
+    SparkSession.builder().sparkContext(sparkContext).getOrCreate().sqlContext
+  }
+
+  /**
+   * Changes the SQLContext that will be returned in this thread and its children when
+   * SQLContext.getOrCreate() is called. This can be used to ensure that a given thread receives
+   * a SQLContext with an isolated session, instead of the global (first created) context.
+   *
+   * @since 1.6.0
+   */
+  @deprecated("Use SparkSession.setActiveSession instead", "2.0.0")
+  def setActive(sqlContext: SQLContext): Unit = {
+    SparkSession.setActiveSession(sqlContext.sparkSession)
+  }
+
+  /**
+   * Clears the active SQLContext for current thread. Subsequent calls to getOrCreate will
+   * return the first created context instead of a thread-local override.
+   *
+   * @since 1.6.0
+   */
+  @deprecated("Use SparkSession.clearActiveSession instead", "2.0.0")
+  def clearActive(): Unit = {
+    SparkSession.clearActiveSession()
+  }
+
   /**
    * Converts an iterator of Java Beans to InternalRow using the provided
    * bean info & schema. This is not related to the singleton, but is a static
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index abefb348cafc7..f89e58c6e971c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
 
 import java.io.Closeable
 import java.util.concurrent.TimeUnit._
-import java.util.concurrent.atomic.AtomicReference
+import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference}
 
 import scala.collection.JavaConverters._
 import scala.reflect.runtime.universe.TypeTag
@@ -29,6 +29,7 @@ import org.apache.spark.{SPARK_VERSION, SparkConf, SparkContext, TaskContext}
 import org.apache.spark.annotation.{DeveloperApi, Experimental, Stable, Unstable}
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.EXECUTOR_ALLOW_SPARK_CONTEXT
 import org.apache.spark.rdd.RDD
 import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
 import org.apache.spark.sql.catalog.Catalog
@@ -49,7 +50,6 @@ import org.apache.spark.sql.types.{DataType, StructType}
 import org.apache.spark.sql.util.ExecutionListenerManager
 import org.apache.spark.util.{CallSite, Utils}
 
-
 /**
  * The entry point to programming Spark with the Dataset and DataFrame API.
  *
@@ -293,8 +293,7 @@ class SparkSession private(
    *
    * @since 2.0.0
    */
-  def createDataFrame[A <: Product : TypeTag](rdd: RDD[A]): DataFrame = {
-    SparkSession.setActiveSession(this)
+  def createDataFrame[A <: Product : TypeTag](rdd: RDD[A]): DataFrame = withActive {
     val encoder = Encoders.product[A]
     Dataset.ofRows(self, ExternalRDD(rdd, self)(encoder))
   }
@@ -304,8 +303,7 @@ class SparkSession private(
    *
    * @since 2.0.0
    */
-  def createDataFrame[A <: Product : TypeTag](data: Seq[A]): DataFrame = {
-    SparkSession.setActiveSession(this)
+  def createDataFrame[A <: Product : TypeTag](data: Seq[A]): DataFrame = withActive {
     val schema = ScalaReflection.schemaFor[A].dataType.asInstanceOf[StructType]
     val attributeSeq = schema.toAttributes
     Dataset.ofRows(self, LocalRelation.fromProduct(attributeSeq, data))
@@ -343,11 +341,12 @@ class SparkSession private(
    * @since 2.0.0
    */
   @DeveloperApi
-  def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame = {
+  def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame = withActive {
     // TODO: use MutableProjection when rowRDD is another DataFrame and the applied
     // schema differs from the existing schema on any field data type.
     val encoder = RowEncoder(schema)
-    val catalystRows = rowRDD.map(encoder.toRow)
+    val toRow = encoder.createSerializer()
+    val catalystRows = rowRDD.map(toRow)
     internalCreateDataFrame(catalystRows.setName(rowRDD.name), schema)
   }
 
@@ -373,7 +372,7 @@ class SparkSession private(
    * @since 2.0.0
    */
   @DeveloperApi
-  def createDataFrame(rows: java.util.List[Row], schema: StructType): DataFrame = {
+  def createDataFrame(rows: java.util.List[Row], schema: StructType): DataFrame = withActive {
     Dataset.ofRows(self, LocalRelation.fromExternalRows(schema.toAttributes, rows.asScala))
   }
 
@@ -385,7 +384,7 @@ class SparkSession private(
    *
    * @since 2.0.0
    */
-  def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame = {
+  def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame = withActive {
     val attributeSeq: Seq[AttributeReference] = getSchema(beanClass)
     val className = beanClass.getName
     val rowRdd = rdd.mapPartitions { iter =>
@@ -414,7 +413,7 @@ class SparkSession private(
    *          SELECT * queries will return the columns in an undefined order.
    * @since 1.6.0
    */
-  def createDataFrame(data: java.util.List[_], beanClass: Class[_]): DataFrame = {
+  def createDataFrame(data: java.util.List[_], beanClass: Class[_]): DataFrame = withActive {
     val attrSeq = getSchema(beanClass)
     val rows = SQLContext.beansToRows(data.asScala.iterator, beanClass, attrSeq)
     Dataset.ofRows(self, LocalRelation(attrSeq, rows.toSeq))
@@ -462,8 +461,9 @@ class SparkSession private(
    */
   def createDataset[T : Encoder](data: Seq[T]): Dataset[T] = {
     val enc = encoderFor[T]
+    val toRow = enc.createSerializer()
     val attributes = enc.schema.toAttributes
-    val encoded = data.map(d => enc.toRow(d).copy())
+    val encoded = data.map(d => toRow(d).copy())
     val plan = new LocalRelation(attributes, encoded)
     Dataset[T](self, plan)
   }
@@ -595,11 +595,11 @@ class SparkSession private(
 
   /**
    * Executes a SQL query using Spark, returning the result as a `DataFrame`.
-   * The dialect that is used for SQL parsing can be configured with 'spark.sql.dialect'.
+   * This API eagerly runs DDL/DML commands, but not for SELECT queries.
    *
    * @since 2.0.0
    */
-  def sql(sqlText: String): DataFrame = {
+  def sql(sqlText: String): DataFrame = withActive {
     val tracker = new QueryPlanningTracker
     val plan = tracker.measurePhase(QueryPlanningTracker.PARSING) {
       sessionState.sqlParser.parsePlan(sqlText)
@@ -751,6 +751,20 @@ class SparkSession private(
     }
   }
 
+  /**
+   * Execute a block of code with the this session set as the active session, and restore the
+   * previous session on completion.
+   */
+  private[sql] def withActive[T](block: => T): T = {
+    // Use the active session thread local directly to make sure we get the session that is actually
+    // set and not the default session. This to prevent that we promote the default session to the
+    // active session once we are done.
+    val old = SparkSession.activeThreadSession.get()
+    SparkSession.setActiveSession(this)
+    try block finally {
+      SparkSession.setActiveSession(old)
+    }
+  }
 }
 
 
@@ -881,20 +895,23 @@ object SparkSession extends Logging {
      * SparkSession exists, the method creates a new SparkSession and assigns the
      * newly created SparkSession as the global default.
      *
-     * In case an existing SparkSession is returned, the config options specified in
+     * In case an existing SparkSession is returned, the non-static config options specified in
      * this builder will be applied to the existing SparkSession.
      *
      * @since 2.0.0
      */
     def getOrCreate(): SparkSession = synchronized {
-      assertOnDriver()
+      val sparkConf = new SparkConf()
+      options.foreach { case (k, v) => sparkConf.set(k, v) }
+
+      if (!sparkConf.get(EXECUTOR_ALLOW_SPARK_CONTEXT)) {
+        assertOnDriver()
+      }
+
       // Get the session from current thread's active session.
       var session = activeThreadSession.get()
       if ((session ne null) && !session.sparkContext.isStopped) {
-        options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
-        if (options.nonEmpty) {
-          logWarning("Using an existing SparkSession; some configuration may not take effect.")
-        }
+        applyModifiableSettings(session)
         return session
       }
 
@@ -903,18 +920,12 @@ object SparkSession extends Logging {
         // If the current thread does not have an active session, get it from the global session.
         session = defaultSession.get()
         if ((session ne null) && !session.sparkContext.isStopped) {
-          options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
-          if (options.nonEmpty) {
-            logWarning("Using an existing SparkSession; some configuration may not take effect.")
-          }
+          applyModifiableSettings(session)
           return session
         }
 
         // No active nor global default session. Create a new one.
         val sparkContext = userSuppliedContext.getOrElse {
-          val sparkConf = new SparkConf()
-          options.foreach { case (k, v) => sparkConf.set(k, v) }
-
           // set a random app name if not given.
           if (!sparkConf.contains("spark.app.name")) {
             sparkConf.setAppName(java.util.UUID.randomUUID().toString)
@@ -932,19 +943,27 @@ object SparkSession extends Logging {
         options.foreach { case (k, v) => session.initialSessionOptions.put(k, v) }
         setDefaultSession(session)
         setActiveSession(session)
-
-        // Register a successfully instantiated context to the singleton. This should be at the
-        // end of the class definition so that the singleton is updated only if there is no
-        // exception in the construction of the instance.
-        sparkContext.addSparkListener(new SparkListener {
-          override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit = {
-            defaultSession.set(null)
-          }
-        })
+        registerContextListener(sparkContext)
       }
 
       return session
     }
+
+    private def applyModifiableSettings(session: SparkSession): Unit = {
+      val (staticConfs, otherConfs) =
+        options.partition(kv => SQLConf.staticConfKeys.contains(kv._1))
+
+      otherConfs.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
+
+      if (staticConfs.nonEmpty) {
+        logWarning("Using an existing SparkSession; the static sql configurations will not take" +
+          " effect.")
+      }
+      if (otherConfs.nonEmpty) {
+        logWarning("Using an existing SparkSession; some spark core configurations may not take" +
+          " effect.")
+      }
+    }
   }
 
   /**
@@ -1040,6 +1059,20 @@ object SparkSession extends Logging {
   // Private methods from now on
   ////////////////////////////////////////////////////////////////////////////////////////
 
+  private val listenerRegistered: AtomicBoolean = new AtomicBoolean(false)
+
+  /** Register the AppEnd listener onto the Context  */
+  private def registerContextListener(sparkContext: SparkContext): Unit = {
+    if (!listenerRegistered.get()) {
+      sparkContext.addSparkListener(new SparkListener {
+        override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit = {
+          defaultSession.set(null)
+        }
+      })
+      listenerRegistered.set(true)
+    }
+  }
+
   /** The active SparkSession for the current thread. */
   private val activeThreadSession = new InheritableThreadLocal[SparkSession]
 
@@ -1057,7 +1090,7 @@ object SparkSession extends Logging {
   }
 
   private def assertOnDriver(): Unit = {
-    if (Utils.isTesting && TaskContext.get != null) {
+    if (TaskContext.get != null) {
       // we're accessing it during task execution, fail.
       throw new IllegalStateException(
         "SparkSession should only be created and accessed on the driver.")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
index 1c2bf9e7c2a57..bd870fb8dcae6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.ColumnarRule
+import org.apache.spark.sql.execution.{ColumnarRule, SparkPlan}
 
 /**
  * :: Experimental ::
@@ -44,6 +44,7 @@ import org.apache.spark.sql.execution.ColumnarRule
  * <li>Customized Parser.</li>
  * <li>(External) Catalog listeners.</li>
  * <li>Columnar Rules.</li>
+ * <li>Adaptive Query Stage Preparation Rules.</li>
  * </ul>
  *
  * The extensions can be used by calling `withExtensions` on the [[SparkSession.Builder]], for
@@ -96,8 +97,10 @@ class SparkSessionExtensions {
   type ParserBuilder = (SparkSession, ParserInterface) => ParserInterface
   type FunctionDescription = (FunctionIdentifier, ExpressionInfo, FunctionBuilder)
   type ColumnarRuleBuilder = SparkSession => ColumnarRule
+  type QueryStagePrepRuleBuilder = SparkSession => Rule[SparkPlan]
 
   private[this] val columnarRuleBuilders = mutable.Buffer.empty[ColumnarRuleBuilder]
+  private[this] val queryStagePrepRuleBuilders = mutable.Buffer.empty[QueryStagePrepRuleBuilder]
 
   /**
    * Build the override rules for columnar execution.
@@ -106,6 +109,13 @@ class SparkSessionExtensions {
     columnarRuleBuilders.map(_.apply(session))
   }
 
+  /**
+   * Build the override rules for the query stage preparation phase of adaptive query execution.
+   */
+  private[sql] def buildQueryStagePrepRules(session: SparkSession): Seq[Rule[SparkPlan]] = {
+    queryStagePrepRuleBuilders.map(_.apply(session)).toSeq
+  }
+
   /**
    * Inject a rule that can override the columnar execution of an executor.
    */
@@ -113,6 +123,14 @@ class SparkSessionExtensions {
     columnarRuleBuilders += builder
   }
 
+  /**
+   * Inject a rule that can override the the query stage preparation phase of adaptive query
+   * execution.
+   */
+  def injectQueryStagePrepRule(builder: QueryStagePrepRuleBuilder): Unit = {
+    queryStagePrepRuleBuilders += builder
+  }
+
   private[this] val resolutionRuleBuilders = mutable.Buffer.empty[RuleBuilder]
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
index 0f08e10c00d22..ced4af46c3f30 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
@@ -125,7 +125,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
     (0 to 22).foreach { x =>
       val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"})
       val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _)
-      val inputSchemas = (1 to x).foldRight("Nil")((i, s) => {s"Try(ScalaReflection.schemaFor[A$i]).toOption :: $s"})
+      val inputEncoders = (1 to x).foldRight("Nil")((i, s) => {s"Try(ExpressionEncoder[A$i]()).toOption :: $s"})
       println(s"""
         |/**
         | * Registers a deterministic Scala closure of $x arguments as user-defined function (UDF).
@@ -134,8 +134,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
         | */
         |def register[$typeTags](name: String, func: Function$x[$types]): UserDefinedFunction = {
         |  val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-        |  val inputSchemas: Seq[Option[ScalaReflection.Schema]] = $inputSchemas
-        |  val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+        |  val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = $inputEncoders
+        |  val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
         |  val finalUdf = if (nullable) udf else udf.asNonNullable()
         |  def builder(e: Seq[Expression]) = if (e.length == $x) {
         |    finalUdf.createScalaUDF(e)
@@ -163,7 +163,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
         |def register(name: String, f: UDF$i[$extTypeArgs], returnType: DataType): Unit = {
         |  val func = $funcCall
         |  def builder(e: Seq[Expression]) = if (e.length == $i) {
-        |    ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+        |    ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
         |  } else {
         |    throw new AnalysisException("Invalid number of arguments for function " + name +
         |      ". Expected: $i; Found: " + e.length)
@@ -180,8 +180,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag](name: String, func: Function0[RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 0) {
       finalUdf.createScalaUDF(e)
@@ -200,8 +200,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 1) {
       finalUdf.createScalaUDF(e)
@@ -220,8 +220,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func: Function2[A1, A2, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 2) {
       finalUdf.createScalaUDF(e)
@@ -240,8 +240,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name: String, func: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 3) {
       finalUdf.createScalaUDF(e)
@@ -260,8 +260,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 4) {
       finalUdf.createScalaUDF(e)
@@ -280,8 +280,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 5) {
       finalUdf.createScalaUDF(e)
@@ -300,8 +300,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 6) {
       finalUdf.createScalaUDF(e)
@@ -320,8 +320,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 7) {
       finalUdf.createScalaUDF(e)
@@ -340,8 +340,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String, func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 8) {
       finalUdf.createScalaUDF(e)
@@ -360,8 +360,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name: String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 9) {
       finalUdf.createScalaUDF(e)
@@ -380,8 +380,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 10) {
       finalUdf.createScalaUDF(e)
@@ -400,8 +400,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 11) {
       finalUdf.createScalaUDF(e)
@@ -420,8 +420,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 12) {
       finalUdf.createScalaUDF(e)
@@ -440,8 +440,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 13) {
       finalUdf.createScalaUDF(e)
@@ -460,8 +460,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String, func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 14) {
       finalUdf.createScalaUDF(e)
@@ -480,8 +480,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 15) {
       finalUdf.createScalaUDF(e)
@@ -500,8 +500,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 16) {
       finalUdf.createScalaUDF(e)
@@ -520,8 +520,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 17) {
       finalUdf.createScalaUDF(e)
@@ -540,8 +540,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 18) {
       finalUdf.createScalaUDF(e)
@@ -560,8 +560,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 19) {
       finalUdf.createScalaUDF(e)
@@ -580,8 +580,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name: String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Try(ScalaReflection.schemaFor[A20]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Try(ExpressionEncoder[A20]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 20) {
       finalUdf.createScalaUDF(e)
@@ -600,8 +600,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Try(ScalaReflection.schemaFor[A20]).toOption :: Try(ScalaReflection.schemaFor[A21]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Try(ExpressionEncoder[A20]()).toOption :: Try(ExpressionEncoder[A21]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 21) {
       finalUdf.createScalaUDF(e)
@@ -620,8 +620,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Try(ScalaReflection.schemaFor[A20]).toOption :: Try(ScalaReflection.schemaFor[A21]).toOption :: Try(ScalaReflection.schemaFor[A22]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Try(ExpressionEncoder[A20]()).toOption :: Try(ExpressionEncoder[A21]()).toOption :: Try(ExpressionEncoder[A22]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 22) {
       finalUdf.createScalaUDF(e)
@@ -731,7 +731,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF0[_], returnType: DataType): Unit = {
     val func = () => f.asInstanceOf[UDF0[Any]].call()
     def builder(e: Seq[Expression]) = if (e.length == 0) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 0; Found: " + e.length)
@@ -746,7 +746,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF1[_, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF1[Any, Any]].call(_: Any)
     def builder(e: Seq[Expression]) = if (e.length == 1) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 1; Found: " + e.length)
@@ -761,7 +761,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF2[_, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 2) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 2; Found: " + e.length)
@@ -776,7 +776,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF3[_, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 3) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 3; Found: " + e.length)
@@ -791,7 +791,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF4[_, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 4) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 4; Found: " + e.length)
@@ -806,7 +806,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF5[_, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 5) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 5; Found: " + e.length)
@@ -821,7 +821,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF6[_, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 6) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 6; Found: " + e.length)
@@ -836,7 +836,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 7) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 7; Found: " + e.length)
@@ -851,7 +851,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 8) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 8; Found: " + e.length)
@@ -866,7 +866,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 9) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 9; Found: " + e.length)
@@ -881,7 +881,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 10) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 10; Found: " + e.length)
@@ -896,7 +896,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 11) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 11; Found: " + e.length)
@@ -911,7 +911,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 12) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 12; Found: " + e.length)
@@ -926,7 +926,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 13) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 13; Found: " + e.length)
@@ -941,7 +941,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 14) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 14; Found: " + e.length)
@@ -956,7 +956,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 15) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 15; Found: " + e.length)
@@ -971,7 +971,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 16) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 16; Found: " + e.length)
@@ -986,7 +986,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF17[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 17) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 17; Found: " + e.length)
@@ -1001,7 +1001,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF18[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 18) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 18; Found: " + e.length)
@@ -1016,7 +1016,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF19[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 19) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 19; Found: " + e.length)
@@ -1031,7 +1031,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF20[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 20) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 20; Found: " + e.length)
@@ -1046,7 +1046,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF21[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 21) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 21; Found: " + e.length)
@@ -1061,7 +1061,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF22[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 22) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 22; Found: " + e.length)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
index b232aa18c816e..38254609e8f67 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
@@ -20,18 +20,23 @@ package org.apache.spark.sql.api.python
 import java.io.InputStream
 import java.nio.channels.Channels
 
+import scala.util.control.NonFatal
+
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.api.python.PythonRDDServer
+import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{DataFrame, Dataset, SQLContext}
+import org.apache.spark.sql.{DataFrame, SQLContext}
+import org.apache.spark.sql.catalyst.ScalaReflection
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.execution.{ExplainMode, QueryExecution}
 import org.apache.spark.sql.execution.arrow.ArrowConverters
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.sql.types.DataType
 
-private[sql] object PythonSQLUtils {
+private[sql] object PythonSQLUtils extends Logging {
   def parseDataType(typeText: String): DataType = CatalystSqlParser.parseDataType(typeText)
 
   // This is needed when generating SQL documentation for built-in functions.
@@ -39,6 +44,30 @@ private[sql] object PythonSQLUtils {
     FunctionRegistry.functionSet.flatMap(f => FunctionRegistry.builtin.lookupFunction(f)).toArray
   }
 
+  private def listAllSQLConfigs(): Seq[(String, String, String, String)] = {
+    val conf = new SQLConf()
+    // Force to build static SQL configurations
+    StaticSQLConf
+    // Force to build SQL configurations from Hive module
+    try {
+      val symbol = ScalaReflection.mirror.staticModule("org.apache.spark.sql.hive.HiveUtils")
+      ScalaReflection.mirror.reflectModule(symbol).instance
+    } catch {
+      case NonFatal(e) =>
+        logWarning("Cannot generated sql configurations from hive module", e)
+    }
+    conf.getAllDefinedConfs
+  }
+
+  def listRuntimeSQLConfigs(): Array[(String, String, String, String)] = {
+    // Py4J doesn't seem to translate Seq well, so we convert to an Array.
+    listAllSQLConfigs().filterNot(p => SQLConf.staticConfKeys.contains(p._1)).toArray
+  }
+
+  def listStaticSQLConfigs(): Array[(String, String, String, String)] = {
+    listAllSQLConfigs().filter(p => SQLConf.staticConfKeys.contains(p._1)).toArray
+  }
+
   /**
    * Python callable function to read a file in Arrow stream format and create a [[RDD]]
    * using each serialized ArrowRecordBatch as a partition.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala
index 318cc629e7a34..60738e6d4ef9e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalog
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.annotation.Stable
+import org.apache.spark.annotation.{Evolving, Experimental, Stable}
 import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.storage.StorageLevel
@@ -208,6 +208,20 @@ abstract class Catalog {
    */
   def functionExists(dbName: String, functionName: String): Boolean
 
+  /**
+   * Creates a table from the given path and returns the corresponding DataFrame.
+   * It will use the default data source configured by spark.sql.sources.default.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(tableName: String, path: String): DataFrame = {
+    createTable(tableName, path)
+  }
+
   /**
    * Creates a table from the given path and returns the corresponding DataFrame.
    * It will use the default data source configured by spark.sql.sources.default.
@@ -219,6 +233,20 @@ abstract class Catalog {
    */
   def createTable(tableName: String, path: String): DataFrame
 
+  /**
+   * Creates a table from the given path based on a data source and returns the corresponding
+   * DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(tableName: String, path: String, source: String): DataFrame = {
+    createTable(tableName, path, source)
+  }
+
   /**
    * Creates a table from the given path based on a data source and returns the corresponding
    * DataFrame.
@@ -230,6 +258,23 @@ abstract class Catalog {
    */
   def createTable(tableName: String, path: String, source: String): DataFrame
 
+  /**
+   * Creates a table from the given path based on a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      options: java.util.Map[String, String]): DataFrame = {
+    createTable(tableName, source, options)
+  }
+
   /**
    * Creates a table based on the dataset in a data source and a set of options.
    * Then, returns the corresponding DataFrame.
@@ -246,6 +291,24 @@ abstract class Catalog {
     createTable(tableName, source, options.asScala.toMap)
   }
 
+  /**
+   * (Scala-specific)
+   * Creates a table from the given path based on a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      options: Map[String, String]): DataFrame = {
+    createTable(tableName, source, options)
+  }
+
   /**
    * (Scala-specific)
    * Creates a table based on the dataset in a data source and a set of options.
@@ -261,6 +324,24 @@ abstract class Catalog {
       source: String,
       options: Map[String, String]): DataFrame
 
+  /**
+   * Create a table from the given path based on a data source, a schema and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      options: java.util.Map[String, String]): DataFrame = {
+    createTable(tableName, source, schema, options)
+  }
+
   /**
    * Create a table based on the dataset in a data source, a schema and a set of options.
    * Then, returns the corresponding DataFrame.
@@ -278,6 +359,25 @@ abstract class Catalog {
     createTable(tableName, source, schema, options.asScala.toMap)
   }
 
+  /**
+   * (Scala-specific)
+   * Create a table from the given path based on a data source, a schema and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      options: Map[String, String]): DataFrame = {
+    createTable(tableName, source, schema, options)
+  }
+
   /**
    * (Scala-specific)
    * Create a table based on the dataset in a data source, a schema and a set of options.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 486e7f1f84b46..9a8d2f0a142d4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, CatalogUtils}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogPlugin, CatalogV2Util, Identifier, LookupCatalog, SupportsNamespaces, TableCatalog, TableChange, V1Table}
+import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogPlugin, CatalogV2Util, LookupCatalog, SupportsNamespaces, TableCatalog, TableChange, V1Table}
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.execution.command._
 import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, RefreshTable}
@@ -39,7 +39,8 @@ import org.apache.spark.sql.types.{HIVE_TYPE_STRING, HiveStringType, MetadataBui
 class ResolveSessionCatalog(
     val catalogManager: CatalogManager,
     conf: SQLConf,
-    isView: Seq[String] => Boolean)
+    isTempView: Seq[String] => Boolean,
+    isTempFunction: String => Boolean)
   extends Rule[LogicalPlan] with LookupCatalog {
   import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
   import org.apache.spark.sql.connector.catalog.CatalogV2Util._
@@ -49,6 +50,9 @@ class ResolveSessionCatalog(
          nameParts @ SessionCatalogAndTable(catalog, tbl), cols) =>
       loadTable(catalog, tbl.asIdentifier).collect {
         case v1Table: V1Table =>
+          if (!DDLUtils.isHiveTable(v1Table.v1Table)) {
+            cols.foreach(c => failCharType(c.dataType))
+          }
           cols.foreach { c =>
             assertTopLevelColumn(c.name, "AlterTableAddColumnsCommand")
             if (!c.nullable) {
@@ -58,6 +62,7 @@ class ResolveSessionCatalog(
           }
           AlterTableAddColumnsCommand(tbl.asTableIdentifier, cols.map(convertToStructField))
       }.getOrElse {
+        cols.foreach(c => failCharType(c.dataType))
         val changes = cols.map { col =>
           TableChange.addColumn(
             col.name.toArray,
@@ -73,14 +78,14 @@ class ResolveSessionCatalog(
          nameParts @ SessionCatalogAndTable(catalog, tbl), _, _, _, _, _) =>
       loadTable(catalog, tbl.asIdentifier).collect {
         case v1Table: V1Table =>
+          if (!DDLUtils.isHiveTable(v1Table.v1Table)) {
+            a.dataType.foreach(failCharType)
+          }
+
           if (a.column.length > 1) {
             throw new AnalysisException(
               "ALTER COLUMN with qualified column is only supported with v2 tables.")
           }
-          if (a.dataType.isEmpty) {
-            throw new AnalysisException(
-              "ALTER COLUMN with v1 tables must specify new data type.")
-          }
           if (a.nullable.isDefined) {
             throw new AnalysisException(
               "ALTER COLUMN with v1 tables cannot specify NOT NULL.")
@@ -92,18 +97,29 @@ class ResolveSessionCatalog(
           val builder = new MetadataBuilder
           // Add comment to metadata
           a.comment.map(c => builder.putString("comment", c))
+          val colName = a.column(0)
+          val dataType = a.dataType.getOrElse {
+            v1Table.schema.findNestedField(Seq(colName), resolver = conf.resolver)
+              .map(_._2.dataType)
+              .getOrElse {
+                throw new AnalysisException(
+                  s"ALTER COLUMN cannot find column ${quoteIfNeeded(colName)} in v1 table. " +
+                    s"Available: ${v1Table.schema.fieldNames.mkString(", ")}")
+              }
+          }
           // Add Hive type string to metadata.
-          val cleanedDataType = HiveStringType.replaceCharType(a.dataType.get)
-          if (a.dataType.get != cleanedDataType) {
-            builder.putString(HIVE_TYPE_STRING, a.dataType.get.catalogString)
+          val cleanedDataType = HiveStringType.replaceCharType(dataType)
+          if (dataType != cleanedDataType) {
+            builder.putString(HIVE_TYPE_STRING, dataType.catalogString)
           }
           val newColumn = StructField(
-            a.column(0),
+            colName,
             cleanedDataType,
             nullable = true,
             builder.build())
-          AlterTableChangeColumnCommand(tbl.asTableIdentifier, a.column(0), newColumn)
+          AlterTableChangeColumnCommand(tbl.asTableIdentifier, colName, newColumn)
       }.getOrElse {
+        a.dataType.foreach(failCharType)
         val colName = a.column.toArray
         val typeChange = a.dataType.map { newDataType =>
           TableChange.updateColumnType(colName, newDataType)
@@ -209,8 +225,9 @@ class ResolveSessionCatalog(
       }
       AlterDatabaseSetLocationCommand(ns.head, location)
 
-    case RenameTableStatement(SessionCatalogAndTable(_, oldName), newNameParts, isView) =>
-      AlterTableRenameCommand(oldName.asTableIdentifier, newNameParts.asTableIdentifier, isView)
+    // v1 RENAME TABLE supports temp view.
+    case RenameTableStatement(TempViewOrV1Table(oldName), newName, isView) =>
+      AlterTableRenameCommand(oldName.asTableIdentifier, newName.asTableIdentifier, isView)
 
     case DescribeRelation(ResolvedTable(_, ident, _: V1Table), partitionSpec, isExtended) =>
       DescribeTableCommand(ident.asTableIdentifier, partitionSpec, isExtended)
@@ -219,45 +236,42 @@ class ResolveSessionCatalog(
     case DescribeRelation(ResolvedView(ident), partitionSpec, isExtended) =>
       DescribeTableCommand(ident.asTableIdentifier, partitionSpec, isExtended)
 
-    case DescribeColumnStatement(
-        SessionCatalogAndTable(catalog, tbl), colNameParts, isExtended) =>
-      loadTable(catalog, tbl.asIdentifier).collect {
-        case v1Table: V1Table =>
-          DescribeColumnCommand(tbl.asTableIdentifier, colNameParts, isExtended)
-      }.getOrElse {
-        if (isView(tbl)) {
-          DescribeColumnCommand(tbl.asTableIdentifier, colNameParts, isExtended)
-        } else {
-          throw new AnalysisException("Describing columns is not supported for v2 tables.")
-        }
-      }
+    case DescribeColumnStatement(tbl, colNameParts, isExtended) =>
+      val name = parseTempViewOrV1Table(tbl, "Describing columns")
+      DescribeColumnCommand(name.asTableIdentifier, colNameParts, isExtended)
 
     // For CREATE TABLE [AS SELECT], we should use the v1 command if the catalog is resolved to the
     // session catalog and the table provider is not v2.
     case c @ CreateTableStatement(
          SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      if (!isV2Provider(c.provider)) {
+      val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+      if (!isV2Provider(provider)) {
+        if (!DDLUtils.isHiveTable(Some(provider))) {
+          assertNoCharTypeInSchema(c.tableSchema)
+        }
         val tableDesc = buildCatalogTable(tbl.asTableIdentifier, c.tableSchema,
-          c.partitioning, c.bucketSpec, c.properties, c.provider, c.options, c.location,
+          c.partitioning, c.bucketSpec, c.properties, provider, c.options, c.location,
           c.comment, c.ifNotExists)
         val mode = if (c.ifNotExists) SaveMode.Ignore else SaveMode.ErrorIfExists
         CreateTable(tableDesc, mode, None)
       } else {
+        assertNoCharTypeInSchema(c.tableSchema)
         CreateV2Table(
           catalog.asTableCatalog,
           tbl.asIdentifier,
           c.tableSchema,
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
-          convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
           ignoreIfExists = c.ifNotExists)
       }
 
     case c @ CreateTableAsSelectStatement(
-         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      if (!isV2Provider(c.provider)) {
+         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
+      val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+      if (!isV2Provider(provider)) {
         val tableDesc = buildCatalogTable(tbl.asTableIdentifier, new StructType,
-          c.partitioning, c.bucketSpec, c.properties, c.provider, c.options, c.location,
+          c.partitioning, c.bucketSpec, c.properties, provider, c.options, c.location,
           c.comment, c.ifNotExists)
         val mode = if (c.ifNotExists) SaveMode.Ignore else SaveMode.ErrorIfExists
         CreateTable(tableDesc, mode, Some(c.asSelect))
@@ -268,34 +282,38 @@ class ResolveSessionCatalog(
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
           c.asSelect,
-          convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
-          writeOptions = c.options,
+          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
+          writeOptions = c.writeOptions,
           ignoreIfExists = c.ifNotExists)
       }
 
-    case RefreshTableStatement(SessionCatalogAndTable(_, tbl)) =>
-      RefreshTable(tbl.asTableIdentifier)
+    // v1 REFRESH TABLE supports temp view.
+    case RefreshTableStatement(TempViewOrV1Table(name)) =>
+      RefreshTable(name.asTableIdentifier)
 
     // For REPLACE TABLE [AS SELECT], we should fail if the catalog is resolved to the
     // session catalog and the table provider is not v2.
     case c @ ReplaceTableStatement(
          SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      if (!isV2Provider(c.provider)) {
+      val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+      if (!isV2Provider(provider)) {
         throw new AnalysisException("REPLACE TABLE is only supported with v2 tables.")
       } else {
+        assertNoCharTypeInSchema(c.tableSchema)
         ReplaceTable(
           catalog.asTableCatalog,
           tbl.asIdentifier,
           c.tableSchema,
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
-          convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
           orCreate = c.orCreate)
       }
 
     case c @ ReplaceTableAsSelectStatement(
-         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      if (!isV2Provider(c.provider)) {
+         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
+      val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+      if (!isV2Provider(provider)) {
         throw new AnalysisException("REPLACE TABLE AS SELECT is only supported with v2 tables.")
       } else {
         ReplaceTableAsSelect(
@@ -304,16 +322,18 @@ class ResolveSessionCatalog(
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
           c.asSelect,
-          convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
-          writeOptions = c.options,
+          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
+          writeOptions = c.writeOptions,
           orCreate = c.orCreate)
       }
 
-    case DropTableStatement(SessionCatalogAndTable(catalog, tbl), ifExists, purge) =>
-      DropTableCommand(tbl.asTableIdentifier, ifExists, isView = false, purge = purge)
+    // v1 DROP TABLE supports temp view.
+    case DropTableStatement(TempViewOrV1Table(name), ifExists, purge) =>
+      DropTableCommand(name.asTableIdentifier, ifExists, isView = false, purge = purge)
 
-    case DropViewStatement(SessionCatalogAndTable(catalog, viewName), ifExists) =>
-      DropTableCommand(viewName.asTableIdentifier, ifExists, isView = true, purge = false)
+    // v1 DROP TABLE supports temp view.
+    case DropViewStatement(TempViewOrV1Table(name), ifExists) =>
+      DropTableCommand(name.asTableIdentifier, ifExists, isView = true, purge = false)
 
     case c @ CreateNamespaceStatement(CatalogAndNamespace(catalog, ns), _, _)
         if isSessionCatalog(catalog) =>
@@ -360,7 +380,7 @@ class ResolveSessionCatalog(
       }
 
     case AnalyzeColumnStatement(tbl, columnNames, allColumns) =>
-      val v1TableName = parseV1Table(tbl, "ANALYZE TABLE")
+      val v1TableName = parseTempViewOrV1Table(tbl, "ANALYZE TABLE")
       AnalyzeColumnCommand(v1TableName.asTableIdentifier, columnNames, allColumns)
 
     case RepairTableStatement(tbl) =>
@@ -379,20 +399,26 @@ class ResolveSessionCatalog(
         partition)
 
     case ShowCreateTableStatement(tbl, asSerde) if !asSerde =>
-      val v1TableName = parseV1Table(tbl, "SHOW CREATE TABLE")
-      ShowCreateTableCommand(v1TableName.asTableIdentifier)
+      val name = parseTempViewOrV1Table(tbl, "SHOW CREATE TABLE")
+      ShowCreateTableCommand(name.asTableIdentifier)
 
     case ShowCreateTableStatement(tbl, asSerde) if asSerde =>
       val v1TableName = parseV1Table(tbl, "SHOW CREATE TABLE AS SERDE")
       ShowCreateTableAsSerdeCommand(v1TableName.asTableIdentifier)
 
     case CacheTableStatement(tbl, plan, isLazy, options) =>
-      val v1TableName = parseV1Table(tbl, "CACHE TABLE")
-      CacheTableCommand(v1TableName.asTableIdentifier, plan, isLazy, options)
+      val name = if (plan.isDefined) {
+        // CACHE TABLE ... AS SELECT creates a temp view with the input query.
+        // Temp view doesn't belong to any catalog and we shouldn't resolve catalog in the name.
+        tbl
+      } else {
+        parseTempViewOrV1Table(tbl, "CACHE TABLE")
+      }
+      CacheTableCommand(name.asTableIdentifier, plan, isLazy, options)
 
     case UncacheTableStatement(tbl, ifExists) =>
-      val v1TableName = parseV1Table(tbl, "UNCACHE TABLE")
-      UncacheTableCommand(v1TableName.asTableIdentifier, ifExists)
+      val name = parseTempViewOrV1Table(tbl, "UNCACHE TABLE")
+      UncacheTableCommand(name.asTableIdentifier, ifExists)
 
     case TruncateTableStatement(tbl, partitionSpec) =>
       val v1TableName = parseV1Table(tbl, "TRUNCATE TABLE")
@@ -407,24 +433,27 @@ class ResolveSessionCatalog(
         partitionSpec)
 
     case ShowColumnsStatement(tbl, ns) =>
+      if (ns.isDefined && ns.get.length > 1) {
+        throw new AnalysisException(
+          s"Namespace name should have only one part if specified: ${ns.get.quoted}")
+      }
+      // Use namespace only if table name doesn't specify it. If namespace is already specified
+      // in the table name, it's checked against the given namespace below.
+      val nameParts = if (ns.isDefined && tbl.length == 1) {
+        ns.get ++ tbl
+      } else {
+        tbl
+      }
       val sql = "SHOW COLUMNS"
-      val v1TableName = parseV1Table(tbl, sql).asTableIdentifier
+      val v1TableName = parseTempViewOrV1Table(nameParts, sql).asTableIdentifier
       val resolver = conf.resolver
       val db = ns match {
-        case Some(db) if (v1TableName.database.exists(!resolver(_, db.head))) =>
+        case Some(db) if v1TableName.database.exists(!resolver(_, db.head)) =>
           throw new AnalysisException(
             s"SHOW COLUMNS with conflicting databases: " +
               s"'${db.head}' != '${v1TableName.database.get}'")
         case _ => ns.map(_.head)
       }
-      if (ns.isDefined && ns.get.length > 1) {
-        throw new AnalysisException(
-          s"Namespace name should have only one part if specified: ${ns.get.quoted}")
-      }
-      if (tbl.length > 2) {
-        throw new AnalysisException(
-          s"Table name should have at most two parts: ${tbl.quoted}")
-      }
       ShowColumnsCommand(db, v1TableName)
 
     case AlterTableRecoverPartitionsStatement(tbl) =>
@@ -464,10 +493,10 @@ class ResolveSessionCatalog(
         serdeProperties,
         partitionSpec)
 
-    case AlterViewAsStatement(tbl, originalText, query) =>
-      val v1TableName = parseV1Table(tbl, "ALTER VIEW QUERY")
+    case AlterViewAsStatement(name, originalText, query) =>
+      val viewName = parseTempViewOrV1Table(name, "ALTER VIEW QUERY")
       AlterViewAsCommand(
-        v1TableName.asTableIdentifier,
+        viewName.asTableIdentifier,
         originalText,
         query)
 
@@ -475,7 +504,12 @@ class ResolveSessionCatalog(
       tbl, userSpecifiedColumns, comment, properties,
       originalText, child, allowExisting, replace, viewType) =>
 
-      val v1TableName = parseV1Table(tbl, "CREATE VIEW")
+      val v1TableName = if (viewType != PersistedView) {
+        // temp view doesn't belong to any catalog and we shouldn't resolve catalog in the name.
+        tbl
+      } else {
+        parseV1Table(tbl, "CREATE VIEW")
+      }
       CreateViewCommand(
         v1TableName.asTableIdentifier,
         userSpecifiedColumns,
@@ -487,59 +521,110 @@ class ResolveSessionCatalog(
         replace,
         viewType)
 
-    case ShowTableProperties(r: ResolvedTable, propertyKey) if isSessionCatalog(r.catalog) =>
+    case ShowViews(resolved: ResolvedNamespace, pattern) =>
+      resolved match {
+        case SessionCatalogAndNamespace(_, ns) =>
+          // Fallback to v1 ShowViewsCommand since there is no view API in v2 catalog
+          assert(ns.nonEmpty)
+          if (ns.length != 1) {
+            throw new AnalysisException(s"The database name is not valid: ${ns.quoted}")
+          }
+          ShowViewsCommand(ns.head, pattern)
+        case _ =>
+          throw new AnalysisException(s"Catalog ${resolved.catalog.name} doesn't support " +
+            "SHOW VIEWS, only SessionCatalog supports this command.")
+      }
+
+    case ShowTableProperties(
+        r @ ResolvedTable(_, _, _: V1Table), propertyKey) if isSessionCatalog(r.catalog) =>
       ShowTablePropertiesCommand(r.identifier.asTableIdentifier, propertyKey)
 
-    case DescribeFunctionStatement(CatalogAndIdentifier(catalog, ident), extended) =>
+    case ShowTableProperties(r: ResolvedView, propertyKey) =>
+      ShowTablePropertiesCommand(r.identifier.asTableIdentifier, propertyKey)
+
+    case DescribeFunctionStatement(nameParts, extended) =>
       val functionIdent =
-        parseSessionCatalogFunctionIdentifier("DESCRIBE FUNCTION", catalog, ident)
+        parseSessionCatalogFunctionIdentifier(nameParts, "DESCRIBE FUNCTION")
       DescribeFunctionCommand(functionIdent, extended)
 
     case ShowFunctionsStatement(userScope, systemScope, pattern, fun) =>
       val (database, function) = fun match {
-        case Some(CatalogAndIdentifier(catalog, ident)) =>
+        case Some(nameParts) =>
           val FunctionIdentifier(fn, db) =
-            parseSessionCatalogFunctionIdentifier("SHOW FUNCTIONS", catalog, ident)
+            parseSessionCatalogFunctionIdentifier(nameParts, "SHOW FUNCTIONS")
           (db, Some(fn))
         case None => (None, pattern)
       }
       ShowFunctionsCommand(database, function, userScope, systemScope)
 
-    case DropFunctionStatement(CatalogAndIdentifier(catalog, ident), ifExists, isTemp) =>
+    case DropFunctionStatement(nameParts, ifExists, isTemp) =>
       val FunctionIdentifier(function, database) =
-        parseSessionCatalogFunctionIdentifier("DROP FUNCTION", catalog, ident)
+        parseSessionCatalogFunctionIdentifier(nameParts, "DROP FUNCTION")
       DropFunctionCommand(database, function, ifExists, isTemp)
 
-    case CreateFunctionStatement(CatalogAndIdentifier(catalog, ident),
+    case CreateFunctionStatement(nameParts,
       className, resources, isTemp, ignoreIfExists, replace) =>
-      val FunctionIdentifier(function, database) =
-        parseSessionCatalogFunctionIdentifier("CREATE FUNCTION", catalog, ident)
-      CreateFunctionCommand(database, function, className, resources, isTemp, ignoreIfExists,
-        replace)
+      if (isTemp) {
+        // temp func doesn't belong to any catalog and we shouldn't resolve catalog in the name.
+        val database = if (nameParts.length > 2) {
+          throw new AnalysisException(s"Unsupported function name '${nameParts.quoted}'")
+        } else if (nameParts.length == 2) {
+          Some(nameParts.head)
+        } else {
+          None
+        }
+        CreateFunctionCommand(
+          database,
+          nameParts.last,
+          className,
+          resources,
+          isTemp,
+          ignoreIfExists,
+          replace)
+      } else {
+        val FunctionIdentifier(function, database) =
+          parseSessionCatalogFunctionIdentifier(nameParts, "CREATE FUNCTION")
+        CreateFunctionCommand(database, function, className, resources, isTemp, ignoreIfExists,
+          replace)
+      }
   }
 
+  // TODO: move function related v2 statements to the new framework.
   private def parseSessionCatalogFunctionIdentifier(
-      sql: String,
-      catalog: CatalogPlugin,
-      functionIdent: Identifier): FunctionIdentifier = {
-    if (isSessionCatalog(catalog)) {
-      functionIdent.asMultipartIdentifier match {
-        case Seq(db, fn) => FunctionIdentifier(fn, Some(db))
-        case Seq(fn) => FunctionIdentifier(fn, None)
-        case _ =>
-          throw new AnalysisException(s"Unsupported function name '${functionIdent.quoted}'")
-      }
-    } else {
-      throw new AnalysisException(s"$sql is only supported in v1 catalog")
+      nameParts: Seq[String],
+      sql: String): FunctionIdentifier = {
+    if (nameParts.length == 1 && isTempFunction(nameParts.head)) {
+      return FunctionIdentifier(nameParts.head)
     }
-  }
 
-  private def parseV1Table(tableName: Seq[String], sql: String): Seq[String] = {
-    val CatalogAndIdentifier(catalog, ident) = tableName
-    if (!isSessionCatalog(catalog)) {
-      throw new AnalysisException(s"$sql is only supported with v1 tables.")
+    nameParts match {
+      case SessionCatalogAndIdentifier(_, ident) =>
+        if (nameParts.length == 1) {
+          // If there is only one name part, it means the current catalog is the session catalog.
+          // Here we don't fill the default database, to keep the error message unchanged for
+          // v1 commands.
+          FunctionIdentifier(nameParts.head, None)
+        } else {
+          ident.namespace match {
+            case Array(db) => FunctionIdentifier(ident.name, Some(db))
+            case _ =>
+              throw new AnalysisException(s"Unsupported function name '$ident'")
+          }
+        }
+
+      case _ => throw new AnalysisException(s"$sql is only supported in v1 catalog")
     }
-    ident.asMultipartIdentifier
+  }
+
+  private def parseV1Table(tableName: Seq[String], sql: String): Seq[String] = tableName match {
+    case SessionCatalogAndTable(_, tbl) => tbl
+    case _ => throw new AnalysisException(s"$sql is only supported with v1 tables.")
+  }
+
+  private def parseTempViewOrV1Table(
+      nameParts: Seq[String], sql: String): Seq[String] = nameParts match {
+    case TempViewOrV1Table(name) => name
+    case _ => throw new AnalysisException(s"$sql is only supported with temp views or v1 tables.")
   }
 
   private def buildCatalogTable(
@@ -583,6 +668,14 @@ class ResolveSessionCatalog(
     }
   }
 
+  object TempViewOrV1Table {
+    def unapply(nameParts: Seq[String]): Option[Seq[String]] = nameParts match {
+      case _ if isTempView(nameParts) => Some(nameParts)
+      case SessionCatalogAndIdentifier(_, tbl) => Some(tbl.asMultipartIdentifier)
+      case _ => None
+    }
+  }
+
   object SessionCatalogAndNamespace {
     def unapply(resolved: ResolvedNamespace): Option[(CatalogPlugin, Seq[String])] =
       if (isSessionCatalog(resolved.catalog)) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
index 413bd7b29cf45..cee8585e387a6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.{Dataset, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, SubqueryExpression}
 import org.apache.spark.sql.catalyst.optimizer.EliminateResolvedHint
 import org.apache.spark.sql.catalyst.plans.logical.{IgnoreCachedData, LogicalPlan, ResolvedHint}
+import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.columnar.InMemoryRelation
 import org.apache.spark.sql.execution.command.CommandUtils
 import org.apache.spark.sql.execution.datasources.{FileIndex, HadoopFsRelation, LogicalRelation}
@@ -45,7 +46,7 @@ case class CachedData(plan: LogicalPlan, cachedRepresentation: InMemoryRelation)
  *
  * Internal to Spark SQL.
  */
-class CacheManager extends Logging {
+class CacheManager extends Logging with AdaptiveSparkPlanHelper {
 
   /**
    * Maintains the list of cached plans as an immutable sequence.  Any updates to the list
@@ -79,20 +80,16 @@ class CacheManager extends Logging {
     if (lookupCachedData(planToCache).nonEmpty) {
       logWarning("Asked to cache already cached data.")
     } else {
-      val sparkSession = query.sparkSession
-      val qe = sparkSession.sessionState.executePlan(planToCache)
-      val originalValue = sparkSession.sessionState.conf.getConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED)
-      val inMemoryRelation = try {
-        // Avoiding changing the output partitioning, here disable AQE.
-        sparkSession.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
+      // Turn off AQE so that the outputPartitioning of the underlying plan can be leveraged.
+      val sessionWithAqeOff = getOrCloneSessionWithAqeOff(query.sparkSession)
+      val inMemoryRelation = sessionWithAqeOff.withActive {
+        val qe = sessionWithAqeOff.sessionState.executePlan(planToCache)
         InMemoryRelation(
-          sparkSession.sessionState.conf.useCompression,
-          sparkSession.sessionState.conf.columnBatchSize, storageLevel,
+          sessionWithAqeOff.sessionState.conf.useCompression,
+          sessionWithAqeOff.sessionState.conf.columnBatchSize, storageLevel,
           qe.executedPlan,
           tableName,
           optimizedPlan = qe.optimizedPlan)
-      } finally {
-        sparkSession.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, originalValue)
       }
 
       this.synchronized {
@@ -192,12 +189,16 @@ class CacheManager extends Logging {
       // Remove the cache entry before creating a new ones.
       cachedData = cachedData.filterNot(cd => needToRecache.exists(_ eq cd))
     }
-    needToRecache.map { cd =>
+    needToRecache.foreach { cd =>
       cd.cachedRepresentation.cacheBuilder.clearCache()
-      val qe = spark.sessionState.executePlan(cd.plan)
-      val newCache = InMemoryRelation(
-        cacheBuilder = cd.cachedRepresentation.cacheBuilder.copy(cachedPlan = qe.executedPlan),
-        optimizedPlan = qe.optimizedPlan)
+      // Turn off AQE so that the outputPartitioning of the underlying plan can be leveraged.
+      val sessionWithAqeOff = getOrCloneSessionWithAqeOff(spark)
+      val newCache = sessionWithAqeOff.withActive {
+        val qe = sessionWithAqeOff.sessionState.executePlan(cd.plan)
+        InMemoryRelation(
+          cacheBuilder = cd.cachedRepresentation.cacheBuilder.copy(cachedPlan = qe.executedPlan),
+          optimizedPlan = qe.optimizedPlan)
+      }
       val recomputedPlan = cd.copy(cachedRepresentation = newCache)
       this.synchronized {
         if (lookupCachedData(recomputedPlan.plan).nonEmpty) {
@@ -247,12 +248,17 @@ class CacheManager extends Logging {
    * `HadoopFsRelation` node(s) as part of its logical plan.
    */
   def recacheByPath(spark: SparkSession, resourcePath: String): Unit = {
-    val (fs, qualifiedPath) = {
-      val path = new Path(resourcePath)
-      val fs = path.getFileSystem(spark.sessionState.newHadoopConf())
-      (fs, fs.makeQualified(path))
-    }
+    val path = new Path(resourcePath)
+    val fs = path.getFileSystem(spark.sessionState.newHadoopConf())
+    recacheByPath(spark, path, fs)
+  }
 
+  /**
+   * Tries to re-cache all the cache entries that contain `resourcePath` in one or more
+   * `HadoopFsRelation` node(s) as part of its logical plan.
+   */
+  def recacheByPath(spark: SparkSession, resourcePath: Path, fs: FileSystem): Unit = {
+    val qualifiedPath = fs.makeQualified(resourcePath)
     recacheByCondition(spark, _.plan.find(lookupAndRefresh(_, fs, qualifiedPath)).isDefined)
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala
index e482bc9941ea9..e1b9c8f430c56 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala
@@ -87,7 +87,7 @@ object CollectMetricsExec {
    * Recursively collect all collected metrics from a query tree.
    */
   def collect(plan: SparkPlan): Map[String, Row] = {
-    val metrics = plan.collectInPlanAndSubqueries {
+    val metrics = plan.collectWithSubqueries {
       case collector: CollectMetricsExec => collector.name -> collector.collectedMetrics
     }
     metrics.toMap
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 0d759085a7e2c..e2bf132a2e18f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -75,10 +75,10 @@ trait DataSourceScanExec extends LeafExecNode {
     }
 
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Output: ${producedAttributes.mkString("[", ", ", "]")}
+       |$formattedNodeName
+       |${ExplainUtils.generateFieldString("Output", output)}
        |${metadataStr.mkString("\n")}
-     """.stripMargin
+       |""".stripMargin
   }
 
   /**
@@ -175,7 +175,7 @@ case class FileSourceScanExec(
 
   private lazy val needsUnsafeRowConversion: Boolean = {
     if (relation.fileFormat.isInstanceOf[ParquetSource]) {
-      SparkSession.getActiveSession.get.sessionState.conf.parquetVectorizedReaderEnabled
+      sqlContext.conf.parquetVectorizedReaderEnabled
     } else {
       false
     }
@@ -209,9 +209,6 @@ case class FileSourceScanExec(
     val ret =
       relation.location.listFiles(
         partitionFilters.filterNot(isDynamicPruningFilter), dataFilters)
-    if (relation.partitionSchemaOption.isDefined) {
-      driverMetrics("numPartitions") = ret.length
-    }
     setFilesNumAndSizeMetric(ret, true)
     val timeTakenMs = NANOSECONDS.toMillis(
       (System.nanoTime() - startTime) + optimizerMetadataTimeNs)
@@ -253,79 +250,83 @@ case class FileSourceScanExec(
     partitionFilters.exists(ExecSubqueryExpression.hasSubquery)
   }
 
-  override lazy val (outputPartitioning, outputOrdering): (Partitioning, Seq[SortOrder]) = {
-    val bucketSpec = if (relation.sparkSession.sessionState.conf.bucketingEnabled) {
-      relation.bucketSpec
+  private def toAttribute(colName: String): Option[Attribute] =
+    output.find(_.name == colName)
+
+  // exposed for testing
+  lazy val bucketedScan: Boolean = {
+    if (relation.sparkSession.sessionState.conf.bucketingEnabled && relation.bucketSpec.isDefined) {
+      val spec = relation.bucketSpec.get
+      val bucketColumns = spec.bucketColumnNames.flatMap(n => toAttribute(n))
+      bucketColumns.size == spec.bucketColumnNames.size
     } else {
-      None
+      false
     }
-    bucketSpec match {
-      case Some(spec) =>
-        // For bucketed columns:
-        // -----------------------
-        // `HashPartitioning` would be used only when:
-        // 1. ALL the bucketing columns are being read from the table
-        //
-        // For sorted columns:
-        // ---------------------
-        // Sort ordering should be used when ALL these criteria's match:
-        // 1. `HashPartitioning` is being used
-        // 2. A prefix (or all) of the sort columns are being read from the table.
-        //
-        // Sort ordering would be over the prefix subset of `sort columns` being read
-        // from the table.
-        // eg.
-        // Assume (col0, col2, col3) are the columns read from the table
-        // If sort columns are (col0, col1), then sort ordering would be considered as (col0)
-        // If sort columns are (col1, col0), then sort ordering would be empty as per rule #2
-        // above
-
-        def toAttribute(colName: String): Option[Attribute] =
-          output.find(_.name == colName)
-
-        val bucketColumns = spec.bucketColumnNames.flatMap(n => toAttribute(n))
-        if (bucketColumns.size == spec.bucketColumnNames.size) {
-          val partitioning = HashPartitioning(bucketColumns, spec.numBuckets)
-          val sortColumns =
-            spec.sortColumnNames.map(x => toAttribute(x)).takeWhile(x => x.isDefined).map(_.get)
-          val shouldCalculateSortOrder =
-            conf.getConf(SQLConf.LEGACY_BUCKETED_TABLE_SCAN_OUTPUT_ORDERING) &&
-              sortColumns.nonEmpty &&
-              !hasPartitionsAvailableAtRunTime
-
-          val sortOrder = if (shouldCalculateSortOrder) {
-            // In case of bucketing, its possible to have multiple files belonging to the
-            // same bucket in a given relation. Each of these files are locally sorted
-            // but those files combined together are not globally sorted. Given that,
-            // the RDD partition will not be sorted even if the relation has sort columns set
-            // Current solution is to check if all the buckets have a single file in it
-
-            val files = selectedPartitions.flatMap(partition => partition.files)
-            val bucketToFilesGrouping =
-              files.map(_.getPath.getName).groupBy(file => BucketingUtils.getBucketId(file))
-            val singleFilePartitions = bucketToFilesGrouping.forall(p => p._2.length <= 1)
-
-            if (singleFilePartitions) {
-              // TODO Currently Spark does not support writing columns sorting in descending order
-              // so using Ascending order. This can be fixed in future
-              sortColumns.map(attribute => SortOrder(attribute, Ascending))
-            } else {
-              Nil
-            }
-          } else {
-            Nil
-          }
-          (partitioning, sortOrder)
+  }
+
+  override lazy val (outputPartitioning, outputOrdering): (Partitioning, Seq[SortOrder]) = {
+    if (bucketedScan) {
+      // For bucketed columns:
+      // -----------------------
+      // `HashPartitioning` would be used only when:
+      // 1. ALL the bucketing columns are being read from the table
+      //
+      // For sorted columns:
+      // ---------------------
+      // Sort ordering should be used when ALL these criteria's match:
+      // 1. `HashPartitioning` is being used
+      // 2. A prefix (or all) of the sort columns are being read from the table.
+      //
+      // Sort ordering would be over the prefix subset of `sort columns` being read
+      // from the table.
+      // eg.
+      // Assume (col0, col2, col3) are the columns read from the table
+      // If sort columns are (col0, col1), then sort ordering would be considered as (col0)
+      // If sort columns are (col1, col0), then sort ordering would be empty as per rule #2
+      // above
+      val spec = relation.bucketSpec.get
+      val bucketColumns = spec.bucketColumnNames.flatMap(n => toAttribute(n))
+      val partitioning = HashPartitioning(bucketColumns, spec.numBuckets)
+      val sortColumns =
+        spec.sortColumnNames.map(x => toAttribute(x)).takeWhile(x => x.isDefined).map(_.get)
+      val shouldCalculateSortOrder =
+        conf.getConf(SQLConf.LEGACY_BUCKETED_TABLE_SCAN_OUTPUT_ORDERING) &&
+          sortColumns.nonEmpty &&
+          !hasPartitionsAvailableAtRunTime
+
+      val sortOrder = if (shouldCalculateSortOrder) {
+        // In case of bucketing, its possible to have multiple files belonging to the
+        // same bucket in a given relation. Each of these files are locally sorted
+        // but those files combined together are not globally sorted. Given that,
+        // the RDD partition will not be sorted even if the relation has sort columns set
+        // Current solution is to check if all the buckets have a single file in it
+
+        val files = selectedPartitions.flatMap(partition => partition.files)
+        val bucketToFilesGrouping =
+          files.map(_.getPath.getName).groupBy(file => BucketingUtils.getBucketId(file))
+        val singleFilePartitions = bucketToFilesGrouping.forall(p => p._2.length <= 1)
+
+        if (singleFilePartitions) {
+          // TODO Currently Spark does not support writing columns sorting in descending order
+          // so using Ascending order. This can be fixed in future
+          sortColumns.map(attribute => SortOrder(attribute, Ascending))
         } else {
-          (UnknownPartitioning(0), Nil)
+          Nil
         }
-      case _ =>
-        (UnknownPartitioning(0), Nil)
+      } else {
+        Nil
+      }
+      (partitioning, sortOrder)
+    } else {
+      (UnknownPartitioning(0), Nil)
     }
   }
 
   @transient
-  private lazy val pushedDownFilters = dataFilters.flatMap(DataSourceStrategy.translateFilter)
+  private lazy val pushedDownFilters = {
+    val supportNestedPredicatePushdown = DataSourceUtils.supportNestedPredicatePushdown(relation)
+    dataFilters.flatMap(DataSourceStrategy.translateFilter(_, supportNestedPredicatePushdown))
+  }
 
   override lazy val metadata: Map[String, String] = {
     def seqToString(seq: Seq[Any]) = seq.mkString("[", ", ", "]")
@@ -376,10 +377,10 @@ case class FileSourceScanExec(
     }
 
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Output: ${producedAttributes.mkString("[", ", ", "]")}
+       |$formattedNodeName
+       |${ExplainUtils.generateFieldString("Output", output)}
        |${metadataStr.mkString("\n")}
-     """.stripMargin
+       |""".stripMargin
   }
 
   lazy val inputRDD: RDD[InternalRow] = {
@@ -393,11 +394,11 @@ case class FileSourceScanExec(
         options = relation.options,
         hadoopConf = relation.sparkSession.sessionState.newHadoopConfWithOptions(relation.options))
 
-    val readRDD = relation.bucketSpec match {
-      case Some(bucketing) if relation.sparkSession.sessionState.conf.bucketingEnabled =>
-        createBucketedReadRDD(bucketing, readFile, dynamicallySelectedPartitions, relation)
-      case _ =>
-        createNonBucketedReadRDD(readFile, dynamicallySelectedPartitions, relation)
+    val readRDD = if (bucketedScan) {
+      createBucketedReadRDD(relation.bucketSpec.get, readFile, dynamicallySelectedPartitions,
+        relation)
+    } else {
+      createNonBucketedReadRDD(readFile, dynamicallySelectedPartitions, relation)
     }
     sendDriverMetrics()
     readRDD
@@ -408,7 +409,7 @@ case class FileSourceScanExec(
   }
 
   /** SQL metrics generated only for scans using dynamic partition pruning. */
-  private lazy val staticMetrics = if (partitionFilters.filter(isDynamicPruningFilter).nonEmpty) {
+  private lazy val staticMetrics = if (partitionFilters.exists(isDynamicPruningFilter)) {
     Map("staticFilesNum" -> SQLMetrics.createMetric(sparkContext, "static number of files read"),
       "staticFilesSize" -> SQLMetrics.createSizeMetric(sparkContext, "static size of files read"))
   } else {
@@ -421,22 +422,23 @@ case class FileSourceScanExec(
       static: Boolean): Unit = {
     val filesNum = partitions.map(_.files.size.toLong).sum
     val filesSize = partitions.map(_.files.map(_.getLen).sum).sum
-    if (!static || partitionFilters.filter(isDynamicPruningFilter).isEmpty) {
+    if (!static || !partitionFilters.exists(isDynamicPruningFilter)) {
       driverMetrics("numFiles") = filesNum
       driverMetrics("filesSize") = filesSize
     } else {
       driverMetrics("staticFilesNum") = filesNum
       driverMetrics("staticFilesSize") = filesSize
     }
+    if (relation.partitionSchemaOption.isDefined) {
+      driverMetrics("numPartitions") = partitions.length
+    }
   }
 
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
     "numFiles" -> SQLMetrics.createMetric(sparkContext, "number of files read"),
     "metadataTime" -> SQLMetrics.createTimingMetric(sparkContext, "metadata time"),
-    "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read"),
-    "pruningTime" ->
-      SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time")
+    "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read")
   ) ++ {
     // Tracking scan time has overhead, we can't afford to do it for each row, and can only do
     // it for each batch.
@@ -447,9 +449,12 @@ case class FileSourceScanExec(
     }
   } ++ {
     if (relation.partitionSchemaOption.isDefined) {
-      Some("numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"))
+      Map(
+        "numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"),
+        "pruningTime" ->
+          SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time"))
     } else {
-      None
+      Map.empty[String, SQLMetric]
     }
   } ++ staticMetrics
 
@@ -585,12 +590,20 @@ case class FileSourceScanExec(
     new FileScanRDD(fsRelation.sparkSession, readFile, partitions)
   }
 
+  // Filters unused DynamicPruningExpression expressions - one which has been replaced
+  // with DynamicPruningExpression(Literal.TrueLiteral) during Physical Planning
+  private def filterUnusedDynamicPruningExpressions(
+      predicates: Seq[Expression]): Seq[Expression] = {
+    predicates.filterNot(_ == DynamicPruningExpression(Literal.TrueLiteral))
+  }
+
   override def doCanonicalize(): FileSourceScanExec = {
     FileSourceScanExec(
       relation,
       output.map(QueryPlan.normalizeExpressions(_, output)),
       requiredSchema,
-      QueryPlan.normalizePredicates(partitionFilters, output),
+      QueryPlan.normalizePredicates(
+        filterUnusedDynamicPruningExpressions(partitionFilters), output),
       optionalBucketSet,
       QueryPlan.normalizePredicates(dataFilters, output),
       None)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
index d4fe272f8c95f..b54bd6a579b66 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
@@ -23,9 +23,9 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.{Expression, PlanExpression}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
-import org.apache.spark.sql.catalyst.trees.TreeNodeTag
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, QueryStageExec}
 
-object ExplainUtils {
+object ExplainUtils extends AdaptiveSparkPlanHelper {
   /**
    * Given a input physical plan, performs the following tasks.
    *   1. Computes the operator id for current operator and records it in the operaror
@@ -145,15 +145,26 @@ object ExplainUtils {
       case p: WholeStageCodegenExec =>
       case p: InputAdapter =>
       case other: QueryPlan[_] =>
-        if (!other.getTagValue(QueryPlan.OP_ID_TAG).isDefined) {
+
+        def setOpId(): Unit = if (other.getTagValue(QueryPlan.OP_ID_TAG).isEmpty) {
           currentOperationID += 1
           other.setTagValue(QueryPlan.OP_ID_TAG, currentOperationID)
           operatorIDs += ((currentOperationID, other))
         }
-        other.innerChildren.foreach { plan =>
-          currentOperationID = generateOperatorIDs(plan,
-            currentOperationID,
-            operatorIDs)
+
+        other match {
+          case p: AdaptiveSparkPlanExec =>
+            currentOperationID =
+              generateOperatorIDs(p.executedPlan, currentOperationID, operatorIDs)
+            setOpId()
+          case p: QueryStageExec =>
+            currentOperationID = generateOperatorIDs(p.plan, currentOperationID, operatorIDs)
+            setOpId()
+          case _ =>
+            setOpId()
+            other.innerChildren.foldLeft(currentOperationID) {
+              (curId, plan) => generateOperatorIDs(plan, curId, operatorIDs)
+            }
         }
     }
     currentOperationID
@@ -164,24 +175,39 @@ object ExplainUtils {
    * whole stage code gen id in the plan via setting a tag.
    */
   private def generateWholeStageCodegenIds(plan: QueryPlan[_]): Unit = {
+    var currentCodegenId = -1
+
+    def setCodegenId(p: QueryPlan[_], children: Seq[QueryPlan[_]]): Unit = {
+      if (currentCodegenId != -1) {
+        p.setTagValue(QueryPlan.CODEGEN_ID_TAG, currentCodegenId)
+      }
+      children.foreach(generateWholeStageCodegenIds)
+    }
+
     // Skip the subqueries as they are not printed as part of main query block.
     if (plan.isInstanceOf[BaseSubqueryExec]) {
       return
     }
-    var currentCodegenId = -1
     plan.foreach {
       case p: WholeStageCodegenExec => currentCodegenId = p.codegenStageId
-      case p: InputAdapter => currentCodegenId = -1
-      case other: QueryPlan[_] =>
-        if (currentCodegenId != -1) {
-          other.setTagValue(QueryPlan.CODEGEN_ID_TAG, currentCodegenId)
-        }
-        other.innerChildren.foreach { plan =>
-          generateWholeStageCodegenIds(plan)
-        }
+      case _: InputAdapter => currentCodegenId = -1
+      case p: AdaptiveSparkPlanExec => setCodegenId(p, Seq(p.executedPlan))
+      case p: QueryStageExec => setCodegenId(p, Seq(p.plan))
+      case other: QueryPlan[_] => setCodegenId(other, other.innerChildren)
     }
   }
 
+  /**
+   * Generate detailed field string with different format based on type of input value
+   */
+  def generateFieldString(fieldName: String, values: Any): String = values match {
+    case iter: Iterable[_] if (iter.size == 0) => s"${fieldName}: []"
+    case iter: Iterable[_] => s"${fieldName} [${iter.size}]: ${iter.mkString("[", ", ", "]")}"
+    case str: String if (str == null || str.isEmpty) => s"${fieldName}: None"
+    case str: String => s"${fieldName}: ${str}"
+    case _ => throw new IllegalArgumentException(s"Unsupported type for argument values: $values")
+  }
+
   /**
    * Given a input plan, returns an array of tuples comprising of :
    *  1. Hosting opeator id.
@@ -207,28 +233,23 @@ object ExplainUtils {
 
   /**
    * Returns the operator identifier for the supplied plan by retrieving the
-   * `operationId` tag value.`
+   * `operationId` tag value.
    */
   def getOpId(plan: QueryPlan[_]): String = {
     plan.getTagValue(QueryPlan.OP_ID_TAG).map(v => s"$v").getOrElse("unknown")
   }
 
-  /**
-   * Returns the operator identifier for the supplied plan by retrieving the
-   * `codegenId` tag value.`
-   */
-  def getCodegenId(plan: QueryPlan[_]): String = {
-    plan.getTagValue(QueryPlan.CODEGEN_ID_TAG).map(v => s"[codegen id : $v]").getOrElse("")
-  }
-
   def removeTags(plan: QueryPlan[_]): Unit = {
+    def remove(p: QueryPlan[_], children: Seq[QueryPlan[_]]): Unit = {
+      p.unsetTagValue(QueryPlan.OP_ID_TAG)
+      p.unsetTagValue(QueryPlan.CODEGEN_ID_TAG)
+      children.foreach(removeTags)
+    }
+
     plan foreach {
-      case plan: QueryPlan[_] =>
-        plan.unsetTagValue(QueryPlan.OP_ID_TAG)
-        plan.unsetTagValue(QueryPlan.CODEGEN_ID_TAG)
-        plan.innerChildren.foreach { p =>
-          removeTags(p)
-        }
+      case p: AdaptiveSparkPlanExec => remove(p, Seq(p.executedPlan))
+      case p: QueryStageExec => remove(p, Seq(p.plan))
+      case plan: QueryPlan[_] => remove(plan, plan.innerChildren)
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index bbe47a63f4d61..9f99bf5011569 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -19,10 +19,12 @@ package org.apache.spark.sql.execution
 
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate, ZoneOffset}
 
 import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
-import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand}
+import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, LegacyDateFormats, TimestampFormatter}
+import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand, ShowViewsCommand}
+import org.apache.spark.sql.execution.datasources.v2.{DescribeTableExec, ShowTablesExec}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
@@ -37,18 +39,21 @@ object HiveResult {
    */
   def hiveResultString(executedPlan: SparkPlan): Seq[String] = executedPlan match {
     case ExecutedCommandExec(_: DescribeCommandBase) =>
-      // If it is a describe command for a Hive table, we want to have the output format
-      // be similar with Hive.
-      executedPlan.executeCollectPublic().map {
-        case Row(name: String, dataType: String, comment) =>
-          Seq(name, dataType,
-            Option(comment.asInstanceOf[String]).getOrElse(""))
-            .map(s => String.format(s"%-20s", s))
-            .mkString("\t")
-      }
-    // SHOW TABLES in Hive only output table names, while ours output database, table name, isTemp.
+      formatDescribeTableOutput(executedPlan.executeCollectPublic())
+    case _: DescribeTableExec =>
+      formatDescribeTableOutput(executedPlan.executeCollectPublic())
+    // SHOW TABLES in Hive only output table names while our v1 command outputs
+    // database, table name, isTemp.
     case command @ ExecutedCommandExec(s: ShowTablesCommand) if !s.isExtended =>
       command.executeCollect().map(_.getString(1))
+    // SHOW TABLES in Hive only output table names while our v2 command outputs
+    // namespace and table name.
+    case command : ShowTablesExec =>
+      command.executeCollect().map(_.getString(1))
+    // SHOW VIEWS in Hive only outputs view names while our v1 command outputs
+    // namespace, viewName, and isTemporary.
+    case command @ ExecutedCommandExec(_: ShowViewsCommand) =>
+      command.executeCollect().map(_.getString(1))
     case other =>
       val result: Seq[Seq[Any]] = other.executeCollectPublic().map(_.toSeq).toSeq
       // We need the types so we can output struct field names
@@ -58,17 +63,42 @@ object HiveResult {
         .map(_.mkString("\t"))
   }
 
-  private lazy val zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)
-  private lazy val dateFormatter = DateFormatter(zoneId)
-  private lazy val timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId)
+  private def formatDescribeTableOutput(rows: Array[Row]): Seq[String] = {
+    rows.map {
+      case Row(name: String, dataType: String, comment) =>
+        Seq(name, dataType, Option(comment.asInstanceOf[String]).getOrElse(""))
+          .map(s => String.format(s"%-20s", s))
+          .mkString("\t")
+    }
+  }
+
+  // We can create the date formatter only once because it does not depend on Spark's
+  // session time zone controlled by the SQL config `spark.sql.session.timeZone`.
+  // The `zoneId` parameter is used only in parsing of special date values like `now`,
+  // `yesterday` and etc. but not in date formatting. While formatting of:
+  // - `java.time.LocalDate`, zone id is not used by `DateTimeFormatter` at all.
+  // - `java.sql.Date`, the date formatter delegates formatting to the legacy formatter
+  //   which uses the default system time zone `TimeZone.getDefault`. This works correctly
+  //   due to `DateTimeUtils.toJavaDate` which is based on the system time zone too.
+  private val dateFormatter = DateFormatter(
+    format = DateFormatter.defaultPattern,
+    // We can set any time zone id. UTC was taken for simplicity.
+    zoneId = ZoneOffset.UTC,
+    locale = DateFormatter.defaultLocale,
+    // Use `FastDateFormat` as the legacy formatter because it is thread-safe.
+    legacyFormat = LegacyDateFormats.FAST_DATE_FORMAT,
+    isParsing = false)
+  private def timestampFormatter = TimestampFormatter.getFractionFormatter(
+    DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone))
 
   /** Formats a datum (based on the given data type) and returns the string representation. */
   def toHiveString(a: (Any, DataType), nested: Boolean = false): String = a match {
     case (null, _) => if (nested) "null" else "NULL"
     case (b, BooleanType) => b.toString
-    case (d: Date, DateType) => dateFormatter.format(DateTimeUtils.fromJavaDate(d))
-    case (t: Timestamp, TimestampType) =>
-      timestampFormatter.format(DateTimeUtils.fromJavaTimestamp(t))
+    case (d: Date, DateType) => dateFormatter.format(d)
+    case (ld: LocalDate, DateType) => dateFormatter.format(ld)
+    case (t: Timestamp, TimestampType) => timestampFormatter.format(t)
+    case (i: Instant, TimestampType) => timestampFormatter.format(i)
     case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8)
     case (decimal: java.math.BigDecimal, DecimalType()) => decimal.toPlainString
     case (n, _: NumericType) => n.toString
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala
index 1b5115f2e29a3..b452213cd6cc7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala
@@ -45,10 +45,14 @@ case class LocalTableScanExec(
     }
   }
 
-  private lazy val numParallelism: Int = math.min(math.max(unsafeRows.length, 1),
-    sqlContext.sparkContext.defaultParallelism)
-
-  private lazy val rdd = sqlContext.sparkContext.parallelize(unsafeRows, numParallelism)
+  @transient private lazy val rdd: RDD[InternalRow] = {
+    if (rows.isEmpty) {
+      sqlContext.sparkContext.emptyRDD
+    } else {
+      val numSlices = math.min(unsafeRows.length, sqlContext.sparkContext.defaultParallelism)
+      sqlContext.sparkContext.parallelize(unsafeRows, numSlices)
+    }
+  }
 
   protected override def doExecute(): RDD[InternalRow] = {
     val numOutputRows = longMetric("numOutputRows")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
index 45e5f415e8da1..492d177c7c773 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
@@ -117,7 +117,7 @@ case class OptimizeMetadataOnlyQuery(catalog: SessionCatalog) extends Rule[Logic
         case a: AttributeReference =>
           a.withName(relation.output.find(_.semanticEquals(a)).get.name)
       }
-    }
+    }.filterNot(SubqueryExpression.hasSubquery)
 
     child transform {
       case plan if plan eq relation =>
@@ -133,10 +133,14 @@ case class OptimizeMetadataOnlyQuery(catalog: SessionCatalog) extends Rule[Logic
               CaseInsensitiveMap(relation.tableMeta.storage.properties)
             val timeZoneId = caseInsensitiveProperties.get(DateTimeUtils.TIMEZONE_OPTION)
               .getOrElse(SQLConf.get.sessionLocalTimeZone)
-            val partitions = if (partFilters.nonEmpty) {
-              catalog.listPartitionsByFilter(relation.tableMeta.identifier, normalizedFilters)
-            } else {
-              catalog.listPartitions(relation.tableMeta.identifier)
+            val partitions = relation.prunedPartitions match {
+              // for the case where partitions have already been pruned by PruneHiveTablePartitions
+              case Some(parts) => parts
+              case None => if (partFilters.nonEmpty) {
+                catalog.listPartitionsByFilter(relation.tableMeta.identifier, normalizedFilters)
+              } else {
+                catalog.listPartitions(relation.tableMeta.identifier)
+              }
             }
 
             val partitionData = partitions.map { p =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 38ef66682c413..ed36c78a77460 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -32,8 +32,8 @@ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, ReturnAnswer}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.catalyst.util.StringUtils.PlanStringConcat
 import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.dynamicpruning.PlanDynamicPruningFilters
 import org.apache.spark.sql.execution.adaptive.{AdaptiveExecutionContext, InsertAdaptiveSparkPlan}
+import org.apache.spark.sql.execution.dynamicpruning.PlanDynamicPruningFilters
 import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReuseExchange}
 import org.apache.spark.sql.execution.streaming.{IncrementalExecution, OffsetSeqMetadata}
 import org.apache.spark.sql.internal.SQLConf
@@ -63,13 +63,12 @@ class QueryExecution(
     }
   }
 
-  lazy val analyzed: LogicalPlan = tracker.measurePhase(QueryPlanningTracker.ANALYSIS) {
-    SparkSession.setActiveSession(sparkSession)
+  lazy val analyzed: LogicalPlan = executePhase(QueryPlanningTracker.ANALYSIS) {
     // We can't clone `logical` here, which will reset the `_analyzed` flag.
     sparkSession.sessionState.analyzer.executeAndCheck(logical, tracker)
   }
 
-  lazy val withCachedData: LogicalPlan = {
+  lazy val withCachedData: LogicalPlan = sparkSession.withActive {
     assertAnalyzed()
     assertSupported()
     // clone the plan to avoid sharing the plan instance between different stages like analyzing,
@@ -77,23 +76,41 @@ class QueryExecution(
     sparkSession.sharedState.cacheManager.useCachedData(analyzed.clone())
   }
 
-  lazy val optimizedPlan: LogicalPlan = tracker.measurePhase(QueryPlanningTracker.OPTIMIZATION) {
+  lazy val optimizedPlan: LogicalPlan = executePhase(QueryPlanningTracker.OPTIMIZATION) {
     // clone the plan to avoid sharing the plan instance between different stages like analyzing,
     // optimizing and planning.
-    sparkSession.sessionState.optimizer.executeAndTrack(withCachedData.clone(), tracker)
+    val plan = sparkSession.sessionState.optimizer.executeAndTrack(withCachedData.clone(), tracker)
+    // We do not want optimized plans to be re-analyzed as literals that have been constant folded
+    // and such can cause issues during analysis. While `clone` should maintain the `analyzed` state
+    // of the LogicalPlan, we set the plan as analyzed here as well out of paranoia.
+    plan.setAnalyzed()
+    plan
   }
 
-  lazy val sparkPlan: SparkPlan = tracker.measurePhase(QueryPlanningTracker.PLANNING) {
-    // Clone the logical plan here, in case the planner rules change the states of the logical plan.
-    QueryExecution.createSparkPlan(sparkSession, planner, optimizedPlan.clone())
+  private def assertOptimized(): Unit = optimizedPlan
+
+  lazy val sparkPlan: SparkPlan = {
+    // We need to materialize the optimizedPlan here because sparkPlan is also tracked under
+    // the planning phase
+    assertOptimized()
+    executePhase(QueryPlanningTracker.PLANNING) {
+      // Clone the logical plan here, in case the planner rules change the states of the logical
+      // plan.
+      QueryExecution.createSparkPlan(sparkSession, planner, optimizedPlan.clone())
+    }
   }
 
   // executedPlan should not be used to initialize any SparkPlan. It should be
   // only used for execution.
-  lazy val executedPlan: SparkPlan = tracker.measurePhase(QueryPlanningTracker.PLANNING) {
-    // clone the plan to avoid sharing the plan instance between different stages like analyzing,
-    // optimizing and planning.
-    QueryExecution.prepareForExecution(preparations, sparkPlan.clone())
+  lazy val executedPlan: SparkPlan = {
+    // We need to materialize the optimizedPlan here, before tracking the planning phase, to ensure
+    // that the optimization time is not counted as part of the planning phase.
+    assertOptimized()
+    executePhase(QueryPlanningTracker.PLANNING) {
+      // clone the plan to avoid sharing the plan instance between different stages like analyzing,
+      // optimizing and planning.
+      QueryExecution.prepareForExecution(preparations, sparkPlan.clone())
+    }
   }
 
   /**
@@ -113,7 +130,12 @@ class QueryExecution(
   def observedMetrics: Map[String, Row] = CollectMetricsExec.collect(executedPlan)
 
   protected def preparations: Seq[Rule[SparkPlan]] = {
-    QueryExecution.preparations(sparkSession)
+    QueryExecution.preparations(sparkSession,
+      Option(InsertAdaptiveSparkPlan(AdaptiveExecutionContext(sparkSession, this))))
+  }
+
+  protected def executePhase[T](phase: String)(block: => T): T = sparkSession.withActive {
+    tracker.measurePhase(phase)(block)
   }
 
   def simpleString: String = simpleString(false)
@@ -271,20 +293,26 @@ object QueryExecution {
    * are correct, insert whole stage code gen, and try to reduce the work done by reusing exchanges
    * and subqueries.
    */
-  private[execution] def preparations(sparkSession: SparkSession): Seq[Rule[SparkPlan]] =
+  private[execution] def preparations(
+      sparkSession: SparkSession,
+      adaptiveExecutionRule: Option[InsertAdaptiveSparkPlan] = None): Seq[Rule[SparkPlan]] = {
+    // `AdaptiveSparkPlanExec` is a leaf node. If inserted, all the following rules will be no-op
+    // as the original plan is hidden behind `AdaptiveSparkPlanExec`.
+    adaptiveExecutionRule.toSeq ++
     Seq(
-      // `AdaptiveSparkPlanExec` is a leaf node. If inserted, all the following rules will be no-op
-      // as the original plan is hidden behind `AdaptiveSparkPlanExec`.
-      InsertAdaptiveSparkPlan(AdaptiveExecutionContext(sparkSession)),
       PlanDynamicPruningFilters(sparkSession),
       PlanSubqueries(sparkSession),
       EnsureRequirements(sparkSession.sessionState.conf),
+      // `RemoveRedundantSorts` needs to be added before `EnsureRequirements` to guarantee the same
+      // number of partitions when instantiating PartitioningCollection.
+      RemoveRedundantSorts(sparkSession.sessionState.conf),
       ApplyColumnarRulesAndInsertTransitions(sparkSession.sessionState.conf,
         sparkSession.sessionState.columnarRules),
       CollapseCodegenStages(sparkSession.sessionState.conf),
       ReuseExchange(sparkSession.sessionState.conf),
       ReuseSubquery(sparkSession.sessionState.conf)
     )
+  }
 
   /**
    * Prepares a planned [[SparkPlan]] for execution by inserting shuffle operations and internal
@@ -305,7 +333,6 @@ object QueryExecution {
       sparkSession: SparkSession,
       planner: SparkPlanner,
       plan: LogicalPlan): SparkPlan = {
-    SparkSession.setActiveSession(sparkSession)
     // TODO: We use next(), i.e. take the first plan returned by the planner, here for now,
     //       but we will implement to choose the best plan.
     planner.plan(ReturnAnswer(plan)).next()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala
new file mode 100644
index 0000000000000..71f36c8c1dd5a
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.catalyst.expressions.SortOrder
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Remove redundant SortExec node from the spark plan. A sort node is redundant when
+ * its child satisfies both its sort orders and its required child distribution. Note
+ * this rule differs from the Optimizer rule EliminateSorts in that this rule also checks
+ * if the child satisfies the required distribution so that it is safe to remove not only a
+ * local sort but also a global sort when its child already satisfies required sort orders.
+ */
+case class RemoveRedundantSorts(conf: SQLConf) extends Rule[SparkPlan] {
+  def apply(plan: SparkPlan): SparkPlan = {
+    if (!conf.getConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED)) {
+      plan
+    } else {
+      removeSorts(plan)
+    }
+  }
+
+  private def removeSorts(plan: SparkPlan): SparkPlan = plan transform {
+    case s @ SortExec(orders, _, child, _)
+        if SortOrder.orderingSatisfies(child.outputOrdering, orders) &&
+          child.outputPartitioning.satisfies(s.requiredChildDistribution.head) =>
+      child
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
index 995d94ef5eac7..5e4f30a5edaf1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
@@ -17,11 +17,9 @@
 
 package org.apache.spark.sql.execution
 
-import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.{ConcurrentHashMap, ExecutorService, Future => JFuture}
 import java.util.concurrent.atomic.AtomicLong
 
-import scala.concurrent.{ExecutionContext, Future}
-
 import org.apache.spark.SparkContext
 import org.apache.spark.internal.config.Tests.IS_TESTING
 import org.apache.spark.sql.SparkSession
@@ -62,9 +60,9 @@ object SQLExecution {
    * we can connect them with an execution.
    */
   def withNewExecutionId[T](
-      sparkSession: SparkSession,
       queryExecution: QueryExecution,
-      name: Option[String] = None)(body: => T): T = {
+      name: Option[String] = None)(body: => T): T = queryExecution.sparkSession.withActive {
+    val sparkSession = queryExecution.sparkSession
     val sc = sparkSession.sparkContext
     val oldExecutionId = sc.getLocalProperty(EXECUTION_ID_KEY)
     val executionId = SQLExecution.nextExecutionId
@@ -172,14 +170,24 @@ object SQLExecution {
    * SparkContext local properties are forwarded to execution thread
    */
   def withThreadLocalCaptured[T](
-      sparkSession: SparkSession, exec: ExecutionContext)(body: => T): Future[T] = {
+      sparkSession: SparkSession, exec: ExecutorService) (body: => T): JFuture[T] = {
     val activeSession = sparkSession
     val sc = sparkSession.sparkContext
     val localProps = Utils.cloneProperties(sc.getLocalProperties)
-    Future {
+    exec.submit(() => {
+      val originalSession = SparkSession.getActiveSession
+      val originalLocalProps = sc.getLocalProperties
       SparkSession.setActiveSession(activeSession)
       sc.setLocalProperties(localProps)
-      body
-    }(exec)
+      val res = body
+      // reset active session and local props.
+      sc.setLocalProperties(originalLocalProps)
+      if (originalSession.nonEmpty) {
+        SparkSession.setActiveSession(originalSession.get)
+      } else {
+        SparkSession.clearActiveSession()
+      }
+      res
+    })
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
index efa493923ccc1..53ab0493f47eb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
@@ -26,17 +26,28 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleReadMetricsReporter}
 import org.apache.spark.sql.internal.SQLConf
 
+sealed trait ShufflePartitionSpec
+
+// A partition that reads data of one or more reducers, from `startReducerIndex` (inclusive) to
+// `endReducerIndex` (exclusive).
+case class CoalescedPartitionSpec(
+  startReducerIndex: Int, endReducerIndex: Int) extends ShufflePartitionSpec
+
+// A partition that reads partial data of one reducer, from `startMapIndex` (inclusive) to
+// `endMapIndex` (exclusive).
+case class PartialReducerPartitionSpec(
+  reducerIndex: Int, startMapIndex: Int, endMapIndex: Int) extends ShufflePartitionSpec
+
+// A partition that reads partial data of one mapper, from `startReducerIndex` (inclusive) to
+// `endReducerIndex` (exclusive).
+case class PartialMapperPartitionSpec(
+  mapIndex: Int, startReducerIndex: Int, endReducerIndex: Int) extends ShufflePartitionSpec
+
 /**
- * The [[Partition]] used by [[ShuffledRowRDD]]. A post-shuffle partition
- * (identified by `postShufflePartitionIndex`) contains a range of pre-shuffle partitions
- * (`startPreShufflePartitionIndex` to `endPreShufflePartitionIndex - 1`, inclusive).
+ * The [[Partition]] used by [[ShuffledRowRDD]].
  */
-private final class ShuffledRowRDDPartition(
-    val postShufflePartitionIndex: Int,
-    val startPreShufflePartitionIndex: Int,
-    val endPreShufflePartitionIndex: Int) extends Partition {
-  override val index: Int = postShufflePartitionIndex
-}
+private final case class ShuffledRowRDDPartition(
+  index: Int, spec: ShufflePartitionSpec) extends Partition
 
 /**
  * A dummy partitioner for use with records whose partition ids have been pre-computed (i.e. for
@@ -94,8 +105,7 @@ class CoalescedPartitioner(val parent: Partitioner, val partitionStartIndices: A
  * interfaces / internals.
  *
  * This RDD takes a [[ShuffleDependency]] (`dependency`),
- * and an optional array of partition start indices as input arguments
- * (`specifiedPartitionStartIndices`).
+ * and an array of [[ShufflePartitionSpec]] as input arguments.
  *
  * The `dependency` has the parent RDD of this RDD, which represents the dataset before shuffle
  * (i.e. map output). Elements of this RDD are (partitionId, Row) pairs.
@@ -103,78 +113,97 @@ class CoalescedPartitioner(val parent: Partitioner, val partitionStartIndices: A
  * `dependency.partitioner` is the original partitioner used to partition
  * map output, and `dependency.partitioner.numPartitions` is the number of pre-shuffle partitions
  * (i.e. the number of partitions of the map output).
- *
- * When `specifiedPartitionStartIndices` is defined, `specifiedPartitionStartIndices.length`
- * will be the number of post-shuffle partitions. For this case, the `i`th post-shuffle
- * partition includes `specifiedPartitionStartIndices[i]` to
- * `specifiedPartitionStartIndices[i+1] - 1` (inclusive).
- *
- * When `specifiedPartitionStartIndices` is not defined, there will be
- * `dependency.partitioner.numPartitions` post-shuffle partitions. For this case,
- * a post-shuffle partition is created for every pre-shuffle partition.
  */
 class ShuffledRowRDD(
     var dependency: ShuffleDependency[Int, InternalRow, InternalRow],
     metrics: Map[String, SQLMetric],
-    specifiedPartitionIndices: Option[Array[(Int, Int)]] = None)
+    partitionSpecs: Array[ShufflePartitionSpec])
   extends RDD[InternalRow](dependency.rdd.context, Nil) {
 
-  if (SQLConf.get.fetchShuffleBlocksInBatchEnabled) {
-    dependency.rdd.context.setLocalProperty(
-      SortShuffleManager.FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED_KEY, "true")
+  def this(
+      dependency: ShuffleDependency[Int, InternalRow, InternalRow],
+      metrics: Map[String, SQLMetric]) = {
+    this(dependency, metrics,
+      Array.tabulate(dependency.partitioner.numPartitions)(i => CoalescedPartitionSpec(i, i + 1)))
   }
 
-  private[this] val numPreShufflePartitions = dependency.partitioner.numPartitions
-
-  private[this] val partitionStartIndices: Array[Int] = specifiedPartitionIndices match {
-    case Some(indices) => indices.map(_._1)
-    case None =>
-      // When specifiedPartitionStartIndices is not defined, every post-shuffle partition
-      // corresponds to a pre-shuffle partition.
-      (0 until numPreShufflePartitions).toArray
+  if (SQLConf.get.fetchShuffleBlocksInBatch) {
+    dependency.rdd.context.setLocalProperty(
+      SortShuffleManager.FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED_KEY, "true")
   }
 
-  private[this] val part: Partitioner =
-    new CoalescedPartitioner(dependency.partitioner, partitionStartIndices)
-
   override def getDependencies: Seq[Dependency[_]] = List(dependency)
 
-  override val partitioner: Option[Partitioner] = Some(part)
+  override val partitioner: Option[Partitioner] =
+    if (partitionSpecs.forall(_.isInstanceOf[CoalescedPartitionSpec])) {
+      val indices = partitionSpecs.map(_.asInstanceOf[CoalescedPartitionSpec].startReducerIndex)
+      // TODO this check is based on assumptions of callers' behavior but is sufficient for now.
+      if (indices.toSet.size == partitionSpecs.length) {
+        Some(new CoalescedPartitioner(dependency.partitioner, indices))
+      } else {
+        None
+      }
+    } else {
+      None
+    }
 
   override def getPartitions: Array[Partition] = {
-    specifiedPartitionIndices match {
-      case Some(indices) =>
-        Array.tabulate[Partition](indices.length) { i =>
-          new ShuffledRowRDDPartition(i, indices(i)._1, indices(i)._2)
-        }
-      case None =>
-        Array.tabulate[Partition](numPreShufflePartitions) { i =>
-          new ShuffledRowRDDPartition(i, i, i + 1)
-        }
+    Array.tabulate[Partition](partitionSpecs.length) { i =>
+      ShuffledRowRDDPartition(i, partitionSpecs(i))
     }
   }
 
   override def getPreferredLocations(partition: Partition): Seq[String] = {
     val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
-    val dep = dependencies.head.asInstanceOf[ShuffleDependency[_, _, _]]
-    tracker.getPreferredLocationsForShuffle(dep, partition.index)
+    partition.asInstanceOf[ShuffledRowRDDPartition].spec match {
+      case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
+        // TODO order by partition size.
+        startReducerIndex.until(endReducerIndex).flatMap { reducerIndex =>
+          tracker.getPreferredLocationsForShuffle(dependency, reducerIndex)
+        }
+
+      case PartialReducerPartitionSpec(_, startMapIndex, endMapIndex) =>
+        tracker.getMapLocation(dependency, startMapIndex, endMapIndex)
+
+      case PartialMapperPartitionSpec(mapIndex, _, _) =>
+        tracker.getMapLocation(dependency, mapIndex, mapIndex + 1)
+    }
   }
 
   override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
-    val shuffledRowPartition = split.asInstanceOf[ShuffledRowRDDPartition]
     val tempMetrics = context.taskMetrics().createTempShuffleReadMetrics()
     // `SQLShuffleReadMetricsReporter` will update its own metrics for SQL exchange operator,
     // as well as the `tempMetrics` for basic shuffle metrics.
     val sqlMetricsReporter = new SQLShuffleReadMetricsReporter(tempMetrics, metrics)
-    // The range of pre-shuffle partitions that we are fetching at here is
-    // [startPreShufflePartitionIndex, endPreShufflePartitionIndex - 1].
-    val reader =
-      SparkEnv.get.shuffleManager.getReader(
-        dependency.shuffleHandle,
-        shuffledRowPartition.startPreShufflePartitionIndex,
-        shuffledRowPartition.endPreShufflePartitionIndex,
-        context,
-        sqlMetricsReporter)
+    val reader = split.asInstanceOf[ShuffledRowRDDPartition].spec match {
+      case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
+        SparkEnv.get.shuffleManager.getReader(
+          dependency.shuffleHandle,
+          startReducerIndex,
+          endReducerIndex,
+          context,
+          sqlMetricsReporter)
+
+      case PartialReducerPartitionSpec(reducerIndex, startMapIndex, endMapIndex) =>
+        SparkEnv.get.shuffleManager.getReaderForRange(
+          dependency.shuffleHandle,
+          startMapIndex,
+          endMapIndex,
+          reducerIndex,
+          reducerIndex + 1,
+          context,
+          sqlMetricsReporter)
+
+      case PartialMapperPartitionSpec(mapIndex, startReducerIndex, endReducerIndex) =>
+        SparkEnv.get.shuffleManager.getReaderForRange(
+          dependency.shuffleHandle,
+          mapIndex,
+          mapIndex + 1,
+          startReducerIndex,
+          endReducerIndex,
+          context,
+          sqlMetricsReporter)
+    }
     reader.read().asInstanceOf[Iterator[Product2[Int, InternalRow]]].map(_._2)
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
index 013d94768a2a8..33b86a2b5340c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
@@ -23,10 +23,10 @@ import org.apache.spark.sql.catalyst.optimizer._
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
-import org.apache.spark.sql.dynamicpruning.{CleanupDynamicPruningFilters, PartitionPruning}
 import org.apache.spark.sql.execution.datasources.PruneFileSourcePartitions
 import org.apache.spark.sql.execution.datasources.SchemaPruning
 import org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown
+import org.apache.spark.sql.execution.dynamicpruning.{CleanupDynamicPruningFilters, PartitionPruning}
 import org.apache.spark.sql.execution.python.{ExtractGroupingPythonUDFFromAggregate, ExtractPythonUDFFromAggregate, ExtractPythonUDFs}
 
 class SparkOptimizer(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
index 3301e9b5ab180..062aa69b3adb3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.vectorized.ColumnarBatch
 
 object SparkPlan {
@@ -134,7 +135,12 @@ abstract class SparkPlan extends QueryPlan[SparkPlan] with Logging with Serializ
   def longMetric(name: String): SQLMetric = metrics(name)
 
   // TODO: Move to `DistributedPlan`
-  /** Specifies how data is partitioned across different nodes in the cluster. */
+  /**
+   * Specifies how data is partitioned across different nodes in the cluster.
+   * Note this method may fail if it is invoked before `EnsureRequirements` is applied
+   * since `PartitioningCollection` requires all its partitionings to have
+   * the same number of partitions.
+   */
   def outputPartitioning: Partitioning = UnknownPartitioning(0) // TODO: WRONG WIDTH!
 
   /**
@@ -512,10 +518,21 @@ trait LeafExecNode extends SparkPlan {
   override final def children: Seq[SparkPlan] = Nil
   override def producedAttributes: AttributeSet = outputSet
   override def verboseStringWithOperatorId(): String = {
-    s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Output: ${producedAttributes.mkString("[", ", ", "]")}
-     """.stripMargin
+    val argumentString = argString(SQLConf.get.maxToStringFields)
+    val outputStr = s"${ExplainUtils.generateFieldString("Output", output)}"
+
+    if (argumentString.nonEmpty) {
+      s"""
+         |$formattedNodeName
+         |$outputStr
+         |Arguments: $argumentString
+         |""".stripMargin
+    } else {
+      s"""
+         |$formattedNodeName
+         |$outputStr
+         |""".stripMargin
+    }
   }
 }
 
@@ -531,10 +548,21 @@ trait UnaryExecNode extends SparkPlan {
 
   override final def children: Seq[SparkPlan] = child :: Nil
   override def verboseStringWithOperatorId(): String = {
-    s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Input: ${child.output.mkString("[", ", ", "]")}
-     """.stripMargin
+    val argumentString = argString(SQLConf.get.maxToStringFields)
+    val inputStr = s"${ExplainUtils.generateFieldString("Input", child.output)}"
+
+    if (argumentString.nonEmpty) {
+      s"""
+         |$formattedNodeName
+         |$inputStr
+         |Arguments: $argumentString
+         |""".stripMargin
+    } else {
+      s"""
+         |$formattedNodeName
+         |$inputStr
+         |""".stripMargin
+    }
   }
 }
 
@@ -544,10 +572,23 @@ trait BinaryExecNode extends SparkPlan {
 
   override final def children: Seq[SparkPlan] = Seq(left, right)
   override def verboseStringWithOperatorId(): String = {
-    s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Left output: ${left.output.mkString("[", ", ", "]")}
-       |Right output: ${right.output.mkString("[", ", ", "]")}
-     """.stripMargin
+    val argumentString = argString(SQLConf.get.maxToStringFields)
+    val leftOutputStr = s"${ExplainUtils.generateFieldString("Left output", left.output)}"
+    val rightOutputStr = s"${ExplainUtils.generateFieldString("Right output", right.output)}"
+
+    if (argumentString.nonEmpty) {
+      s"""
+         |$formattedNodeName
+         |$leftOutputStr
+         |$rightOutputStr
+         |Arguments: $argumentString
+         |""".stripMargin
+    } else {
+      s"""
+         |$formattedNodeName
+         |$leftOutputStr
+         |$rightOutputStr
+         |""".stripMargin
+    }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
index 5b72ec058e127..357820a9d63d0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.execution
 
 import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, LocalShuffleReaderExec, QueryStageExec}
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
 import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
 import org.apache.spark.sql.execution.metric.SQLMetricInfo
 import org.apache.spark.sql.internal.SQLConf
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index aa139cb6b0c3b..3e4bc48553c5a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -196,7 +196,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
         operationNotAllowed("CREATE TEMPORARY TABLE IF NOT EXISTS", ctx)
       }
 
-      val (_, _, _, options, _, _) = visitCreateTableClauses(ctx.createTableClauses())
+      val (_, _, _, options, location, _) = visitCreateTableClauses(ctx.createTableClauses())
       val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText).getOrElse(
         throw new ParseException("CREATE TEMPORARY TABLE without a provider is not allowed.", ctx))
       val schema = Option(ctx.colTypeList()).map(createSchema)
@@ -205,7 +205,9 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
           "CREATE TEMPORARY VIEW ... USING ... instead")
 
       val table = tableIdentifier(ident, "CREATE TEMPORARY VIEW", ctx)
-      CreateTempViewUsing(table, schema, replace = false, global = false, provider, options)
+      val optionsWithLocation = location.map(l => options + ("path" -> l)).getOrElse(options)
+      CreateTempViewUsing(table, schema, replace = false, global = false, provider,
+        optionsWithLocation)
     }
   }
 
@@ -587,10 +589,6 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
    */
   override def visitRowFormatDelimited(
       ctx: RowFormatDelimitedContext): CatalogStorageFormat = withOrigin(ctx) {
-    // Collect the entries if any.
-    def entry(key: String, value: Token): Seq[(String, String)] = {
-      Option(value).toSeq.map(x => key -> string(x))
-    }
     // TODO we need proper support for the NULL format.
     val entries =
       entry("field.delim", ctx.fieldsTerminatedBy) ++
@@ -689,9 +687,6 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
         // expects a seq of pairs in which the old parsers' token names are used as keys.
         // Transforming the result of visitRowFormatDelimited would be quite a bit messier than
         // retrieving the key value pairs ourselves.
-        def entry(key: String, value: Token): Seq[(String, String)] = {
-          Option(value).map(t => key -> t.getText).toSeq
-        }
         val entries = entry("TOK_TABLEROWFORMATFIELD", c.fieldsTerminatedBy) ++
           entry("TOK_TABLEROWFORMATCOLLITEMS", c.collectionItemsTerminatedBy) ++
           entry("TOK_TABLEROWFORMATMAPKEYS", c.keysTerminatedBy) ++
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index bd2684d92a1d2..689d1eb62ffa5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -244,7 +244,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
       //   4. Pick cartesian product if join type is inner like.
       //   5. Pick broadcast nested loop join as the final solution. It may OOM but we don't have
       //      other choice.
-      case ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, condition, left, right, hint) =>
+      case j @ ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, nonEquiCond, left, right, hint) =>
         def createBroadcastHashJoin(buildLeft: Boolean, buildRight: Boolean) = {
           val wantToBuildLeft = canBuildLeft(joinType) && buildLeft
           val wantToBuildRight = canBuildRight(joinType) && buildRight
@@ -254,7 +254,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               rightKeys,
               joinType,
               buildSide,
-              condition,
+              nonEquiCond,
               planLater(left),
               planLater(right)))
           }
@@ -269,7 +269,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               rightKeys,
               joinType,
               buildSide,
-              condition,
+              nonEquiCond,
               planLater(left),
               planLater(right)))
           }
@@ -278,7 +278,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
         def createSortMergeJoin() = {
           if (RowOrdering.isOrderable(leftKeys)) {
             Some(Seq(joins.SortMergeJoinExec(
-              leftKeys, rightKeys, joinType, condition, planLater(left), planLater(right))))
+              leftKeys, rightKeys, joinType, nonEquiCond, planLater(left), planLater(right))))
           } else {
             None
           }
@@ -286,7 +286,9 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
         def createCartesianProduct() = {
           if (joinType.isInstanceOf[InnerLike]) {
-            Some(Seq(joins.CartesianProductExec(planLater(left), planLater(right), condition)))
+            // `CartesianProductExec` can't implicitly evaluate equal join condition, here we should
+            // pass the original condition which includes both equal and non-equal conditions.
+            Some(Seq(joins.CartesianProductExec(planLater(left), planLater(right), j.condition)))
           } else {
             None
           }
@@ -311,7 +313,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               // This join could be very slow or OOM
               val buildSide = getSmallerSide(left, right)
               Seq(joins.BroadcastNestedLoopJoinExec(
-                planLater(left), planLater(right), buildSide, joinType, condition))
+                planLater(left), planLater(right), buildSide, joinType, nonEquiCond))
             }
         }
 
@@ -515,7 +517,8 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
         val (functionsWithDistinct, functionsWithoutDistinct) =
           aggregateExpressions.partition(_.isDistinct)
-        if (functionsWithDistinct.map(_.aggregateFunction.children.toSet).distinct.length > 1) {
+        if (functionsWithDistinct.map(
+          _.aggregateFunction.children.filterNot(_.foldable).toSet).distinct.length > 1) {
           // This is a sanity check. We should not reach here when we have multiple distinct
           // column sets. Our `RewriteDistinctAggregates` should take care this case.
           sys.error("You hit a query analyzer bug. Please report your query to " +
@@ -527,6 +530,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
         val normalizedGroupingExpressions = groupingExpressions.map { e =>
           NormalizeFloatingNumbers.normalize(e) match {
             case n: NamedExpression => n
+            // Keep the name of the original expression.
             case other => Alias(other, e.name)(exprId = e.exprId)
           }
         }
@@ -539,10 +543,35 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               resultExpressions,
               planLater(child))
           } else {
+            // functionsWithDistinct is guaranteed to be non-empty. Even though it may contain
+            // more than one DISTINCT aggregate function, all of those functions will have the
+            // same column expressions. For example, it would be valid for functionsWithDistinct
+            // to be [COUNT(DISTINCT foo), MAX(DISTINCT foo)], but
+            // [COUNT(DISTINCT bar), COUNT(DISTINCT foo)] is disallowed because those two distinct
+            // aggregates have different column expressions.
+            val distinctExpressions =
+              functionsWithDistinct.head.aggregateFunction.children.filterNot(_.foldable)
+            val normalizedNamedDistinctExpressions = distinctExpressions.map { e =>
+              // Ideally this should be done in `NormalizeFloatingNumbers`, but we do it here
+              // because `distinctExpressions` is not extracted during logical phase.
+              NormalizeFloatingNumbers.normalize(e) match {
+                case ne: NamedExpression => ne
+                case other =>
+                  // Keep the name of the original expression.
+                  val name = e match {
+                    case ne: NamedExpression => ne.name
+                    case _ => e.toString
+                  }
+                  Alias(other, name)()
+              }
+            }
+
             AggUtils.planAggregateWithOneDistinct(
               normalizedGroupingExpressions,
               functionsWithDistinct,
               functionsWithoutDistinct,
+              distinctExpressions,
+              normalizedNamedDistinctExpressions,
               resultExpressions,
               planLater(child))
           }
@@ -656,7 +685,8 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
       case MemoryPlan(sink, output) =>
         val encoder = RowEncoder(StructType.fromAttributes(output))
-        LocalTableScanExec(output, sink.allData.map(r => encoder.toRow(r).copy())) :: Nil
+        val toRow = encoder.createSerializer()
+        LocalTableScanExec(output, sink.allData.map(r => toRow(r).copy())) :: Nil
 
       case logical.Distinct(child) =>
         throw new IllegalStateException(
@@ -724,7 +754,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
       case logical.Repartition(numPartitions, shuffle, child) =>
         if (shuffle) {
           ShuffleExchangeExec(RoundRobinPartitioning(numPartitions),
-            planLater(child), canChangeNumPartitions = false) :: Nil
+            planLater(child), noUserSpecifiedNumPartition = false) :: Nil
         } else {
           execution.CoalesceExec(numPartitions, planLater(child)) :: Nil
         }
@@ -758,7 +788,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
         execution.RangeExec(r) :: Nil
       case r: logical.RepartitionByExpression =>
         exchange.ShuffleExchangeExec(
-          r.partitioning, planLater(r.child), canChangeNumPartitions = false) :: Nil
+          r.partitioning, planLater(r.child), noUserSpecifiedNumPartition = false) :: Nil
       case ExternalRDD(outputObjAttr, rdd) => ExternalRDDScanExec(outputObjAttr, rdd) :: Nil
       case r: LogicalRDD =>
         RDDScanExec(r.output, r.rdd, "ExistingRDD", r.outputPartitioning, r.outputOrdering) :: Nil
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 3f20b59361988..187827ca6005e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -37,7 +37,6 @@ import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec._
 import org.apache.spark.sql.execution.exchange._
-import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.execution.ui.{SparkListenerSQLAdaptiveExecutionUpdate, SparkListenerSQLAdaptiveSQLMetricUpdates, SQLPlanMetric}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.util.ThreadUtils
@@ -67,6 +66,15 @@ case class AdaptiveSparkPlanExec(
 
   @transient private val lock = new Object()
 
+  @transient private val logOnLevel: ( => String) => Unit = conf.adaptiveExecutionLogLevel match {
+    case "TRACE" => logTrace(_)
+    case "DEBUG" => logDebug(_)
+    case "INFO" => logInfo(_)
+    case "WARN" => logWarning(_)
+    case "ERROR" => logError(_)
+    case _ => logDebug(_)
+  }
+
   // The logical plan optimizer for re-optimizing the current logical plan.
   @transient private val optimizer = new RuleExecutor[LogicalPlan] {
     // TODO add more optimization rules
@@ -75,27 +83,31 @@ case class AdaptiveSparkPlanExec(
     )
   }
 
+  @transient private val removeRedundantSorts = RemoveRedundantSorts(conf)
   @transient private val ensureRequirements = EnsureRequirements(conf)
 
   // A list of physical plan rules to be applied before creation of query stages. The physical
   // plan should reach a final status of query stages (i.e., no more addition or removal of
   // Exchange nodes) after running these rules.
   private def queryStagePreparationRules: Seq[Rule[SparkPlan]] = Seq(
-    ensureRequirements
-  )
+    ensureRequirements,
+    removeRedundantSorts
+  ) ++ context.session.sessionState.queryStagePrepRules
 
   // A list of physical optimizer rules to be applied to a new stage before its execution. These
   // optimizations should be stage-independent.
   @transient private val queryStageOptimizerRules: Seq[Rule[SparkPlan]] = Seq(
     ReuseAdaptiveSubquery(conf, context.subqueryCache),
-    // Here the 'OptimizeSkewedJoin' rule should be executed
-    // before 'ReduceNumShufflePartitions', as the skewed partition handled
-    // in 'OptimizeSkewedJoin' rule, should be omitted in 'ReduceNumShufflePartitions'.
+    CoalesceShufflePartitions(context.session),
+    // The following two rules need to make use of 'CustomShuffleReaderExec.partitionSpecs'
+    // added by `CoalesceShufflePartitions`. So they must be executed after it.
     OptimizeSkewedJoin(conf),
-    ReduceNumShufflePartitions(conf),
-    // The rule of 'OptimizeLocalShuffleReader' need to make use of the 'partitionStartIndices'
-    // in 'ReduceNumShufflePartitions' rule. So it must be after 'ReduceNumShufflePartitions' rule.
-    OptimizeLocalShuffleReader(conf),
+    OptimizeLocalShuffleReader(conf)
+  )
+
+  // A list of physical optimizer rules to be applied right after a new stage is created. The input
+  // plan to these rules has exchange as its root node.
+  @transient private val postStageCreationRules = Seq(
     ApplyColumnarRulesAndInsertTransitions(conf, context.session.sessionState.columnarRules),
     CollapseCodegenStages(conf)
   )
@@ -133,39 +145,33 @@ case class AdaptiveSparkPlanExec(
     executedPlan.resetMetrics()
   }
 
-  private def collectSQLMetrics(plan: SparkPlan): Seq[SQLMetric] = {
-    val metrics = new mutable.ArrayBuffer[SQLMetric]()
-    plan.foreach {
-      case p: ShuffleQueryStageExec if (p.resultOption.isEmpty) =>
-        collectSQLMetrics(p.plan).foreach(metrics += _)
-      case p: BroadcastQueryStageExec if (p.resultOption.isEmpty) =>
-        collectSQLMetrics(p.plan).foreach(metrics += _)
-      case p: SparkPlan =>
-        p.metrics.foreach { case metric =>
-          metrics += metric._2
-        }
-    }
-    metrics
+  private def getExecutionId: Option[Long] = {
+    // If the `QueryExecution` does not match the current execution ID, it means the execution ID
+    // belongs to another (parent) query, and we should not call update UI in this query.
+    Option(context.session.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY))
+      .map(_.toLong).filter(SQLExecution.getQueryExecution(_) eq context.qe)
   }
 
   private def getFinalPhysicalPlan(): SparkPlan = lock.synchronized {
-    if (!isFinalPlan) {
-      // Subqueries do not have their own execution IDs and therefore rely on the main query to
-      // update UI.
-      val executionId = Option(context.session.sparkContext.getLocalProperty(
-        SQLExecution.EXECUTION_ID_KEY)).map(_.toLong)
+    if (isFinalPlan) return currentPhysicalPlan
+
+    // In case of this adaptive plan being executed out of `withActive` scoped functions, e.g.,
+    // `plan.queryExecution.rdd`, we need to set active session here as new plan nodes can be
+    // created in the middle of the execution.
+    context.session.withActive {
+      val executionId = getExecutionId
       var currentLogicalPlan = currentPhysicalPlan.logicalLink.get
       var result = createQueryStages(currentPhysicalPlan)
       val events = new LinkedBlockingQueue[StageMaterializationEvent]()
-      val errors = new mutable.ArrayBuffer[SparkException]()
+      val errors = new mutable.ArrayBuffer[Throwable]()
       var stagesToReplace = Seq.empty[QueryStageExec]
       while (!result.allChildStagesMaterialized) {
         currentPhysicalPlan = result.newPlan
         if (result.newStages.nonEmpty) {
           stagesToReplace = result.newStages ++ stagesToReplace
-          executionId.foreach(onUpdatePlan)
+          executionId.foreach(onUpdatePlan(_, result.newStages.map(_.plan)))
 
-          // Start materialization of all new stages.
+          // Start materialization of all new stages and fail fast if any stages failed eagerly
           result.newStages.foreach { stage =>
             try {
               stage.materialize().onComplete { res =>
@@ -176,7 +182,8 @@ case class AdaptiveSparkPlanExec(
                 }
               }(AdaptiveSparkPlanExec.executionContext)
             } catch {
-              case e: Throwable => events.offer(StageFailure(stage, e))
+              case e: Throwable =>
+                cleanUpAndThrowException(Seq(e), Some(stage.id))
             }
           }
         }
@@ -191,14 +198,12 @@ case class AdaptiveSparkPlanExec(
           case StageSuccess(stage, res) =>
             stage.resultOption = Some(res)
           case StageFailure(stage, ex) =>
-            errors.append(
-              new SparkException(s"Failed to materialize query stage: ${stage.treeString}." +
-                s" and the cause is ${ex.getMessage}", ex))
+            errors.append(ex)
         }
 
         // In case of errors, we cancel all running stages and throw exception.
         if (errors.nonEmpty) {
-          cleanUpAndThrowException(errors)
+          cleanUpAndThrowException(errors, None)
         }
 
         // Try re-optimizing and re-planning. Adopt the new plan if its cost is equal to or less
@@ -218,6 +223,7 @@ case class AdaptiveSparkPlanExec(
         val newCost = costEvaluator.evaluateCost(newPhysicalPlan)
         if (newCost < origCost ||
             (newCost == origCost && currentPhysicalPlan != newPhysicalPlan)) {
+          logOnLevel(s"Plan changed from $currentPhysicalPlan to $newPhysicalPlan")
           cleanUpTempTags(newPhysicalPlan)
           currentPhysicalPlan = newPhysicalPlan
           currentLogicalPlan = newLogicalPlan
@@ -228,34 +234,50 @@ case class AdaptiveSparkPlanExec(
       }
 
       // Run the final plan when there's no more unfinished stages.
-      currentPhysicalPlan = applyPhysicalRules(result.newPlan, queryStageOptimizerRules)
+      currentPhysicalPlan = applyPhysicalRules(
+        result.newPlan, queryStageOptimizerRules ++ postStageCreationRules)
       isFinalPlan = true
-      executionId.foreach(onUpdatePlan)
-      logDebug(s"Final plan: $currentPhysicalPlan")
+      executionId.foreach(onUpdatePlan(_, Seq(currentPhysicalPlan)))
+      currentPhysicalPlan
+    }
+  }
+
+  // Use a lazy val to avoid this being called more than once.
+  @transient private lazy val finalPlanUpdate: Unit = {
+    // Subqueries that don't belong to any query stage of the main query will execute after the
+    // last UI update in `getFinalPhysicalPlan`, so we need to update UI here again to make sure
+    // the newly generated nodes of those subqueries are updated.
+    if (!isSubquery && currentPhysicalPlan.find(_.subqueries.nonEmpty).isDefined) {
+      getExecutionId.foreach(onUpdatePlan(_, Seq.empty))
     }
-    currentPhysicalPlan
+    logOnLevel(s"Final plan: $currentPhysicalPlan")
   }
 
   override def executeCollect(): Array[InternalRow] = {
-    getFinalPhysicalPlan().executeCollect()
+    val rdd = getFinalPhysicalPlan().executeCollect()
+    finalPlanUpdate
+    rdd
   }
 
   override def executeTake(n: Int): Array[InternalRow] = {
-    getFinalPhysicalPlan().executeTake(n)
+    val rdd = getFinalPhysicalPlan().executeTake(n)
+    finalPlanUpdate
+    rdd
   }
 
   override def executeTail(n: Int): Array[InternalRow] = {
-    getFinalPhysicalPlan().executeTail(n)
+    val rdd = getFinalPhysicalPlan().executeTail(n)
+    finalPlanUpdate
+    rdd
   }
 
   override def doExecute(): RDD[InternalRow] = {
-    getFinalPhysicalPlan().execute()
+    val rdd = getFinalPhysicalPlan().execute()
+    finalPlanUpdate
+    rdd
   }
 
-  override def verboseString(maxFields: Int): String = simpleString(maxFields)
-
-  override def simpleString(maxFields: Int): String =
-    s"AdaptiveSparkPlan(isFinalPlan=$isFinalPlan)"
+  protected override def stringArgs: Iterator[Any] = Iterator(s"isFinalPlan=$isFinalPlan")
 
   override def generateTreeString(
       depth: Int,
@@ -361,10 +383,22 @@ case class AdaptiveSparkPlanExec(
   private def newQueryStage(e: Exchange): QueryStageExec = {
     val optimizedPlan = applyPhysicalRules(e.child, queryStageOptimizerRules)
     val queryStage = e match {
-      case s: ShuffleExchangeExec =>
-        ShuffleQueryStageExec(currentStageId, s.copy(child = optimizedPlan))
-      case b: BroadcastExchangeExec =>
-        BroadcastQueryStageExec(currentStageId, b.copy(child = optimizedPlan))
+      case s: ShuffleExchangeLike =>
+        val newShuffle = applyPhysicalRules(
+          s.withNewChildren(Seq(optimizedPlan)), postStageCreationRules)
+        if (!newShuffle.isInstanceOf[ShuffleExchangeLike]) {
+          throw new IllegalStateException(
+            "Custom columnar rules cannot transform shuffle node to something else.")
+        }
+        ShuffleQueryStageExec(currentStageId, newShuffle)
+      case b: BroadcastExchangeLike =>
+        val newBroadcast = applyPhysicalRules(
+          b.withNewChildren(Seq(optimizedPlan)), postStageCreationRules)
+        if (!newBroadcast.isInstanceOf[BroadcastExchangeLike]) {
+          throw new IllegalStateException(
+            "Custom columnar rules cannot transform broadcast node to something else.")
+        }
+        BroadcastQueryStageExec(currentStageId, newBroadcast)
     }
     currentStageId += 1
     setLogicalLinkForNewQueryStage(queryStage, e)
@@ -467,7 +501,6 @@ case class AdaptiveSparkPlanExec(
   private def reOptimize(logicalPlan: LogicalPlan): (SparkPlan, LogicalPlan) = {
     logicalPlan.invalidateStatsCache()
     val optimized = optimizer.execute(logicalPlan)
-    SparkSession.setActiveSession(context.session)
     val sparkPlan = context.session.sessionState.planner.plan(ReturnAnswer(optimized)).next()
     val newPlan = applyPhysicalRules(sparkPlan, preprocessingRules ++ queryStagePreparationRules)
     (newPlan, optimized)
@@ -495,62 +528,58 @@ case class AdaptiveSparkPlanExec(
   /**
    * Notify the listeners of the physical plan change.
    */
-  private def onUpdatePlan(executionId: Long): Unit = {
+  private def onUpdatePlan(executionId: Long, newSubPlans: Seq[SparkPlan]): Unit = {
     if (isSubquery) {
       // When executing subqueries, we can't update the query plan in the UI as the
       // UI doesn't support partial update yet. However, the subquery may have been
       // optimized into a different plan and we must let the UI know the SQL metrics
       // of the new plan nodes, so that it can track the valid accumulator updates later
       // and display SQL metrics correctly.
-      onUpdateSQLMetrics(collectSQLMetrics(currentPhysicalPlan), executionId)
+      val newMetrics = newSubPlans.flatMap { p =>
+        p.flatMap(_.metrics.values.map(m => SQLPlanMetric(m.name.get, m.id, m.metricType)))
+      }
+      context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveSQLMetricUpdates(
+        executionId.toLong, newMetrics))
     } else {
       context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveExecutionUpdate(
         executionId,
-        SQLExecution.getQueryExecution(executionId).toString,
-        SparkPlanInfo.fromSparkPlan(this)))
-    }
-  }
-
-  private def onUpdateSQLMetrics(sqlMetrics: Seq[SQLMetric], executionId: Long): Unit = {
-    val sqlPlanMetrics = sqlMetrics.map { case sqlMetric =>
-      SQLPlanMetric(sqlMetric.name.get, sqlMetric.id, sqlMetric.metricType)
+        context.qe.toString,
+        SparkPlanInfo.fromSparkPlan(context.qe.executedPlan)))
     }
-    context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveSQLMetricUpdates(
-      executionId.toLong, sqlPlanMetrics))
   }
 
   /**
    * Cancel all running stages with best effort and throw an Exception containing all stage
    * materialization errors and stage cancellation errors.
    */
-  private def cleanUpAndThrowException(errors: Seq[SparkException]): Unit = {
-    val runningStages = currentPhysicalPlan.collect {
-      case s: QueryStageExec => s
-    }
-    val cancelErrors = new mutable.ArrayBuffer[SparkException]()
-    try {
-      runningStages.foreach { s =>
+  private def cleanUpAndThrowException(
+       errors: Seq[Throwable],
+       earlyFailedStage: Option[Int]): Unit = {
+    currentPhysicalPlan.foreach {
+      // earlyFailedStage is the stage which failed before calling doMaterialize,
+      // so we should avoid calling cancel on it to re-trigger the failure again.
+      case s: QueryStageExec if !earlyFailedStage.contains(s.id) =>
         try {
           s.cancel()
         } catch {
           case NonFatal(t) =>
-            cancelErrors.append(
-              new SparkException(s"Failed to cancel query stage: ${s.treeString}", t))
+            logError(s"Exception in cancelling query stage: ${s.treeString}", t)
         }
-      }
-    } finally {
-      val ex = new SparkException(
-        "Adaptive execution failed due to stage materialization failures." +
-          s" and the cause is ${errors.head.getMessage}", errors.head)
-      errors.tail.foreach(ex.addSuppressed)
-      cancelErrors.foreach(ex.addSuppressed)
-      throw ex
+      case _ =>
+    }
+    val e = if (errors.size == 1) {
+      errors.head
+    } else {
+      val se = new SparkException("Multiple failures in stage materialization.", errors.head)
+      errors.tail.foreach(se.addSuppressed)
+      se
     }
+    throw e
   }
 }
 
 object AdaptiveSparkPlanExec {
-  private val executionContext = ExecutionContext.fromExecutorService(
+  private[adaptive] val executionContext = ExecutionContext.fromExecutorService(
     ThreadUtils.newDaemonCachedThreadPool("QueryStageCreator", 16))
 
   /**
@@ -576,7 +605,7 @@ object AdaptiveSparkPlanExec {
 /**
  * The execution context shared between the main query and all sub-queries.
  */
-case class AdaptiveExecutionContext(session: SparkSession) {
+case class AdaptiveExecutionContext(session: SparkSession, qe: QueryExecution) {
 
   /**
    * The subquery-reuse map shared across the entire query.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
index 61ae6cb14ccd3..3cf6a13a4a892 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
@@ -17,7 +17,9 @@
 
 package org.apache.spark.sql.execution.adaptive
 
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.internal.SQLConf
 
 /**
  * This class provides utility methods related to tree traversal of an [[AdaptiveSparkPlanExec]]
@@ -109,7 +111,7 @@ trait AdaptiveSparkPlanHelper {
    * Returns a sequence containing the result of applying a partial function to all elements in this
    * plan, also considering all the plans in its (nested) subqueries
    */
-  def collectInPlanAndSubqueries[B](p: SparkPlan)(f: PartialFunction[SparkPlan, B]): Seq[B] = {
+  def collectWithSubqueries[B](p: SparkPlan)(f: PartialFunction[SparkPlan, B]): Seq[B] = {
     (p +: subqueriesAll(p)).flatMap(collect(_)(f))
   }
 
@@ -135,4 +137,18 @@ trait AdaptiveSparkPlanHelper {
     case a: AdaptiveSparkPlanExec => a.executedPlan
     case other => other
   }
- }
+
+  /**
+   * Returns a cloned [[SparkSession]] with adaptive execution disabled, or the original
+   * [[SparkSession]] if its adaptive execution is already disabled.
+   */
+  def getOrCloneSessionWithAqeOff[T](session: SparkSession): SparkSession = {
+    if (!session.sessionState.conf.adaptiveExecutionEnabled) {
+      session
+    } else {
+      val newSession = session.cloneSession()
+      newSession.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
+      newSession
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
new file mode 100644
index 0000000000000..096d65f16e42f
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * A rule to coalesce the shuffle partitions based on the map output statistics, which can
+ * avoid many small reduce tasks that hurt performance.
+ */
+case class CoalesceShufflePartitions(session: SparkSession) extends Rule[SparkPlan] {
+  import CoalesceShufflePartitions._
+  private def conf = session.sessionState.conf
+
+  override def apply(plan: SparkPlan): SparkPlan = {
+    if (!conf.coalesceShufflePartitionsEnabled) {
+      return plan
+    }
+    if (!plan.collectLeaves().forall(_.isInstanceOf[QueryStageExec])
+        || plan.find(_.isInstanceOf[CustomShuffleReaderExec]).isDefined) {
+      // If not all leaf nodes are query stages, it's not safe to reduce the number of
+      // shuffle partitions, because we may break the assumption that all children of a spark plan
+      // have same number of output partitions.
+      return plan
+    }
+
+    def collectShuffleStages(plan: SparkPlan): Seq[ShuffleQueryStageExec] = plan match {
+      case stage: ShuffleQueryStageExec => Seq(stage)
+      case _ => plan.children.flatMap(collectShuffleStages)
+    }
+
+    val shuffleStages = collectShuffleStages(plan)
+    // ShuffleExchanges introduced by repartition do not support changing the number of partitions.
+    // We change the number of partitions in the stage only if all the ShuffleExchanges support it.
+    if (!shuffleStages.forall(_.shuffle.canChangeNumPartitions)) {
+      plan
+    } else {
+      // `ShuffleQueryStageExec#mapStats` returns None when the input RDD has 0 partitions,
+      // we should skip it when calculating the `partitionStartIndices`.
+      val validMetrics = shuffleStages.flatMap(_.mapStats)
+
+      // We may have different pre-shuffle partition numbers, don't reduce shuffle partition number
+      // in that case. For example when we union fully aggregated data (data is arranged to a single
+      // partition) and a result of a SortMergeJoin (multiple partitions).
+      val distinctNumPreShufflePartitions =
+        validMetrics.map(stats => stats.bytesByPartitionId.length).distinct
+      if (validMetrics.nonEmpty && distinctNumPreShufflePartitions.length == 1) {
+        // We fall back to Spark default parallelism if the minimum number of coalesced partitions
+        // is not set, so to avoid perf regressions compared to no coalescing.
+        val minPartitionNum = conf.getConf(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM)
+          .getOrElse(session.sparkContext.defaultParallelism)
+        val partitionSpecs = ShufflePartitionsUtil.coalescePartitions(
+          validMetrics.toArray,
+          advisoryTargetSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES),
+          minNumPartitions = minPartitionNum)
+        // This transformation adds new nodes, so we must use `transformUp` here.
+        val stageIds = shuffleStages.map(_.id).toSet
+        plan.transformUp {
+          // even for shuffle exchange whose input RDD has 0 partition, we should still update its
+          // `partitionStartIndices`, so that all the leaf shuffles in a stage have the same
+          // number of output partitions.
+          case stage: ShuffleQueryStageExec if stageIds.contains(stage.id) =>
+            CustomShuffleReaderExec(stage, partitionSpecs, COALESCED_SHUFFLE_READER_DESCRIPTION)
+        }
+      } else {
+        plan
+      }
+    }
+  }
+}
+
+object CoalesceShufflePartitions {
+  val COALESCED_SHUFFLE_READER_DESCRIPTION = "coalesced"
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
new file mode 100644
index 0000000000000..8fd572088b620
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
+import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
+import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec, ShuffleExchangeLike}
+import org.apache.spark.sql.vectorized.ColumnarBatch
+
+
+/**
+ * A wrapper of shuffle query stage, which follows the given partition arrangement.
+ *
+ * @param child           It is usually `ShuffleQueryStageExec`, but can be the shuffle exchange
+ *                        node during canonicalization.
+ * @param partitionSpecs  The partition specs that defines the arrangement.
+ * @param description     The string description of this shuffle reader.
+ */
+case class CustomShuffleReaderExec private(
+    child: SparkPlan,
+    partitionSpecs: Seq[ShufflePartitionSpec],
+    description: String) extends UnaryExecNode {
+
+  override def supportsColumnar: Boolean = child.supportsColumnar
+
+  override def output: Seq[Attribute] = child.output
+  override lazy val outputPartitioning: Partitioning = {
+    // If it is a local shuffle reader with one mapper per task, then the output partitioning is
+    // the same as the plan before shuffle.
+    // TODO this check is based on assumptions of callers' behavior but is sufficient for now.
+    if (partitionSpecs.forall(_.isInstanceOf[PartialMapperPartitionSpec]) &&
+        partitionSpecs.map(_.asInstanceOf[PartialMapperPartitionSpec].mapIndex).toSet.size ==
+          partitionSpecs.length) {
+      child match {
+        case ShuffleQueryStageExec(_, s: ShuffleExchangeLike) =>
+          s.child.outputPartitioning
+        case ShuffleQueryStageExec(_, r @ ReusedExchangeExec(_, s: ShuffleExchangeLike)) =>
+          s.child.outputPartitioning match {
+            case e: Expression => r.updateAttr(e).asInstanceOf[Partitioning]
+            case other => other
+          }
+        case _ =>
+          throw new IllegalStateException("operating on canonicalization plan")
+      }
+    } else {
+      UnknownPartitioning(partitionSpecs.length)
+    }
+  }
+
+  override def stringArgs: Iterator[Any] = Iterator(description)
+
+  private def shuffleStage = child match {
+    case stage: ShuffleQueryStageExec => Some(stage)
+    case _ => None
+  }
+
+  private lazy val shuffleRDD: RDD[_] = {
+    shuffleStage.map { stage =>
+      stage.shuffle.getShuffleRDD(partitionSpecs.toArray)
+    }.getOrElse {
+      throw new IllegalStateException("operating on canonicalized plan")
+    }
+  }
+
+  override protected def doExecute(): RDD[InternalRow] = {
+    shuffleRDD.asInstanceOf[RDD[InternalRow]]
+  }
+
+  override protected def doExecuteColumnar(): RDD[ColumnarBatch] = {
+    shuffleRDD.asInstanceOf[RDD[ColumnarBatch]]
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala
index e5642991c59a3..0f2868e41cc39 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import org.apache.spark.MapOutputStatistics
 import org.apache.spark.sql.catalyst.plans.logical.{HintInfo, Join, LogicalPlan, NO_BROADCAST_HASH}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.internal.SQLConf
@@ -30,10 +29,10 @@ case class DemoteBroadcastHashJoin(conf: SQLConf) extends Rule[LogicalPlan] {
 
   private def shouldDemote(plan: LogicalPlan): Boolean = plan match {
     case LogicalQueryStage(_, stage: ShuffleQueryStageExec) if stage.resultOption.isDefined
-      && stage.resultOption.get != null =>
-      val mapOutputStatistics = stage.resultOption.get.asInstanceOf[MapOutputStatistics]
-      val partitionCnt = mapOutputStatistics.bytesByPartitionId.length
-      val nonZeroCnt = mapOutputStatistics.bytesByPartitionId.count(_ > 0)
+      && stage.mapStats.isDefined =>
+      val mapStats = stage.mapStats.get
+      val partitionCnt = mapStats.bytesByPartitionId.length
+      val nonZeroCnt = mapStats.bytesByPartitionId.count(_ > 0)
       partitionCnt > 0 && nonZeroCnt > 0 &&
         (nonZeroCnt * 1.0 / partitionCnt) < conf.nonEmptyPartitionRatioForBroadcastJoin
     case _ => false
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
index 04696209ce10e..754225dd3fe95 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
@@ -20,13 +20,13 @@ package org.apache.spark.sql.execution.adaptive
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, DynamicPruningSubquery, ListQuery, Literal, SubqueryExpression}
+import org.apache.spark.sql.catalyst.expressions.{DynamicPruningSubquery, ListQuery, SubqueryExpression}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.plans.physical.UnspecifiedDistribution
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.command.ExecutedCommandExec
+import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec}
+import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
 import org.apache.spark.sql.execution.exchange.Exchange
 import org.apache.spark.sql.internal.SQLConf
 
@@ -41,49 +41,62 @@ case class InsertAdaptiveSparkPlan(
 
   private val conf = adaptiveExecutionContext.session.sessionState.conf
 
-  def containShuffle(plan: SparkPlan): Boolean = {
-    plan.find {
-      case _: Exchange => true
-      case s: SparkPlan => !s.requiredChildDistribution.forall(_ == UnspecifiedDistribution)
-    }.isDefined
-  }
-
-  def containSubQuery(plan: SparkPlan): Boolean = {
-    plan.find(_.expressions.exists(_.find {
-      case _: SubqueryExpression => true
-      case _ => false
-    }.isDefined)).isDefined
-  }
-
   override def apply(plan: SparkPlan): SparkPlan = applyInternal(plan, false)
 
   private def applyInternal(plan: SparkPlan, isSubquery: Boolean): SparkPlan = plan match {
+    case _ if !conf.adaptiveExecutionEnabled => plan
     case _: ExecutedCommandExec => plan
-    case _ if conf.adaptiveExecutionEnabled && supportAdaptive(plan)
-      && (isSubquery || containShuffle(plan) || containSubQuery(plan)) =>
-      try {
-        // Plan sub-queries recursively and pass in the shared stage cache for exchange reuse. Fall
-        // back to non-adaptive mode if adaptive execution is supported in any of the sub-queries.
-        val subqueryMap = buildSubqueryMap(plan)
-        val planSubqueriesRule = PlanAdaptiveSubqueries(subqueryMap)
-        val preprocessingRules = Seq(
-          planSubqueriesRule)
-        // Run pre-processing rules.
-        val newPlan = AdaptiveSparkPlanExec.applyPhysicalRules(plan, preprocessingRules)
-        logDebug(s"Adaptive execution enabled for plan: $plan")
-        AdaptiveSparkPlanExec(newPlan, adaptiveExecutionContext, preprocessingRules, isSubquery)
-      } catch {
-        case SubqueryAdaptiveNotSupportedException(subquery) =>
-          logWarning(s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} is enabled " +
-            s"but is not supported for sub-query: $subquery.")
-          plan
-      }
-    case _ =>
-      if (conf.adaptiveExecutionEnabled) {
+    case c: DataWritingCommandExec => c.copy(child = apply(c.child))
+    case c: V2CommandExec => c.withNewChildren(c.children.map(apply))
+    case _ if shouldApplyAQE(plan, isSubquery) =>
+      if (supportAdaptive(plan)) {
+        try {
+          // Plan sub-queries recursively and pass in the shared stage cache for exchange reuse.
+          // Fall back to non-AQE mode if AQE is not supported in any of the sub-queries.
+          val subqueryMap = buildSubqueryMap(plan)
+          val planSubqueriesRule = PlanAdaptiveSubqueries(subqueryMap)
+          val preprocessingRules = Seq(
+            planSubqueriesRule)
+          // Run pre-processing rules.
+          val newPlan = AdaptiveSparkPlanExec.applyPhysicalRules(plan, preprocessingRules)
+          logDebug(s"Adaptive execution enabled for plan: $plan")
+          AdaptiveSparkPlanExec(newPlan, adaptiveExecutionContext, preprocessingRules, isSubquery)
+        } catch {
+          case SubqueryAdaptiveNotSupportedException(subquery) =>
+            logWarning(s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} is enabled " +
+              s"but is not supported for sub-query: $subquery.")
+            plan
+        }
+      } else {
         logWarning(s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} is enabled " +
           s"but is not supported for query: $plan.")
+        plan
       }
-      plan
+
+    case _ => plan
+  }
+
+  // AQE is only useful when the query has exchanges or sub-queries. This method returns true if
+  // one of the following conditions is satisfied:
+  //   - The config ADAPTIVE_EXECUTION_FORCE_APPLY is true.
+  //   - The input query is from a sub-query. When this happens, it means we've already decided to
+  //     apply AQE for the main query and we must continue to do it.
+  //   - The query contains exchanges.
+  //   - The query may need to add exchanges. It's an overkill to run `EnsureRequirements` here, so
+  //     we just check `SparkPlan.requiredChildDistribution` and see if it's possible that the
+  //     the query needs to add exchanges later.
+  //   - The query contains sub-query.
+  private def shouldApplyAQE(plan: SparkPlan, isSubquery: Boolean): Boolean = {
+    conf.getConf(SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY) || isSubquery || {
+      plan.find {
+        case _: Exchange => true
+        case p if !p.requiredChildDistribution.forall(_ == UnspecifiedDistribution) => true
+        case p => p.expressions.exists(_.find {
+          case _: SubqueryExpression => true
+          case _ => false
+        }.isDefined)
+      }.isDefined
+    }
   }
 
   private def supportAdaptive(plan: SparkPlan): Boolean = {
@@ -102,36 +115,25 @@ case class InsertAdaptiveSparkPlan(
    * For each sub-query, generate the adaptive execution plan for each sub-query by applying this
    * rule, or reuse the execution plan from another sub-query of the same semantics if possible.
    */
-  private def buildSubqueryMap(plan: SparkPlan): mutable.HashMap[Long, ExecSubqueryExpression] = {
-    val subqueryMap = mutable.HashMap.empty[Long, ExecSubqueryExpression]
+  private def buildSubqueryMap(plan: SparkPlan): Map[Long, SubqueryExec] = {
+    val subqueryMap = mutable.HashMap.empty[Long, SubqueryExec]
     plan.foreach(_.expressions.foreach(_.foreach {
       case expressions.ScalarSubquery(p, _, exprId)
           if !subqueryMap.contains(exprId.id) =>
         val executedPlan = compileSubquery(p)
         verifyAdaptivePlan(executedPlan, p)
-        val scalarSubquery = execution.ScalarSubquery(
-          SubqueryExec(s"subquery${exprId.id}", executedPlan), exprId)
-        subqueryMap.put(exprId.id, scalarSubquery)
-      case expressions.InSubquery(values, ListQuery(query, _, exprId, _))
+        val subquery = SubqueryExec(s"subquery#${exprId.id}", executedPlan)
+        subqueryMap.put(exprId.id, subquery)
+      case expressions.InSubquery(_, ListQuery(query, _, exprId, _))
           if !subqueryMap.contains(exprId.id) =>
         val executedPlan = compileSubquery(query)
         verifyAdaptivePlan(executedPlan, query)
-        val expr = if (values.length == 1) {
-          values.head
-        } else {
-          CreateNamedStruct(
-            values.zipWithIndex.flatMap { case (v, index) =>
-              Seq(Literal(s"col_$index"), v)
-            }
-          )
-        }
-        val inSubquery = InSubqueryExec(expr,
-          SubqueryExec(s"subquery#${exprId.id}", executedPlan), exprId)
-        subqueryMap.put(exprId.id, inSubquery)
+        val subquery = SubqueryExec(s"subquery#${exprId.id}", executedPlan)
+        subqueryMap.put(exprId.id, subquery)
       case _ =>
     }))
 
-    subqueryMap
+    subqueryMap.toMap
   }
 
   def compileSubquery(plan: LogicalPlan): SparkPlan = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LocalShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LocalShuffledRowRDD.scala
deleted file mode 100644
index 19b78f5e36c9b..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LocalShuffledRowRDD.scala
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.adaptive
-
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleReadMetricsReporter}
-
-/**
- * The [[Partition]] used by [[LocalShuffledRowRDD]].
- * @param mapIndex the index of mapper.
- * @param startPartition the start partition ID in mapIndex mapper.
- * @param endPartition the end partition ID in mapIndex mapper.
- */
-private final class LocalShuffledRowRDDPartition(
-    override val index: Int,
-    val mapIndex: Int,
-    val startPartition: Int,
-    val endPartition: Int) extends Partition {
-}
-
-/**
- * This is a specialized version of [[org.apache.spark.sql.execution.ShuffledRowRDD]]. This is used
- * in Spark SQL adaptive execution when a shuffle join is converted to broadcast join at runtime
- * because the map output of one input table is small enough for broadcast. This RDD represents the
- * data of another input table of the join that reads from shuffle. Each partition of the RDD reads
- * the whole data from just one mapper output locally. So actually there is no data transferred
- * from the network.
- *
- * This RDD takes a [[ShuffleDependency]] (`dependency`).
- *
- * The `dependency` has the parent RDD of this RDD, which represents the dataset before shuffle
- * (i.e. map output). Elements of this RDD are (partitionId, Row) pairs.
- * Partition ids should be in the range [0, numPartitions - 1].
- * `dependency.partitioner.numPartitions` is the number of pre-shuffle partitions. (i.e. the number
- * of partitions of the map output). The post-shuffle partition number is the same to the parent
- * RDD's partition number.
- *
- * `partitionStartIndicesPerMapper` specifies how to split the shuffle blocks of each mapper into
- * one or more partitions. For a mapper `i`, the `j`th partition includes shuffle blocks from
- * `partitionStartIndicesPerMapper[i][j]` to `partitionStartIndicesPerMapper[i][j+1]` (exclusive).
- */
-class LocalShuffledRowRDD(
-     var dependency: ShuffleDependency[Int, InternalRow, InternalRow],
-     metrics: Map[String, SQLMetric],
-     partitionStartIndicesPerMapper: Array[Array[Int]])
-  extends RDD[InternalRow](dependency.rdd.context, Nil) {
-
-  private[this] val numReducers = dependency.partitioner.numPartitions
-  private[this] val numMappers = dependency.rdd.partitions.length
-
-  override def getDependencies: Seq[Dependency[_]] = List(dependency)
-
-  override def getPartitions: Array[Partition] = {
-    val partitions = ArrayBuffer[LocalShuffledRowRDDPartition]()
-    for (mapIndex <- 0 until numMappers) {
-      (partitionStartIndicesPerMapper(mapIndex) :+ numReducers).sliding(2, 1).foreach {
-        case Array(start, end) =>
-          partitions += new LocalShuffledRowRDDPartition(partitions.length, mapIndex, start, end)
-      }
-    }
-    partitions.toArray
-  }
-
-  override def getPreferredLocations(partition: Partition): Seq[String] = {
-    val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
-    tracker.getMapLocation(dependency, partition.index, partition.index + 1)
-  }
-
-  override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
-    val localRowPartition = split.asInstanceOf[LocalShuffledRowRDDPartition]
-    val mapIndex = localRowPartition.mapIndex
-    val tempMetrics = context.taskMetrics().createTempShuffleReadMetrics()
-    // `SQLShuffleReadMetricsReporter` will update its own metrics for SQL exchange operator,
-    // as well as the `tempMetrics` for basic shuffle metrics.
-    val sqlMetricsReporter = new SQLShuffleReadMetricsReporter(tempMetrics, metrics)
-
-    val reader = SparkEnv.get.shuffleManager.getReaderForRange(
-      dependency.shuffleHandle,
-      mapIndex,
-      mapIndex + 1,
-      localRowPartition.startPartition,
-      localRowPartition.endPartition,
-      context,
-      sqlMetricsReporter)
-    reader.read().asInstanceOf[Iterator[Product2[Int, InternalRow]]].map(_._2)
-  }
-
-  override def clearDependencies() {
-    super.clearDependencies()
-    dependency = null
-  }
-}
-
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
index a8d8f358ab660..31d1f34b64a65 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
@@ -17,13 +17,9 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
-import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
-import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReusedExchangeExec, ShuffleExchangeExec}
+import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ShuffleExchangeExec}
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildLeft, BuildRight, BuildSide}
 import org.apache.spark.sql.internal.SQLConf
 
@@ -67,27 +63,33 @@ case class OptimizeLocalShuffleReader(conf: SQLConf) extends Rule[SparkPlan] {
     }
   }
 
-  private def createLocalReader(plan: SparkPlan): LocalShuffleReaderExec = {
+  private def createLocalReader(plan: SparkPlan): CustomShuffleReaderExec = {
     plan match {
-      case c @ CoalescedShuffleReaderExec(s: ShuffleQueryStageExec, _) =>
-        LocalShuffleReaderExec(
-          s, getPartitionStartIndices(s, Some(c.partitionIndices.length)))
+      case c @ CustomShuffleReaderExec(s: ShuffleQueryStageExec, _, _) =>
+        CustomShuffleReaderExec(
+          s, getPartitionSpecs(s, Some(c.partitionSpecs.length)), LOCAL_SHUFFLE_READER_DESCRIPTION)
       case s: ShuffleQueryStageExec =>
-        LocalShuffleReaderExec(s, getPartitionStartIndices(s, None))
+        CustomShuffleReaderExec(s, getPartitionSpecs(s, None), LOCAL_SHUFFLE_READER_DESCRIPTION)
     }
   }
 
   // TODO: this method assumes all shuffle blocks are the same data size. We should calculate the
   //       partition start indices based on block size to avoid data skew.
-  private def getPartitionStartIndices(
+  private def getPartitionSpecs(
       shuffleStage: ShuffleQueryStageExec,
-      advisoryParallelism: Option[Int]): Array[Array[Int]] = {
-    val shuffleDep = shuffleStage.shuffle.shuffleDependency
-    val numReducers = shuffleDep.partitioner.numPartitions
+      advisoryParallelism: Option[Int]): Seq[ShufflePartitionSpec] = {
+    val numMappers = shuffleStage.shuffle.numMappers
+    val numReducers = shuffleStage.shuffle.numPartitions
     val expectedParallelism = advisoryParallelism.getOrElse(numReducers)
-    val numMappers = shuffleDep.rdd.getNumPartitions
-    Array.fill(numMappers) {
-      equallyDivide(numReducers, math.max(1, expectedParallelism / numMappers)).toArray
+    val splitPoints = if (numMappers == 0) {
+      Seq.empty
+    } else {
+      equallyDivide(numReducers, math.max(1, expectedParallelism / numMappers))
+    }
+    (0 until numMappers).flatMap { mapIndex =>
+      (splitPoints :+ numReducers).sliding(2).map {
+        case Seq(start, end) => PartialMapperPartitionSpec(mapIndex, start, end)
+      }
     }
   }
 
@@ -120,6 +122,8 @@ case class OptimizeLocalShuffleReader(conf: SQLConf) extends Rule[SparkPlan] {
 
 object OptimizeLocalShuffleReader {
 
+  val LOCAL_SHUFFLE_READER_DESCRIPTION: String = "local"
+
   object BroadcastJoinWithShuffleLeft {
     def unapply(plan: SparkPlan): Option[(SparkPlan, BuildSide)] = plan match {
       case join: BroadcastHashJoinExec if canUseLocalShuffleReader(join.left) =>
@@ -137,61 +141,10 @@ object OptimizeLocalShuffleReader {
   }
 
   def canUseLocalShuffleReader(plan: SparkPlan): Boolean = plan match {
-    case s: ShuffleQueryStageExec => s.shuffle.canChangeNumPartitions
-    case CoalescedShuffleReaderExec(s: ShuffleQueryStageExec, _) => s.shuffle.canChangeNumPartitions
+    case s: ShuffleQueryStageExec =>
+      s.shuffle.canChangeNumPartitions && s.mapStats.isDefined
+    case CustomShuffleReaderExec(s: ShuffleQueryStageExec, _, _) =>
+      s.shuffle.canChangeNumPartitions && s.mapStats.isDefined
     case _ => false
   }
 }
-
-/**
- * A wrapper of shuffle query stage, which submits one or more reduce tasks per mapper to read the
- * shuffle files written by one mapper. By doing this, it's very likely to read the shuffle files
- * locally, as the shuffle files that a reduce task needs to read are in one node.
- *
- * @param child It's usually `ShuffleQueryStageExec`, but can be the shuffle exchange node during
- *              canonicalization.
- * @param partitionStartIndicesPerMapper A mapper usually writes many shuffle blocks, and it's
- *                                       better to launch multiple tasks to read shuffle blocks of
- *                                       one mapper. This array contains the partition start
- *                                       indices for each mapper.
- */
-case class LocalShuffleReaderExec(
-    child: SparkPlan,
-    partitionStartIndicesPerMapper: Array[Array[Int]]) extends UnaryExecNode {
-
-  override def output: Seq[Attribute] = child.output
-
-  override lazy val outputPartitioning: Partitioning = {
-    // when we read one mapper per task, then the output partitioning is the same as the plan
-    // before shuffle.
-    if (partitionStartIndicesPerMapper.forall(_.length == 1)) {
-      child match {
-        case ShuffleQueryStageExec(_, s: ShuffleExchangeExec) =>
-          s.child.outputPartitioning
-        case ShuffleQueryStageExec(_, r @ ReusedExchangeExec(_, s: ShuffleExchangeExec)) =>
-          s.child.outputPartitioning match {
-            case e: Expression => r.updateAttr(e).asInstanceOf[Partitioning]
-            case other => other
-          }
-        case _ =>
-          throw new IllegalStateException("operating on canonicalization plan")
-      }
-    } else {
-      UnknownPartitioning(partitionStartIndicesPerMapper.map(_.length).sum)
-    }
-  }
-
-  private var cachedShuffleRDD: RDD[InternalRow] = null
-
-  override protected def doExecute(): RDD[InternalRow] = {
-    if (cachedShuffleRDD == null) {
-      cachedShuffleRDD = child match {
-        case stage: ShuffleQueryStageExec =>
-          stage.shuffle.createLocalShuffleRDD(partitionStartIndicesPerMapper)
-        case _ =>
-          throw new IllegalStateException("operating on canonicalization plan")
-      }
-    }
-    cachedShuffleRDD
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index 74b7fbd317fc8..b3b3eb2151f5e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -18,20 +18,41 @@
 package org.apache.spark.sql.execution.adaptive
 
 import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
+
+import org.apache.commons.io.FileUtils
 
 import org.apache.spark.{MapOutputStatistics, MapOutputTrackerMaster, SparkEnv}
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans._
-import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ShuffleExchangeExec}
 import org.apache.spark.sql.execution.joins.SortMergeJoinExec
 import org.apache.spark.sql.internal.SQLConf
 
+/**
+ * A rule to optimize skewed joins to avoid straggler tasks whose share of data are significantly
+ * larger than those of the rest of the tasks.
+ *
+ * The general idea is to divide each skew partition into smaller partitions and replicate its
+ * matching partition on the other side of the join so that they can run in parallel tasks.
+ * Note that when matching partitions from the left side and the right side both have skew,
+ * it will become a cartesian product of splits from left and right joining together.
+ *
+ * For example, assume the Sort-Merge join has 4 partitions:
+ * left:  [L1, L2, L3, L4]
+ * right: [R1, R2, R3, R4]
+ *
+ * Let's say L2, L4 and R3, R4 are skewed, and each of them get split into 2 sub-partitions. This
+ * is scheduled to run 4 tasks at the beginning: (L1, R1), (L2, R2), (L3, R3), (L4, R4).
+ * This rule expands it to 9 tasks to increase parallelism:
+ * (L1, R1),
+ * (L2-1, R2), (L2-2, R2),
+ * (L3, R3-1), (L3, R3-2),
+ * (L4-1, R4-1), (L4-2, R4-1), (L4-1, R4-2), (L4-2, R4-2)
+ *
+ * Note that, when this rule is enabled, it also coalesces non-skewed partitions like
+ * `CoalesceShufflePartitions` does.
+ */
 case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 
   private val ensureRequirements = EnsureRequirements(conf)
@@ -41,21 +62,17 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 
   /**
    * A partition is considered as a skewed partition if its size is larger than the median
-   * partition size * spark.sql.adaptive.skewedPartitionFactor and also larger than
-   * spark.sql.adaptive.skewedPartitionSizeThreshold.
+   * partition size * ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR and also larger than
+   * ADVISORY_PARTITION_SIZE_IN_BYTES.
    */
-  private def isSkewed(
-      stats: MapOutputStatistics,
-      partitionId: Int,
-      medianSize: Long): Boolean = {
-    val size = stats.bytesByPartitionId(partitionId)
-    size > medianSize * conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR) &&
-      size > conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD)
+  private def isSkewed(size: Long, medianSize: Long): Boolean = {
+    size > medianSize * conf.getConf(SQLConf.SKEW_JOIN_SKEWED_PARTITION_FACTOR) &&
+      size > conf.getConf(SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD)
   }
 
-  private def medianSize(stats: MapOutputStatistics): Long = {
-    val numPartitions = stats.bytesByPartitionId.length
-    val bytes = stats.bytesByPartitionId.sorted
+  private def medianSize(sizes: Seq[Long]): Long = {
+    val numPartitions = sizes.length
+    val bytes = sizes.sorted
     numPartitions match {
       case _ if (numPartitions % 2 == 0) =>
         math.max((bytes(numPartitions / 2) + bytes(numPartitions / 2 - 1)) / 2, 1)
@@ -63,6 +80,19 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
     }
   }
 
+  /**
+   * The goal of skew join optimization is to make the data distribution more even. The target size
+   * to split skewed partitions is the average size of non-skewed partition, or the
+   * advisory partition size if avg size is smaller than it.
+   */
+  private def targetSize(sizes: Seq[Long], medianSize: Long): Long = {
+    val advisorySize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES)
+    val nonSkewSizes = sizes.filterNot(isSkewed(_, medianSize))
+    // It's impossible that all the partitions are skewed, as we use median size to define skew.
+    assert(nonSkewSizes.nonEmpty)
+    math.max(advisorySize, nonSkewSizes.sum / nonSkewSizes.length)
+  }
+
   /**
    * Get the map size of the specific reduce shuffle Id.
    */
@@ -72,57 +102,43 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   }
 
   /**
-   * Split the skewed partition based on the map size and the max split number.
+   * Splits the skewed partition based on the map size and the target partition size
+   * after split, and create a list of `PartialMapperPartitionSpec`. Returns None if can't split.
    */
-  private def getMapStartIndices(stage: ShuffleQueryStageExec, partitionId: Int): Array[Int] = {
-    val shuffleId = stage.shuffle.shuffleDependency.shuffleHandle.shuffleId
-    val mapPartitionSizes = getMapSizesForReduceId(shuffleId, partitionId)
-    val maxSplits = math.min(conf.getConf(
-      SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_MAX_SPLITS), mapPartitionSizes.length)
-    val avgPartitionSize = mapPartitionSizes.sum / maxSplits
-    val advisoryPartitionSize = math.max(avgPartitionSize,
-      conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD))
-    val partitionStartIndices = ArrayBuffer[Int]()
-    partitionStartIndices += 0
-    var i = 0
-    var postMapPartitionSize = 0L
-    while (i < mapPartitionSizes.length) {
-      val nextMapPartitionSize = mapPartitionSizes(i)
-      if (i > 0 && postMapPartitionSize + nextMapPartitionSize > advisoryPartitionSize) {
-        partitionStartIndices += i
-        postMapPartitionSize = nextMapPartitionSize
-      } else {
-        postMapPartitionSize += nextMapPartitionSize
-      }
-      i += 1
+  private def createSkewPartitionSpecs(
+      shuffleId: Int,
+      reducerId: Int,
+      targetSize: Long): Option[Seq[PartialReducerPartitionSpec]] = {
+    val mapPartitionSizes = getMapSizesForReduceId(shuffleId, reducerId)
+    val mapStartIndices = ShufflePartitionsUtil.splitSizeListByTargetSize(
+      mapPartitionSizes, targetSize)
+    if (mapStartIndices.length > 1) {
+      Some(mapStartIndices.indices.map { i =>
+        val startMapIndex = mapStartIndices(i)
+        val endMapIndex = if (i == mapStartIndices.length - 1) {
+          mapPartitionSizes.length
+        } else {
+          mapStartIndices(i + 1)
+        }
+        PartialReducerPartitionSpec(reducerId, startMapIndex, endMapIndex)
+      })
+    } else {
+      None
     }
-
-    if (partitionStartIndices.size > maxSplits) {
-      partitionStartIndices.take(maxSplits).toArray
-    } else partitionStartIndices.toArray
-  }
-
-  private def getStatistics(stage: ShuffleQueryStageExec): MapOutputStatistics = {
-    assert(stage.resultOption.isDefined, "ShuffleQueryStageExec should" +
-      " already be ready when executing OptimizeSkewedPartitions rule")
-    stage.resultOption.get.asInstanceOf[MapOutputStatistics]
   }
 
-  private def supportSplitOnLeftPartition(joinType: JoinType) = {
+  private def canSplitLeftSide(joinType: JoinType) = {
     joinType == Inner || joinType == Cross || joinType == LeftSemi ||
       joinType == LeftAnti || joinType == LeftOuter
   }
 
-  private def supportSplitOnRightPartition(joinType: JoinType) = {
+  private def canSplitRightSide(joinType: JoinType) = {
     joinType == Inner || joinType == Cross || joinType == RightOuter
   }
 
-  private def getNumMappers(stage: ShuffleQueryStageExec): Int = {
-    stage.shuffle.shuffleDependency.rdd.partitions.length
-  }
-
-  private def getSizeInfo(medianSize: Long, maxSize: Long): String = {
-    s"median size: $medianSize, max size: ${maxSize}"
+  private def getSizeInfo(medianSize: Long, sizes: Seq[Long]): String = {
+    s"median size: $medianSize, max size: ${sizes.max}, min size: ${sizes.min}, avg size: " +
+      sizes.sum / sizes.length
   }
 
   /*
@@ -130,88 +146,108 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
    * 1. Check whether the shuffle partition is skewed based on the median size
    *    and the skewed partition threshold in origin smj.
    * 2. Assuming partition0 is skewed in left side, and it has 5 mappers (Map0, Map1...Map4).
-   *    And we will split the 5 Mappers into 3 mapper ranges [(Map0, Map1), (Map2, Map3), (Map4)]
+   *    And we may split the 5 Mappers into 3 mapper ranges [(Map0, Map1), (Map2, Map3), (Map4)]
    *    based on the map size and the max split number.
-   * 3. Create the 3 smjs with separately reading the above mapper ranges and then join with
-   *    the Partition0 in right side.
-   * 4. Finally union the above 3 split smjs and the origin smj.
+   * 3. Wrap the join left child with a special shuffle reader that reads each mapper range with one
+   *    task, so total 3 tasks.
+   * 4. Wrap the join right child with a special shuffle reader that reads partition0 3 times by
+   *    3 tasks separately.
    */
   def optimizeSkewJoin(plan: SparkPlan): SparkPlan = plan.transformUp {
-    case smj @ SortMergeJoinExec(leftKeys, rightKeys, joinType, condition,
-        s1 @ SortExec(_, _, left: ShuffleQueryStageExec, _),
-        s2 @ SortExec(_, _, right: ShuffleQueryStageExec, _), _)
-      if (supportedJoinTypes.contains(joinType)) =>
-      val leftStats = getStatistics(left)
-      val rightStats = getStatistics(right)
-      val numPartitions = leftStats.bytesByPartitionId.length
-
-      val leftMedSize = medianSize(leftStats)
-      val rightMedSize = medianSize(rightStats)
+    case smj @ SortMergeJoinExec(_, _, joinType, _,
+        s1 @ SortExec(_, _, ShuffleStage(left: ShuffleStageInfo), _),
+        s2 @ SortExec(_, _, ShuffleStage(right: ShuffleStageInfo), _), _)
+        if supportedJoinTypes.contains(joinType) =>
+      assert(left.partitionsWithSizes.length == right.partitionsWithSizes.length)
+      val numPartitions = left.partitionsWithSizes.length
+      // Use the median size of the actual (coalesced) partition sizes to detect skewed partitions.
+      val leftMedSize = medianSize(left.partitionsWithSizes.map(_._2))
+      val rightMedSize = medianSize(right.partitionsWithSizes.map(_._2))
       logDebug(
         s"""
-          |Try to optimize skewed join.
-          |Left side partition size:
-          |${getSizeInfo(leftMedSize, leftStats.bytesByPartitionId.max)}
-          |Right side partition size:
-          |${getSizeInfo(rightMedSize, rightStats.bytesByPartitionId.max)}
+          |Optimizing skewed join.
+          |Left side partitions size info:
+          |${getSizeInfo(leftMedSize, left.partitionsWithSizes.map(_._2))}
+          |Right side partitions size info:
+          |${getSizeInfo(rightMedSize, right.partitionsWithSizes.map(_._2))}
         """.stripMargin)
-
-      val skewedPartitions = mutable.HashSet[Int]()
-      val subJoins = mutable.ArrayBuffer[SparkPlan]()
-      for (partitionId <- 0 until numPartitions) {
-        val isLeftSkew = isSkewed(leftStats, partitionId, leftMedSize)
-        val isRightSkew = isSkewed(rightStats, partitionId, rightMedSize)
-        val leftMapIdStartIndices = if (isLeftSkew && supportSplitOnLeftPartition(joinType)) {
-          getMapStartIndices(left, partitionId)
+      val canSplitLeft = canSplitLeftSide(joinType)
+      val canSplitRight = canSplitRightSide(joinType)
+      // We use the actual partition sizes (may be coalesced) to calculate target size, so that
+      // the final data distribution is even (coalesced partitions + split partitions).
+      val leftActualSizes = left.partitionsWithSizes.map(_._2)
+      val rightActualSizes = right.partitionsWithSizes.map(_._2)
+      val leftTargetSize = targetSize(leftActualSizes, leftMedSize)
+      val rightTargetSize = targetSize(rightActualSizes, rightMedSize)
+
+      val leftSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
+      val rightSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
+      val leftSkewDesc = new SkewDesc
+      val rightSkewDesc = new SkewDesc
+      for (partitionIndex <- 0 until numPartitions) {
+        val isLeftSkew = isSkewed(leftActualSizes(partitionIndex), leftMedSize) && canSplitLeft
+        val leftPartSpec = left.partitionsWithSizes(partitionIndex)._1
+        val isLeftCoalesced = leftPartSpec.startReducerIndex + 1 < leftPartSpec.endReducerIndex
+
+        val isRightSkew = isSkewed(rightActualSizes(partitionIndex), rightMedSize) && canSplitRight
+        val rightPartSpec = right.partitionsWithSizes(partitionIndex)._1
+        val isRightCoalesced = rightPartSpec.startReducerIndex + 1 < rightPartSpec.endReducerIndex
+
+        // A skewed partition should never be coalesced, but skip it here just to be safe.
+        val leftParts = if (isLeftSkew && !isLeftCoalesced) {
+          val reducerId = leftPartSpec.startReducerIndex
+          val skewSpecs = createSkewPartitionSpecs(
+            left.mapStats.shuffleId, reducerId, leftTargetSize)
+          if (skewSpecs.isDefined) {
+            logDebug(s"Left side partition $partitionIndex is skewed, split it into " +
+              s"${skewSpecs.get.length} parts.")
+            leftSkewDesc.addPartitionSize(leftActualSizes(partitionIndex))
+          }
+          skewSpecs.getOrElse(Seq(leftPartSpec))
         } else {
-          Array(0)
+          Seq(leftPartSpec)
         }
-        val rightMapIdStartIndices = if (isRightSkew && supportSplitOnRightPartition(joinType)) {
-          getMapStartIndices(right, partitionId)
+
+        // A skewed partition should never be coalesced, but skip it here just to be safe.
+        val rightParts = if (isRightSkew && !isRightCoalesced) {
+          val reducerId = rightPartSpec.startReducerIndex
+          val skewSpecs = createSkewPartitionSpecs(
+            right.mapStats.shuffleId, reducerId, rightTargetSize)
+          if (skewSpecs.isDefined) {
+            logDebug(s"Right side partition $partitionIndex is skewed, split it into " +
+              s"${skewSpecs.get.length} parts.")
+            rightSkewDesc.addPartitionSize(rightActualSizes(partitionIndex))
+          }
+          skewSpecs.getOrElse(Seq(rightPartSpec))
         } else {
-          Array(0)
+          Seq(rightPartSpec)
         }
 
-        if (leftMapIdStartIndices.length > 1 || rightMapIdStartIndices.length > 1) {
-          skewedPartitions += partitionId
-          for (i <- 0 until leftMapIdStartIndices.length;
-               j <- 0 until rightMapIdStartIndices.length) {
-            val leftEndMapId = if (i == leftMapIdStartIndices.length - 1) {
-              getNumMappers(left)
-            } else {
-              leftMapIdStartIndices(i + 1)
-            }
-            val rightEndMapId = if (j == rightMapIdStartIndices.length - 1) {
-              getNumMappers(right)
-            } else {
-              rightMapIdStartIndices(j + 1)
-            }
-            // TODO: we may can optimize the sort merge join to broad cast join after
-            //       obtaining the raw data size of per partition,
-            val leftSkewedReader = SkewedPartitionReaderExec(
-              left, partitionId, leftMapIdStartIndices(i), leftEndMapId)
-            val rightSkewedReader = SkewedPartitionReaderExec(right, partitionId,
-              rightMapIdStartIndices(j), rightEndMapId)
-            subJoins += SortMergeJoinExec(leftKeys, rightKeys, joinType, condition,
-              s1.copy(child = leftSkewedReader), s2.copy(child = rightSkewedReader), true)
-          }
+        for {
+          leftSidePartition <- leftParts
+          rightSidePartition <- rightParts
+        } {
+          leftSidePartitions += leftSidePartition
+          rightSidePartitions += rightSidePartition
         }
       }
-      logDebug(s"number of skewed partitions is ${skewedPartitions.size}")
-      if (skewedPartitions.nonEmpty) {
-        val optimizedSmj = smj.copy(
-          left = s1.copy(child = PartialShuffleReaderExec(left, skewedPartitions.toSet)),
-          right = s2.copy(child = PartialShuffleReaderExec(right, skewedPartitions.toSet)),
-          isPartial = true)
-        subJoins += optimizedSmj
-        UnionExec(subJoins)
+
+      logDebug("number of skewed partitions: " +
+        s"left ${leftSkewDesc.numPartitions}, right ${rightSkewDesc.numPartitions}")
+      if (leftSkewDesc.numPartitions > 0 || rightSkewDesc.numPartitions > 0) {
+        val newLeft = CustomShuffleReaderExec(
+          left.shuffleStage, leftSidePartitions, leftSkewDesc.toString)
+        val newRight = CustomShuffleReaderExec(
+          right.shuffleStage, rightSidePartitions, rightSkewDesc.toString)
+        smj.copy(
+          left = s1.copy(child = newLeft), right = s2.copy(child = newRight), isSkewJoin = true)
       } else {
         smj
       }
   }
 
   override def apply(plan: SparkPlan): SparkPlan = {
-    if (!conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_JOIN_ENABLED)) {
+    if (!conf.getConf(SQLConf.SKEW_JOIN_ENABLED)) {
       return plan
     }
 
@@ -224,7 +260,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 
     if (shuffleStages.length == 2) {
       // When multi table join, there will be too many complex combination to consider.
-      // Currently we only handle 2 table join like following two use cases.
+      // Currently we only handle 2 table join like following use case.
       // SMJ
       //   Sort
       //     Shuffle
@@ -248,81 +284,71 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   }
 }
 
-/**
- * A wrapper of shuffle query stage, which submits one reduce task to read a single
- * shuffle partition 'partitionIndex' produced by the mappers in range [startMapIndex, endMapIndex).
- * This is used to increase the parallelism when reading skewed partitions.
- *
- * @param child It's usually `ShuffleQueryStageExec`, but can be the shuffle exchange
- *              node during canonicalization.
- * @param partitionIndex The pre shuffle partition index.
- * @param startMapIndex The start map index.
- * @param endMapIndex The end map index.
- */
-case class SkewedPartitionReaderExec(
-    child: QueryStageExec,
-    partitionIndex: Int,
-    startMapIndex: Int,
-    endMapIndex: Int) extends LeafExecNode {
-
-  override def output: Seq[Attribute] = child.output
-
-  override def outputPartitioning: Partitioning = {
-    UnknownPartitioning(1)
-  }
-  private var cachedSkewedShuffleRDD: SkewedShuffledRowRDD = null
-
-  override def doExecute(): RDD[InternalRow] = {
-    if (cachedSkewedShuffleRDD == null) {
-      cachedSkewedShuffleRDD = child match {
-        case stage: ShuffleQueryStageExec =>
-          stage.shuffle.createSkewedShuffleRDD(partitionIndex, startMapIndex, endMapIndex)
-        case _ =>
-          throw new IllegalStateException("operating on canonicalization plan")
+private object ShuffleStage {
+  def unapply(plan: SparkPlan): Option[ShuffleStageInfo] = plan match {
+    case s: ShuffleQueryStageExec if s.mapStats.isDefined =>
+      val mapStats = s.mapStats.get
+      val sizes = mapStats.bytesByPartitionId
+      val partitions = sizes.zipWithIndex.map {
+        case (size, i) => CoalescedPartitionSpec(i, i + 1) -> size
       }
-    }
-    cachedSkewedShuffleRDD
+      Some(ShuffleStageInfo(s, mapStats, partitions))
+
+    case CustomShuffleReaderExec(s: ShuffleQueryStageExec, partitionSpecs, _)
+      if s.mapStats.isDefined && partitionSpecs.nonEmpty =>
+      val mapStats = s.mapStats.get
+      val sizes = mapStats.bytesByPartitionId
+      val partitions = partitionSpecs.map {
+        case spec @ CoalescedPartitionSpec(start, end) =>
+          var sum = 0L
+          var i = start
+          while (i < end) {
+            sum += sizes(i)
+            i += 1
+          }
+          spec -> sum
+        case other => throw new IllegalArgumentException(
+          s"Expect CoalescedPartitionSpec but got $other")
+      }
+      Some(ShuffleStageInfo(s, mapStats, partitions))
+
+    case _ => None
   }
 }
 
-/**
- * A wrapper of shuffle query stage, which skips some partitions when reading the shuffle blocks.
- *
- * @param child It's usually `ShuffleQueryStageExec`, but can be the shuffle exchange node during
- *              canonicalization.
- * @param excludedPartitions The partitions to skip when reading.
- */
-case class PartialShuffleReaderExec(
-    child: QueryStageExec,
-    excludedPartitions: Set[Int]) extends UnaryExecNode {
-
-  override def output: Seq[Attribute] = child.output
+private case class ShuffleStageInfo(
+    shuffleStage: ShuffleQueryStageExec,
+    mapStats: MapOutputStatistics,
+    partitionsWithSizes: Seq[(CoalescedPartitionSpec, Long)])
 
-  override def outputPartitioning: Partitioning = {
-    UnknownPartitioning(1)
-  }
+private class SkewDesc {
+  private[this] var numSkewedPartitions: Int = 0
+  private[this] var totalSize: Long = 0
+  private[this] var maxSize: Long = 0
+  private[this] var minSize: Long = 0
 
-  private def shuffleExchange(): ShuffleExchangeExec = child match {
-    case stage: ShuffleQueryStageExec => stage.shuffle
-    case _ =>
-      throw new IllegalStateException("operating on canonicalization plan")
-  }
+  def numPartitions: Int = numSkewedPartitions
 
-  private def getPartitionIndexRanges(): Array[(Int, Int)] = {
-    val length = shuffleExchange().shuffleDependency.partitioner.numPartitions
-    (0 until length).filterNot(excludedPartitions.contains).map(i => (i, i + 1)).toArray
+  def addPartitionSize(size: Long): Unit = {
+    if (numSkewedPartitions == 0) {
+      maxSize = size
+      minSize = size
+    }
+    numSkewedPartitions += 1
+    totalSize += size
+    if (size > maxSize) maxSize = size
+    if (size < minSize) minSize = size
   }
 
-  private var cachedShuffleRDD: RDD[InternalRow] = null
-
-  override def doExecute(): RDD[InternalRow] = {
-    if (cachedShuffleRDD == null) {
-      cachedShuffleRDD = if (excludedPartitions.isEmpty) {
-        child.execute()
-      } else {
-        shuffleExchange().createShuffledRDD(Some(getPartitionIndexRanges()))
-      }
+  override def toString: String = {
+    if (numSkewedPartitions == 0) {
+      "no skewed partition"
+    } else {
+      val maxSizeStr = FileUtils.byteCountToDisplaySize(maxSize)
+      val minSizeStr = FileUtils.byteCountToDisplaySize(minSize)
+      val avgSizeStr = FileUtils.byteCountToDisplaySize(totalSize / numSkewedPartitions)
+      s"$numSkewedPartitions skewed partitions with " +
+        s"size(max=$maxSizeStr, min=$minSizeStr, avg=$avgSizeStr)"
     }
-    cachedShuffleRDD
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala
index 91d4359224a6a..f845b6b16ee3a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala
@@ -18,19 +18,28 @@
 package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.catalyst.expressions.ListQuery
+import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, ListQuery, Literal}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.{ExecSubqueryExpression, SparkPlan}
+import org.apache.spark.sql.execution
+import org.apache.spark.sql.execution.{InSubqueryExec, SparkPlan, SubqueryExec}
 
-case class PlanAdaptiveSubqueries(
-    subqueryMap: scala.collection.Map[Long, ExecSubqueryExpression]) extends Rule[SparkPlan] {
+case class PlanAdaptiveSubqueries(subqueryMap: Map[Long, SubqueryExec]) extends Rule[SparkPlan] {
 
   def apply(plan: SparkPlan): SparkPlan = {
     plan.transformAllExpressions {
       case expressions.ScalarSubquery(_, _, exprId) =>
-        subqueryMap(exprId.id)
-      case expressions.InSubquery(_, ListQuery(_, _, exprId, _)) =>
-        subqueryMap(exprId.id)
+        execution.ScalarSubquery(subqueryMap(exprId.id), exprId)
+      case expressions.InSubquery(values, ListQuery(_, _, exprId, _)) =>
+        val expr = if (values.length == 1) {
+          values.head
+        } else {
+          CreateNamedStruct(
+            values.zipWithIndex.flatMap { case (v, index) =>
+              Seq(Literal(s"col_$index"), v)
+            }
+          )
+        }
+        InSubqueryExec(expr, subqueryMap(exprId.id), exprId)
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
index d5dc1be63f06e..74fe1eaab6f64 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
@@ -17,17 +17,23 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import scala.concurrent.Future
+import java.util.concurrent.TimeUnit
 
-import org.apache.spark.{FutureAction, MapOutputStatistics}
+import scala.concurrent.{Future, Promise}
+
+import org.apache.spark.{FutureAction, MapOutputStatistics, SparkException}
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.errors.attachTree
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.logical.Statistics
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.exchange._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.vectorized.ColumnarBatch
+import org.apache.spark.util.ThreadUtils
 
 /**
  * A query stage is an independent subgraph of the query plan. Query stage materializes its output
@@ -75,6 +81,11 @@ abstract class QueryStageExec extends LeafExecNode {
 
   def newReuseInstance(newStageId: Int, newOutput: Seq[Attribute]): QueryStageExec
 
+  /**
+   * Returns the runtime statistics after stage materialization.
+   */
+  def getRuntimeStatistics: Statistics
+
   /**
    * Compute the statistics of the query stage if executed, otherwise None.
    */
@@ -100,8 +111,10 @@ abstract class QueryStageExec extends LeafExecNode {
   override def executeTail(n: Int): Array[InternalRow] = plan.executeTail(n)
   override def executeToIterator(): Iterator[InternalRow] = plan.executeToIterator()
 
-  override def doPrepare(): Unit = plan.prepare()
-  override def doExecute(): RDD[InternalRow] = plan.execute()
+  protected override def doPrepare(): Unit = plan.prepare()
+  protected override def doExecute(): RDD[InternalRow] = plan.execute()
+  override def supportsColumnar: Boolean = plan.supportsColumnar
+  protected override def doExecuteColumnar(): RDD[ColumnarBatch] = plan.executeColumnar()
   override def doExecuteBroadcast[T](): Broadcast[T] = plan.executeBroadcast()
   override def doCanonicalize(): SparkPlan = plan.canonicalized
 
@@ -130,20 +143,20 @@ abstract class QueryStageExec extends LeafExecNode {
 }
 
 /**
- * A shuffle query stage whose child is a [[ShuffleExchangeExec]] or [[ReusedExchangeExec]].
+ * A shuffle query stage whose child is a [[ShuffleExchangeLike]] or [[ReusedExchangeExec]].
  */
 case class ShuffleQueryStageExec(
     override val id: Int,
     override val plan: SparkPlan) extends QueryStageExec {
 
   @transient val shuffle = plan match {
-    case s: ShuffleExchangeExec => s
-    case ReusedExchangeExec(_, s: ShuffleExchangeExec) => s
+    case s: ShuffleExchangeLike => s
+    case ReusedExchangeExec(_, s: ShuffleExchangeLike) => s
     case _ =>
       throw new IllegalStateException("wrong plan for shuffle stage:\n " + plan.treeString)
   }
 
-  override def doMaterialize(): Future[Any] = {
+  override def doMaterialize(): Future[Any] = attachTree(this, "execute") {
     shuffle.mapOutputStatisticsFuture
   }
 
@@ -161,24 +174,52 @@ case class ShuffleQueryStageExec(
       case _ =>
     }
   }
+
+  /**
+   * Returns the Option[MapOutputStatistics]. If the shuffle map stage has no partition,
+   * this method returns None, as there is no map statistics.
+   */
+  def mapStats: Option[MapOutputStatistics] = {
+    assert(resultOption.isDefined, "ShuffleQueryStageExec should already be ready")
+    val stats = resultOption.get.asInstanceOf[MapOutputStatistics]
+    Option(stats)
+  }
+
+  override def getRuntimeStatistics: Statistics = shuffle.runtimeStatistics
 }
 
 /**
- * A broadcast query stage whose child is a [[BroadcastExchangeExec]] or [[ReusedExchangeExec]].
+ * A broadcast query stage whose child is a [[BroadcastExchangeLike]] or [[ReusedExchangeExec]].
  */
 case class BroadcastQueryStageExec(
     override val id: Int,
     override val plan: SparkPlan) extends QueryStageExec {
 
   @transient val broadcast = plan match {
-    case b: BroadcastExchangeExec => b
-    case ReusedExchangeExec(_, b: BroadcastExchangeExec) => b
+    case b: BroadcastExchangeLike => b
+    case ReusedExchangeExec(_, b: BroadcastExchangeLike) => b
     case _ =>
       throw new IllegalStateException("wrong plan for broadcast stage:\n " + plan.treeString)
   }
 
+  @transient private lazy val materializeWithTimeout = {
+    val broadcastFuture = broadcast.completionFuture
+    val timeout = SQLConf.get.broadcastTimeout
+    val promise = Promise[Any]()
+    val fail = BroadcastQueryStageExec.scheduledExecutor.schedule(new Runnable() {
+      override def run(): Unit = {
+        promise.tryFailure(new SparkException(s"Could not execute broadcast in $timeout secs. " +
+          s"You can increase the timeout for broadcasts via ${SQLConf.BROADCAST_TIMEOUT.key} or " +
+          s"disable broadcast join by setting ${SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key} to -1"))
+      }
+    }, timeout, TimeUnit.SECONDS)
+    broadcastFuture.onComplete(_ => fail.cancel(false))(AdaptiveSparkPlanExec.executionContext)
+    Future.firstCompletedOf(
+      Seq(broadcastFuture, promise.future))(AdaptiveSparkPlanExec.executionContext)
+  }
+
   override def doMaterialize(): Future[Any] = {
-    broadcast.completionFuture
+    materializeWithTimeout
   }
 
   override def newReuseInstance(newStageId: Int, newOutput: Seq[Attribute]): QueryStageExec = {
@@ -193,4 +234,11 @@ case class BroadcastQueryStageExec(
       broadcast.relationFuture.cancel(true)
     }
   }
+
+  override def getRuntimeStatistics: Statistics = broadcast.runtimeStatistics
+}
+
+object BroadcastQueryStageExec {
+  private val scheduledExecutor =
+    ThreadUtils.newDaemonSingleThreadScheduledExecutor("BroadcastStageTimeout")
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala
deleted file mode 100644
index 2c50b638b4d45..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.adaptive
-
-import scala.collection.mutable.{ArrayBuffer, HashSet}
-
-import org.apache.spark.MapOutputStatistics
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.{ShuffledRowRDD, SparkPlan, UnaryExecNode}
-import org.apache.spark.sql.internal.SQLConf
-
-/**
- * A rule to adjust the post shuffle partitions based on the map output statistics.
- *
- * The strategy used to determine the number of post-shuffle partitions is described as follows.
- * To determine the number of post-shuffle partitions, we have a target input size for a
- * post-shuffle partition. Once we have size statistics of all pre-shuffle partitions, we will do
- * a pass of those statistics and pack pre-shuffle partitions with continuous indices to a single
- * post-shuffle partition until adding another pre-shuffle partition would cause the size of a
- * post-shuffle partition to be greater than the target size.
- *
- * For example, we have two stages with the following pre-shuffle partition size statistics:
- * stage 1: [100 MiB, 20 MiB, 100 MiB, 10MiB, 30 MiB]
- * stage 2: [10 MiB,  10 MiB, 70 MiB,  5 MiB, 5 MiB]
- * assuming the target input size is 128 MiB, we will have four post-shuffle partitions,
- * which are:
- *  - post-shuffle partition 0: pre-shuffle partition 0 (size 110 MiB)
- *  - post-shuffle partition 1: pre-shuffle partition 1 (size 30 MiB)
- *  - post-shuffle partition 2: pre-shuffle partition 2 (size 170 MiB)
- *  - post-shuffle partition 3: pre-shuffle partition 3 and 4 (size 50 MiB)
- */
-case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
-
-  override def apply(plan: SparkPlan): SparkPlan = {
-    if (!conf.reducePostShufflePartitionsEnabled) {
-      return plan
-    }
-    // 'SkewedShufflePartitionReader' is added by us, so it's safe to ignore it when changing
-    // number of reducers.
-    val leafNodes = plan.collectLeaves().filter(!_.isInstanceOf[SkewedPartitionReaderExec])
-    if (!leafNodes.forall(_.isInstanceOf[QueryStageExec])) {
-      // If not all leaf nodes are query stages, it's not safe to reduce the number of
-      // shuffle partitions, because we may break the assumption that all children of a spark plan
-      // have same number of output partitions.
-      return plan
-    }
-
-    def collectShuffles(plan: SparkPlan): Seq[SparkPlan] = plan match {
-      case _: LocalShuffleReaderExec => Nil
-      case p: PartialShuffleReaderExec => Seq(p)
-      case stage: ShuffleQueryStageExec => Seq(stage)
-      case _ => plan.children.flatMap(collectShuffles)
-    }
-
-    val shuffles = collectShuffles(plan)
-    val shuffleStages = shuffles.map {
-      case PartialShuffleReaderExec(s: ShuffleQueryStageExec, _) => s
-      case s: ShuffleQueryStageExec => s
-    }
-    // ShuffleExchanges introduced by repartition do not support changing the number of partitions.
-    // We change the number of partitions in the stage only if all the ShuffleExchanges support it.
-    if (!shuffleStages.forall(_.shuffle.canChangeNumPartitions)) {
-      plan
-    } else {
-      val shuffleMetrics = shuffleStages.map { stage =>
-        assert(stage.resultOption.isDefined, "ShuffleQueryStageExec should already be ready")
-        stage.resultOption.get.asInstanceOf[MapOutputStatistics]
-      }
-
-      // `ShuffleQueryStageExec` gives null mapOutputStatistics when the input RDD has 0 partitions,
-      // we should skip it when calculating the `partitionStartIndices`.
-      val validMetrics = shuffleMetrics.filter(_ != null)
-      // We may have different pre-shuffle partition numbers, don't reduce shuffle partition number
-      // in that case. For example when we union fully aggregated data (data is arranged to a single
-      // partition) and a result of a SortMergeJoin (multiple partitions).
-      val distinctNumPreShufflePartitions =
-        validMetrics.map(stats => stats.bytesByPartitionId.length).distinct
-      val distinctExcludedPartitions = shuffles.map {
-        case PartialShuffleReaderExec(_, excludedPartitions) => excludedPartitions
-        case _: ShuffleQueryStageExec => Set.empty[Int]
-      }.distinct
-      if (validMetrics.nonEmpty && distinctNumPreShufflePartitions.length == 1
-        && distinctExcludedPartitions.length == 1) {
-        val excludedPartitions = distinctExcludedPartitions.head
-        val partitionIndices = estimatePartitionStartAndEndIndices(
-          validMetrics.toArray, excludedPartitions)
-        // This transformation adds new nodes, so we must use `transformUp` here.
-        // Even for shuffle exchange whose input RDD has 0 partition, we should still update its
-        // `partitionStartIndices`, so that all the leaf shuffles in a stage have the same
-        // number of output partitions.
-        val visitedStages = HashSet.empty[Int]
-        plan.transformDown {
-          // Replace `PartialShuffleReaderExec` with `CoalescedShuffleReaderExec`, which keeps the
-          // "excludedPartition" requirement and also merges some partitions.
-          case PartialShuffleReaderExec(stage: ShuffleQueryStageExec, _) =>
-            visitedStages.add(stage.id)
-            CoalescedShuffleReaderExec(stage, partitionIndices)
-
-          // We are doing `transformDown`, so the `ShuffleQueryStageExec` may already be optimized
-          // and wrapped by `CoalescedShuffleReaderExec`.
-          case stage: ShuffleQueryStageExec if !visitedStages.contains(stage.id) =>
-            visitedStages.add(stage.id)
-            CoalescedShuffleReaderExec(stage, partitionIndices)
-        }
-      } else {
-        plan
-      }
-    }
-  }
-
-  /**
-   * Estimates partition start and end indices for post-shuffle partitions based on
-   * mapOutputStatistics provided by all pre-shuffle stages and skip the omittedPartitions
-   * already handled in skewed partition optimization.
-   */
-  // visible for testing.
-  private[sql] def estimatePartitionStartAndEndIndices(
-      mapOutputStatistics: Array[MapOutputStatistics],
-      excludedPartitions: Set[Int] = Set.empty): Array[(Int, Int)] = {
-    val minNumPostShufflePartitions = conf.minNumPostShufflePartitions - excludedPartitions.size
-    val advisoryTargetPostShuffleInputSize = conf.targetPostShuffleInputSize
-    // If minNumPostShufflePartitions is defined, it is possible that we need to use a
-    // value less than advisoryTargetPostShuffleInputSize as the target input size of
-    // a post shuffle task.
-    val totalPostShuffleInputSize = mapOutputStatistics.map(_.bytesByPartitionId.sum).sum
-    // The max at here is to make sure that when we have an empty table, we
-    // only have a single post-shuffle partition.
-    // There is no particular reason that we pick 16. We just need a number to
-    // prevent maxPostShuffleInputSize from being set to 0.
-    val maxPostShuffleInputSize = math.max(
-      math.ceil(totalPostShuffleInputSize / minNumPostShufflePartitions.toDouble).toLong, 16)
-    val targetPostShuffleInputSize =
-      math.min(maxPostShuffleInputSize, advisoryTargetPostShuffleInputSize)
-
-    logInfo(
-      s"advisoryTargetPostShuffleInputSize: $advisoryTargetPostShuffleInputSize, " +
-        s"targetPostShuffleInputSize $targetPostShuffleInputSize.")
-
-    // Make sure we do get the same number of pre-shuffle partitions for those stages.
-    val distinctNumPreShufflePartitions =
-      mapOutputStatistics.map(stats => stats.bytesByPartitionId.length).distinct
-    // The reason that we are expecting a single value of the number of pre-shuffle partitions
-    // is that when we add Exchanges, we set the number of pre-shuffle partitions
-    // (i.e. map output partitions) using a static setting, which is the value of
-    // spark.sql.shuffle.partitions. Even if two input RDDs are having different
-    // number of partitions, they will have the same number of pre-shuffle partitions
-    // (i.e. map output partitions).
-    assert(
-      distinctNumPreShufflePartitions.length == 1,
-      "There should be only one distinct value of the number pre-shuffle partitions " +
-        "among registered Exchange operator.")
-
-    val partitionStartIndices = ArrayBuffer[Int]()
-    val partitionEndIndices = ArrayBuffer[Int]()
-    val numPartitions = distinctNumPreShufflePartitions.head
-    val includedPartitions = (0 until numPartitions).filter(!excludedPartitions.contains(_))
-    val firstStartIndex = includedPartitions(0)
-    partitionStartIndices += firstStartIndex
-    var postShuffleInputSize = mapOutputStatistics.map(_.bytesByPartitionId(firstStartIndex)).sum
-    var i = firstStartIndex
-    includedPartitions.drop(1).foreach { nextPartitionIndex =>
-        val nextShuffleInputSize =
-          mapOutputStatistics.map(_.bytesByPartitionId(nextPartitionIndex)).sum
-        // If nextPartitionIndices is skewed and omitted, or including
-        // the nextShuffleInputSize would exceed the target partition size,
-        // then start a new partition.
-        if (nextPartitionIndex != i + 1 ||
-          (postShuffleInputSize + nextShuffleInputSize > targetPostShuffleInputSize)) {
-          partitionEndIndices += i + 1
-          partitionStartIndices += nextPartitionIndex
-          // reset postShuffleInputSize.
-          postShuffleInputSize = nextShuffleInputSize
-          i = nextPartitionIndex
-        } else {
-          postShuffleInputSize += nextShuffleInputSize
-          i += 1
-        }
-    }
-    partitionEndIndices += i + 1
-    partitionStartIndices.zip(partitionEndIndices).toArray
-  }
-}
-
-/**
- * A wrapper of shuffle query stage, which submits fewer reduce task as one reduce task may read
- * multiple shuffle partitions. This can avoid many small reduce tasks that hurt performance.
- *
- * @param child It's usually `ShuffleQueryStageExec`, but can be the shuffle exchange node during
- *              canonicalization.
- */
-case class CoalescedShuffleReaderExec(
-    child: SparkPlan,
-    partitionIndices: Array[(Int, Int)]) extends UnaryExecNode {
-
-  override def output: Seq[Attribute] = child.output
-
-  override def outputPartitioning: Partitioning = {
-    UnknownPartitioning(partitionIndices.length)
-  }
-
-  private var cachedShuffleRDD: ShuffledRowRDD = null
-
-  override protected def doExecute(): RDD[InternalRow] = {
-    if (cachedShuffleRDD == null) {
-      cachedShuffleRDD = child match {
-        case stage: ShuffleQueryStageExec =>
-          stage.shuffle.createShuffledRDD(Some(partitionIndices))
-        case _ =>
-          throw new IllegalStateException("operating on canonicalization plan")
-      }
-    }
-    cachedShuffleRDD
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
new file mode 100644
index 0000000000000..e10ed4f481cf7
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.MapOutputStatistics
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.execution.{CoalescedPartitionSpec, ShufflePartitionSpec}
+
+object ShufflePartitionsUtil extends Logging {
+  final val SMALL_PARTITION_FACTOR = 0.2
+  final val MERGED_PARTITION_FACTOR = 1.2
+
+  /**
+   * Coalesce the partitions from multiple shuffles. This method assumes that all the shuffles
+   * have the same number of partitions, and the partitions of same index will be read together
+   * by one task.
+   *
+   * The strategy used to determine the number of coalesced partitions is described as follows.
+   * To determine the number of coalesced partitions, we have a target size for a coalesced
+   * partition. Once we have size statistics of all shuffle partitions, we will do
+   * a pass of those statistics and pack shuffle partitions with continuous indices to a single
+   * coalesced partition until adding another shuffle partition would cause the size of a
+   * coalesced partition to be greater than the target size.
+   *
+   * For example, we have two shuffles with the following partition size statistics:
+   *  - shuffle 1 (5 partitions): [100 MiB, 20 MiB, 100 MiB, 10MiB, 30 MiB]
+   *  - shuffle 2 (5 partitions): [10 MiB,  10 MiB, 70 MiB,  5 MiB, 5 MiB]
+   * Assuming the target size is 128 MiB, we will have 4 coalesced partitions, which are:
+   *  - coalesced partition 0: shuffle partition 0 (size 110 MiB)
+   *  - coalesced partition 1: shuffle partition 1 (size 30 MiB)
+   *  - coalesced partition 2: shuffle partition 2 (size 170 MiB)
+   *  - coalesced partition 3: shuffle partition 3 and 4 (size 50 MiB)
+   *
+   *  @return A sequence of [[CoalescedPartitionSpec]]s. For example, if partitions [0, 1, 2, 3, 4]
+   *          split at indices [0, 2, 3], the returned partition specs will be:
+   *          CoalescedPartitionSpec(0, 2), CoalescedPartitionSpec(2, 3) and
+   *          CoalescedPartitionSpec(3, 5).
+   */
+  def coalescePartitions(
+      mapOutputStatistics: Array[MapOutputStatistics],
+      advisoryTargetSize: Long,
+      minNumPartitions: Int): Seq[ShufflePartitionSpec] = {
+    // If `minNumPartitions` is very large, it is possible that we need to use a value less than
+    // `advisoryTargetSize` as the target size of a coalesced task.
+    val totalPostShuffleInputSize = mapOutputStatistics.map(_.bytesByPartitionId.sum).sum
+    // The max at here is to make sure that when we have an empty table, we only have a single
+    // coalesced partition.
+    // There is no particular reason that we pick 16. We just need a number to prevent
+    // `maxTargetSize` from being set to 0.
+    val maxTargetSize = math.max(
+      math.ceil(totalPostShuffleInputSize / minNumPartitions.toDouble).toLong, 16)
+    val targetSize = math.min(maxTargetSize, advisoryTargetSize)
+
+    val shuffleIds = mapOutputStatistics.map(_.shuffleId).mkString(", ")
+    logInfo(s"For shuffle($shuffleIds), advisory target size: $advisoryTargetSize, " +
+      s"actual target size $targetSize.")
+
+    // Make sure these shuffles have the same number of partitions.
+    val distinctNumShufflePartitions =
+      mapOutputStatistics.map(stats => stats.bytesByPartitionId.length).distinct
+    // The reason that we are expecting a single value of the number of shuffle partitions
+    // is that when we add Exchanges, we set the number of shuffle partitions
+    // (i.e. map output partitions) using a static setting, which is the value of
+    // `spark.sql.shuffle.partitions`. Even if two input RDDs are having different
+    // number of partitions, they will have the same number of shuffle partitions
+    // (i.e. map output partitions).
+    assert(
+      distinctNumShufflePartitions.length == 1,
+      "There should be only one distinct value of the number of shuffle partitions " +
+        "among registered Exchange operators.")
+
+    val numPartitions = distinctNumShufflePartitions.head
+    val partitionSpecs = ArrayBuffer[CoalescedPartitionSpec]()
+    var latestSplitPoint = 0
+    var coalescedSize = 0L
+    var i = 0
+    while (i < numPartitions) {
+      // We calculate the total size of i-th shuffle partitions from all shuffles.
+      var totalSizeOfCurrentPartition = 0L
+      var j = 0
+      while (j < mapOutputStatistics.length) {
+        totalSizeOfCurrentPartition += mapOutputStatistics(j).bytesByPartitionId(i)
+        j += 1
+      }
+
+      // If including the `totalSizeOfCurrentPartition` would exceed the target size, then start a
+      // new coalesced partition.
+      if (i > latestSplitPoint && coalescedSize + totalSizeOfCurrentPartition > targetSize) {
+        partitionSpecs += CoalescedPartitionSpec(latestSplitPoint, i)
+        latestSplitPoint = i
+        // reset postShuffleInputSize.
+        coalescedSize = totalSizeOfCurrentPartition
+      } else {
+        coalescedSize += totalSizeOfCurrentPartition
+      }
+      i += 1
+    }
+    partitionSpecs += CoalescedPartitionSpec(latestSplitPoint, numPartitions)
+
+    partitionSpecs
+  }
+
+  /**
+   * Given a list of size, return an array of indices to split the list into multiple partitions,
+   * so that the size sum of each partition is close to the target size. Each index indicates the
+   * start of a partition.
+   */
+  def splitSizeListByTargetSize(sizes: Seq[Long], targetSize: Long): Array[Int] = {
+    val partitionStartIndices = ArrayBuffer[Int]()
+    partitionStartIndices += 0
+    var i = 0
+    var currentPartitionSize = 0L
+    var lastPartitionSize = -1L
+
+    def tryMergePartitions() = {
+      // When we are going to start a new partition, it's possible that the current partition or
+      // the previous partition is very small and it's better to merge the current partition into
+      // the previous partition.
+      val shouldMergePartitions = lastPartitionSize > -1 &&
+        ((currentPartitionSize + lastPartitionSize) < targetSize * MERGED_PARTITION_FACTOR ||
+        (currentPartitionSize < targetSize * SMALL_PARTITION_FACTOR ||
+          lastPartitionSize < targetSize * SMALL_PARTITION_FACTOR))
+      if (shouldMergePartitions) {
+        // We decide to merge the current partition into the previous one, so the start index of
+        // the current partition should be removed.
+        partitionStartIndices.remove(partitionStartIndices.length - 1)
+        lastPartitionSize += currentPartitionSize
+      } else {
+        lastPartitionSize = currentPartitionSize
+      }
+    }
+
+    while (i < sizes.length) {
+      // If including the next size in the current partition exceeds the target size, package the
+      // current partition and start a new partition.
+      if (i > 0 && currentPartitionSize + sizes(i) > targetSize) {
+        tryMergePartitions()
+        partitionStartIndices += i
+        currentPartitionSize = sizes(i)
+      } else {
+        currentPartitionSize += sizes(i)
+      }
+      i += 1
+    }
+    tryMergePartitions()
+    partitionStartIndices.toArray
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/SkewedShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/SkewedShuffledRowRDD.scala
deleted file mode 100644
index 52f793b24aa17..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/SkewedShuffledRowRDD.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.adaptive
-
-import org.apache.spark._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleReadMetricsReporter}
-
-/**
- * The [[Partition]] used by [[SkewedShuffledRowRDD]].
- */
-class SkewedShuffledRowRDDPartition(override val index: Int) extends Partition
-
-/**
- * This is a specialized version of [[org.apache.spark.sql.execution.ShuffledRowRDD]]. This is used
- * in Spark SQL adaptive execution to solve data skew issues. This RDD includes rearranged
- * partitions from mappers.
- *
- * This RDD takes a [[ShuffleDependency]] (`dependency`), a partitionIndex
- * and the range of startMapIndex to endMapIndex.
- */
-class SkewedShuffledRowRDD(
-     var dependency: ShuffleDependency[Int, InternalRow, InternalRow],
-     partitionIndex: Int,
-     startMapIndex: Int,
-     endMapIndex: Int,
-     metrics: Map[String, SQLMetric])
-  extends RDD[InternalRow](dependency.rdd.context, Nil) {
-
-  override def getDependencies: Seq[Dependency[_]] = List(dependency)
-
-  override def getPartitions: Array[Partition] = {
-    Array(new SkewedShuffledRowRDDPartition(0))
-  }
-
-  override def getPreferredLocations(partition: Partition): Seq[String] = {
-    val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
-    tracker.getMapLocation(dependency, startMapIndex, endMapIndex)
-  }
-
-  override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
-    val tempMetrics = context.taskMetrics().createTempShuffleReadMetrics()
-    // `SQLShuffleReadMetricsReporter` will update its own metrics for SQL exchange operator,
-    // as well as the `tempMetrics` for basic shuffle metrics.
-    val sqlMetricsReporter = new SQLShuffleReadMetricsReporter(tempMetrics, metrics)
-
-    val reader = SparkEnv.get.shuffleManager.getReaderForRange(
-      dependency.shuffleHandle,
-      startMapIndex,
-      endMapIndex,
-      partitionIndex,
-      partitionIndex + 1,
-      context,
-      sqlMetricsReporter)
-    reader.read().asInstanceOf[Iterator[Product2[Int, InternalRow]]].map(_._2)
-  }
-
-  override def clearDependencies() {
-    super.clearDependencies()
-    dependency = null
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala
index 67cd720bb5b33..cdc57dbc7dcc2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
+import org.apache.spark.sql.execution.exchange.{ShuffleExchangeExec, ShuffleExchangeLike}
 
 /**
  * A simple implementation of [[Cost]], which takes a number of [[Long]] as the cost value.
@@ -35,13 +35,13 @@ case class SimpleCost(value: Long) extends Cost {
 
 /**
  * A simple implementation of [[CostEvaluator]], which counts the number of
- * [[ShuffleExchangeExec]] nodes in the plan.
+ * [[ShuffleExchangeLike]] nodes in the plan.
  */
 object SimpleCostEvaluator extends CostEvaluator {
 
   override def evaluateCost(plan: SparkPlan): Cost = {
     val cost = plan.collect {
-      case s: ShuffleExchangeExec => s
+      case s: ShuffleExchangeLike => s
     }.size
     SimpleCost(cost)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
index 56a287d4d0279..761ac20e84744 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
@@ -135,20 +135,12 @@ object AggUtils {
       groupingExpressions: Seq[NamedExpression],
       functionsWithDistinct: Seq[AggregateExpression],
       functionsWithoutDistinct: Seq[AggregateExpression],
+      distinctExpressions: Seq[Expression],
+      normalizedNamedDistinctExpressions: Seq[NamedExpression],
       resultExpressions: Seq[NamedExpression],
       child: SparkPlan): Seq[SparkPlan] = {
 
-    // functionsWithDistinct is guaranteed to be non-empty. Even though it may contain more than one
-    // DISTINCT aggregate function, all of those functions will have the same column expressions.
-    // For example, it would be valid for functionsWithDistinct to be
-    // [COUNT(DISTINCT foo), MAX(DISTINCT foo)], but [COUNT(DISTINCT bar), COUNT(DISTINCT foo)] is
-    // disallowed because those two distinct aggregates have different column expressions.
-    val distinctExpressions = functionsWithDistinct.head.aggregateFunction.children
-    val namedDistinctExpressions = distinctExpressions.map {
-      case ne: NamedExpression => ne
-      case other => Alias(other, other.toString)()
-    }
-    val distinctAttributes = namedDistinctExpressions.map(_.toAttribute)
+    val distinctAttributes = normalizedNamedDistinctExpressions.map(_.toAttribute)
     val groupingAttributes = groupingExpressions.map(_.toAttribute)
 
     // 1. Create an Aggregate Operator for partial aggregations.
@@ -159,7 +151,7 @@ object AggUtils {
       // DISTINCT column. For example, for AVG(DISTINCT value) GROUP BY key, the grouping
       // expressions will be [key, value].
       createAggregate(
-        groupingExpressions = groupingExpressions ++ namedDistinctExpressions,
+        groupingExpressions = groupingExpressions ++ normalizedNamedDistinctExpressions,
         aggregateExpressions = aggregateExpressions,
         aggregateAttributes = aggregateAttributes,
         resultExpressions = groupingAttributes ++ distinctAttributes ++
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
new file mode 100644
index 0000000000000..f1e053f7fb2a5
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.aggregate
+
+import org.apache.spark.sql.catalyst.expressions.{Attribute, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Final, PartialMerge}
+import org.apache.spark.sql.execution.{ExplainUtils, UnaryExecNode}
+
+/**
+ * Holds common logic for aggregate operators
+ */
+trait BaseAggregateExec extends UnaryExecNode {
+  def groupingExpressions: Seq[NamedExpression]
+  def aggregateExpressions: Seq[AggregateExpression]
+  def aggregateAttributes: Seq[Attribute]
+  def resultExpressions: Seq[NamedExpression]
+
+  override def verboseStringWithOperatorId(): String = {
+    s"""
+       |$formattedNodeName
+       |${ExplainUtils.generateFieldString("Input", child.output)}
+       |${ExplainUtils.generateFieldString("Keys", groupingExpressions)}
+       |${ExplainUtils.generateFieldString("Functions", aggregateExpressions)}
+       |${ExplainUtils.generateFieldString("Aggregate Attributes", aggregateAttributes)}
+       |${ExplainUtils.generateFieldString("Results", resultExpressions)}
+       |""".stripMargin
+  }
+
+  protected def inputAttributes: Seq[Attribute] = {
+    val modes = aggregateExpressions.map(_.mode).distinct
+    if (modes.contains(Final) || modes.contains(PartialMerge)) {
+      // SPARK-31620: when planning aggregates, the partial aggregate uses aggregate function's
+      // `inputAggBufferAttributes` as its output. And Final and PartialMerge aggregate rely on the
+      // output to bind references for `DeclarativeAggregate.mergeExpressions`. But if we copy the
+      // aggregate function somehow after aggregate planning, like `PlanSubqueries`, the
+      // `DeclarativeAggregate` will be replaced by a new instance with new
+      // `inputAggBufferAttributes` and `mergeExpressions`. Then Final and PartialMerge aggregate
+      // can't bind the `mergeExpressions` with the output of the partial aggregate, as they use
+      // the `inputAggBufferAttributes` of the original `DeclarativeAggregate` before copy. Instead,
+      // we shall use `inputAggBufferAttributes` after copy to match the new `mergeExpressions`.
+      val aggAttrs = aggregateExpressions
+        // there're exactly four cases needs `inputAggBufferAttributes` from child according to the
+        // agg planning in `AggUtils`: Partial -> Final, PartialMerge -> Final,
+        // Partial -> PartialMerge, PartialMerge -> PartialMerge.
+        .filter(a => a.mode == Final || a.mode == PartialMerge).map(_.aggregateFunction)
+        .flatMap(_.inputAggBufferAttributes)
+      child.output.dropRight(aggAttrs.length) ++ aggAttrs
+    } else {
+      child.output
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
index f73e214a6b41f..56641bee2111b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
@@ -53,7 +53,7 @@ case class HashAggregateExec(
     initialInputBufferOffset: Int,
     resultExpressions: Seq[NamedExpression],
     child: SparkPlan)
-  extends UnaryExecNode with BlockingOperatorWithCodegen with AliasAwareOutputPartitioning {
+  extends BaseAggregateExec with BlockingOperatorWithCodegen with AliasAwareOutputPartitioning {
 
   private[this] val aggregateBufferAttributes = {
     aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)
@@ -127,7 +127,7 @@ case class HashAggregateExec(
             resultExpressions,
             (expressions, inputSchema) =>
               MutableProjection.create(expressions, inputSchema),
-            child.output,
+            inputAttributes,
             iter,
             testFallbackStartsAt,
             numOutputRows,
@@ -332,7 +332,7 @@ case class HashAggregateExec(
   private def doConsumeWithoutKeys(ctx: CodegenContext, input: Seq[ExprCode]): String = {
     // only have DeclarativeAggregate
     val functions = aggregateExpressions.map(_.aggregateFunction.asInstanceOf[DeclarativeAggregate])
-    val inputAttrs = functions.flatMap(_.aggBufferAttributes) ++ child.output
+    val inputAttrs = functions.flatMap(_.aggBufferAttributes) ++ inputAttributes
     // To individually generate code for each aggregate function, an element in `updateExprs` holds
     // all the expressions for the buffer of an aggregation function.
     val updateExprs = aggregateExpressions.map { e =>
@@ -367,10 +367,10 @@ case class HashAggregateExec(
          """.stripMargin
       }
       code"""
-         |// do aggregate for ${aggNames(i)}
-         |// evaluate aggregate function
+         |${ctx.registerComment(s"do aggregate for ${aggNames(i)}")}
+         |${ctx.registerComment("evaluate aggregate function")}
          |${evaluateVariables(bufferEvalsForOneFunc)}
-         |// update aggregation buffers
+         |${ctx.registerComment("update aggregation buffers")}
          |${updates.mkString("\n").trim}
        """.stripMargin
     }
@@ -929,7 +929,7 @@ case class HashAggregateExec(
       }
     }
 
-    val inputAttr = aggregateBufferAttributes ++ child.output
+    val inputAttr = aggregateBufferAttributes ++ inputAttributes
     // Here we set `currentVars(0)` to `currentVars(numBufferSlots)` to null, so that when
     // generating code for buffer columns, we use `INPUT_ROW`(will be the buffer row), while
     // generating input columns, we use `currentVars`.
@@ -975,9 +975,9 @@ case class HashAggregateExec(
           CodeGenerator.updateColumn(unsafeRowBuffer, dt, bufferOffset + j, ev, nullable)
         }
         code"""
-           |// evaluate aggregate function for ${aggNames(i)}
+           |${ctx.registerComment(s"evaluate aggregate function for ${aggNames(i)}")}
            |${evaluateVariables(rowBufferEvalsForOneFunc)}
-           |// update unsafe row buffer
+           |${ctx.registerComment("update unsafe row buffer")}
            |${updateRowBuffers.mkString("\n").trim}
          """.stripMargin
       }
@@ -1030,9 +1030,9 @@ case class HashAggregateExec(
                 isVectorized = true)
             }
             code"""
-               |// evaluate aggregate function for ${aggNames(i)}
+               |${ctx.registerComment(s"evaluate aggregate function for ${aggNames(i)}")}
                |${evaluateVariables(fastRowEvalsForOneFunc)}
-               |// update fast row
+               |${ctx.registerComment("update fast row")}
                |${updateRowBuffer.mkString("\n").trim}
              """.stripMargin
           }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
index 4376f6b6edd57..f1c0719ff8948 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
@@ -67,7 +67,7 @@ case class ObjectHashAggregateExec(
     initialInputBufferOffset: Int,
     resultExpressions: Seq[NamedExpression],
     child: SparkPlan)
-  extends UnaryExecNode with AliasAwareOutputPartitioning {
+  extends BaseAggregateExec with AliasAwareOutputPartitioning {
 
   private[this] val aggregateBufferAttributes = {
     aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)
@@ -123,7 +123,7 @@ case class ObjectHashAggregateExec(
             resultExpressions,
             (expressions, inputSchema) =>
               MutableProjection.create(expressions, inputSchema),
-            child.output,
+            inputAttributes,
             iter,
             fallbackCountThreshold,
             numOutputRows)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
index b6e684e62ea5c..c23e19d3b6262 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.execution.{AliasAwareOutputPartitioning, SparkPlan, UnaryExecNode}
+import org.apache.spark.sql.execution.{AliasAwareOutputPartitioning, SparkPlan}
 import org.apache.spark.sql.execution.metric.SQLMetrics
 
 /**
@@ -38,7 +38,7 @@ case class SortAggregateExec(
     initialInputBufferOffset: Int,
     resultExpressions: Seq[NamedExpression],
     child: SparkPlan)
-  extends UnaryExecNode with AliasAwareOutputPartitioning {
+  extends BaseAggregateExec with AliasAwareOutputPartitioning {
 
   private[this] val aggregateBufferAttributes = {
     aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)
@@ -86,7 +86,7 @@ case class SortAggregateExec(
         val outputIter = new SortBasedAggregationIterator(
           partIndex,
           groupingExpressions,
-          child.output,
+          inputAttributes,
           iter,
           aggregateExpressions,
           aggregateAttributes,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
index dfae5c07e0373..44bc9c2e3a9d0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
@@ -27,6 +27,8 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete}
 import org.apache.spark.sql.catalyst.expressions.aggregate.{ImperativeAggregate, TypedImperativeAggregate}
 import org.apache.spark.sql.catalyst.expressions.codegen.{GenerateMutableProjection, GenerateSafeProjection}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.expressions.{Aggregator, MutableAggregationBuffer, UserDefinedAggregateFunction}
 import org.apache.spark.sql.types._
 
@@ -458,7 +460,8 @@ case class ScalaUDAF(
 case class ScalaAggregator[IN, BUF, OUT](
     children: Seq[Expression],
     agg: Aggregator[IN, BUF, OUT],
-    inputEncoderNR: ExpressionEncoder[IN],
+    inputEncoder: ExpressionEncoder[IN],
+    bufferEncoder: ExpressionEncoder[BUF],
     nullable: Boolean = true,
     isDeterministic: Boolean = true,
     mutableAggBufferOffset: Int = 0,
@@ -469,10 +472,12 @@ case class ScalaAggregator[IN, BUF, OUT](
   with ImplicitCastInputTypes
   with Logging {
 
-  private[this] lazy val inputEncoder = inputEncoderNR.resolveAndBind()
-  private[this] lazy val bufferEncoder =
-    agg.bufferEncoder.asInstanceOf[ExpressionEncoder[BUF]].resolveAndBind()
+  // input and buffer encoders are resolved by ResolveEncodersInScalaAgg
+  private[this] lazy val inputDeserializer = inputEncoder.createDeserializer()
+  private[this] lazy val bufferSerializer = bufferEncoder.createSerializer()
+  private[this] lazy val bufferDeserializer = bufferEncoder.createDeserializer()
   private[this] lazy val outputEncoder = agg.outputEncoder.asInstanceOf[ExpressionEncoder[OUT]]
+  private[this] lazy val outputSerializer = outputEncoder.createSerializer()
 
   def dataType: DataType = outputEncoder.objSerializer.dataType
 
@@ -491,26 +496,41 @@ case class ScalaAggregator[IN, BUF, OUT](
   def createAggregationBuffer(): BUF = agg.zero
 
   def update(buffer: BUF, input: InternalRow): BUF =
-    agg.reduce(buffer, inputEncoder.fromRow(inputProjection(input)))
+    agg.reduce(buffer, inputDeserializer(inputProjection(input)))
 
   def merge(buffer: BUF, input: BUF): BUF = agg.merge(buffer, input)
 
   def eval(buffer: BUF): Any = {
-    val row = outputEncoder.toRow(agg.finish(buffer))
+    val row = outputSerializer(agg.finish(buffer))
     if (outputEncoder.isSerializedAsStruct) row else row.get(0, dataType)
   }
 
   private[this] lazy val bufferRow = new UnsafeRow(bufferEncoder.namedExpressions.length)
 
   def serialize(agg: BUF): Array[Byte] =
-    bufferEncoder.toRow(agg).asInstanceOf[UnsafeRow].getBytes()
+    bufferSerializer(agg).asInstanceOf[UnsafeRow].getBytes()
 
   def deserialize(storageFormat: Array[Byte]): BUF = {
     bufferRow.pointTo(storageFormat, storageFormat.length)
-    bufferEncoder.fromRow(bufferRow)
+    bufferDeserializer(bufferRow)
   }
 
   override def toString: String = s"""${nodeName}(${children.mkString(",")})"""
 
   override def nodeName: String = agg.getClass.getSimpleName
 }
+
+/**
+ * An extension rule to resolve encoder expressions from a [[ScalaAggregator]]
+ */
+object ResolveEncodersInScalaAgg extends Rule[LogicalPlan] {
+  override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    case p if !p.resolved => p
+    case p => p.transformExpressionsUp {
+      case agg: ScalaAggregator[_, _, _] =>
+        agg.copy(
+          inputEncoder = agg.inputEncoder.resolveAndBind(),
+          bufferEncoder = agg.bufferEncoder.resolveAndBind())
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala
index 614d6c2846bfa..136f7c47f5341 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala
@@ -76,6 +76,8 @@ class DetectAmbiguousSelfJoin(conf: SQLConf) extends Rule[LogicalPlan] {
     // We always remove the special metadata from `AttributeReference` at the end of this rule, so
     // Dataset column reference only exists in the root node via Dataset transformations like
     // `Dataset#select`.
+    if (plan.find(_.isInstanceOf[Join]).isEmpty) return stripColumnReferenceMetadataInPlan(plan)
+
     val colRefAttrs = plan.expressions.flatMap(_.collect {
       case a: AttributeReference if isColumnReference(a) => a
     })
@@ -153,6 +155,10 @@ class DetectAmbiguousSelfJoin(conf: SQLConf) extends Rule[LogicalPlan] {
       }
     }
 
+    stripColumnReferenceMetadataInPlan(plan)
+  }
+
+  private def stripColumnReferenceMetadataInPlan(plan: LogicalPlan): LogicalPlan = {
     plan.transformExpressions {
       case a: AttributeReference if isColumnReference(a) =>
         // Remove the special metadata from this `AttributeReference`, as the detection is done.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
index c35c48496e1c9..036b02953a430 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
@@ -17,10 +17,11 @@
 
 package org.apache.spark.sql.execution
 
+import java.util.concurrent.{Future => JFuture}
 import java.util.concurrent.TimeUnit._
 
 import scala.collection.mutable
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.{ExecutionContext}
 import scala.concurrent.duration.Duration
 
 import org.apache.spark.{InterruptibleIterator, Partition, SparkContext, TaskContext}
@@ -85,10 +86,10 @@ case class ProjectExec(projectList: Seq[NamedExpression], child: SparkPlan)
 
   override def verboseStringWithOperatorId(): String = {
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Output    : ${projectList.mkString("[", ", ", "]")}
-       |Input     : ${child.output.mkString("[", ", ", "]")}
-     """.stripMargin
+       |$formattedNodeName
+       |${ExplainUtils.generateFieldString("Output", projectList)}
+       |${ExplainUtils.generateFieldString("Input", child.output)}
+       |""".stripMargin
   }
 }
 
@@ -242,10 +243,10 @@ case class FilterExec(condition: Expression, child: SparkPlan)
 
   override def verboseStringWithOperatorId(): String = {
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Input     : ${child.output.mkString("[", ", ", "]")}
+       |$formattedNodeName
+       |${ExplainUtils.generateFieldString("Input", child.output)}
        |Condition : ${condition}
-     """.stripMargin
+       |""".stripMargin
   }
 }
 
@@ -746,7 +747,7 @@ case class SubqueryExec(name: String, child: SparkPlan)
     "collectTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to collect"))
 
   @transient
-  private lazy val relationFuture: Future[Array[InternalRow]] = {
+  private lazy val relationFuture: JFuture[Array[InternalRow]] = {
     // relationFuture is used in "doExecute". Therefore we can get the execution id correctly here.
     val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
     SQLExecution.withThreadLocalCaptured[Array[InternalRow]](
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala
index 00a1d54b41709..3cc59af9b7ce3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala
@@ -318,7 +318,8 @@ private[columnar] case object RunLengthEncoding extends CompressionScheme {
       var valueCountLocal = 0
       var currentValueLocal: Long = 0
 
-      while (valueCountLocal < runLocal || (pos < capacity)) {
+      while (pos < capacity) {
+        assert(valueCountLocal <= runLocal)
         if (pos != nextNullIndex) {
           if (valueCountLocal == runLocal) {
             currentValueLocal = getFunction(buffer)
@@ -616,7 +617,6 @@ private[columnar] case object BooleanBitSet extends CompressionScheme {
     override def hasNext: Boolean = visited < count
 
     override def decompress(columnVector: WritableColumnVector, capacity: Int): Unit = {
-      val countLocal = count
       var currentWordLocal: Long = 0
       var visitedLocal: Int = 0
       val nullsBuffer = buffer.duplicate().order(ByteOrder.nativeOrder())
@@ -626,7 +626,7 @@ private[columnar] case object BooleanBitSet extends CompressionScheme {
       var pos = 0
       var seenNulls = 0
 
-      while (visitedLocal < countLocal) {
+      while (pos < capacity) {
         if (pos != nextNullIndex) {
           val bit = visitedLocal % BITS_PER_LONG
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala
new file mode 100644
index 0000000000000..dedace4af4d14
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.util.SchemaUtils
+
+/**
+ * Checks legitimization of various execution commands.
+ */
+case class CommandCheck(conf: SQLConf) extends (LogicalPlan => Unit) {
+
+  override def apply(plan: LogicalPlan): Unit = {
+    plan.foreach {
+      case AnalyzeColumnCommand(_, colsOpt, allColumns) if !allColumns =>
+        colsOpt.foreach(SchemaUtils.checkColumnNameDuplication(
+          _, "in analyze columns.", conf.caseSensitiveAnalysis))
+
+      case _ =>
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
index b229b238238fd..34e0ac954010e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
@@ -385,4 +385,12 @@ object CommandUtils extends Logging {
   private def isDataPath(path: Path, stagingDir: String): Boolean = {
     !path.getName.startsWith(stagingDir) && DataSourceUtils.isDataPath(path)
   }
+
+  def uncacheTableOrView(sparkSession: SparkSession, name: String): Unit = {
+    try {
+      sparkSession.catalog.uncacheTable(name)
+    } catch {
+      case NonFatal(e) => logWarning(s"Exception when attempting to uncache $name", e)
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
index 39b08e2894dcd..3dc1d52697714 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
@@ -115,14 +115,19 @@ case class SetCommand(kv: Option[(String, Option[String])]) extends RunnableComm
     case Some(("-v", None)) =>
       val runFunc = (sparkSession: SparkSession) => {
         sparkSession.sessionState.conf.getAllDefinedConfs.sorted.map {
-          case (key, defaultValue, doc) =>
-            Row(key, Option(defaultValue).getOrElse("<undefined>"), doc)
+          case (key, defaultValue, doc, version) =>
+            Row(
+              key,
+              Option(defaultValue).getOrElse("<undefined>"),
+              doc,
+              Option(version).getOrElse("<unknown>"))
         }
       }
       val schema = StructType(
         StructField("key", StringType, nullable = false) ::
           StructField("value", StringType, nullable = false) ::
-          StructField("meaning", StringType, nullable = false) :: Nil)
+          StructField("meaning", StringType, nullable = false) ::
+          StructField("Since version", StringType, nullable = false) :: Nil)
       (schema.toAttributes, runFunc)
 
     // Queries the deprecated "mapred.reduce.tasks" property.
@@ -157,7 +162,8 @@ object SetCommand {
 }
 
 /**
- * This command is for resetting SQLConf to the default values. Command that runs
+ * This command is for resetting SQLConf to the default values. Any configurations that were set
+ * via [[SetCommand]] will get reset to default value. Command that runs
  * {{{
  *   reset;
  * }}}
@@ -165,7 +171,11 @@ object SetCommand {
 case object ResetCommand extends RunnableCommand with IgnoreCachedData {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    sparkSession.sessionState.conf.clear()
+    val conf = sparkSession.sessionState.conf
+    conf.clear()
+    sparkSession.sparkContext.conf.getAll.foreach { case (k, v) =>
+      conf.setConfString(k, v)
+    }
     Seq.empty[Row]
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index f41c4eca203af..588f52d2e69b1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -484,17 +484,18 @@ case class AlterTableAddPartitionCommand(
       catalog.createPartitions(table.identifier, batch, ignoreIfExists = ifNotExists)
     }
 
-    if (table.stats.nonEmpty) {
-      if (sparkSession.sessionState.conf.autoSizeUpdateEnabled) {
-        val addedSize = CommandUtils.calculateTotalLocationSize(sparkSession, table.identifier,
-          parts.map(_.storage.locationUri))
-        if (addedSize > 0) {
-          val newStats = CatalogStatistics(sizeInBytes = table.stats.get.sizeInBytes + addedSize)
-          catalog.alterTableStats(table.identifier, Some(newStats))
-        }
-      } else {
-        catalog.alterTableStats(table.identifier, None)
+    sparkSession.catalog.refreshTable(table.identifier.quotedString)
+    if (table.stats.nonEmpty && sparkSession.sessionState.conf.autoSizeUpdateEnabled) {
+      // Updating table stats only if new partition is not empty
+      val addedSize = CommandUtils.calculateTotalLocationSize(sparkSession, table.identifier,
+        parts.map(_.storage.locationUri))
+      if (addedSize > 0) {
+        val newStats = CatalogStatistics(sizeInBytes = table.stats.get.sizeInBytes + addedSize)
+        catalog.alterTableStats(table.identifier, Some(newStats))
       }
+    } else {
+      // Re-calculating of table size including all partitions
+      CommandUtils.updateTableStats(sparkSession, table)
     }
     Seq.empty[Row]
   }
@@ -535,6 +536,7 @@ case class AlterTableRenamePartitionCommand(
 
     catalog.renamePartitions(
       tableName, Seq(normalizedOldPartition), Seq(normalizedNewPartition))
+    sparkSession.catalog.refreshTable(table.identifier.quotedString)
     Seq.empty[Row]
   }
 
@@ -580,6 +582,7 @@ case class AlterTableDropPartitionCommand(
       table.identifier, normalizedSpecs, ignoreIfNotExists = ifExists, purge = purge,
       retainData = retainData)
 
+    sparkSession.catalog.refreshTable(table.identifier.quotedString)
     CommandUtils.updateTableStats(sparkSession, table)
 
     Seq.empty[Row]
@@ -672,7 +675,7 @@ case class AlterTableRecoverPartitionsCommand(
     // This is always the case for Hive format tables, but is not true for Datasource tables created
     // before Spark 2.1 unless they are converted via `msck repair table`.
     spark.sessionState.catalog.alterTable(table.copy(tracksPartitionsInCatalog = true))
-    catalog.refreshTable(tableName)
+    spark.catalog.refreshTable(tableIdentWithDB)
     logInfo(s"Recovered all partitions ($total).")
     Seq.empty[Row]
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
index 6fdc7f4a58195..d55d6967a0291 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
@@ -88,7 +88,9 @@ case class CreateFunctionCommand(
       } else {
         // For a permanent, we will store the metadata into underlying external catalog.
         // This function will be loaded into the FunctionRegistry when a query uses it.
-        // We do not load it into FunctionRegistry right now.
+        // We do not load it into FunctionRegistry right now, to avoid loading the resource and
+        // UDF class immediately, as the Spark application to create the function may not have
+        // access to the resource and/or UDF class.
         catalog.createFunction(func, ignoreIfExists)
       }
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
index 1119e5cb1d288..549477dbae6ba 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
@@ -47,7 +47,7 @@ case class AddJarCommand(path: String) extends RunnableCommand {
  */
 case class AddFileCommand(path: String) extends RunnableCommand {
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    val recursive = sparkSession.sessionState.conf.addDirectoryRecursiveEnabled
+    val recursive = !sparkSession.sessionState.conf.addSingleFileInAddFile
     sparkSession.sparkContext.addFile(path, recursive)
     Seq.empty[Row]
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 468ca505cce1f..42f4a7245031c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -19,12 +19,12 @@ package org.apache.spark.sql.execution.command
 
 import java.net.{URI, URISyntaxException}
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
-import scala.util.Try
 import scala.util.control.NonFatal
 
 import org.apache.hadoop.fs.{FileContext, FsConstants, Path}
-import org.apache.hadoop.fs.permission.{AclEntry, FsPermission}
+import org.apache.hadoop.fs.permission.{AclEntry, AclEntryScope, AclEntryType, FsAction, FsPermission}
 
 import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
 import org.apache.spark.sql.catalyst.TableIdentifier
@@ -123,7 +123,8 @@ case class CreateTableLikeCommand(
         provider = newProvider,
         partitionColumnNames = sourceTableDesc.partitionColumnNames,
         bucketSpec = sourceTableDesc.bucketSpec,
-        properties = properties)
+        properties = properties,
+        tracksPartitionsInCatalog = sourceTableDesc.tracksPartitionsInCatalog)
 
     catalog.createTable(newTableDesc, ifNotExists)
     Seq.empty[Row]
@@ -190,22 +191,19 @@ case class AlterTableRenameCommand(
     } else {
       val table = catalog.getTableMetadata(oldName)
       DDLUtils.verifyAlterTableType(catalog, table, isView)
-      // If an exception is thrown here we can just assume the table is uncached;
-      // this can happen with Hive tables when the underlying catalog is in-memory.
-      val wasCached = Try(sparkSession.catalog.isCached(oldName.unquotedString)).getOrElse(false)
-      if (wasCached) {
-        try {
-          sparkSession.catalog.uncacheTable(oldName.unquotedString)
-        } catch {
-          case NonFatal(e) => log.warn(e.toString, e)
-        }
+      // If `optStorageLevel` is defined, the old table was cached.
+      val optCachedData = sparkSession.sharedState.cacheManager.lookupCachedData(
+        sparkSession.table(oldName.unquotedString))
+      val optStorageLevel = optCachedData.map(_.cachedRepresentation.cacheBuilder.storageLevel)
+      if (optStorageLevel.isDefined) {
+        CommandUtils.uncacheTableOrView(sparkSession, oldName.unquotedString)
       }
       // Invalidate the table last, otherwise uncaching the table would load the logical plan
       // back into the hive metastore cache
       catalog.refreshTable(oldName)
       catalog.renameTable(oldName, newName)
-      if (wasCached) {
-        sparkSession.catalog.cacheTable(newName.unquotedString)
+      optStorageLevel.foreach { storageLevel =>
+        sparkSession.catalog.cacheTable(newName.unquotedString, storageLevel)
       }
     }
     Seq.empty[Row]
@@ -228,12 +226,7 @@ case class AlterTableAddColumnsCommand(
     val catalog = sparkSession.sessionState.catalog
     val catalogTable = verifyAlterTableAddColumn(sparkSession.sessionState.conf, catalog, table)
 
-    try {
-      sparkSession.catalog.uncacheTable(table.quotedString)
-    } catch {
-      case NonFatal(e) =>
-        log.warn(s"Exception when attempting to uncache table ${table.quotedString}", e)
-    }
+    CommandUtils.uncacheTableOrView(sparkSession, table.quotedString)
     catalog.refreshTable(table)
 
     SchemaUtils.checkColumnNameDuplication(
@@ -506,8 +499,8 @@ case class TruncateTableCommand(
           var optPermission: Option[FsPermission] = None
           var optAcls: Option[java.util.List[AclEntry]] = None
           if (!ignorePermissionAcl) {
-            val fileStatus = fs.getFileStatus(path)
             try {
+              val fileStatus = fs.getFileStatus(path)
               optPermission = Some(fileStatus.getPermission())
             } catch {
               case NonFatal(_) => // do nothing
@@ -538,12 +531,27 @@ case class TruncateTableCommand(
               }
             }
             optAcls.foreach { acls =>
+              val aclEntries = acls.asScala.filter(_.getName != null).asJava
+
+              // If the path doesn't have default ACLs, `setAcl` API will throw an error
+              // as it expects user/group/other permissions must be in ACL entries.
+              // So we need to add tradition user/group/other permission
+              // in the form of ACL.
+              optPermission.map { permission =>
+                aclEntries.add(newAclEntry(AclEntryScope.ACCESS,
+                  AclEntryType.USER, permission.getUserAction()))
+                aclEntries.add(newAclEntry(AclEntryScope.ACCESS,
+                  AclEntryType.GROUP, permission.getGroupAction()))
+                aclEntries.add(newAclEntry(AclEntryScope.ACCESS,
+                  AclEntryType.OTHER, permission.getOtherAction()))
+              }
+
               try {
-                fs.setAcl(path, acls)
+                fs.setAcl(path, aclEntries)
               } catch {
                 case NonFatal(e) =>
                   throw new SecurityException(
-                    s"Failed to set original ACL $acls back to " +
+                    s"Failed to set original ACL $aclEntries back to " +
                       s"the created path: $path. Exception: ${e.getMessage}")
               }
             }
@@ -574,6 +582,16 @@ case class TruncateTableCommand(
     }
     Seq.empty[Row]
   }
+
+  private def newAclEntry(
+      scope: AclEntryScope,
+      aclType: AclEntryType,
+      permission: FsAction): AclEntry = {
+    new AclEntry.Builder()
+      .setScope(scope)
+      .setType(aclType)
+      .setPermission(permission).build()
+  }
 }
 
 abstract class DescribeCommandBase extends RunnableCommand {
@@ -866,12 +884,20 @@ case class ShowTablesCommand(
       //
       // Note: tableIdentifierPattern should be non-empty, otherwise a [[ParseException]]
       // should have been thrown by the sql parser.
-      val tableIdent = TableIdentifier(tableIdentifierPattern.get, Some(db))
-      val table = catalog.getTableMetadata(tableIdent).identifier
-      val partition = catalog.getPartition(tableIdent, partitionSpec.get)
-      val database = table.database.getOrElse("")
-      val tableName = table.table
-      val isTemp = catalog.isTemporaryTable(table)
+      val table = catalog.getTableMetadata(TableIdentifier(tableIdentifierPattern.get, Some(db)))
+
+      DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "SHOW TABLE EXTENDED")
+
+      val tableIdent = table.identifier
+      val normalizedSpec = PartitioningUtils.normalizePartitionSpec(
+        partitionSpec.get,
+        table.partitionColumnNames,
+        tableIdent.quotedString,
+        sparkSession.sessionState.conf.resolver)
+      val partition = catalog.getPartition(tableIdent, normalizedSpec)
+      val database = tableIdent.database.getOrElse("")
+      val tableName = tableIdent.table
+      val isTemp = catalog.isTemporaryTable(tableIdent)
       val information = partition.simpleString
       Seq(Row(database, tableName, isTemp, s"$information\n"))
     }
@@ -900,15 +926,20 @@ case class ShowTablePropertiesCommand(table: TableIdentifier, propertyKey: Optio
   }
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    val catalogTable = sparkSession.sessionState.catalog.getTableMetadata(table)
-    propertyKey match {
-      case Some(p) =>
-        val propValue = catalogTable
-          .properties
-          .getOrElse(p, s"Table ${catalogTable.qualifiedName} does not have property: $p")
-        Seq(Row(propValue))
-      case None =>
-        catalogTable.properties.map(p => Row(p._1, p._2)).toSeq
+    val catalog = sparkSession.sessionState.catalog
+    if (catalog.isTemporaryTable(table)) {
+      Seq.empty[Row]
+    } else {
+      val catalogTable = catalog.getTableMetadata(table)
+      propertyKey match {
+        case Some(p) =>
+          val propValue = catalogTable
+            .properties
+            .getOrElse(p, s"Table ${catalogTable.qualifiedName} does not have property: $p")
+          Seq(Row(propValue))
+        case None =>
+          catalogTable.properties.map(p => Row(p._1, p._2)).toSeq
+      }
     }
   }
 }
@@ -984,20 +1015,18 @@ case class ShowPartitionsCommand(
     DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "SHOW PARTITIONS")
 
     /**
-     * Validate the partitioning spec by making sure all the referenced columns are
+     * Normalizes the partition spec w.r.t the partition columns and case sensitivity settings,
+     * and validates the spec by making sure all the referenced columns are
      * defined as partitioning columns in table definition. An AnalysisException exception is
      * thrown if the partitioning spec is invalid.
      */
-    if (spec.isDefined) {
-      val badColumns = spec.get.keySet.filterNot(table.partitionColumnNames.contains)
-      if (badColumns.nonEmpty) {
-        val badCols = badColumns.mkString("[", ", ", "]")
-        throw new AnalysisException(
-          s"Non-partitioning column(s) $badCols are specified for SHOW PARTITIONS")
-      }
-    }
+    val normalizedSpec = spec.map(partitionSpec => PartitioningUtils.normalizePartitionSpec(
+      partitionSpec,
+      table.partitionColumnNames,
+      table.identifier.quotedString,
+      sparkSession.sessionState.conf.resolver))
 
-    val partNames = catalog.listPartitionNames(tableName, spec)
+    val partNames = catalog.listPartitionNames(tableName, normalizedSpec)
     partNames.map(Row(_))
   }
 }
@@ -1039,6 +1068,42 @@ trait ShowCreateTableCommandBase {
   protected def concatByMultiLines(iter: Iterable[String]): String = {
     iter.mkString("(\n  ", ",\n  ", ")\n")
   }
+
+  protected def showCreateView(metadata: CatalogTable, builder: StringBuilder): Unit = {
+    showViewDataColumns(metadata, builder)
+    showTableComment(metadata, builder)
+    showViewProperties(metadata, builder)
+    showViewText(metadata, builder)
+  }
+
+  private def showViewDataColumns(metadata: CatalogTable, builder: StringBuilder): Unit = {
+    if (metadata.schema.nonEmpty) {
+      val viewColumns = metadata.schema.map { f =>
+        val comment = f.getComment()
+          .map(escapeSingleQuotedString)
+          .map(" COMMENT '" + _ + "'")
+
+        // view columns shouldn't have data type info
+        s"${quoteIdentifier(f.name)}${comment.getOrElse("")}"
+      }
+      builder ++= concatByMultiLines(viewColumns)
+    }
+  }
+
+  private def showViewProperties(metadata: CatalogTable, builder: StringBuilder): Unit = {
+    val viewProps = metadata.properties.filterKeys(!_.startsWith(CatalogTable.VIEW_PREFIX))
+    if (viewProps.nonEmpty) {
+      val props = viewProps.map { case (key, value) =>
+        s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
+      }
+
+      builder ++= s"TBLPROPERTIES ${concatByMultiLines(props)}"
+    }
+  }
+
+  private def showViewText(metadata: CatalogTable, builder: StringBuilder): Unit = {
+    builder ++= metadata.viewText.mkString("AS ", "", "\n")
+  }
 }
 
 /**
@@ -1076,23 +1141,39 @@ case class ShowCreateTableCommand(table: TableIdentifier)
             "Failed to execute SHOW CREATE TABLE against table " +
               s"${tableMetadata.identifier}, which is created by Hive and uses the " +
               "following unsupported feature(s)\n" +
-              tableMetadata.unsupportedFeatures.map(" - " + _).mkString("\n")
+              tableMetadata.unsupportedFeatures.map(" - " + _).mkString("\n") + ". " +
+              s"Please use `SHOW CREATE TABLE ${tableMetadata.identifier} AS SERDE` " +
+              "to show Hive DDL instead."
           )
         }
 
-        if (tableMetadata.tableType == VIEW) {
-          throw new AnalysisException("Hive view isn't supported by SHOW CREATE TABLE")
-        }
-
         if ("true".equalsIgnoreCase(tableMetadata.properties.getOrElse("transactional", "false"))) {
           throw new AnalysisException(
-            "SHOW CREATE TABLE doesn't support transactional Hive table")
+            "SHOW CREATE TABLE doesn't support transactional Hive table. " +
+              s"Please use `SHOW CREATE TABLE ${tableMetadata.identifier} AS SERDE` " +
+              "to show Hive DDL instead.")
         }
 
-        convertTableMetadata(tableMetadata)
+        if (tableMetadata.tableType == VIEW) {
+          tableMetadata
+        } else {
+          convertTableMetadata(tableMetadata)
+        }
       }
 
-      val stmt = showCreateDataSourceTable(metadata)
+      val builder = StringBuilder.newBuilder
+
+      val stmt = if (tableMetadata.tableType == VIEW) {
+        builder ++= s"CREATE VIEW ${table.quotedString} "
+        showCreateView(metadata, builder)
+
+        builder.toString()
+      } else {
+        builder ++= s"CREATE TABLE ${table.quotedString} "
+
+        showCreateDataSourceTable(metadata, builder)
+        builder.toString()
+      }
 
       Seq(Row(stmt))
     }
@@ -1172,18 +1253,13 @@ case class ShowCreateTableCommand(table: TableIdentifier)
     }
   }
 
-  private def showCreateDataSourceTable(metadata: CatalogTable): String = {
-    val builder = StringBuilder.newBuilder
-
-    builder ++= s"CREATE TABLE ${table.quotedString} "
+  private def showCreateDataSourceTable(metadata: CatalogTable, builder: StringBuilder): Unit = {
     showDataSourceTableDataColumns(metadata, builder)
     showDataSourceTableOptions(metadata, builder)
     showDataSourceTableNonDataColumns(metadata, builder)
     showTableComment(metadata, builder)
     showTableLocation(metadata, builder)
     showTableProperties(metadata, builder)
-
-    builder.toString()
   }
 }
 
@@ -1242,10 +1318,7 @@ case class ShowCreateTableAsSerdeCommand(table: TableIdentifier)
     builder ++= s"CREATE$tableTypeString ${table.quotedString}"
 
     if (metadata.tableType == VIEW) {
-      showViewDataColumns(metadata, builder)
-      showTableComment(metadata, builder)
-      showViewProperties(metadata, builder)
-      showViewText(metadata, builder)
+      showCreateView(metadata, builder)
     } else {
       showHiveTableHeader(metadata, builder)
       showTableComment(metadata, builder)
@@ -1258,35 +1331,6 @@ case class ShowCreateTableAsSerdeCommand(table: TableIdentifier)
     builder.toString()
   }
 
-  private def showViewDataColumns(metadata: CatalogTable, builder: StringBuilder): Unit = {
-    if (metadata.schema.nonEmpty) {
-      val viewColumns = metadata.schema.map { f =>
-        val comment = f.getComment()
-          .map(escapeSingleQuotedString)
-          .map(" COMMENT '" + _ + "'")
-
-        // view columns shouldn't have data type info
-        s"${quoteIdentifier(f.name)}${comment.getOrElse("")}"
-      }
-      builder ++= concatByMultiLines(viewColumns)
-    }
-  }
-
-  private def showViewProperties(metadata: CatalogTable, builder: StringBuilder): Unit = {
-    val viewProps = metadata.properties.filterKeys(!_.startsWith(CatalogTable.VIEW_PREFIX))
-    if (viewProps.nonEmpty) {
-      val props = viewProps.map { case (key, value) =>
-        s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
-      }
-
-      builder ++= s"TBLPROPERTIES ${concatByMultiLines(props)}"
-    }
-  }
-
-  private def showViewText(metadata: CatalogTable, builder: StringBuilder): Unit = {
-    builder ++= metadata.viewText.mkString("AS ", "", "\n")
-  }
-
   private def showHiveTableHeader(metadata: CatalogTable, builder: StringBuilder): Unit = {
     val columns = metadata.schema.filterNot { column =>
       metadata.partitionColumnNames.contains(column.name)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 38481dda428a5..0ba76ee94a860 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -23,10 +23,12 @@ import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.{GlobalTempView, LocalTempView, PersistedView, UnresolvedFunction, UnresolvedRelation, ViewType}
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, SessionCatalog}
-import org.apache.spark.sql.catalyst.expressions.{Alias, SubqueryExpression}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, SubqueryExpression}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, View}
-import org.apache.spark.sql.types.MetadataBuilder
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
+import org.apache.spark.sql.internal.StaticSQLConf
+import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StringType}
 import org.apache.spark.sql.util.SchemaUtils
 
 /**
@@ -108,9 +110,21 @@ case class CreateViewCommand(
     verifyTemporaryObjectsNotExists(catalog)
 
     if (viewType == LocalTempView) {
+      if (replace && catalog.getTempView(name.table).isDefined &&
+          !catalog.getTempView(name.table).get.sameResult(child)) {
+        logInfo(s"Try to uncache ${name.quotedString} before replacing.")
+        CommandUtils.uncacheTableOrView(sparkSession, name.quotedString)
+      }
       val aliasedPlan = aliasPlan(sparkSession, analyzedPlan)
       catalog.createTempView(name.table, aliasedPlan, overrideIfExists = replace)
     } else if (viewType == GlobalTempView) {
+      if (replace && catalog.getGlobalTempView(name.table).isDefined &&
+          !catalog.getGlobalTempView(name.table).get.sameResult(child)) {
+        val db = sparkSession.sessionState.conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE)
+        val globalTempView = TableIdentifier(name.table, Option(db))
+        logInfo(s"Try to uncache ${globalTempView.quotedString} before replacing.")
+        CommandUtils.uncacheTableOrView(sparkSession, globalTempView.quotedString)
+      }
       val aliasedPlan = aliasPlan(sparkSession, analyzedPlan)
       catalog.createGlobalTempView(name.table, aliasedPlan, overrideIfExists = replace)
     } else if (catalog.tableExists(name)) {
@@ -125,6 +139,10 @@ case class CreateViewCommand(
         val viewIdent = tableMetadata.identifier
         checkCyclicViewReference(analyzedPlan, Seq(viewIdent), viewIdent)
 
+        // uncache the cached data before replacing an exists view
+        logDebug(s"Try to uncache ${viewIdent.quotedString} before replacing.")
+        CommandUtils.uncacheTableOrView(sparkSession, viewIdent.quotedString)
+
         // Handles `CREATE OR REPLACE VIEW v0 AS SELECT ...`
         // Nothing we need to retain from the old view, so just drop and create a new one
         catalog.dropTable(viewIdent, ignoreIfNotExists = false, purge = false)
@@ -280,6 +298,40 @@ case class AlterViewAsCommand(
   }
 }
 
+/**
+ * A command for users to get views in the given database.
+ * If a databaseName is not given, the current database will be used.
+ * The syntax of using this command in SQL is:
+ * {{{
+ *   SHOW VIEWS [(IN|FROM) database_name] [[LIKE] 'identifier_with_wildcards'];
+ * }}}
+ */
+case class ShowViewsCommand(
+    databaseName: String,
+    tableIdentifierPattern: Option[String]) extends RunnableCommand {
+
+  // The result of SHOW VIEWS has three basic columns: namespace, viewName and isTemporary.
+  override val output: Seq[Attribute] = Seq(
+    AttributeReference("namespace", StringType, nullable = false)(),
+    AttributeReference("viewName", StringType, nullable = false)(),
+    AttributeReference("isTemporary", BooleanType, nullable = false)())
+
+  override def run(sparkSession: SparkSession): Seq[Row] = {
+    val catalog = sparkSession.sessionState.catalog
+
+    // Show the information of views.
+    val views = tableIdentifierPattern.map(catalog.listViews(databaseName, _))
+      .getOrElse(catalog.listViews(databaseName, "*"))
+    views.map { tableIdent =>
+      val namespace = tableIdent.database.toArray.quoted
+      val tableName = tableIdent.table
+      val isTemp = catalog.isTemporaryTable(tableIdent)
+
+      Row(namespace, tableName, isTemp)
+    }
+  }
+}
+
 object ViewHelper {
 
   import CatalogTable._
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
index 3615afcf86c7a..54faadf73b25f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
@@ -67,8 +67,9 @@ import org.apache.spark.util.Utils
  * metadata.  For example, when reading a partitioned table from a file system, partition columns
  * will be inferred from the directory layout even if they are not specified.
  *
- * @param paths A list of file system paths that hold data.  These will be globbed before and
- *              qualified. This option only works when reading from a [[FileFormat]].
+ * @param paths A list of file system paths that hold data. These will be globbed before if
+ *              the "__globPaths__" option is true, and will be qualified. This option only works
+ *              when reading from a [[FileFormat]].
  * @param userSpecifiedSchema An optional specification of the schema of the data. When present
  *                            we skip attempting to infer the schema.
  * @param partitionColumns A list of column names that the relation is partitioned by. This list is
@@ -109,10 +110,22 @@ case class DataSource(
 
   private def providingInstance() = providingClass.getConstructor().newInstance()
 
+  private def newHadoopConfiguration(): Configuration =
+    sparkSession.sessionState.newHadoopConfWithOptions(options)
+
   lazy val sourceInfo: SourceInfo = sourceSchema()
   private val caseInsensitiveOptions = CaseInsensitiveMap(options)
   private val equality = sparkSession.sessionState.conf.resolver
 
+  /**
+   * Whether or not paths should be globbed before being used to access files.
+   */
+  def globPaths: Boolean = {
+    options.get(DataSource.GLOB_PATHS_KEY)
+      .map(_ == "true")
+      .getOrElse(true)
+  }
+
   bucketSpec.map { bucket =>
     SchemaUtils.checkColumnNameDuplication(
       bucket.bucketColumnNames, "in the bucket definition", equality)
@@ -187,9 +200,11 @@ case class DataSource(
     val dataSchema = userSpecifiedSchema.map { schema =>
       StructType(schema.filterNot(f => partitionSchema.exists(p => equality(p.name, f.name))))
     }.orElse {
+      // Remove "path" option so that it is not added to the paths returned by
+      // `tempFileIndex.allFiles()`.
       format.inferSchema(
         sparkSession,
-        caseInsensitiveOptions,
+        caseInsensitiveOptions - "path",
         tempFileIndex.allFiles())
     }.getOrElse {
       throw new AnalysisException(
@@ -229,8 +244,8 @@ case class DataSource(
         // For glob pattern, we do not check it because the glob pattern might only make sense
         // once the streaming job starts and some upstream source starts dropping data.
         val hdfsPath = new Path(path)
-        if (!SparkHadoopUtil.get.isGlobPath(hdfsPath)) {
-          val fs = hdfsPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
+        if (!globPaths || !SparkHadoopUtil.get.isGlobPath(hdfsPath)) {
+          val fs = hdfsPath.getFileSystem(newHadoopConfiguration())
           if (!fs.exists(hdfsPath)) {
             throw new AnalysisException(s"Path does not exist: $path")
           }
@@ -357,15 +372,17 @@ case class DataSource(
       case (format: FileFormat, _)
           if FileStreamSink.hasMetadata(
             caseInsensitiveOptions.get("path").toSeq ++ paths,
-            sparkSession.sessionState.newHadoopConf(),
+            newHadoopConfiguration(),
             sparkSession.sessionState.conf) =>
         val basePath = new Path((caseInsensitiveOptions.get("path").toSeq ++ paths).head)
         val fileCatalog = new MetadataLogFileIndex(sparkSession, basePath,
           caseInsensitiveOptions, userSpecifiedSchema)
         val dataSchema = userSpecifiedSchema.orElse {
+          // Remove "path" option so that it is not added to the paths returned by
+          // `fileCatalog.allFiles()`.
           format.inferSchema(
             sparkSession,
-            caseInsensitiveOptions,
+            caseInsensitiveOptions - "path",
             fileCatalog.allFiles())
         }.getOrElse {
           throw new AnalysisException(
@@ -449,7 +466,7 @@ case class DataSource(
     val allPaths = paths ++ caseInsensitiveOptions.get("path")
     val outputPath = if (allPaths.length == 1) {
       val path = new Path(allPaths.head)
-      val fs = path.getFileSystem(sparkSession.sessionState.newHadoopConf())
+      val fs = path.getFileSystem(newHadoopConfiguration())
       path.makeQualified(fs.getUri, fs.getWorkingDirectory)
     } else {
       throw new IllegalArgumentException("Expected exactly one path to be specified, but " +
@@ -569,10 +586,8 @@ case class DataSource(
       checkEmptyGlobPath: Boolean,
       checkFilesExist: Boolean): Seq[Path] = {
     val allPaths = caseInsensitiveOptions.get("path") ++ paths
-    val hadoopConf = sparkSession.sessionState.newHadoopConf()
-
-    DataSource.checkAndGlobPathIfNecessary(allPaths.toSeq, hadoopConf,
-      checkEmptyGlobPath, checkFilesExist)
+    DataSource.checkAndGlobPathIfNecessary(allPaths.toSeq, newHadoopConfiguration(),
+      checkEmptyGlobPath, checkFilesExist, enableGlobbing = globPaths)
   }
 }
 
@@ -735,6 +750,11 @@ object DataSource extends Logging {
     }
   }
 
+  /**
+   * The key in the "options" map for deciding whether or not to glob paths before use.
+   */
+  val GLOB_PATHS_KEY = "__globPaths__"
+
   /**
    * Checks and returns files in all the paths.
    */
@@ -742,12 +762,17 @@ object DataSource extends Logging {
       paths: Seq[String],
       hadoopConf: Configuration,
       checkEmptyGlobPath: Boolean,
-      checkFilesExist: Boolean): Seq[Path] = {
+      checkFilesExist: Boolean,
+      enableGlobbing: Boolean): Seq[Path] = {
     val allGlobPath = paths.flatMap { path =>
       val hdfsPath = new Path(path)
       val fs = hdfsPath.getFileSystem(hadoopConf)
       val qualified = hdfsPath.makeQualified(fs.getUri, fs.getWorkingDirectory)
-      val globPath = SparkHadoopUtil.get.globPathIfNecessary(fs, qualified)
+      val globPath = if (enableGlobbing) {
+        SparkHadoopUtil.get.globPathIfNecessary(fs, qualified)
+      } else {
+        qualified :: Nil
+      }
 
       if (checkEmptyGlobPath && globPath.isEmpty) {
         throw new AnalysisException(s"Path does not exist: $qualified")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
index e3a0a0a6c34e5..23454d7d5e7f3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
@@ -39,6 +39,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.{RowDataSourceScanExec, SparkPlan}
 import org.apache.spark.sql.execution.command._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -104,7 +105,17 @@ case class DataSourceAnalysis(conf: SQLConf) extends Rule[LogicalPlan] with Cast
         None
       } else if (potentialSpecs.size == 1) {
         val partValue = potentialSpecs.head._2
-        Some(Alias(cast(Literal(partValue), field.dataType), field.name)())
+        conf.storeAssignmentPolicy match {
+          // SPARK-30844: try our best to follow StoreAssignmentPolicy for static partition
+          // values but not completely follow because we can't do static type checking due to
+          // the reason that the parser has erased the type info of static partition values
+          // and converted them to string.
+          case StoreAssignmentPolicy.ANSI | StoreAssignmentPolicy.STRICT =>
+            Some(Alias(AnsiCast(Literal(partValue), field.dataType,
+              Option(conf.sessionLocalTimeZone)), field.name)())
+          case _ =>
+            Some(Alias(cast(Literal(partValue), field.dataType), field.name)())
+        }
       } else {
         throw new AnalysisException(
           s"Partition column ${field.name} have multiple values specified, " +
@@ -437,61 +448,63 @@ object DataSourceStrategy {
     }
   }
 
-  private def translateLeafNodeFilter(predicate: Expression): Option[Filter] = predicate match {
-    case expressions.EqualTo(a: Attribute, Literal(v, t)) =>
-      Some(sources.EqualTo(a.name, convertToScala(v, t)))
-    case expressions.EqualTo(Literal(v, t), a: Attribute) =>
-      Some(sources.EqualTo(a.name, convertToScala(v, t)))
-
-    case expressions.EqualNullSafe(a: Attribute, Literal(v, t)) =>
-      Some(sources.EqualNullSafe(a.name, convertToScala(v, t)))
-    case expressions.EqualNullSafe(Literal(v, t), a: Attribute) =>
-      Some(sources.EqualNullSafe(a.name, convertToScala(v, t)))
-
-    case expressions.GreaterThan(a: Attribute, Literal(v, t)) =>
-      Some(sources.GreaterThan(a.name, convertToScala(v, t)))
-    case expressions.GreaterThan(Literal(v, t), a: Attribute) =>
-      Some(sources.LessThan(a.name, convertToScala(v, t)))
-
-    case expressions.LessThan(a: Attribute, Literal(v, t)) =>
-      Some(sources.LessThan(a.name, convertToScala(v, t)))
-    case expressions.LessThan(Literal(v, t), a: Attribute) =>
-      Some(sources.GreaterThan(a.name, convertToScala(v, t)))
-
-    case expressions.GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
-      Some(sources.GreaterThanOrEqual(a.name, convertToScala(v, t)))
-    case expressions.GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
-      Some(sources.LessThanOrEqual(a.name, convertToScala(v, t)))
-
-    case expressions.LessThanOrEqual(a: Attribute, Literal(v, t)) =>
-      Some(sources.LessThanOrEqual(a.name, convertToScala(v, t)))
-    case expressions.LessThanOrEqual(Literal(v, t), a: Attribute) =>
-      Some(sources.GreaterThanOrEqual(a.name, convertToScala(v, t)))
-
-    case expressions.InSet(a: Attribute, set) =>
-      val toScala = CatalystTypeConverters.createToScalaConverter(a.dataType)
-      Some(sources.In(a.name, set.toArray.map(toScala)))
+  private def translateLeafNodeFilter(
+      predicate: Expression,
+      pushableColumn: PushableColumnBase): Option[Filter] = predicate match {
+    case expressions.EqualTo(pushableColumn(name), Literal(v, t)) =>
+      Some(sources.EqualTo(name, convertToScala(v, t)))
+    case expressions.EqualTo(Literal(v, t), pushableColumn(name)) =>
+      Some(sources.EqualTo(name, convertToScala(v, t)))
+
+    case expressions.EqualNullSafe(pushableColumn(name), Literal(v, t)) =>
+      Some(sources.EqualNullSafe(name, convertToScala(v, t)))
+    case expressions.EqualNullSafe(Literal(v, t), pushableColumn(name)) =>
+      Some(sources.EqualNullSafe(name, convertToScala(v, t)))
+
+    case expressions.GreaterThan(pushableColumn(name), Literal(v, t)) =>
+      Some(sources.GreaterThan(name, convertToScala(v, t)))
+    case expressions.GreaterThan(Literal(v, t), pushableColumn(name)) =>
+      Some(sources.LessThan(name, convertToScala(v, t)))
+
+    case expressions.LessThan(pushableColumn(name), Literal(v, t)) =>
+      Some(sources.LessThan(name, convertToScala(v, t)))
+    case expressions.LessThan(Literal(v, t), pushableColumn(name)) =>
+      Some(sources.GreaterThan(name, convertToScala(v, t)))
+
+    case expressions.GreaterThanOrEqual(pushableColumn(name), Literal(v, t)) =>
+      Some(sources.GreaterThanOrEqual(name, convertToScala(v, t)))
+    case expressions.GreaterThanOrEqual(Literal(v, t), pushableColumn(name)) =>
+      Some(sources.LessThanOrEqual(name, convertToScala(v, t)))
+
+    case expressions.LessThanOrEqual(pushableColumn(name), Literal(v, t)) =>
+      Some(sources.LessThanOrEqual(name, convertToScala(v, t)))
+    case expressions.LessThanOrEqual(Literal(v, t), pushableColumn(name)) =>
+      Some(sources.GreaterThanOrEqual(name, convertToScala(v, t)))
+
+    case expressions.InSet(e @ pushableColumn(name), set) =>
+      val toScala = CatalystTypeConverters.createToScalaConverter(e.dataType)
+      Some(sources.In(name, set.toArray.map(toScala)))
 
     // Because we only convert In to InSet in Optimizer when there are more than certain
     // items. So it is possible we still get an In expression here that needs to be pushed
     // down.
-    case expressions.In(a: Attribute, list) if list.forall(_.isInstanceOf[Literal]) =>
+    case expressions.In(e @ pushableColumn(name), list) if list.forall(_.isInstanceOf[Literal]) =>
       val hSet = list.map(_.eval(EmptyRow))
-      val toScala = CatalystTypeConverters.createToScalaConverter(a.dataType)
-      Some(sources.In(a.name, hSet.toArray.map(toScala)))
+      val toScala = CatalystTypeConverters.createToScalaConverter(e.dataType)
+      Some(sources.In(name, hSet.toArray.map(toScala)))
 
-    case expressions.IsNull(a: Attribute) =>
-      Some(sources.IsNull(a.name))
-    case expressions.IsNotNull(a: Attribute) =>
-      Some(sources.IsNotNull(a.name))
-    case expressions.StartsWith(a: Attribute, Literal(v: UTF8String, StringType)) =>
-      Some(sources.StringStartsWith(a.name, v.toString))
+    case expressions.IsNull(pushableColumn(name)) =>
+      Some(sources.IsNull(name))
+    case expressions.IsNotNull(pushableColumn(name)) =>
+      Some(sources.IsNotNull(name))
+    case expressions.StartsWith(pushableColumn(name), Literal(v: UTF8String, StringType)) =>
+      Some(sources.StringStartsWith(name, v.toString))
 
-    case expressions.EndsWith(a: Attribute, Literal(v: UTF8String, StringType)) =>
-      Some(sources.StringEndsWith(a.name, v.toString))
+    case expressions.EndsWith(pushableColumn(name), Literal(v: UTF8String, StringType)) =>
+      Some(sources.StringEndsWith(name, v.toString))
 
-    case expressions.Contains(a: Attribute, Literal(v: UTF8String, StringType)) =>
-      Some(sources.StringContains(a.name, v.toString))
+    case expressions.Contains(pushableColumn(name), Literal(v: UTF8String, StringType)) =>
+      Some(sources.StringContains(name, v.toString))
 
     case expressions.Literal(true, BooleanType) =>
       Some(sources.AlwaysTrue)
@@ -507,8 +520,9 @@ object DataSourceStrategy {
    *
    * @return a `Some[Filter]` if the input [[Expression]] is convertible, otherwise a `None`.
    */
-  protected[sql] def translateFilter(predicate: Expression): Option[Filter] = {
-    translateFilterWithMapping(predicate, None)
+  protected[sql] def translateFilter(
+      predicate: Expression, supportNestedPredicatePushdown: Boolean): Option[Filter] = {
+    translateFilterWithMapping(predicate, None, supportNestedPredicatePushdown)
   }
 
   /**
@@ -518,11 +532,13 @@ object DataSourceStrategy {
    * @param translatedFilterToExpr An optional map from leaf node filter expressions to its
    *                               translated [[Filter]]. The map is used for rebuilding
    *                               [[Expression]] from [[Filter]].
+   * @param nestedPredicatePushdownEnabled Whether nested predicate pushdown is enabled.
    * @return a `Some[Filter]` if the input [[Expression]] is convertible, otherwise a `None`.
    */
   protected[sql] def translateFilterWithMapping(
       predicate: Expression,
-      translatedFilterToExpr: Option[mutable.HashMap[sources.Filter, Expression]])
+      translatedFilterToExpr: Option[mutable.HashMap[sources.Filter, Expression]],
+      nestedPredicatePushdownEnabled: Boolean)
     : Option[Filter] = {
     predicate match {
       case expressions.And(left, right) =>
@@ -536,21 +552,26 @@ object DataSourceStrategy {
         // Pushing one leg of AND down is only safe to do at the top level.
         // You can see ParquetFilters' createFilter for more details.
         for {
-          leftFilter <- translateFilterWithMapping(left, translatedFilterToExpr)
-          rightFilter <- translateFilterWithMapping(right, translatedFilterToExpr)
+          leftFilter <- translateFilterWithMapping(
+            left, translatedFilterToExpr, nestedPredicatePushdownEnabled)
+          rightFilter <- translateFilterWithMapping(
+            right, translatedFilterToExpr, nestedPredicatePushdownEnabled)
         } yield sources.And(leftFilter, rightFilter)
 
       case expressions.Or(left, right) =>
         for {
-          leftFilter <- translateFilterWithMapping(left, translatedFilterToExpr)
-          rightFilter <- translateFilterWithMapping(right, translatedFilterToExpr)
+          leftFilter <- translateFilterWithMapping(
+            left, translatedFilterToExpr, nestedPredicatePushdownEnabled)
+          rightFilter <- translateFilterWithMapping(
+            right, translatedFilterToExpr, nestedPredicatePushdownEnabled)
         } yield sources.Or(leftFilter, rightFilter)
 
       case expressions.Not(child) =>
-        translateFilterWithMapping(child, translatedFilterToExpr).map(sources.Not)
+        translateFilterWithMapping(child, translatedFilterToExpr, nestedPredicatePushdownEnabled)
+          .map(sources.Not)
 
       case other =>
-        val filter = translateLeafNodeFilter(other)
+        val filter = translateLeafNodeFilter(other, PushableColumn(nestedPredicatePushdownEnabled))
         if (filter.isDefined && translatedFilterToExpr.isDefined) {
           translatedFilterToExpr.get(filter.get) = predicate
         }
@@ -597,8 +618,9 @@ object DataSourceStrategy {
 
     // A map from original Catalyst expressions to corresponding translated data source filters.
     // If a predicate is not in this map, it means it cannot be pushed down.
+    val supportNestedPredicatePushdown = DataSourceUtils.supportNestedPredicatePushdown(relation)
     val translatedMap: Map[Expression, Filter] = predicates.flatMap { p =>
-      translateFilter(p).map(f => p -> f)
+      translateFilter(p, supportNestedPredicatePushdown).map(f => p -> f)
     }.toMap
 
     val pushedFilters: Seq[Filter] = translatedMap.values.toSeq
@@ -626,12 +648,53 @@ object DataSourceStrategy {
       output: Seq[Attribute],
       rdd: RDD[Row]): RDD[InternalRow] = {
     if (relation.needConversion) {
-      val converters = RowEncoder(StructType.fromAttributes(output))
+      val toRow = RowEncoder(StructType.fromAttributes(output)).createSerializer()
       rdd.mapPartitions { iterator =>
-        iterator.map(converters.toRow)
+        iterator.map(toRow)
       }
     } else {
       rdd.asInstanceOf[RDD[InternalRow]]
     }
   }
 }
+
+/**
+ * Find the column name of an expression that can be pushed down.
+ */
+abstract class PushableColumnBase {
+  val nestedPredicatePushdownEnabled: Boolean
+
+  def unapply(e: Expression): Option[String] = {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+    def helper(e: Expression): Option[Seq[String]] = e match {
+      case a: Attribute =>
+        if (nestedPredicatePushdownEnabled || !a.name.contains(".")) {
+          Some(Seq(a.name))
+        } else {
+          None
+        }
+      case s: GetStructField if nestedPredicatePushdownEnabled =>
+        helper(s.child).map(_ :+ s.childSchema(s.ordinal).name)
+      case _ => None
+    }
+    helper(e).map(_.quoted)
+  }
+}
+
+object PushableColumn {
+  def apply(nestedPredicatePushdownEnabled: Boolean): PushableColumnBase = {
+    if (nestedPredicatePushdownEnabled) {
+      PushableColumnAndNestedColumn
+    } else {
+      PushableColumnWithoutNestedColumn
+    }
+  }
+}
+
+object PushableColumnAndNestedColumn extends PushableColumnBase {
+  override val nestedPredicatePushdownEnabled = true
+}
+
+object PushableColumnWithoutNestedColumn extends PushableColumnBase {
+  override val nestedPredicatePushdownEnabled = false
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
index bd56635084c34..abb74d8d09ec6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
@@ -17,12 +17,21 @@
 
 package org.apache.spark.sql.execution.datasources
 
+import java.util.Locale
+
 import org.apache.hadoop.fs.Path
 import org.json4s.NoTypeHints
 import org.json4s.jackson.Serialization
 
+import org.apache.spark.SparkUpgradeException
+import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.util.RebaseDateTime
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
+import org.apache.spark.sql.sources.BaseRelation
 import org.apache.spark.sql.types._
+import org.apache.spark.util.Utils
 
 
 object DataSourceUtils {
@@ -64,4 +73,121 @@ object DataSourceUtils {
 
   private[sql] def isDataFile(fileName: String) =
     !(fileName.startsWith("_") || fileName.startsWith("."))
+
+  /**
+   * Returns if the given relation's V1 datasource provider supports nested predicate pushdown.
+   */
+  private[sql] def supportNestedPredicatePushdown(relation: BaseRelation): Boolean =
+    relation match {
+      case hs: HadoopFsRelation =>
+        val supportedDatasources =
+          Utils.stringToSeq(SQLConf.get.getConf(SQLConf.NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST)
+            .toLowerCase(Locale.ROOT))
+        supportedDatasources.contains(hs.toString)
+      case _ => false
+    }
+
+  def datetimeRebaseMode(
+      lookupFileMeta: String => String,
+      modeByConfig: String): LegacyBehaviorPolicy.Value = {
+    if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") {
+      return LegacyBehaviorPolicy.CORRECTED
+    }
+    // If there is no version, we return the mode specified by the config.
+    Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version =>
+      // Files written by Spark 2.4 and earlier follow the legacy hybrid calendar and we need to
+      // rebase the datetime values.
+      // Files written by Spark 3.0 and latter may also need the rebase if they were written with
+      // the "LEGACY" rebase mode.
+      if (version < "3.0.0" || lookupFileMeta(SPARK_LEGACY_DATETIME) != null) {
+        LegacyBehaviorPolicy.LEGACY
+      } else {
+        LegacyBehaviorPolicy.CORRECTED
+      }
+    }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig))
+  }
+
+  def newRebaseExceptionInRead(format: String): SparkUpgradeException = {
+    val config = if (format == "Parquet") {
+      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key
+    } else if (format == "Avro") {
+      SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key
+    } else {
+      throw new IllegalStateException("unrecognized format " + format)
+    }
+    new SparkUpgradeException("3.0", "reading dates before 1582-10-15 or timestamps before " +
+      s"1900-01-01T00:00:00Z from $format files can be ambiguous, as the files may be written by " +
+      "Spark 2.x or legacy versions of Hive, which uses a legacy hybrid calendar that is " +
+      "different from Spark 3.0+'s Proleptic Gregorian calendar. See more details in " +
+      s"SPARK-31404. You can set $config to 'LEGACY' to rebase the datetime values w.r.t. " +
+      s"the calendar difference during reading. Or set $config to 'CORRECTED' to read the " +
+      "datetime values as it is.", null)
+  }
+
+  def newRebaseExceptionInWrite(format: String): SparkUpgradeException = {
+    val config = if (format == "Parquet") {
+      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key
+    } else if (format == "Avro") {
+      SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key
+    } else {
+      throw new IllegalStateException("unrecognized format " + format)
+    }
+    new SparkUpgradeException("3.0", "writing dates before 1582-10-15 or timestamps before " +
+      s"1900-01-01T00:00:00Z into $format files can be dangerous, as the files may be read by " +
+      "Spark 2.x or legacy versions of Hive later, which uses a legacy hybrid calendar that is " +
+      "different from Spark 3.0+'s Proleptic Gregorian calendar. See more details in " +
+      s"SPARK-31404. You can set $config to 'LEGACY' to rebase the datetime values w.r.t. " +
+      "the calendar difference during writing, to get maximum interoperability. Or set " +
+      s"$config to 'CORRECTED' to write the datetime values as it is, if you are 100% sure that " +
+      "the written files will only be read by Spark 3.0+ or other systems that use Proleptic " +
+      "Gregorian calendar.", null)
+  }
+
+  def creteDateRebaseFuncInRead(
+      rebaseMode: LegacyBehaviorPolicy.Value,
+      format: String): Int => Int = rebaseMode match {
+    case LegacyBehaviorPolicy.EXCEPTION => days: Int =>
+      if (days < RebaseDateTime.lastSwitchJulianDay) {
+        throw DataSourceUtils.newRebaseExceptionInRead(format)
+      }
+      days
+    case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianDays
+    case LegacyBehaviorPolicy.CORRECTED => identity[Int]
+  }
+
+  def creteDateRebaseFuncInWrite(
+      rebaseMode: LegacyBehaviorPolicy.Value,
+      format: String): Int => Int = rebaseMode match {
+    case LegacyBehaviorPolicy.EXCEPTION => days: Int =>
+      if (days < RebaseDateTime.lastSwitchGregorianDay) {
+        throw DataSourceUtils.newRebaseExceptionInWrite(format)
+      }
+      days
+    case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianDays
+    case LegacyBehaviorPolicy.CORRECTED => identity[Int]
+  }
+
+  def creteTimestampRebaseFuncInRead(
+      rebaseMode: LegacyBehaviorPolicy.Value,
+      format: String): Long => Long = rebaseMode match {
+    case LegacyBehaviorPolicy.EXCEPTION => micros: Long =>
+      if (micros < RebaseDateTime.lastSwitchJulianTs) {
+        throw DataSourceUtils.newRebaseExceptionInRead(format)
+      }
+      micros
+    case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianMicros
+    case LegacyBehaviorPolicy.CORRECTED => identity[Long]
+  }
+
+  def creteTimestampRebaseFuncInWrite(
+      rebaseMode: LegacyBehaviorPolicy.Value,
+      format: String): Long => Long = rebaseMode match {
+    case LegacyBehaviorPolicy.EXCEPTION => micros: Long =>
+      if (micros < RebaseDateTime.lastSwitchGregorianTs) {
+        throw DataSourceUtils.newRebaseExceptionInWrite(format)
+      }
+      micros
+    case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianMicros
+    case LegacyBehaviorPolicy.CORRECTED => identity[Long]
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
new file mode 100644
index 0000000000000..a04c2fcbbac12
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources
+
+import java.io.{DataInput, DataOutput, IOException}
+import java.sql.Date
+
+import org.apache.hadoop.hive.serde2.io.DateWritable
+import org.apache.hadoop.io.WritableUtils
+
+import org.apache.spark.sql.catalyst.util.RebaseDateTime.{rebaseGregorianToJulianDays, rebaseJulianToGregorianDays}
+
+/**
+ * The class accepts/returns days in Gregorian calendar and rebase them
+ * via conversion to local date in Julian calendar for dates before 1582-10-15
+ * in read/write for backward compatibility with Spark 2.4 and earlier versions.
+ *
+ * @param gregorianDays The number of days since the epoch 1970-01-01 in
+ *                      Gregorian calendar.
+ * @param julianDays The number of days since the epoch 1970-01-01 in
+ *                   Julian calendar.
+ */
+class DaysWritable(
+    var gregorianDays: Int,
+    var julianDays: Int)
+  extends DateWritable {
+
+  def this() = this(0, 0)
+  def this(gregorianDays: Int) =
+    this(gregorianDays, rebaseGregorianToJulianDays(gregorianDays))
+  def this(dateWritable: DateWritable) = {
+    this(
+      gregorianDays = dateWritable match {
+        case daysWritable: DaysWritable => daysWritable.gregorianDays
+        case dateWritable: DateWritable =>
+        rebaseJulianToGregorianDays(dateWritable.getDays)
+      },
+      julianDays = dateWritable.getDays)
+  }
+
+  override def getDays: Int = julianDays
+  override def get: Date = {
+    new Date(DateWritable.daysToMillis(julianDays))
+  }
+  override def get(doesTimeMatter: Boolean): Date = {
+    new Date(DateWritable.daysToMillis(julianDays, doesTimeMatter))
+  }
+
+  override def set(d: Int): Unit = {
+    gregorianDays = d
+    julianDays = rebaseGregorianToJulianDays(d)
+  }
+
+  @throws[IOException]
+  override def write(out: DataOutput): Unit = {
+    WritableUtils.writeVInt(out, julianDays)
+  }
+
+  @throws[IOException]
+  override def readFields(in: DataInput): Unit = {
+    julianDays = WritableUtils.readVInt(in)
+    gregorianDays = rebaseJulianToGregorianDays(julianDays)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
index 50c4f6cd57a96..edb49d3f90ca3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
@@ -182,8 +182,7 @@ class DynamicPartitionDataWriter(
       val partitionName = ScalaUDF(
         ExternalCatalogUtils.getPartitionPathString _,
         StringType,
-        Seq(Literal(c.name), Cast(c, StringType, Option(description.timeZoneId))),
-        Seq(false, false))
+        Seq(Literal(c.name), Cast(c, StringType, Option(description.timeZoneId))))
       if (i == 0) Seq(partitionName) else Seq(Literal(Path.SEPARATOR), partitionName)
     })
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
index 219c778b9164a..a71aeb47872ce 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
@@ -39,6 +39,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCo
 import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
 import org.apache.spark.sql.execution.{ProjectExec, SortExec, SparkPlan, SQLExecution}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StringType
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.{SerializableConfiguration, Utils}
@@ -132,7 +133,7 @@ object FileFormatWriter extends Logging {
       fileFormat.prepareWrite(sparkSession, job, caseInsensitiveOptions, dataSchema)
 
     val description = new WriteJobDescription(
-      uuid = UUID.randomUUID().toString,
+      uuid = UUID.randomUUID.toString,
       serializableHadoopConf = new SerializableConfiguration(job.getConfiguration),
       outputWriterFactory = outputWriterFactory,
       allColumns = outputSpec.outputColumns,
@@ -163,6 +164,10 @@ object FileFormatWriter extends Logging {
 
     SQLExecution.checkSQLExecutionId(sparkSession)
 
+    // propagate the decription UUID into the jobs, so that committers
+    // get an ID guaranteed to be unique.
+    job.getConfiguration.set("spark.sql.sources.writeJobUUID", description.uuid)
+
     // This call shouldn't be put into the `try` block below because it only initializes and
     // prepares the job, any exception thrown from here shouldn't cause abortJob() to be called.
     committer.setupJob(job)
@@ -283,7 +288,7 @@ object FileFormatWriter extends Logging {
     } catch {
       case e: FetchFailedException =>
         throw e
-      case f: FileAlreadyExistsException =>
+      case f: FileAlreadyExistsException if SQLConf.get.fastFailFileFormatOutput =>
         // If any output file to write already exists, it does not make sense to re-run this task.
         // We throw the exception and let Executor throw ExceptionFailure to abort the job.
         throw new TaskOutputFileAlreadyExistException(f)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
index 542c996a5342d..fc59336d6107c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
@@ -21,7 +21,7 @@ import java.io.{FileNotFoundException, IOException}
 
 import org.apache.parquet.io.ParquetDecodingException
 
-import org.apache.spark.{Partition => RDDPartition, TaskContext}
+import org.apache.spark.{Partition => RDDPartition, SparkUpgradeException, TaskContext}
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.rdd.{InputFileBlockHolder, RDD}
 import org.apache.spark.sql.SparkSession
@@ -178,7 +178,9 @@ class FileScanRDD(
                 s"Expected: ${e.getLogicalType}, Found: ${e.getPhysicalType}"
               throw new QueryExecutionException(message, e)
             case e: ParquetDecodingException =>
-              if (e.getMessage.contains("Can not read value at")) {
+              if (e.getCause.isInstanceOf[SparkUpgradeException]) {
+                throw e.getCause
+              } else if (e.getMessage.contains("Can not read value at")) {
                 val message = "Encounter error while reading parquet files. " +
                   "One possible cause: Parquet column cannot be converted in the " +
                   "corresponding files. Details: "
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
index f45495121a980..4b9e0c6da11dd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
@@ -89,9 +89,8 @@ object FileSourceStrategy extends Strategy with Logging {
       case expressions.In(a: Attribute, list)
         if list.forall(_.isInstanceOf[Literal]) && a.name == bucketColumnName =>
         getBucketSetFromIterable(a, list.map(e => e.eval(EmptyRow)))
-      case expressions.InSet(a: Attribute, hset)
-        if hset.forall(_.isInstanceOf[Literal]) && a.name == bucketColumnName =>
-        getBucketSetFromIterable(a, hset.map(e => expressions.Literal(e).eval(EmptyRow)))
+      case expressions.InSet(a: Attribute, hset) if a.name == bucketColumnName =>
+        getBucketSetFromIterable(a, hset)
       case expressions.IsNull(a: Attribute) if a.name == bucketColumnName =>
         getBucketSetFromValue(a, null)
       case expressions.And(left, right) =>
@@ -154,11 +153,15 @@ object FileSourceStrategy extends Strategy with Logging {
         l.resolve(
           fsRelation.partitionSchema, fsRelation.sparkSession.sessionState.analyzer.resolver)
       val partitionSet = AttributeSet(partitionColumns)
-      val partitionKeyFilters =
-        ExpressionSet(normalizedFilters
+      val partitionKeyFilters = if (partitionColumns.isEmpty) {
+        ExpressionSet(Nil)
+      } else {
+        val predicates = ExpressionSet(normalizedFilters
           .filter(_.references.subsetOf(partitionSet)))
+        logInfo(s"Pruning directories with: ${predicates.mkString(",")}")
+        predicates
+      }
 
-      logInfo(s"Pruning directories with: ${partitionKeyFilters.mkString(",")}")
 
       // subquery expressions are filtered out because they can't be used to prune buckets or pushed
       // down as data filters, yet they would be executed
@@ -178,8 +181,11 @@ object FileSourceStrategy extends Strategy with Logging {
       // Partition keys are not available in the statistics of the files.
       val dataFilters =
         normalizedFiltersWithoutSubqueries.filter(_.references.intersect(partitionSet).isEmpty)
-      logInfo(s"Pushed Filters: " +
-        s"${dataFilters.flatMap(DataSourceStrategy.translateFilter).mkString(",")}")
+      val supportNestedPredicatePushdown =
+        DataSourceUtils.supportNestedPredicatePushdown(fsRelation)
+      val pushedFilters = dataFilters
+        .flatMap(DataSourceStrategy.translateFilter(_, supportNestedPredicatePushdown))
+      logInfo(s"Pushed Filters: ${pushedFilters.mkString(",")}")
 
       // Predicates with both partition keys and attributes need to be evaluated after the scan.
       val afterScanFilters = filterSet -- partitionKeyFilters.filter(_.references.nonEmpty)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
index f11972115e09f..fe733f4238e1a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
@@ -192,7 +192,7 @@ case class InsertIntoHadoopFsRelationCommand(
       // refresh cached files in FileIndex
       fileIndex.foreach(_.refresh())
       // refresh data cache if table is cached
-      sparkSession.catalog.refreshByPath(outputPath.toString)
+      sparkSession.sharedState.cacheManager.recacheByPath(sparkSession, outputPath, fs)
 
       if (catalogTable.nonEmpty) {
         CommandUtils.updateTableStats(sparkSession, catalogTable.get)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
index 2e09c729529a6..5341e22f5e670 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
@@ -56,14 +56,17 @@ abstract class PartitioningAwareFileIndex(
 
   protected def leafDirToChildrenFiles: Map[Path, Array[FileStatus]]
 
-  protected lazy val pathGlobFilter = parameters.get("pathGlobFilter").map(new GlobFilter(_))
+  private val caseInsensitiveMap = CaseInsensitiveMap(parameters)
+
+  protected lazy val pathGlobFilter: Option[GlobFilter] =
+    caseInsensitiveMap.get("pathGlobFilter").map(new GlobFilter(_))
 
   protected def matchGlobPattern(file: FileStatus): Boolean = {
     pathGlobFilter.forall(_.accept(file.getPath))
   }
 
-  protected lazy val recursiveFileLookup = {
-    parameters.getOrElse("recursiveFileLookup", "false").toBoolean
+  protected lazy val recursiveFileLookup: Boolean = {
+    caseInsensitiveMap.getOrElse("recursiveFileLookup", "false").toBoolean
   }
 
   override def listFiles(
@@ -215,7 +218,7 @@ abstract class PartitioningAwareFileIndex(
    * and the returned DataFrame will have the column of `something`.
    */
   private def basePaths: Set[Path] = {
-    parameters.get(BASE_PATH_PARAM).map(new Path(_)) match {
+    caseInsensitiveMap.get(BASE_PATH_PARAM).map(new Path(_)) match {
       case Some(userDefinedBasePath) =>
         val fs = userDefinedBasePath.getFileSystem(hadoopConf)
         if (!fs.isDirectory(userDefinedBasePath)) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
index fdad43b23c5aa..5846d46e146ed 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
@@ -60,7 +60,7 @@ object PartitionSpec {
 
 object PartitioningUtils {
 
-  val timestampPartitionPattern = "uuuu-MM-dd HH:mm:ss[.S]"
+  val timestampPartitionPattern = "yyyy-MM-dd HH:mm:ss[.S]"
 
   private[datasources] case class PartitionValues(columnNames: Seq[String], literals: Seq[Literal])
   {
@@ -131,7 +131,10 @@ object PartitioningUtils {
     }
 
     val dateFormatter = DateFormatter(zoneId)
-    val timestampFormatter = TimestampFormatter(timestampPartitionPattern, zoneId)
+    val timestampFormatter = TimestampFormatter(
+      timestampPartitionPattern,
+      zoneId,
+      isParsing = true)
     // First, we need to parse every partition's path and see if we can find partition values.
     val (partitionValues, optDiscoveredBasePaths) = paths.map { path =>
       parsePartition(path, typeInference, basePaths, userSpecifiedDataTypes,
@@ -542,6 +545,9 @@ object PartitioningUtils {
       partitionColumns: Seq[String],
       caseSensitive: Boolean): Unit = {
 
+    SchemaUtils.checkColumnNameDuplication(
+      partitionColumns, partitionColumns.mkString(", "), caseSensitive)
+
     partitionColumnsSchema(schema, partitionColumns, caseSensitive).foreach {
       field => field.dataType match {
         case _: AtomicType => // OK
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala
index 1ea19c187e51a..a7129fb14d1a6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala
@@ -26,6 +26,19 @@ import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2ScanRelation, FileScan}
 import org.apache.spark.sql.types.StructType
 
+/**
+ * Prune the partitions of file source based table using partition filters. Currently, this rule
+ * is applied to [[HadoopFsRelation]] with [[CatalogFileIndex]] and [[DataSourceV2ScanRelation]]
+ * with [[FileScan]].
+ *
+ * For [[HadoopFsRelation]], the location will be replaced by pruned file index, and corresponding
+ * statistics will be updated. And the partition filters will be kept in the filters of returned
+ * logical plan.
+ *
+ * For [[DataSourceV2ScanRelation]], both partition filters and data filters will be added to
+ * its underlying [[FileScan]]. And the partition filters will be removed in the filters of
+ * returned logical plan.
+ */
 private[sql] object PruneFileSourcePartitions extends Rule[LogicalPlan] {
 
   private def getPartitionKeyFiltersAndDataFilters(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala
index 99882b0f7c7b0..28097c35401c9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala
@@ -32,10 +32,12 @@ object SchemaMergeUtils extends Logging {
    */
   def mergeSchemasInParallel(
       sparkSession: SparkSession,
+      parameters: Map[String, String],
       files: Seq[FileStatus],
       schemaReader: (Seq[FileStatus], Configuration, Boolean) => Seq[StructType])
       : Option[StructType] = {
-    val serializedConf = new SerializableConfiguration(sparkSession.sessionState.newHadoopConf())
+    val serializedConf = new SerializableConfiguration(
+      sparkSession.sessionState.newHadoopConfWithOptions(parameters))
 
     // !! HACK ALERT !!
     // Here is a hack for Parquet, but it can be used by Orc as well.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
index 375cec597166c..d0283f39707c3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
@@ -156,7 +156,7 @@ object TextInputCSVDataSource extends CSVDataSource {
           sparkSession,
           paths = paths,
           className = classOf[TextFileFormat].getName,
-          options = options.parameters
+          options = options.parameters.originalMap ++ Map(DataSource.GLOB_PATHS_KEY -> "false")
         ).resolveRelation(checkFilesExist = false))
         .select("value").as[String](Encoders.STRING)
     } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala
index 21fabac472f4b..d8b52c503ad34 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala
@@ -33,11 +33,12 @@ object CSVUtils {
     // with the one below, `filterCommentAndEmpty` but execution path is different. One of them
     // might have to be removed in the near future if possible.
     import lines.sqlContext.implicits._
-    val nonEmptyLines = lines.filter(length(trim($"value")) > 0)
+    val aliased = lines.toDF("value")
+    val nonEmptyLines = aliased.filter(length(trim($"value")) > 0)
     if (options.isCommentSet) {
-      nonEmptyLines.filter(!$"value".startsWith(options.comment.toString))
+      nonEmptyLines.filter(!$"value".startsWith(options.comment.toString)).as[String]
     } else {
-      nonEmptyLines
+      nonEmptyLines.as[String]
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
index c1e1aed83bae5..3f44312fd6985 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
@@ -331,9 +331,9 @@ object JdbcUtils extends Logging {
   def resultSetToRows(resultSet: ResultSet, schema: StructType): Iterator[Row] = {
     val inputMetrics =
       Option(TaskContext.get()).map(_.taskMetrics().inputMetrics).getOrElse(new InputMetrics)
-    val encoder = RowEncoder(schema).resolveAndBind()
+    val fromRow = RowEncoder(schema).resolveAndBind().createDeserializer()
     val internalRows = resultSetToSparkInternalRows(resultSet, schema, inputMetrics)
-    internalRows.map(encoder.fromRow)
+    internalRows.map(fromRow)
   }
 
   private[spark] def resultSetToSparkInternalRows(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
index 7ec2267e3461f..920f83e6bea97 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
@@ -120,7 +120,7 @@ object TextInputJsonDataSource extends JsonDataSource {
         sparkSession,
         paths = inputPaths.map(_.getPath.toString),
         className = classOf[TextFileFormat].getName,
-        options = parsedOptions.parameters
+        options = parsedOptions.parameters.originalMap ++ Map(DataSource.GLOB_PATHS_KEY -> "false")
       ).resolveRelation(checkFilesExist = false))
       .select("value").as(Encoders.STRING)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
index 4fad0a2484cde..8a6c4dce75f30 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability}
 import org.apache.spark.sql.connector.write.{BatchWrite, DataWriter, DataWriterFactory, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
-import org.apache.spark.sql.internal.connector.SimpleTableProvider
+import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStreamingUpdate}
 import org.apache.spark.sql.sources.DataSourceRegister
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -52,8 +52,10 @@ private[noop] object NoopTable extends Table with SupportsWrite {
   }
 }
 
-private[noop] object NoopWriteBuilder extends WriteBuilder with SupportsTruncate {
+private[noop] object NoopWriteBuilder extends WriteBuilder
+  with SupportsTruncate with SupportsStreamingUpdate {
   override def truncate(): WriteBuilder = this
+  override def update(): WriteBuilder = this
   override def buildForBatch(): BatchWrite = NoopBatchWrite
   override def buildForStreaming(): StreamingWrite = NoopStreamingWrite
 }
@@ -61,6 +63,7 @@ private[noop] object NoopWriteBuilder extends WriteBuilder with SupportsTruncate
 private[noop] object NoopBatchWrite extends BatchWrite {
   override def createBatchWriterFactory(info: PhysicalWriteInfo): DataWriterFactory =
     NoopWriterFactory
+  override def useCommitCoordinator(): Boolean = false
   override def commit(messages: Array[WriterCommitMessage]): Unit = {}
   override def abort(messages: Array[WriterCommitMessage]): Unit = {}
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
index 6d52d40d6dd03..4ab009c6bd014 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
@@ -23,6 +23,7 @@ import org.apache.orc.mapred.{OrcList, OrcMap, OrcStruct, OrcTimestamp}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.RebaseDateTime.rebaseJulianToGregorianDays
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -108,7 +109,7 @@ class OrcDeserializer(
         updater.set(ordinal, bytes)
 
       case DateType => (ordinal, value) =>
-        updater.setInt(ordinal, DateTimeUtils.fromJavaDate(OrcShimUtils.getSqlDate(value)))
+        updater.setInt(ordinal, OrcShimUtils.getGregorianDays(value))
 
       case TimestampType => (ordinal, value) =>
         updater.setLong(ordinal, DateTimeUtils.fromJavaTimestamp(value.asInstanceOf[OrcTimestamp]))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
index fd791ce7c5e19..c540007d5aad1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
@@ -153,24 +153,19 @@ class OrcFileFormat
       filters: Seq[Filter],
       options: Map[String, String],
       hadoopConf: Configuration): (PartitionedFile) => Iterator[InternalRow] = {
-    if (sparkSession.sessionState.conf.orcFilterPushDown) {
-      OrcFilters.createFilter(dataSchema, filters).foreach { f =>
-        OrcInputFormat.setSearchArgument(hadoopConf, f, dataSchema.fieldNames)
-      }
-    }
 
     val resultSchema = StructType(requiredSchema.fields ++ partitionSchema.fields)
     val sqlConf = sparkSession.sessionState.conf
     val enableVectorizedReader = supportBatch(sparkSession, resultSchema)
     val capacity = sqlConf.orcVectorizedReaderBatchSize
 
-    val resultSchemaString = OrcUtils.orcTypeDescriptionString(resultSchema)
-    OrcConf.MAPRED_INPUT_SCHEMA.setString(hadoopConf, resultSchemaString)
     OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(hadoopConf, sqlConf.caseSensitiveAnalysis)
 
     val broadcastedConf =
       sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf))
     val isCaseSensitive = sparkSession.sessionState.conf.caseSensitiveAnalysis
+    val orcFilterPushDown = sparkSession.sessionState.conf.orcFilterPushDown
+    val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
 
     (file: PartitionedFile) => {
       val conf = broadcastedConf.value.value
@@ -179,16 +174,27 @@ class OrcFileFormat
 
       val fs = filePath.getFileSystem(conf)
       val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
-      val requestedColIdsOrEmptyFile =
+      val resultedColPruneInfo =
         Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions)) { reader =>
           OrcUtils.requestedColumnIds(
             isCaseSensitive, dataSchema, requiredSchema, reader, conf)
         }
 
-      if (requestedColIdsOrEmptyFile.isEmpty) {
+      if (resultedColPruneInfo.isEmpty) {
         Iterator.empty
       } else {
-        val requestedColIds = requestedColIdsOrEmptyFile.get
+        // ORC predicate pushdown
+        if (orcFilterPushDown) {
+          OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).foreach { fileSchema =>
+            OrcFilters.createFilter(fileSchema, filters).foreach { f =>
+              OrcInputFormat.setSearchArgument(conf, f, fileSchema.fieldNames)
+            }
+          }
+        }
+
+        val (requestedColIds, canPruneCols) = resultedColPruneInfo.get
+        val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols,
+          dataSchema, resultSchema, partitionSchema, conf)
         assert(requestedColIds.length == requiredSchema.length,
           "[BUG] requested column IDs do not match required schema")
         val taskConf = new Configuration(conf)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
index 0b5658715377a..4554899ec2827 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
@@ -17,14 +17,45 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
+import java.util.Locale
+
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.sources.{And, Filter}
-import org.apache.spark.sql.types.{AtomicType, BinaryType, DataType}
+import org.apache.spark.sql.types.{AtomicType, BinaryType, DataType, StructType}
 
 /**
  * Methods that can be shared when upgrading the built-in Hive.
  */
 trait OrcFiltersBase {
 
+  case class OrcPrimitiveField(fieldName: String, fieldType: DataType)
+
+  protected[sql] def getDataTypeMap(
+      schema: StructType,
+      caseSensitive: Boolean): Map[String, OrcPrimitiveField] = {
+    val fields = schema.flatMap { f =>
+      if (isSearchableType(f.dataType)) {
+        Some(quoteIfNeeded(f.name) -> OrcPrimitiveField(quoteIfNeeded(f.name), f.dataType))
+      } else {
+        None
+      }
+    }
+
+    if (caseSensitive) {
+      fields.toMap
+    } else {
+      // Don't consider ambiguity here, i.e. more than one field are matched in case insensitive
+      // mode, just skip pushdown for these fields, they will trigger Exception when reading,
+      // See: SPARK-25175.
+      val dedupPrimitiveFields = fields
+        .groupBy(_._1.toLowerCase(Locale.ROOT))
+        .filter(_._2.size == 1)
+        .mapValues(_.head._2)
+      CaseInsensitiveMap(dedupPrimitiveFields)
+    }
+  }
+
   private[sql] def buildTree(filters: Seq[Filter]): Option[Filter] = {
     filters match {
       case Seq() => None
@@ -36,21 +67,11 @@ trait OrcFiltersBase {
     }
   }
 
-  // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
-  // in order to distinguish predicate pushdown for nested columns.
-  protected[sql] def quoteAttributeNameIfNeeded(name: String) : String = {
-    if (!name.contains("`") && name.contains(".")) {
-      s"`$name`"
-    } else {
-      name
-    }
-  }
-
   /**
    * Return true if this is a searchable type in ORC.
    * Both CharType and VarcharType are cleaned at AstBuilder.
    */
-  protected[sql] def isSearchableType(dataType: DataType) = dataType match {
+  private def isSearchableType(dataType: DataType) = dataType match {
     case BinaryType => false
     case _: AtomicType => true
     case _ => false
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index eea9b2a8f9613..c29287f1789e6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -24,7 +24,7 @@ import scala.collection.JavaConverters._
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, Path}
-import org.apache.orc.{OrcFile, Reader, TypeDescription, Writer}
+import org.apache.orc.{OrcConf, OrcFile, Reader, TypeDescription, Writer}
 
 import org.apache.spark.{SPARK_VERSION_SHORT, SparkException}
 import org.apache.spark.deploy.SparkHadoopUtil
@@ -81,10 +81,10 @@ object OrcUtils extends Logging {
     }
   }
 
-  def readSchema(sparkSession: SparkSession, files: Seq[FileStatus])
+  def readSchema(sparkSession: SparkSession, files: Seq[FileStatus], options: Map[String, String])
       : Option[StructType] = {
     val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
-    val conf = sparkSession.sessionState.newHadoopConf()
+    val conf = sparkSession.sessionState.newHadoopConfWithOptions(options)
     files.toIterator.map(file => readSchema(file.getPath, conf, ignoreCorruptFiles)).collectFirst {
       case Some(schema) =>
         logDebug(s"Reading schema from file $files, got Hive schema string: $schema")
@@ -92,6 +92,20 @@ object OrcUtils extends Logging {
     }
   }
 
+  def readCatalystSchema(
+      file: Path,
+      conf: Configuration,
+      ignoreCorruptFiles: Boolean): Option[StructType] = {
+    readSchema(file, conf, ignoreCorruptFiles) match {
+      case Some(schema) =>
+        Some(CatalystSqlParser.parseDataType(schema.toString).asInstanceOf[StructType])
+
+      case None =>
+        // Field names is empty or `FileFormatException` was thrown but ignoreCorruptFiles is true.
+        None
+    }
+  }
+
   /**
    * Reads ORC file schemas in multi-threaded manner, using native version of ORC.
    * This is visible for testing.
@@ -109,22 +123,24 @@ object OrcUtils extends Logging {
     val orcOptions = new OrcOptions(options, sparkSession.sessionState.conf)
     if (orcOptions.mergeSchema) {
       SchemaMergeUtils.mergeSchemasInParallel(
-        sparkSession, files, OrcUtils.readOrcSchemasInParallel)
+        sparkSession, options, files, OrcUtils.readOrcSchemasInParallel)
     } else {
-      OrcUtils.readSchema(sparkSession, files)
+      OrcUtils.readSchema(sparkSession, files, options)
     }
   }
 
   /**
-   * Returns the requested column ids from the given ORC file. Column id can be -1, which means the
-   * requested column doesn't exist in the ORC file. Returns None if the given ORC file is empty.
+   * @return Returns the combination of requested column ids from the given ORC file and
+   *         boolean flag to find if the pruneCols is allowed or not. Requested Column id can be
+   *         -1, which means the requested column doesn't exist in the ORC file. Returns None
+   *         if the given ORC file is empty.
    */
   def requestedColumnIds(
       isCaseSensitive: Boolean,
       dataSchema: StructType,
       requiredSchema: StructType,
       reader: Reader,
-      conf: Configuration): Option[Array[Int]] = {
+      conf: Configuration): Option[(Array[Int], Boolean)] = {
     val orcFieldNames = reader.getSchema.getFieldNames.asScala
     if (orcFieldNames.isEmpty) {
       // SPARK-8501: Some old empty ORC files always have an empty schema stored in their footer.
@@ -136,6 +152,10 @@ object OrcUtils extends Logging {
         assert(orcFieldNames.length <= dataSchema.length, "The given data schema " +
           s"${dataSchema.catalogString} has less fields than the actual ORC physical schema, " +
           "no idea which columns were dropped, fail to read.")
+        // for ORC file written by Hive, no field names
+        // in the physical schema, there is a need to send the
+        // entire dataSchema instead of required schema.
+        // So pruneCols is not done in this case
         Some(requiredSchema.fieldNames.map { name =>
           val index = dataSchema.fieldIndex(name)
           if (index < orcFieldNames.length) {
@@ -143,7 +163,7 @@ object OrcUtils extends Logging {
           } else {
             -1
           }
-        })
+        }, false)
       } else {
         if (isCaseSensitive) {
           Some(requiredSchema.fieldNames.zipWithIndex.map { case (name, idx) =>
@@ -152,7 +172,7 @@ object OrcUtils extends Logging {
             } else {
               -1
             }
-          })
+          }, true)
         } else {
           // Do case-insensitive resolution only if in case-insensitive mode
           val caseInsensitiveOrcFieldMap = orcFieldNames.groupBy(_.toLowerCase(Locale.ROOT))
@@ -170,7 +190,7 @@ object OrcUtils extends Logging {
                   idx
                 }
               }.getOrElse(-1)
-          })
+          }, true)
         }
       }
     }
@@ -199,4 +219,25 @@ object OrcUtils extends Logging {
       s"map<${orcTypeDescriptionString(m.keyType)},${orcTypeDescriptionString(m.valueType)}>"
     case _ => dt.catalogString
   }
+
+  /**
+   * @return Returns the result schema string based on the canPruneCols flag.
+   *         resultSchemaString will be created using resultsSchema in case of
+   *         canPruneCols is true and for canPruneCols as false value
+   *         resultSchemaString will be created using the actual dataSchema.
+   */
+  def orcResultSchemaString(
+      canPruneCols: Boolean,
+      dataSchema: StructType,
+      resultSchema: StructType,
+      partitionSchema: StructType,
+      conf: Configuration): String = {
+    val resultSchemaString = if (canPruneCols) {
+      OrcUtils.orcTypeDescriptionString(resultSchema)
+    } else {
+      OrcUtils.orcTypeDescriptionString(StructType(dataSchema.fields ++ partitionSchema.fields))
+    }
+    OrcConf.MAPRED_INPUT_SCHEMA.setString(conf, resultSchemaString)
+    resultSchemaString
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
index f52aaf0140e1d..68f49f9442579 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
@@ -295,11 +295,15 @@ class ParquetFileFormat
 
       val convertTz =
         if (timestampConversion && !isCreatedByParquetMr) {
-          Some(DateTimeUtils.getTimeZone(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
+          Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
         } else {
           None
         }
 
+      val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
+        footerFileMetaData.getKeyValueMetaData.get,
+        SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ))
+
       val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
       val hadoopAttemptContext =
         new TaskAttemptContextImpl(broadcastedHadoopConf.value.value, attemptId)
@@ -312,7 +316,10 @@ class ParquetFileFormat
       val taskContext = Option(TaskContext.get())
       if (enableVectorizedReader) {
         val vectorizedReader = new VectorizedParquetRecordReader(
-          convertTz.orNull, enableOffHeapColumnVector && taskContext.isDefined, capacity)
+          convertTz.orNull,
+          datetimeRebaseMode.toString,
+          enableOffHeapColumnVector && taskContext.isDefined,
+          capacity)
         val iter = new RecordReaderIterator(vectorizedReader)
         // SPARK-23457 Register a task completion listener before `initialization`.
         taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close()))
@@ -328,7 +335,8 @@ class ParquetFileFormat
       } else {
         logDebug(s"Falling back to parquet-mr")
         // ParquetRecordReader returns InternalRow
-        val readSupport = new ParquetReadSupport(convertTz, enableVectorizedReader = false)
+        val readSupport = new ParquetReadSupport(
+          convertTz, enableVectorizedReader = false, datetimeRebaseMode)
         val reader = if (pushed.isDefined && enableRecordFilter) {
           val parquetFilter = FilterCompat.get(pushed.get, null)
           new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
@@ -467,6 +475,7 @@ object ParquetFileFormat extends Logging {
    *     S3 nodes).
    */
   def mergeSchemasInParallel(
+      parameters: Map[String, String],
       filesToTouch: Seq[FileStatus],
       sparkSession: SparkSession): Option[StructType] = {
     val assumeBinaryIsString = sparkSession.sessionState.conf.isParquetBinaryAsString
@@ -482,7 +491,7 @@ object ParquetFileFormat extends Logging {
         .map(ParquetFileFormat.readSchemaFromFooter(_, converter))
     }
 
-    SchemaMergeUtils.mergeSchemasInParallel(sparkSession, filesToTouch, reader)
+    SchemaMergeUtils.mergeSchemasInParallel(sparkSession, parameters, filesToTouch, reader)
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index b9b86adb438e6..13dee484bd98a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -20,14 +20,15 @@ package org.apache.spark.sql.execution.datasources.parquet
 import java.lang.{Boolean => JBoolean, Double => JDouble, Float => JFloat, Long => JLong}
 import java.math.{BigDecimal => JBigDecimal}
 import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate}
 import java.util.Locale
 
 import scala.collection.JavaConverters.asScalaBufferConverter
 
 import org.apache.parquet.filter2.predicate._
-import org.apache.parquet.filter2.predicate.FilterApi._
+import org.apache.parquet.filter2.predicate.SparkFilterApi._
 import org.apache.parquet.io.api.Binary
-import org.apache.parquet.schema.{DecimalMetadata, MessageType, OriginalType, PrimitiveComparator}
+import org.apache.parquet.schema.{DecimalMetadata, GroupType, MessageType, OriginalType, PrimitiveComparator, PrimitiveType, Type}
 import org.apache.parquet.schema.OriginalType._
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._
@@ -49,15 +50,35 @@ class ParquetFilters(
     pushDownInFilterThreshold: Int,
     caseSensitive: Boolean) {
   // A map which contains parquet field name and data type, if predicate push down applies.
-  private val nameToParquetField : Map[String, ParquetField] = {
-    // Here we don't flatten the fields in the nested schema but just look up through
-    // root fields. Currently, accessing to nested fields does not push down filters
-    // and it does not support to create filters for them.
-    val primitiveFields =
-    schema.getFields.asScala.filter(_.isPrimitive).map(_.asPrimitiveType()).map { f =>
-      f.getName -> ParquetField(f.getName,
-        ParquetSchemaType(f.getOriginalType,
-          f.getPrimitiveTypeName, f.getTypeLength, f.getDecimalMetadata))
+  //
+  // Each key in `nameToParquetField` represents a column; `dots` are used as separators for
+  // nested columns. If any part of the names contains `dots`, it is quoted to avoid confusion.
+  // See `org.apache.spark.sql.connector.catalog.quote` for implementation details.
+  private val nameToParquetField : Map[String, ParquetPrimitiveField] = {
+    // Recursively traverse the parquet schema to get primitive fields that can be pushed-down.
+    // `parentFieldNames` is used to keep track of the current nested level when traversing.
+    def getPrimitiveFields(
+        fields: Seq[Type],
+        parentFieldNames: Array[String] = Array.empty): Seq[ParquetPrimitiveField] = {
+      fields.flatMap {
+        case p: PrimitiveType =>
+          Some(ParquetPrimitiveField(fieldNames = parentFieldNames :+ p.getName,
+            fieldType = ParquetSchemaType(p.getOriginalType,
+              p.getPrimitiveTypeName, p.getTypeLength, p.getDecimalMetadata)))
+        // Note that when g is a `Struct`, `g.getOriginalType` is `null`.
+        // When g is a `Map`, `g.getOriginalType` is `MAP`.
+        // When g is a `List`, `g.getOriginalType` is `LIST`.
+        case g: GroupType if g.getOriginalType == null =>
+          getPrimitiveFields(g.getFields.asScala, parentFieldNames :+ g.getName)
+        // Parquet only supports push-down for primitive types; as a result, Map and List types
+        // are removed.
+        case _ => None
+      }
+    }
+
+    val primitiveFields = getPrimitiveFields(schema.getFields.asScala).map { field =>
+      import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+      (field.fieldNames.toSeq.quoted, field)
     }
     if (caseSensitive) {
       primitiveFields.toMap
@@ -75,13 +96,13 @@ class ParquetFilters(
   }
 
   /**
-   * Holds a single field information stored in the underlying parquet file.
+   * Holds a single primitive field information stored in the underlying parquet file.
    *
-   * @param fieldName field name in parquet file
+   * @param fieldNames a field name as an array of string multi-identifier in parquet file
    * @param fieldType field type related info in parquet file
    */
-  private case class ParquetField(
-      fieldName: String,
+  private case class ParquetPrimitiveField(
+      fieldNames: Array[String],
       fieldType: ParquetSchemaType)
 
   private case class ParquetSchemaType(
@@ -103,8 +124,14 @@ class ParquetFilters(
   private val ParquetTimestampMicrosType = ParquetSchemaType(TIMESTAMP_MICROS, INT64, 0, null)
   private val ParquetTimestampMillisType = ParquetSchemaType(TIMESTAMP_MILLIS, INT64, 0, null)
 
-  private def dateToDays(date: Date): SQLDate = {
-    DateTimeUtils.fromJavaDate(date)
+  private def dateToDays(date: Any): SQLDate = date match {
+    case d: Date => DateTimeUtils.fromJavaDate(d)
+    case ld: LocalDate => DateTimeUtils.localDateToDays(ld)
+  }
+
+  private def timestampToMicros(v: Any): JLong = v match {
+    case i: Instant => DateTimeUtils.instantToMicros(i)
+    case t: Timestamp => DateTimeUtils.fromJavaTimestamp(t)
   }
 
   private def decimalToInt32(decimal: JBigDecimal): Integer = decimal.unscaledValue().intValue()
@@ -126,264 +153,258 @@ class ParquetFilters(
     Binary.fromConstantByteArray(fixedLengthBytes, 0, numBytes)
   }
 
-  private val makeEq: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private def timestampToMillis(v: Any): JLong = {
+    val micros = timestampToMicros(v)
+    val millis = DateTimeUtils.toMillis(micros)
+    millis.asInstanceOf[JLong]
+  }
+
+  private val makeEq:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetBooleanType =>
-      (n: String, v: Any) => FilterApi.eq(booleanColumn(n), v.asInstanceOf[JBoolean])
+      (n: Array[String], v: Any) => FilterApi.eq(booleanColumn(n), v.asInstanceOf[JBoolean])
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         intColumn(n),
         Option(v).map(_.asInstanceOf[Number].intValue.asInstanceOf[Integer]).orNull)
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.eq(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.eq(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.eq(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.eq(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.eq(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.eq(doubleColumn(n), v.asInstanceOf[JDouble])
 
     // Binary.fromString and Binary.fromByteArray don't accept null values
     case ParquetStringType =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         binaryColumn(n),
         Option(v).map(s => Binary.fromString(s.asInstanceOf[String])).orNull)
     case ParquetBinaryType =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         binaryColumn(n),
         Option(v).map(b => Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]])).orNull)
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         intColumn(n),
-        Option(v).map(date => dateToDays(date.asInstanceOf[Date]).asInstanceOf[Integer]).orNull)
+        Option(v).map(date => dateToDays(date).asInstanceOf[Integer]).orNull)
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
-        Option(v).map(t => DateTimeUtils.fromJavaTimestamp(t.asInstanceOf[Timestamp])
-          .asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMicros).orNull)
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
-        Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMillis).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         intColumn(n),
         Option(v).map(d => decimalToInt32(d.asInstanceOf[JBigDecimal])).orNull)
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
         Option(v).map(d => decimalToInt64(d.asInstanceOf[JBigDecimal])).orNull)
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         binaryColumn(n),
         Option(v).map(d => decimalToByteArray(d.asInstanceOf[JBigDecimal], length)).orNull)
   }
 
-  private val makeNotEq: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeNotEq:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetBooleanType =>
-      (n: String, v: Any) => FilterApi.notEq(booleanColumn(n), v.asInstanceOf[JBoolean])
+      (n: Array[String], v: Any) => FilterApi.notEq(booleanColumn(n), v.asInstanceOf[JBoolean])
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         intColumn(n),
         Option(v).map(_.asInstanceOf[Number].intValue.asInstanceOf[Integer]).orNull)
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.notEq(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.notEq(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.notEq(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.notEq(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.notEq(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.notEq(doubleColumn(n), v.asInstanceOf[JDouble])
 
     case ParquetStringType =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         binaryColumn(n),
         Option(v).map(s => Binary.fromString(s.asInstanceOf[String])).orNull)
     case ParquetBinaryType =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         binaryColumn(n),
         Option(v).map(b => Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]])).orNull)
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         intColumn(n),
-        Option(v).map(date => dateToDays(date.asInstanceOf[Date]).asInstanceOf[Integer]).orNull)
+        Option(v).map(date => dateToDays(date).asInstanceOf[Integer]).orNull)
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
-        Option(v).map(t => DateTimeUtils.fromJavaTimestamp(t.asInstanceOf[Timestamp])
-          .asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMicros).orNull)
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
-        Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMillis).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         intColumn(n),
         Option(v).map(d => decimalToInt32(d.asInstanceOf[JBigDecimal])).orNull)
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
         Option(v).map(d => decimalToInt64(d.asInstanceOf[JBigDecimal])).orNull)
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         binaryColumn(n),
         Option(v).map(d => decimalToByteArray(d.asInstanceOf[JBigDecimal], length)).orNull)
   }
 
-  private val makeLt: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeLt:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(intColumn(n), v.asInstanceOf[Number].intValue.asInstanceOf[Integer])
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.lt(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.lt(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.lt(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.lt(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.lt(doubleColumn(n), v.asInstanceOf[JDouble])
 
     case ParquetStringType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(binaryColumn(n), Binary.fromString(v.asInstanceOf[String]))
     case ParquetBinaryType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) =>
-        FilterApi.lt(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
+      (n: Array[String], v: Any) =>
+        FilterApi.lt(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.lt(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.lt(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(intColumn(n), decimalToInt32(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(longColumn(n), decimalToInt64(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(binaryColumn(n), decimalToByteArray(v.asInstanceOf[JBigDecimal], length))
   }
 
-  private val makeLtEq: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeLtEq:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(intColumn(n), v.asInstanceOf[Number].intValue.asInstanceOf[Integer])
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.ltEq(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.ltEq(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.ltEq(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.ltEq(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.ltEq(doubleColumn(n), v.asInstanceOf[JDouble])
 
     case ParquetStringType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(binaryColumn(n), Binary.fromString(v.asInstanceOf[String]))
     case ParquetBinaryType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) =>
-        FilterApi.ltEq(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
+      (n: Array[String], v: Any) =>
+        FilterApi.ltEq(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.ltEq(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.ltEq(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(intColumn(n), decimalToInt32(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(longColumn(n), decimalToInt64(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(binaryColumn(n), decimalToByteArray(v.asInstanceOf[JBigDecimal], length))
   }
 
-  private val makeGt: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeGt:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(intColumn(n), v.asInstanceOf[Number].intValue.asInstanceOf[Integer])
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.gt(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.gt(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.gt(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.gt(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.gt(doubleColumn(n), v.asInstanceOf[JDouble])
 
     case ParquetStringType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(binaryColumn(n), Binary.fromString(v.asInstanceOf[String]))
     case ParquetBinaryType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) =>
-        FilterApi.gt(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
+      (n: Array[String], v: Any) =>
+        FilterApi.gt(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.gt(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.gt(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(intColumn(n), decimalToInt32(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(longColumn(n), decimalToInt64(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(binaryColumn(n), decimalToByteArray(v.asInstanceOf[JBigDecimal], length))
   }
 
-  private val makeGtEq: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeGtEq:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(intColumn(n), v.asInstanceOf[Number].intValue.asInstanceOf[Integer])
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.gtEq(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.gtEq(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.gtEq(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.gtEq(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.gtEq(doubleColumn(n), v.asInstanceOf[JDouble])
 
     case ParquetStringType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(binaryColumn(n), Binary.fromString(v.asInstanceOf[String]))
     case ParquetBinaryType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) =>
-        FilterApi.gtEq(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
+      (n: Array[String], v: Any) =>
+        FilterApi.gtEq(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.gtEq(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.gtEq(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(intColumn(n), decimalToInt32(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(longColumn(n), decimalToInt64(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(binaryColumn(n), decimalToByteArray(v.asInstanceOf[JBigDecimal], length))
   }
 
@@ -445,9 +466,10 @@ class ParquetFilters(
       case ParquetDoubleType => value.isInstanceOf[JDouble]
       case ParquetStringType => value.isInstanceOf[String]
       case ParquetBinaryType => value.isInstanceOf[Array[Byte]]
-      case ParquetDateType => value.isInstanceOf[Date]
+      case ParquetDateType =>
+        value.isInstanceOf[Date] || value.isInstanceOf[LocalDate]
       case ParquetTimestampMicrosType | ParquetTimestampMillisType =>
-        value.isInstanceOf[Timestamp]
+        value.isInstanceOf[Timestamp] || value.isInstanceOf[Instant]
       case ParquetSchemaType(DECIMAL, INT32, _, decimalMeta) =>
         isDecimalMatched(value, decimalMeta)
       case ParquetSchemaType(DECIMAL, INT64, _, decimalMeta) =>
@@ -466,13 +488,8 @@ class ParquetFilters(
     case _ => false
   }
 
-  // Parquet does not allow dots in the column name because dots are used as a column path
-  // delimiter. Since Parquet 1.8.2 (PARQUET-389), Parquet accepts the filter predicates
-  // with missing columns. The incorrect results could be got from Parquet when we push down
-  // filters for the column having dots in the names. Thus, we do not push down such filters.
-  // See SPARK-20364.
   private def canMakeFilterOn(name: String, value: Any): Boolean = {
-    nameToParquetField.contains(name) && !name.contains(".") && valueCanMakeFilterOn(name, value)
+    nameToParquetField.contains(name) && valueCanMakeFilterOn(name, value)
   }
 
   /**
@@ -503,38 +520,38 @@ class ParquetFilters(
     predicate match {
       case sources.IsNull(name) if canMakeFilterOn(name, null) =>
         makeEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, null))
+          .map(_(nameToParquetField(name).fieldNames, null))
       case sources.IsNotNull(name) if canMakeFilterOn(name, null) =>
         makeNotEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, null))
+          .map(_(nameToParquetField(name).fieldNames, null))
 
       case sources.EqualTo(name, value) if canMakeFilterOn(name, value) =>
         makeEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(nameToParquetField(name).fieldNames, value))
       case sources.Not(sources.EqualTo(name, value)) if canMakeFilterOn(name, value) =>
         makeNotEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(nameToParquetField(name).fieldNames, value))
 
       case sources.EqualNullSafe(name, value) if canMakeFilterOn(name, value) =>
         makeEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(nameToParquetField(name).fieldNames, value))
       case sources.Not(sources.EqualNullSafe(name, value)) if canMakeFilterOn(name, value) =>
         makeNotEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(nameToParquetField(name).fieldNames, value))
 
       case sources.LessThan(name, value) if canMakeFilterOn(name, value) =>
         makeLt.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(nameToParquetField(name).fieldNames, value))
       case sources.LessThanOrEqual(name, value) if canMakeFilterOn(name, value) =>
         makeLtEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(nameToParquetField(name).fieldNames, value))
 
       case sources.GreaterThan(name, value) if canMakeFilterOn(name, value) =>
         makeGt.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(nameToParquetField(name).fieldNames, value))
       case sources.GreaterThanOrEqual(name, value) if canMakeFilterOn(name, value) =>
         makeGtEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(nameToParquetField(name).fieldNames, value))
 
       case sources.And(lhs, rhs) =>
         // At here, it is not safe to just convert one side and remove the other side
@@ -585,13 +602,13 @@ class ParquetFilters(
         && values.distinct.length <= pushDownInFilterThreshold =>
         values.distinct.flatMap { v =>
           makeEq.lift(nameToParquetField(name).fieldType)
-            .map(_(nameToParquetField(name).fieldName, v))
+            .map(_(nameToParquetField(name).fieldNames, v))
         }.reduceLeftOption(FilterApi.or)
 
       case sources.StringStartsWith(name, prefix)
           if pushDownStartWith && canMakeFilterOn(name, prefix) =>
         Option(prefix).map { v =>
-          FilterApi.userDefined(binaryColumn(name),
+          FilterApi.userDefined(binaryColumn(nameToParquetField(name).fieldNames),
             new UserDefinedPredicate[Binary] with Serializable {
               private val strToBinary = Binary.fromReusedByteArray(v.getBytes)
               private val size = strToBinary.length
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
index 69c8bad5f1c83..a30d1c26b3b2d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
-import java.util.{Locale, Map => JMap, TimeZone}
+import java.time.ZoneId
+import java.util.{Locale, Map => JMap}
 
 import scala.collection.JavaConverters._
 
@@ -31,6 +32,7 @@ import org.apache.parquet.schema.Type.Repetition
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 
 /**
@@ -49,16 +51,18 @@ import org.apache.spark.sql.types._
  * Due to this reason, we no longer rely on [[ReadContext]] to pass requested schema from [[init()]]
  * to [[prepareForRead()]], but use a private `var` for simplicity.
  */
-class ParquetReadSupport(val convertTz: Option[TimeZone],
-    enableVectorizedReader: Boolean)
+class ParquetReadSupport(
+    val convertTz: Option[ZoneId],
+    enableVectorizedReader: Boolean,
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value)
   extends ReadSupport[InternalRow] with Logging {
   private var catalystRequestedSchema: StructType = _
 
   def this() {
     // We need a zero-arg constructor for SpecificParquetRecordReaderBase.  But that is only
-    // used in the vectorized reader, where we get the convertTz value directly, and the value here
-    // is ignored.
-    this(None, enableVectorizedReader = true)
+    // used in the vectorized reader, where we get the convertTz/rebaseDateTime value directly,
+    // and the values here are ignored.
+    this(None, enableVectorizedReader = true, datetimeRebaseMode = LegacyBehaviorPolicy.CORRECTED)
   }
 
   /**
@@ -126,7 +130,8 @@ class ParquetReadSupport(val convertTz: Option[TimeZone],
       parquetRequestedSchema,
       ParquetReadSupport.expandUDT(catalystRequestedSchema),
       new ParquetToSparkSchemaConverter(conf),
-      convertTz)
+      convertTz,
+      datetimeRebaseMode)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
index 3098a332d3027..bb528d548b6ef 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
@@ -17,12 +17,13 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
-import java.util.TimeZone
+import java.time.ZoneId
 
 import org.apache.parquet.io.api.{GroupConverter, RecordMaterializer}
 import org.apache.parquet.schema.MessageType
 
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types.StructType
 
 /**
@@ -31,16 +32,20 @@ import org.apache.spark.sql.types.StructType
  * @param parquetSchema Parquet schema of the records to be read
  * @param catalystSchema Catalyst schema of the rows to be constructed
  * @param schemaConverter A Parquet-Catalyst schema converter that helps initializing row converters
+ * @param convertTz the optional time zone to convert to int96 data
+ * @param datetimeRebaseMode the mode of rebasing date/timestamp from Julian to Proleptic Gregorian
+ *                           calendar
  */
 private[parquet] class ParquetRecordMaterializer(
     parquetSchema: MessageType,
     catalystSchema: StructType,
     schemaConverter: ParquetToSparkSchemaConverter,
-    convertTz: Option[TimeZone])
+    convertTz: Option[ZoneId],
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value)
   extends RecordMaterializer[InternalRow] {
 
-  private val rootConverter =
-    new ParquetRowConverter(schemaConverter, parquetSchema, catalystSchema, convertTz, NoopUpdater)
+  private val rootConverter = new ParquetRowConverter(
+    schemaConverter, parquetSchema, catalystSchema, convertTz, datetimeRebaseMode, NoopUpdater)
 
   override def getCurrentRecord: InternalRow = rootConverter.currentRecord
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index 98ac2ecd2955c..9d37f17a24fb7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.parquet
 
 import java.math.{BigDecimal, BigInteger}
 import java.nio.ByteOrder
-import java.util.TimeZone
+import java.time.{ZoneId, ZoneOffset}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
@@ -33,8 +33,11 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.{BINARY, DOUBLE
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, DateTimeUtils, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, CaseInsensitiveMap, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.SQLTimestamp
+import org.apache.spark.sql.execution.datasources.DataSourceUtils
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -118,14 +121,17 @@ private[parquet] class ParquetPrimitiveConverter(val updater: ParentContainerUpd
  * @param parquetType Parquet schema of Parquet records
  * @param catalystType Spark SQL schema that corresponds to the Parquet record type. User-defined
  *        types should have been expanded.
- * @param convertTz the optional time zone to convert to for int96 data
+ * @param convertTz the optional time zone to convert to int96 data
+ * @param datetimeRebaseMode the mode of rebasing date/timestamp from Julian to Proleptic Gregorian
+ *                           calendar
  * @param updater An updater which propagates converted field values to the parent container
  */
 private[parquet] class ParquetRowConverter(
     schemaConverter: ParquetToSparkSchemaConverter,
     parquetType: GroupType,
     catalystType: StructType,
-    convertTz: Option[TimeZone],
+    convertTz: Option[ZoneId],
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value,
     updater: ParentContainerUpdater)
   extends ParquetGroupConverter(updater) with Logging {
 
@@ -154,8 +160,6 @@ private[parquet] class ParquetRowConverter(
        |${catalystType.prettyJson}
      """.stripMargin)
 
-  private[this] val UTC = DateTimeUtils.TimeZoneUTC
-
   /**
    * Updater used together with field converters within a [[ParquetRowConverter]].  It propagates
    * converted filed values to the `ordinal`-th cell in `currentRow`.
@@ -178,10 +182,23 @@ private[parquet] class ParquetRowConverter(
    */
   def currentRecord: InternalRow = currentRow
 
+  private val dateRebaseFunc = DataSourceUtils.creteDateRebaseFuncInRead(
+    datetimeRebaseMode, "Parquet")
+
+  private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInRead(
+    datetimeRebaseMode, "Parquet")
+
   // Converters for each field.
   private[this] val fieldConverters: Array[Converter with HasParentContainerUpdater] = {
+    // (SPARK-31116) Use case insensitive map if spark.sql.caseSensitive is false
+    // to prevent throwing IllegalArgumentException when searching catalyst type's field index
+    val catalystFieldNameToIndex = if (SQLConf.get.caseSensitiveAnalysis) {
+      catalystType.fieldNames.zipWithIndex.toMap
+    } else {
+      CaseInsensitiveMap(catalystType.fieldNames.zipWithIndex.toMap)
+    }
     parquetType.getFields.asScala.map { parquetField =>
-      val fieldIndex = catalystType.fieldIndex(parquetField.getName)
+      val fieldIndex = catalystFieldNameToIndex(parquetField.getName)
       val catalystField = catalystType(fieldIndex)
       // Converted field value should be set to the `fieldIndex`-th cell of `currentRow`
       newConverter(parquetField, catalystField.dataType, new RowUpdater(currentRow, fieldIndex))
@@ -267,14 +284,15 @@ private[parquet] class ParquetRowConverter(
       case TimestampType if parquetType.getOriginalType == OriginalType.TIMESTAMP_MICROS =>
         new ParquetPrimitiveConverter(updater) {
           override def addLong(value: Long): Unit = {
-            updater.setLong(value)
+            updater.setLong(timestampRebaseFunc(value))
           }
         }
 
       case TimestampType if parquetType.getOriginalType == OriginalType.TIMESTAMP_MILLIS =>
         new ParquetPrimitiveConverter(updater) {
           override def addLong(value: Long): Unit = {
-            updater.setLong(DateTimeUtils.fromMillis(value))
+            val micros = DateTimeUtils.fromMillis(value)
+            updater.setLong(timestampRebaseFunc(micros))
           }
         }
 
@@ -292,7 +310,8 @@ private[parquet] class ParquetRowConverter(
             val timeOfDayNanos = buf.getLong
             val julianDay = buf.getInt
             val rawTime = DateTimeUtils.fromJulianDay(julianDay, timeOfDayNanos)
-            val adjTime = convertTz.map(DateTimeUtils.convertTz(rawTime, _, UTC)).getOrElse(rawTime)
+            val adjTime = convertTz.map(DateTimeUtils.convertTz(rawTime, _, ZoneOffset.UTC))
+              .getOrElse(rawTime)
             updater.setLong(adjTime)
           }
         }
@@ -300,8 +319,7 @@ private[parquet] class ParquetRowConverter(
       case DateType =>
         new ParquetPrimitiveConverter(updater) {
           override def addInt(value: Int): Unit = {
-            // DateType is not specialized in `SpecificMutableRow`, have to box it here.
-            updater.set(value.asInstanceOf[DateType#InternalType])
+            updater.set(dateRebaseFunc(value))
           }
         }
 
@@ -349,7 +367,12 @@ private[parquet] class ParquetRowConverter(
           }
         }
         new ParquetRowConverter(
-          schemaConverter, parquetType.asGroupType(), t, convertTz, wrappedUpdater)
+          schemaConverter,
+          parquetType.asGroupType(),
+          t,
+          convertTz,
+          datetimeRebaseMode,
+          wrappedUpdater)
 
       case t =>
         throw new RuntimeException(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
index 7e7dba92f37b5..b91d75c55c513 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
@@ -104,7 +104,7 @@ object ParquetUtils {
           .orElse(filesByType.data.headOption)
           .toSeq
       }
-    ParquetFileFormat.mergeSchemasInParallel(filesToTouch, sparkSession)
+    ParquetFileFormat.mergeSchemasInParallel(parameters, filesToTouch, sparkSession)
   }
 
   case class FileTypes(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
index f6490614ab05b..4e535c0c5ea99 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
@@ -31,11 +31,13 @@ import org.apache.parquet.io.api.{Binary, RecordConsumer}
 
 import org.apache.spark.SPARK_VERSION_SHORT
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY
+import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.execution.datasources.DataSourceUtils
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 
 /**
@@ -77,6 +79,15 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
   private val decimalBuffer =
     new Array[Byte](Decimal.minBytesForPrecision(DecimalType.MAX_PRECISION))
 
+  private val datetimeRebaseMode = LegacyBehaviorPolicy.withName(
+    SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE))
+
+  private val dateRebaseFunc = DataSourceUtils.creteDateRebaseFuncInWrite(
+    datetimeRebaseMode, "Parquet")
+
+  private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInWrite(
+    datetimeRebaseMode, "Parquet")
+
   override def init(configuration: Configuration): WriteContext = {
     val schemaString = configuration.get(ParquetWriteSupport.SPARK_ROW_SCHEMA)
     this.schema = StructType.fromString(schemaString)
@@ -98,7 +109,13 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
     val metadata = Map(
       SPARK_VERSION_METADATA_KEY -> SPARK_VERSION_SHORT,
       ParquetReadSupport.SPARK_METADATA_KEY -> schemaString
-    ).asJava
+    ) ++ {
+      if (datetimeRebaseMode == LegacyBehaviorPolicy.LEGACY) {
+        Some(SPARK_LEGACY_DATETIME -> "")
+      } else {
+        None
+      }
+    }
 
     logInfo(
       s"""Initialized Parquet WriteSupport with Catalyst schema:
@@ -107,7 +124,7 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
          |$messageType
        """.stripMargin)
 
-    new WriteContext(messageType, metadata)
+    new WriteContext(messageType, metadata.asJava)
   }
 
   override def prepareForWrite(recordConsumer: RecordConsumer): Unit = {
@@ -147,7 +164,11 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
         (row: SpecializedGetters, ordinal: Int) =>
           recordConsumer.addInteger(row.getShort(ordinal))
 
-      case IntegerType | DateType =>
+      case DateType =>
+        (row: SpecializedGetters, ordinal: Int) =>
+          recordConsumer.addInteger(dateRebaseFunc(row.getInt(ordinal)))
+
+      case IntegerType =>
         (row: SpecializedGetters, ordinal: Int) =>
           recordConsumer.addInteger(row.getInt(ordinal))
 
@@ -179,11 +200,13 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
 
           case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS =>
             (row: SpecializedGetters, ordinal: Int) =>
-              recordConsumer.addLong(row.getLong(ordinal))
+              val micros = row.getLong(ordinal)
+              recordConsumer.addLong(timestampRebaseFunc(micros))
 
           case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MILLIS =>
             (row: SpecializedGetters, ordinal: Int) =>
-              val millis = DateTimeUtils.toMillis(row.getLong(ordinal))
+              val micros = row.getLong(ordinal)
+              val millis = DateTimeUtils.toMillis(timestampRebaseFunc(micros))
               recordConsumer.addLong(millis)
         }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
index 95343e2872def..888678d6ff36d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
@@ -240,7 +240,7 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
         c.copy(tableDesc = normalizedTable.copy(schema = reorderedSchema))
       }
 
-    case create: V2CreateTablePlan =>
+    case create: V2CreateTablePlan if create.childrenResolved =>
       val schema = create.tableSchema
       val partitioning = create.partitioning
       val identifier = create.tableName
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala
index e4e7887017a1d..c199df676ced3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala
@@ -40,7 +40,7 @@ case class BatchScanExec(
 
   override def hashCode(): Int = batch.hashCode()
 
-  override lazy val partitions: Seq[InputPartition] = batch.planInputPartitions()
+  @transient override lazy val partitions: Seq[InputPartition] = batch.planInputPartitions()
 
   override lazy val readerFactory: PartitionReaderFactory = batch.createReaderFactory()
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index 8f4e2d256c714..877aea178c552 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -130,7 +130,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
       }
 
     case RefreshTable(catalog, ident) =>
-      RefreshTableExec(catalog, ident) :: Nil
+      RefreshTableExec(session, catalog, ident) :: Nil
 
     case ReplaceTable(catalog, ident, schema, parts, props, orCreate) =>
       val propsWithOwner = CatalogV2Util.withDefaultOwnership(props)
@@ -180,8 +180,9 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
     case OverwriteByExpression(r: DataSourceV2Relation, deleteExpr, query, writeOptions, _) =>
       // fail if any filter cannot be converted. correctness depends on removing all matching data.
       val filters = splitConjunctivePredicates(deleteExpr).map {
-        filter => DataSourceStrategy.translateFilter(deleteExpr).getOrElse(
-          throw new AnalysisException(s"Cannot translate expression to source filter: $filter"))
+        filter => DataSourceStrategy.translateFilter(deleteExpr,
+          supportNestedPredicatePushdown = true).getOrElse(
+            throw new AnalysisException(s"Cannot translate expression to source filter: $filter"))
       }.toArray
       r.table.asWritable match {
         case v1 if v1.supports(TableCapability.V1_BATCH_WRITE) =>
@@ -205,7 +206,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
           // correctness depends on removing all matching data.
           val filters = DataSourceStrategy.normalizeExprs(condition.toSeq, output)
               .flatMap(splitConjunctivePredicates(_).map {
-                f => DataSourceStrategy.translateFilter(f).getOrElse(
+                f => DataSourceStrategy.translateFilter(f, true).getOrElse(
                   throw new AnalysisException(s"Exec update failed:" +
                       s" cannot translate expression to source filter: $f"))
               }).toArray
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala
index 64b98fb83b8fa..b4a14c6face31 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala
@@ -34,7 +34,9 @@ case class DescribeNamespaceExec(
     catalog: SupportsNamespaces,
     namespace: Seq[String],
     isExtended: Boolean) extends V2CommandExec {
-  private val encoder = RowEncoder(StructType.fromAttributes(output)).resolveAndBind()
+  private val toRow = {
+    RowEncoder(StructType.fromAttributes(output)).resolveAndBind().createSerializer()
+  }
 
   override protected def run(): Seq[InternalRow] = {
     val rows = new ArrayBuffer[InternalRow]()
@@ -57,6 +59,6 @@ case class DescribeNamespaceExec(
   }
 
   private def toCatalystRow(strs: String*): InternalRow = {
-    encoder.toRow(new GenericRowWithSchema(strs.toArray, schema)).copy()
+    toRow(new GenericRowWithSchema(strs.toArray, schema)).copy()
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala
index 9c280206c548e..bc6bb175f979e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala
@@ -31,7 +31,9 @@ case class DescribeTableExec(
     table: Table,
     isExtended: Boolean) extends V2CommandExec {
 
-  private val encoder = RowEncoder(StructType.fromAttributes(output)).resolveAndBind()
+  private val toRow = {
+    RowEncoder(StructType.fromAttributes(output)).resolveAndBind().createSerializer()
+  }
 
   override protected def run(): Seq[InternalRow] = {
     val rows = new ArrayBuffer[InternalRow]()
@@ -85,6 +87,6 @@ case class DescribeTableExec(
   private def emptyRow(): InternalRow = toCatalystRow("", "", "")
 
   private def toCatalystRow(strs: String*): InternalRow = {
-    encoder.toRow(new GenericRowWithSchema(strs.toArray, schema)).copy()
+    toRow(new GenericRowWithSchema(strs.toArray, schema)).copy()
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
index 30a964d7e643f..fa871fe866fcd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
@@ -18,7 +18,10 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import java.util
 
+import scala.collection.JavaConverters._
+
 import com.fasterxml.jackson.databind.ObjectMapper
+import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 
 import org.apache.spark.sql.SparkSession
@@ -53,14 +56,23 @@ trait FileDataSourceV2 extends TableProvider with DataSourceRegister {
     paths ++ Option(map.get("path")).toSeq
   }
 
-  protected def getTableName(paths: Seq[String]): String = {
-    val name = shortName() + " " + paths.map(qualifiedPathName).mkString(",")
+  protected def getOptionsWithoutPaths(map: CaseInsensitiveStringMap): CaseInsensitiveStringMap = {
+    val withoutPath = map.asCaseSensitiveMap().asScala.filterKeys { k =>
+      !k.equalsIgnoreCase("path") && !k.equalsIgnoreCase("paths")
+    }
+    new CaseInsensitiveStringMap(withoutPath.asJava)
+  }
+
+  protected def getTableName(map: CaseInsensitiveStringMap, paths: Seq[String]): String = {
+    val hadoopConf = sparkSession.sessionState.newHadoopConfWithOptions(
+      map.asCaseSensitiveMap().asScala.toMap)
+    val name = shortName() + " " + paths.map(qualifiedPathName(_, hadoopConf)).mkString(",")
     Utils.redact(sparkSession.sessionState.conf.stringRedactionPattern, name)
   }
 
-  private def qualifiedPathName(path: String): String = {
+  private def qualifiedPathName(path: String, hadoopConf: Configuration): String = {
     val hdfsPath = new Path(path)
-    val fs = hdfsPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
+    val fs = hdfsPath.getFileSystem(hadoopConf)
     hdfsPath.makeQualified(fs.getUri, fs.getWorkingDirectory).toString
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala
index 59dc3ae56bf25..1658a250462b3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala
@@ -53,7 +53,7 @@ abstract class FileTable(
     } else {
       // This is a non-streaming file based datasource.
       val rootPathsSpecified = DataSource.checkAndGlobPathIfNecessary(paths, hadoopConf,
-        checkEmptyGlobPath = true, checkFilesExist = true)
+        checkEmptyGlobPath = true, checkFilesExist = true, enableGlobbing = globPaths)
       val fileStatusCache = FileStatusCache.getOrCreate(sparkSession)
       new InMemoryFileIndex(
         sparkSession, rootPathsSpecified, caseSensitiveMap, userSpecifiedSchema, fileStatusCache)
@@ -139,6 +139,14 @@ abstract class FileTable(
    * 2. Catalog support is required, which is still under development for data source V2.
    */
   def fallbackFileFormat: Class[_ <: FileFormat]
+
+  /**
+   * Whether or not paths should be globbed before being used to access files.
+   */
+  private def globPaths: Boolean = {
+    val entry = options.get(DataSource.GLOB_PATHS_KEY)
+    Option(entry).map(_ == "true").getOrElse(true)
+  }
 }
 
 object FileTable {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala
index d519832c57501..cd62ee7814bf2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala
@@ -16,7 +16,6 @@
  */
 package org.apache.spark.sql.execution.datasources.v2
 
-import java.io.IOException
 import java.util.UUID
 
 import scala.collection.JavaConverters._
@@ -27,7 +26,7 @@ import org.apache.hadoop.mapreduce.Job
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
 
 import org.apache.spark.internal.io.FileCommitProtocol
-import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession}
+import org.apache.spark.sql.{AnalysisException, SparkSession}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
 import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, WriteBuilder}
@@ -46,12 +45,6 @@ abstract class FileWriteBuilder(
   private val schema = info.schema()
   private val queryId = info.queryId()
   private val options = info.options()
-  private var mode: SaveMode = _
-
-  def mode(mode: SaveMode): WriteBuilder = {
-    this.mode = mode
-    this
-  }
 
   override def buildForBatch(): BatchWrite = {
     val sparkSession = SparkSession.active
@@ -68,26 +61,8 @@ abstract class FileWriteBuilder(
     lazy val description =
       createWriteJobDescription(sparkSession, hadoopConf, job, paths.head, options.asScala.toMap)
 
-    val fs = path.getFileSystem(hadoopConf)
-    mode match {
-      case SaveMode.ErrorIfExists if fs.exists(path) =>
-        val qualifiedOutputPath = path.makeQualified(fs.getUri, fs.getWorkingDirectory)
-        throw new AnalysisException(s"path $qualifiedOutputPath already exists.")
-
-      case SaveMode.Ignore if fs.exists(path) =>
-        null
-
-      case SaveMode.Overwrite =>
-        if (fs.exists(path) && !committer.deleteWithJob(fs, path, true)) {
-          throw new IOException(s"Unable to clear directory $path prior to writing to it")
-        }
-        committer.setupJob(job)
-        new FileBatchWrite(job, description, committer)
-
-      case _ =>
-        committer.setupJob(job)
-        new FileBatchWrite(job, description, committer)
-    }
+    committer.setupJob(job)
+    new FileBatchWrite(job, description, committer)
   }
 
   /**
@@ -104,7 +79,6 @@ abstract class FileWriteBuilder(
   private def validateInputs(caseSensitiveAnalysis: Boolean): Unit = {
     assert(schema != null, "Missing input data schema")
     assert(queryId != null, "Missing query ID")
-    assert(mode != null, "Missing save mode")
 
     if (paths.length != 1) {
       throw new IllegalArgumentException("Expected exactly one path to be specified, but " +
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala
index 33338b06565c9..1a6f03f54f2e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala
@@ -48,7 +48,8 @@ object PushDownUtils extends PredicateHelper {
 
         for (filterExpr <- filters) {
           val translated =
-            DataSourceStrategy.translateFilterWithMapping(filterExpr, Some(translatedFilterToExpr))
+            DataSourceStrategy.translateFilterWithMapping(filterExpr, Some(translatedFilterToExpr),
+              nestedPredicatePushdownEnabled = true)
           if (translated.isEmpty) {
             untranslatableExprs += filterExpr
           } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala
index 2a19ff304a9e0..9717b9f954671 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala
@@ -17,15 +17,26 @@
 
 package org.apache.spark.sql.execution.datasources.v2
 
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
+import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog}
 
 case class RefreshTableExec(
+    session: SparkSession,
     catalog: TableCatalog,
     ident: Identifier) extends V2CommandExec {
   override protected def run(): Seq[InternalRow] = {
     catalog.invalidateTable(ident)
+
+    if (catalog.tableExists(ident)) {
+      val table = catalog.loadTable(ident)
+      // invalidate all caches referencing the given table
+      // TODO(SPARK-33437): re-cache the table itself once we support caching a DSv2 table
+      val v2Relation = DataSourceV2Relation.create(table, Some(catalog), Some(ident))
+      session.sharedState.cacheManager.uncacheQuery(session, v2Relation, cascade = true)
+    }
+
     Seq.empty
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala
index 9e6f00e0923ea..b13cea266707b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala
@@ -32,8 +32,8 @@ case class SetCatalogAndNamespaceExec(
   override protected def run(): Seq[InternalRow] = {
     // The catalog is updated first because CatalogManager resets the current namespace
     // when the current catalog is set.
-    catalogName.map(catalogManager.setCurrentCatalog)
-    namespace.map(ns => catalogManager.setCurrentNamespace(ns.toArray))
+    catalogName.foreach(catalogManager.setCurrentCatalog)
+    namespace.foreach(ns => catalogManager.setCurrentNamespace(ns.toArray))
 
     Seq.empty
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala
index 42b80a15080a6..5f7b6f4061467 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala
@@ -31,10 +31,11 @@ case class ShowCurrentNamespaceExec(
     catalogManager: CatalogManager)
   extends V2CommandExec {
   override protected def run(): Seq[InternalRow] = {
-    val encoder = RowEncoder(schema).resolveAndBind()
-    Seq(encoder
-      .toRow(new GenericRowWithSchema(
-        Array(catalogManager.currentCatalog.name, catalogManager.currentNamespace.quoted), schema))
-      .copy())
+    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
+    val result = new GenericRowWithSchema(Array[Any](
+      catalogManager.currentCatalog.name,
+      catalogManager.currentNamespace.quoted),
+      schema)
+    Seq(toRow(result).copy())
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
index fe3ab8023db6f..9188f4eb60d56 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchem
 import org.apache.spark.sql.catalyst.util.StringUtils
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces
+import org.apache.spark.sql.execution.LeafExecNode
 
 /**
  * Physical plan node for showing namespaces.
@@ -33,8 +34,7 @@ case class ShowNamespacesExec(
     output: Seq[Attribute],
     catalog: SupportsNamespaces,
     namespace: Seq[String],
-    pattern: Option[String])
-    extends V2CommandExec {
+    pattern: Option[String]) extends V2CommandExec with LeafExecNode {
 
   override protected def run(): Seq[InternalRow] = {
     val namespaces = if (namespace.nonEmpty) {
@@ -44,13 +44,11 @@ case class ShowNamespacesExec(
     }
 
     val rows = new ArrayBuffer[InternalRow]()
-    val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
 
     namespaces.map(_.quoted).map { ns =>
       if (pattern.map(StringUtils.filterPattern(Seq(ns), _).nonEmpty).getOrElse(true)) {
-        rows += encoder
-          .toRow(new GenericRowWithSchema(Array(ns), schema))
-          .copy()
+        rows += toRow(new GenericRowWithSchema(Array(ns), schema)).copy()
       }
     }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
index 7905c35f55de0..95715fd1af56e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
@@ -19,8 +19,8 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.RowEncoder
-import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchema}
-import org.apache.spark.sql.connector.catalog.Table
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, GenericRowWithSchema}
+import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Table, TableCatalog}
 
 /**
  * Physical plan node for showing table properties.
@@ -30,19 +30,23 @@ case class ShowTablePropertiesExec(
     catalogTable: Table,
     propertyKey: Option[String]) extends V2CommandExec {
 
+  override def producedAttributes: AttributeSet = AttributeSet(output)
+
   override protected def run(): Seq[InternalRow] = {
     import scala.collection.JavaConverters._
-    val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
 
+    // The reserved properties are accessible through DESCRIBE
     val properties = catalogTable.properties.asScala
+      .filter { case (k, v) => !CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(k) }
     propertyKey match {
       case Some(p) =>
         val propValue = properties
           .getOrElse(p, s"Table ${catalogTable.name} does not have property: $p")
-        Seq(encoder.toRow(new GenericRowWithSchema(Array(p, propValue), schema)).copy())
+        Seq(toRow(new GenericRowWithSchema(Array(p, propValue), schema)).copy())
       case None =>
         properties.keys.map(k =>
-          encoder.toRow(new GenericRowWithSchema(Array(k, properties(k)), schema)).copy()).toSeq
+          toRow(new GenericRowWithSchema(Array(k, properties(k)), schema)).copy()).toSeq
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
index 995b00871fc2a..820f5ae8f1b12 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchem
 import org.apache.spark.sql.catalyst.util.StringUtils
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
 import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.execution.LeafExecNode
 
 /**
  * Physical plan node for showing tables.
@@ -33,21 +34,18 @@ case class ShowTablesExec(
     output: Seq[Attribute],
     catalog: TableCatalog,
     namespace: Seq[String],
-    pattern: Option[String])
-    extends V2CommandExec {
+    pattern: Option[String]) extends V2CommandExec with LeafExecNode {
   override protected def run(): Seq[InternalRow] = {
     val rows = new ArrayBuffer[InternalRow]()
-    val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
 
     val tables = catalog.listTables(namespace.toArray)
     tables.map { table =>
       if (pattern.map(StringUtils.filterPattern(Seq(table.name()), _).nonEmpty).getOrElse(true)) {
-        rows += encoder
-          .toRow(
-            new GenericRowWithSchema(
-              Array(table.namespace().quoted, table.name()),
-              schema))
-          .copy()
+        val result = new GenericRowWithSchema(
+          Array(table.namespace().quoted, table.name()),
+          schema)
+        rows += toRow(result).copy()
       }
     }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
index f97300025400d..95082bc466b90 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.execution.datasources.v2
 import java.util.UUID
 
 import org.apache.spark.SparkException
-import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.Dataset
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
@@ -41,7 +40,7 @@ case class AppendDataExecV1(
     writeOptions: CaseInsensitiveStringMap,
     plan: LogicalPlan) extends V1FallbackWriters {
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     writeWithV1(newWriteBuilder().buildForV1Write())
   }
 }
@@ -67,7 +66,7 @@ case class OverwriteByExpressionExecV1(
     filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
   }
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     newWriteBuilder() match {
       case builder: SupportsTruncate if isTruncate(deleteWhere) =>
         writeWithV1(builder.truncate().asV1Builder.buildForV1Write())
@@ -82,7 +81,7 @@ case class OverwriteByExpressionExecV1(
 }
 
 /** Some helper interfaces that use V2 write semantics through the V1 writer interface. */
-sealed trait V1FallbackWriters extends SupportsV1Write {
+sealed trait V1FallbackWriters extends V2CommandExec with SupportsV1Write {
   override def output: Seq[Attribute] = Nil
   override final def children: Seq[SparkPlan] = Nil
 
@@ -112,11 +111,10 @@ sealed trait V1FallbackWriters extends SupportsV1Write {
  * A trait that allows Tables that use V1 Writer interfaces to append data.
  */
 trait SupportsV1Write extends SparkPlan {
-  // TODO: We should be able to work on SparkPlans at this point.
   def plan: LogicalPlan
 
-  protected def writeWithV1(relation: InsertableRelation): RDD[InternalRow] = {
+  protected def writeWithV1(relation: InsertableRelation): Seq[InternalRow] = {
     relation.insert(Dataset.ofRows(sqlContext.sparkSession, plan), overwrite = false)
-    sparkContext.emptyRDD
+    Nil
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala
index a1f685d47a346..4be4a6b30edcd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala
@@ -19,13 +19,13 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.LeafExecNode
+import org.apache.spark.sql.execution.SparkPlan
 
 /**
  * A physical operator that executes run() and saves the result to prevent multiple executions.
  * Any V2 commands that do not require triggering a spark job should extend this class.
  */
-abstract class V2CommandExec extends LeafExecNode {
+abstract class V2CommandExec extends SparkPlan {
 
   /**
    * Abstract method that each concrete command needs to implement to compute the result.
@@ -53,4 +53,6 @@ abstract class V2CommandExec extends LeafExecNode {
   protected override def doExecute(): RDD[InternalRow] = {
     sqlContext.sparkContext.parallelize(result, 1)
   }
+
+  override def children: Seq[SparkPlan] = Nil
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala
index 59089fa6b77e9..b168e848f0b6f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala
@@ -59,7 +59,7 @@ object V2ScanRelationPushDown extends Rule[LogicalPlan] {
 
       val wrappedScan = scan match {
         case v1: V1Scan =>
-          val translated = filters.flatMap(DataSourceStrategy.translateFilter)
+          val translated = filters.flatMap(DataSourceStrategy.translateFilter(_, true))
           V1ScanWrapper(v1, translated, pushedFilters)
         case _ => scan
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala
index cef9b5f675889..2ed33b867183b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala
@@ -167,14 +167,15 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
   }
 
   implicit class TableIdentifierHelper(ident: Identifier) {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
+
     def asTableIdentifier: TableIdentifier = {
       ident.namespace match {
         case Array(db) =>
           TableIdentifier(ident.name, Some(db))
-        case Array() =>
-          TableIdentifier(ident.name, Some(catalog.getCurrentDatabase))
         case _ =>
-          throw new NoSuchTableException(ident)
+          throw new NoSuchTableException(
+            s"V2 session catalog requires a single-part namespace: ${ident.quoted}")
       }
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
index e360a9e656a16..616e18ee85a6b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
@@ -70,10 +70,10 @@ case class CreateTableAsSelectExec(
 
   import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     if (catalog.tableExists(ident)) {
       if (ifNotExists) {
-        return sparkContext.parallelize(Seq.empty, 1)
+        return Nil
       }
 
       throw new TableAlreadyExistsException(ident)
@@ -125,10 +125,10 @@ case class AtomicCreateTableAsSelectExec(
     writeOptions: CaseInsensitiveStringMap,
     ifNotExists: Boolean) extends AtomicTableWriteExec {
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     if (catalog.tableExists(ident)) {
       if (ifNotExists) {
-        return sparkContext.parallelize(Seq.empty, 1)
+        return Nil
       }
 
       throw new TableAlreadyExistsException(ident)
@@ -161,7 +161,7 @@ case class ReplaceTableAsSelectExec(
 
   import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     // Note that this operation is potentially unsafe, but these are the strict semantics of
     // RTAS if the catalog does not support atomic operations.
     //
@@ -225,7 +225,7 @@ case class AtomicReplaceTableAsSelectExec(
     writeOptions: CaseInsensitiveStringMap,
     orCreate: Boolean) extends AtomicTableWriteExec {
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     val schema = query.schema.asNullable
     val staged = if (orCreate) {
       catalog.stageCreateOrReplace(
@@ -255,7 +255,7 @@ case class AppendDataExec(
     writeOptions: CaseInsensitiveStringMap,
     query: SparkPlan) extends V2TableWriteExec with BatchWriteHelper {
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     writeWithV2(newWriteBuilder().buildForBatch())
   }
 }
@@ -280,7 +280,7 @@ case class OverwriteByExpressionExec(
     filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
   }
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     newWriteBuilder() match {
       case builder: SupportsTruncate if isTruncate(deleteWhere) =>
         writeWithV2(builder.truncate().buildForBatch())
@@ -308,7 +308,7 @@ case class OverwritePartitionsDynamicExec(
     writeOptions: CaseInsensitiveStringMap,
     query: SparkPlan) extends V2TableWriteExec with BatchWriteHelper {
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     newWriteBuilder() match {
       case builder: SupportsDynamicOverwrite =>
         writeWithV2(builder.overwriteDynamicPartitions().buildForBatch())
@@ -325,7 +325,7 @@ case class WriteToDataSourceV2Exec(
 
   def writeOptions: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty()
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     writeWithV2(batchWrite)
   }
 }
@@ -350,7 +350,7 @@ trait BatchWriteHelper {
 /**
  * The base physical plan for writing data into data source v2.
  */
-trait V2TableWriteExec extends UnaryExecNode {
+trait V2TableWriteExec extends V2CommandExec with UnaryExecNode {
   def query: SparkPlan
 
   var commitProgress: Option[StreamWriterCommitProgress] = None
@@ -358,7 +358,7 @@ trait V2TableWriteExec extends UnaryExecNode {
   override def child: SparkPlan = query
   override def output: Seq[Attribute] = Nil
 
-  protected def writeWithV2(batchWrite: BatchWrite): RDD[InternalRow] = {
+  protected def writeWithV2(batchWrite: BatchWrite): Seq[InternalRow] = {
     val rdd: RDD[InternalRow] = {
       val tempRdd = query.execute()
       // SPARK-23271 If we are attempting to write a zero partition rdd, create a dummy single
@@ -415,7 +415,7 @@ trait V2TableWriteExec extends UnaryExecNode {
         }
     }
 
-    sparkContext.emptyRDD
+    Nil
   }
 }
 
@@ -485,7 +485,7 @@ private[v2] trait AtomicTableWriteExec extends V2TableWriteExec with SupportsV1W
   protected def writeToStagedTable(
       stagedTable: StagedTable,
       writeOptions: CaseInsensitiveStringMap,
-      ident: Identifier): RDD[InternalRow] = {
+      ident: Identifier): Seq[InternalRow] = {
     Utils.tryWithSafeFinallyAndFailureCallbacks({
       stagedTable match {
         case table: SupportsWrite =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
index 1f99d4282f6da..c577cbf8d756a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
@@ -31,13 +31,15 @@ class CSVDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    CSVTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    CSVTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    CSVTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    CSVTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
index 7a0949e586cd8..cd0eba0ca9c51 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
@@ -31,14 +31,16 @@ class JsonDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    JsonTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    JsonTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    JsonTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    JsonTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
index 8665af33b976a..6303723e4b805 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
@@ -31,14 +31,16 @@ class OrcDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    OrcTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    OrcTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    OrcTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    OrcTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
index 03d58fdcb7f67..b0ddee0a6b336 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
@@ -31,9 +31,10 @@ import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader}
 import org.apache.spark.sql.execution.datasources.PartitionedFile
-import org.apache.spark.sql.execution.datasources.orc.{OrcColumnarBatchReader, OrcDeserializer, OrcUtils}
+import org.apache.spark.sql.execution.datasources.orc.{OrcColumnarBatchReader, OrcDeserializer, OrcFilters, OrcUtils}
 import org.apache.spark.sql.execution.datasources.v2._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.{AtomicType, StructType}
 import org.apache.spark.sql.vectorized.ColumnarBatch
 import org.apache.spark.util.{SerializableConfiguration, Utils}
@@ -52,10 +53,13 @@ case class OrcPartitionReaderFactory(
     broadcastedConf: Broadcast[SerializableConfiguration],
     dataSchema: StructType,
     readDataSchema: StructType,
-    partitionSchema: StructType) extends FilePartitionReaderFactory {
+    partitionSchema: StructType,
+    filters: Array[Filter]) extends FilePartitionReaderFactory {
   private val resultSchema = StructType(readDataSchema.fields ++ partitionSchema.fields)
   private val isCaseSensitive = sqlConf.caseSensitiveAnalysis
   private val capacity = sqlConf.orcVectorizedReaderBatchSize
+  private val orcFilterPushDown = sqlConf.orcFilterPushDown
+  private val ignoreCorruptFiles = sqlConf.ignoreCorruptFiles
 
   override def supportColumnarReads(partition: InputPartition): Boolean = {
     sqlConf.orcVectorizedReaderEnabled && sqlConf.wholeStageEnabled &&
@@ -63,27 +67,39 @@ case class OrcPartitionReaderFactory(
       resultSchema.forall(_.dataType.isInstanceOf[AtomicType])
   }
 
+  private def pushDownPredicates(filePath: Path, conf: Configuration): Unit = {
+    if (orcFilterPushDown) {
+      OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).foreach { fileSchema =>
+        OrcFilters.createFilter(fileSchema, filters).foreach { f =>
+          OrcInputFormat.setSearchArgument(conf, f, fileSchema.fieldNames)
+        }
+      }
+    }
+  }
+
   override def buildReader(file: PartitionedFile): PartitionReader[InternalRow] = {
     val conf = broadcastedConf.value.value
 
-    val resultSchemaString = OrcUtils.orcTypeDescriptionString(resultSchema)
-    OrcConf.MAPRED_INPUT_SCHEMA.setString(conf, resultSchemaString)
     OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(conf, isCaseSensitive)
 
     val filePath = new Path(new URI(file.filePath))
 
+    pushDownPredicates(filePath, conf)
+
     val fs = filePath.getFileSystem(conf)
     val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
-    val requestedColIdsOrEmptyFile =
+    val resultedColPruneInfo =
       Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions)) { reader =>
         OrcUtils.requestedColumnIds(
           isCaseSensitive, dataSchema, readDataSchema, reader, conf)
       }
 
-    if (requestedColIdsOrEmptyFile.isEmpty) {
+    if (resultedColPruneInfo.isEmpty) {
       new EmptyPartitionReader[InternalRow]
     } else {
-      val requestedColIds = requestedColIdsOrEmptyFile.get
+      val (requestedColIds, canPruneCols) = resultedColPruneInfo.get
+      val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols,
+        dataSchema, resultSchema, partitionSchema, conf)
       assert(requestedColIds.length == readDataSchema.length,
         "[BUG] requested column IDs do not match required schema")
 
@@ -112,24 +128,27 @@ case class OrcPartitionReaderFactory(
   override def buildColumnarReader(file: PartitionedFile): PartitionReader[ColumnarBatch] = {
     val conf = broadcastedConf.value.value
 
-    val resultSchemaString = OrcUtils.orcTypeDescriptionString(resultSchema)
-    OrcConf.MAPRED_INPUT_SCHEMA.setString(conf, resultSchemaString)
     OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(conf, isCaseSensitive)
 
     val filePath = new Path(new URI(file.filePath))
 
+    pushDownPredicates(filePath, conf)
+
     val fs = filePath.getFileSystem(conf)
     val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
-    val requestedColIdsOrEmptyFile =
+    val resultedColPruneInfo =
       Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions)) { reader =>
         OrcUtils.requestedColumnIds(
           isCaseSensitive, dataSchema, readDataSchema, reader, conf)
       }
 
-    if (requestedColIdsOrEmptyFile.isEmpty) {
+    if (resultedColPruneInfo.isEmpty) {
       new EmptyPartitionReader
     } else {
-      val requestedColIds = requestedColIdsOrEmptyFile.get ++ Array.fill(partitionSchema.length)(-1)
+      val (requestedDataColIds, canPruneCols) = resultedColPruneInfo.get
+      val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols,
+        dataSchema, resultSchema, partitionSchema, conf)
+      val requestedColIds = requestedDataColIds ++ Array.fill(partitionSchema.length)(-1)
       assert(requestedColIds.length == resultSchema.length,
         "[BUG] requested column IDs do not match required schema")
       val taskConf = new Configuration(conf)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala
index 62894fa7a2538..35e3b1a88e087 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala
@@ -48,7 +48,7 @@ case class OrcScan(
     // The partition values are already truncated in `FileScan.partitions`.
     // We should use `readPartitionSchema` as the partition schema here.
     OrcPartitionReaderFactory(sparkSession.sessionState.conf, broadcastedConf,
-      dataSchema, readDataSchema, readPartitionSchema)
+      dataSchema, readDataSchema, readPartitionSchema, pushedFilters)
   }
 
   override def equals(obj: Any): Boolean = obj match {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
index 1421ffd8b6de4..6a9cb250ca3d9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.connector.read.{Scan, SupportsPushDownFilters}
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
 import org.apache.spark.sql.execution.datasources.orc.OrcFilters
 import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -54,13 +55,10 @@ case class OrcScanBuilder(
 
   override def pushFilters(filters: Array[Filter]): Array[Filter] = {
     if (sparkSession.sessionState.conf.orcFilterPushDown) {
-      OrcFilters.createFilter(schema, filters).foreach { f =>
-        // The pushed filters will be set in `hadoopConf`. After that, we can simply use the
-        // changed `hadoopConf` in executors.
-        OrcInputFormat.setSearchArgument(hadoopConf, f, schema.fieldNames)
-      }
-      val dataTypeMap = schema.map(f => f.name -> f.dataType).toMap
-      _pushedFilters = OrcFilters.convertibleFilters(schema, dataTypeMap, filters).toArray
+      val dataTypeMap = OrcFilters.getDataTypeMap(schema, SQLConf.get.caseSensitiveAnalysis)
+      // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
+      val newFilters = filters.filter(!_.containsNestedColumn)
+      _pushedFilters = OrcFilters.convertibleFilters(schema, dataTypeMap, newFilters).toArray
     }
     filters
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
index 8cb6186c12ff3..4590660bc7b7e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
@@ -31,14 +31,17 @@ class ParquetDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    ParquetTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    ParquetTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    ParquetTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    ParquetTable(
+      tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
index b2fc724057eba..3b482b0c8ab62 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.sql.execution.datasources.v2.parquet
 
 import java.net.URI
-import java.util.TimeZone
+import java.time.ZoneId
 
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.mapreduce._
@@ -33,10 +33,11 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader}
-import org.apache.spark.sql.execution.datasources.{PartitionedFile, RecordReaderIterator}
+import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile, RecordReaderIterator}
 import org.apache.spark.sql.execution.datasources.parquet._
 import org.apache.spark.sql.execution.datasources.v2._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.{AtomicType, StructType}
 import org.apache.spark.sql.vectorized.ColumnarBatch
@@ -116,8 +117,9 @@ case class ParquetPartitionReaderFactory(
   private def buildReaderBase[T](
       file: PartitionedFile,
       buildReaderFunc: (
-        ParquetInputSplit, InternalRow, TaskAttemptContextImpl, Option[FilterPredicate],
-          Option[TimeZone]) => RecordReader[Void, T]): RecordReader[Void, T] = {
+        ParquetInputSplit, InternalRow, TaskAttemptContextImpl,
+          Option[FilterPredicate], Option[ZoneId],
+          LegacyBehaviorPolicy.Value) => RecordReader[Void, T]): RecordReader[Void, T] = {
     val conf = broadcastedConf.value.value
 
     val filePath = new Path(new URI(file.filePath))
@@ -156,7 +158,7 @@ case class ParquetPartitionReaderFactory(
 
     val convertTz =
       if (timestampConversion && !isCreatedByParquetMr) {
-        Some(DateTimeUtils.getTimeZone(conf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
+        Some(DateTimeUtils.getZoneId(conf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
       } else {
         None
       }
@@ -169,8 +171,11 @@ case class ParquetPartitionReaderFactory(
     if (pushed.isDefined) {
       ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get)
     }
-    val reader =
-      buildReaderFunc(split, file.partitionValues, hadoopAttemptContext, pushed, convertTz)
+    val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
+      footerFileMetaData.getKeyValueMetaData.get,
+      SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ))
+    val reader = buildReaderFunc(
+      split, file.partitionValues, hadoopAttemptContext, pushed, convertTz, datetimeRebaseMode)
     reader.initialize(split, hadoopAttemptContext)
     reader
   }
@@ -184,11 +189,13 @@ case class ParquetPartitionReaderFactory(
       partitionValues: InternalRow,
       hadoopAttemptContext: TaskAttemptContextImpl,
       pushed: Option[FilterPredicate],
-      convertTz: Option[TimeZone]): RecordReader[Void, InternalRow] = {
+      convertTz: Option[ZoneId],
+      datetimeRebaseMode: LegacyBehaviorPolicy.Value): RecordReader[Void, InternalRow] = {
     logDebug(s"Falling back to parquet-mr")
     val taskContext = Option(TaskContext.get())
     // ParquetRecordReader returns InternalRow
-    val readSupport = new ParquetReadSupport(convertTz, enableVectorizedReader = false)
+    val readSupport = new ParquetReadSupport(
+      convertTz, enableVectorizedReader = false, datetimeRebaseMode)
     val reader = if (pushed.isDefined && enableRecordFilter) {
       val parquetFilter = FilterCompat.get(pushed.get, null)
       new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
@@ -213,10 +220,14 @@ case class ParquetPartitionReaderFactory(
       partitionValues: InternalRow,
       hadoopAttemptContext: TaskAttemptContextImpl,
       pushed: Option[FilterPredicate],
-      convertTz: Option[TimeZone]): VectorizedParquetRecordReader = {
+      convertTz: Option[ZoneId],
+      datetimeRebaseMode: LegacyBehaviorPolicy.Value): VectorizedParquetRecordReader = {
     val taskContext = Option(TaskContext.get())
     val vectorizedReader = new VectorizedParquetRecordReader(
-      convertTz.orNull, enableOffHeapColumnVector && taskContext.isDefined, capacity)
+      convertTz.orNull,
+      datetimeRebaseMode.toString,
+      enableOffHeapColumnVector && taskContext.isDefined,
+      capacity)
     val iter = new RecordReaderIterator(vectorizedReader)
     // SPARK-23457 Register a task completion listener before `initialization`.
     taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close()))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
index 049c717effa26..f375a128d8177 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
@@ -31,14 +31,16 @@ class TextDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    TextTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    TextTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
-    TextTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val tableName = getTableName(options, paths)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    TextTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
index 6a57ef2cafe23..3cbebca14f7dc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
@@ -107,13 +107,18 @@ package object debug {
    */
   def codegenStringSeq(plan: SparkPlan): Seq[(String, String, ByteCodeStats)] = {
     val codegenSubtrees = new collection.mutable.HashSet[WholeStageCodegenExec]()
-    plan transform {
-      case s: WholeStageCodegenExec =>
-        codegenSubtrees += s
-        s
-      case s => s
+
+    def findSubtrees(plan: SparkPlan): Unit = {
+      plan foreach {
+        case s: WholeStageCodegenExec =>
+          codegenSubtrees += s
+        case s =>
+          s.subqueries.foreach(findSubtrees)
+      }
     }
-    codegenSubtrees.toSeq.map { subtree =>
+
+    findSubtrees(plan)
+    codegenSubtrees.toSeq.sortBy(_.codegenStageId).map { subtree =>
       val (_, source) = subtree.doCodeGen()
       val codeStats = try {
         CodeGenerator.compile(source)._2
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/CleanupDynamicPruningFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/CleanupDynamicPruningFilters.scala
similarity index 97%
rename from sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/CleanupDynamicPruningFilters.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/CleanupDynamicPruningFilters.scala
index 84be2c9cf5a21..1cf55a2f2954d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/CleanupDynamicPruningFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/CleanupDynamicPruningFilters.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.dynamicpruning
+package org.apache.spark.sql.execution.dynamicpruning
 
 import org.apache.spark.sql.catalyst.expressions.{DynamicPruning, PredicateHelper}
 import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PartitionPruning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
similarity index 98%
rename from sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PartitionPruning.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
index 48ba8618f272e..43c6581632687 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PartitionPruning.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.dynamicpruning
+package org.apache.spark.sql.execution.dynamicpruning
 
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
@@ -86,7 +86,7 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
       filteringPlan: LogicalPlan,
       joinKeys: Seq[Expression],
       hasBenefit: Boolean): LogicalPlan = {
-    val reuseEnabled = SQLConf.get.dynamicPartitionPruningReuseBroadcast
+    val reuseEnabled = SQLConf.get.exchangeReuseEnabled
     val index = joinKeys.indexOf(filteringKey)
     if (hasBenefit || reuseEnabled) {
       // insert a DynamicPruning wrapper to identify the subquery during query planning
@@ -96,7 +96,7 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
           filteringPlan,
           joinKeys,
           index,
-          !hasBenefit),
+          !hasBenefit || SQLConf.get.dynamicPartitionPruningReuseBroadcastOnly),
         pruningPlan)
     } else {
       // abort dynamic partition pruning
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PlanDynamicPruningFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala
similarity index 94%
rename from sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PlanDynamicPruningFilters.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala
index 1398dc049dd99..eb091758910cd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PlanDynamicPruningFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.dynamicpruning
+package org.apache.spark.sql.execution.dynamicpruning
 
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.expressions
@@ -36,9 +36,6 @@ import org.apache.spark.sql.internal.SQLConf
 case class PlanDynamicPruningFilters(sparkSession: SparkSession)
     extends Rule[SparkPlan] with PredicateHelper {
 
-  private def reuseBroadcast: Boolean =
-    SQLConf.get.dynamicPartitionPruningReuseBroadcast && SQLConf.get.exchangeReuseEnabled
-
   /**
    * Identify the shape in which keys of a given plan are broadcasted.
    */
@@ -59,7 +56,7 @@ case class PlanDynamicPruningFilters(sparkSession: SparkSession)
           sparkSession, sparkSession.sessionState.planner, buildPlan)
         // Using `sparkPlan` is a little hacky as it is based on the assumption that this rule is
         // the first to be applied (apart from `InsertAdaptiveSparkPlan`).
-        val canReuseExchange = reuseBroadcast && buildKeys.nonEmpty &&
+        val canReuseExchange = SQLConf.get.exchangeReuseEnabled && buildKeys.nonEmpty &&
           plan.find {
             case BroadcastHashJoinExec(_, _, _, BuildLeft, _, left, _) =>
               left.sameResult(sparkPlan)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
index 36f0d173cd0b0..bcdaf61c9c997 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
@@ -29,22 +29,52 @@ import org.apache.spark.launcher.SparkLauncher
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
+import org.apache.spark.sql.catalyst.plans.logical.Statistics
 import org.apache.spark.sql.catalyst.plans.physical.{BroadcastMode, BroadcastPartitioning, Partitioning}
 import org.apache.spark.sql.execution.{SparkPlan, SQLExecution}
 import org.apache.spark.sql.execution.joins.HashedRelation
 import org.apache.spark.sql.execution.metric.SQLMetrics
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
+import org.apache.spark.unsafe.map.BytesToBytesMap
 import org.apache.spark.util.{SparkFatalException, ThreadUtils}
 
+/**
+ * Common trait for all broadcast exchange implementations to facilitate pattern matching.
+ */
+trait BroadcastExchangeLike extends Exchange {
+
+  /**
+   * The broadcast job group ID
+   */
+  def runId: UUID = UUID.randomUUID
+
+  /**
+   * The asynchronous job that prepares the broadcast relation.
+   */
+  def relationFuture: Future[broadcast.Broadcast[Any]]
+
+  /**
+   * For registering callbacks on `relationFuture`.
+   * Note that calling this method may not start the execution of broadcast job.
+   */
+  def completionFuture: scala.concurrent.Future[broadcast.Broadcast[Any]]
+
+  /**
+   * Returns the runtime statistics after broadcast materialization.
+   */
+  def runtimeStatistics: Statistics
+}
+
 /**
  * A [[BroadcastExchangeExec]] collects, transforms and finally broadcasts the result of
  * a transformed SparkPlan.
  */
 case class BroadcastExchangeExec(
     mode: BroadcastMode,
-    child: SparkPlan) extends Exchange {
+    child: SparkPlan) extends BroadcastExchangeLike {
+  import BroadcastExchangeExec._
 
-  private[sql] val runId: UUID = UUID.randomUUID
+  override val runId: UUID = UUID.randomUUID
 
   override lazy val metrics = Map(
     "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"),
@@ -58,6 +88,11 @@ case class BroadcastExchangeExec(
     BroadcastExchangeExec(mode.canonicalized, child.canonicalized)
   }
 
+  override def runtimeStatistics: Statistics = {
+    val dataSize = metrics("dataSize").value
+    Statistics(dataSize)
+  }
+
   @transient
   private lazy val promise = Promise[broadcast.Broadcast[Any]]()
 
@@ -66,20 +101,16 @@ case class BroadcastExchangeExec(
    * Note that calling this field will not start the execution of broadcast job.
    */
   @transient
-  lazy val completionFuture: scala.concurrent.Future[broadcast.Broadcast[Any]] = promise.future
+  override lazy val completionFuture: scala.concurrent.Future[broadcast.Broadcast[Any]] =
+    promise.future
 
   @transient
   private val timeout: Long = SQLConf.get.broadcastTimeout
 
   @transient
-  private[sql] lazy val relationFuture: Future[broadcast.Broadcast[Any]] = {
-    // relationFuture is used in "doExecute". Therefore we can get the execution id correctly here.
-    val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
-    val task = new Callable[broadcast.Broadcast[Any]]() {
-      override def call(): broadcast.Broadcast[Any] = {
-        // This will run in another thread. Set the execution id so that we can connect these jobs
-        // with the correct execution.
-        SQLExecution.withExecutionId(sqlContext.sparkSession, executionId) {
+  override lazy val relationFuture: Future[broadcast.Broadcast[Any]] = {
+    SQLExecution.withThreadLocalCaptured[broadcast.Broadcast[Any]](
+      sqlContext.sparkSession, BroadcastExchangeExec.executionContext) {
           try {
             // Setup a job group here so later it may get cancelled by groupId if necessary.
             sparkContext.setJobGroup(runId.toString, s"broadcast exchange (runId $runId)",
@@ -87,9 +118,9 @@ case class BroadcastExchangeExec(
             val beforeCollect = System.nanoTime()
             // Use executeCollect/executeCollectIterator to avoid conversion to Scala types
             val (numRows, input) = child.executeCollectIterator()
-            if (numRows >= 512000000) {
+            if (numRows >= MAX_BROADCAST_TABLE_ROWS) {
               throw new SparkException(
-                s"Cannot broadcast the table with 512 million or more rows: $numRows rows")
+                s"Cannot broadcast the table over $MAX_BROADCAST_TABLE_ROWS rows: $numRows rows")
             }
 
             val beforeBuild = System.nanoTime()
@@ -109,7 +140,7 @@ case class BroadcastExchangeExec(
             }
 
             longMetric("dataSize") += dataSize
-            if (dataSize >= (8L << 30)) {
+            if (dataSize >= MAX_BROADCAST_TABLE_BYTES) {
               throw new SparkException(
                 s"Cannot broadcast the table that is larger than 8GB: ${dataSize >> 30} GB")
             }
@@ -121,9 +152,9 @@ case class BroadcastExchangeExec(
             val broadcasted = sparkContext.broadcast(relation)
             longMetric("broadcastTime") += NANOSECONDS.toMillis(
               System.nanoTime() - beforeBroadcast)
-
+            val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
             SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toSeq)
-            promise.success(broadcasted)
+            promise.trySuccess(broadcasted)
             broadcasted
           } catch {
             // SPARK-24294: To bypass scala bug: https://github.com/scala/bug/issues/9554, we throw
@@ -136,20 +167,17 @@ case class BroadcastExchangeExec(
                   s"${SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key} to -1 or increase the spark " +
                   s"driver memory by setting ${SparkLauncher.DRIVER_MEMORY} to a higher value.")
                   .initCause(oe.getCause))
-              promise.failure(ex)
+              promise.tryFailure(ex)
               throw ex
             case e if !NonFatal(e) =>
               val ex = new SparkFatalException(e)
-              promise.failure(ex)
+              promise.tryFailure(ex)
               throw ex
             case e: Throwable =>
-              promise.failure(e)
+              promise.tryFailure(e)
               throw e
           }
-        }
-      }
     }
-    BroadcastExchangeExec.executionContext.submit[broadcast.Broadcast[Any]](task)
   }
 
   override protected def doPrepare(): Unit = {
@@ -181,6 +209,13 @@ case class BroadcastExchangeExec(
 }
 
 object BroadcastExchangeExec {
+  // Since the maximum number of keys that BytesToBytesMap supports is 1 << 29,
+  // and only 70% of the slots can be used before growing in HashedRelation,
+  // here the limitation should not be over 341 million.
+  val MAX_BROADCAST_TABLE_ROWS = (BytesToBytesMap.MAX_CAPACITY / 1.5).toLong
+
+  val MAX_BROADCAST_TABLE_BYTES = 8L << 30
+
   private[execution] val executionContext = ExecutionContext.fromExecutorService(
       ThreadUtils.newDaemonCachedThreadPool("broadcast-exchange",
         SQLConf.get.getConf(StaticSQLConf.BROADCAST_EXCHANGE_MAX_THREAD_THRESHOLD)))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
index ab4176cada527..c242320635a59 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
@@ -35,12 +35,6 @@ import org.apache.spark.sql.internal.SQLConf
  * the input partition ordering requirements are met.
  */
 case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
-  private def defaultNumPreShufflePartitions: Int =
-    if (conf.adaptiveExecutionEnabled && conf.reducePostShufflePartitionsEnabled) {
-      conf.maxNumPostShufflePartitions
-    } else {
-      conf.numShufflePartitions
-    }
 
   private def ensureDistributionAndOrdering(operator: SparkPlan): SparkPlan = {
     val requiredChildDistributions: Seq[Distribution] = operator.requiredChildDistribution
@@ -57,7 +51,7 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
         BroadcastExchangeExec(mode, child)
       case (child, distribution) =>
         val numPartitions = distribution.requiredNumPartitions
-          .getOrElse(defaultNumPreShufflePartitions)
+          .getOrElse(conf.numShufflePartitions)
         ShuffleExchangeExec(distribution.createPartitioning(numPartitions), child)
     }
 
@@ -91,11 +85,16 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
         childrenIndexes.map(children).filterNot(_.isInstanceOf[ShuffleExchangeExec])
           .map(_.outputPartitioning.numPartitions)
       val expectedChildrenNumPartitions = if (nonShuffleChildrenNumPartitions.nonEmpty) {
-        // Here we pick the max number of partitions among these non-shuffle children as the
-        // expected number of shuffle partitions. However, if it's smaller than
-        // `conf.numShufflePartitions`, we pick `conf.numShufflePartitions` as the
-        // expected number of shuffle partitions.
-        math.max(nonShuffleChildrenNumPartitions.max, conf.numShufflePartitions)
+        if (nonShuffleChildrenNumPartitions.length == childrenIndexes.length) {
+          // Here we pick the max number of partitions among these non-shuffle children.
+          nonShuffleChildrenNumPartitions.max
+        } else {
+          // Here we pick the max number of partitions among these non-shuffle children as the
+          // expected number of shuffle partitions. However, if it's smaller than
+          // `conf.numShufflePartitions`, we pick `conf.numShufflePartitions` as the
+          // expected number of shuffle partitions.
+          math.max(nonShuffleChildrenNumPartitions.max, conf.defaultNumShufflePartitions)
+        }
       } else {
         childrenNumPartitions.max
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
index 849ff384c130a..c4062879c2727 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
@@ -88,12 +88,11 @@ case class ReusedExchangeExec(override val output: Seq[Attribute], child: Exchan
   }
 
   override def verboseStringWithOperatorId(): String = {
-    val cdgen = ExplainUtils.getCodegenId(this)
     val reuse_op_str = ExplainUtils.getOpId(child)
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${cdgen} [Reuses operator id: $reuse_op_str]
-       |Output : ${output}
-     """.stripMargin
+       |$formattedNodeName [Reuses operator id: $reuse_op_str]
+       |${ExplainUtils.generateFieldString("Output", output)}
+       |""".stripMargin
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
index ffcd6c7783354..24c736951fdc4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
@@ -30,30 +30,69 @@ import org.apache.spark.shuffle.{ShuffleWriteMetricsReporter, ShuffleWriteProces
 import org.apache.spark.shuffle.sort.SortShuffleManager
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.errors._
-import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, UnsafeProjection, UnsafeRow}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Divide, Literal, UnsafeProjection, UnsafeRow}
 import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering
+import org.apache.spark.sql.catalyst.plans.logical.Statistics
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.{LocalShuffledRowRDD, SkewedShuffledRowRDD}
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics, SQLShuffleReadMetricsReporter, SQLShuffleWriteMetricsReporter}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.MutablePair
 import org.apache.spark.util.collection.unsafe.sort.{PrefixComparators, RecordComparator}
 
+/**
+ * Common trait for all shuffle exchange implementations to facilitate pattern matching.
+ */
+trait ShuffleExchangeLike extends Exchange {
+
+  /**
+   * Returns the number of mappers of this shuffle.
+   */
+  def numMappers: Int
+
+  /**
+   * Returns the shuffle partition number.
+   */
+  def numPartitions: Int
+
+  /**
+   * Returns whether the shuffle partition number can be changed.
+   */
+  def canChangeNumPartitions: Boolean
+
+  /**
+   * The asynchronous job that materializes the shuffle.
+   */
+  def mapOutputStatisticsFuture: Future[MapOutputStatistics]
+
+  /**
+   * Returns the shuffle RDD with specified partition specs.
+   */
+  def getShuffleRDD(partitionSpecs: Array[ShufflePartitionSpec]): RDD[_]
+
+  /**
+   * Returns the runtime statistics after shuffle materialization.
+   */
+  def runtimeStatistics: Statistics
+}
+
 /**
  * Performs a shuffle that will result in the desired partitioning.
  */
 case class ShuffleExchangeExec(
     override val outputPartitioning: Partitioning,
     child: SparkPlan,
-    canChangeNumPartitions: Boolean = true) extends Exchange {
+    noUserSpecifiedNumPartition: Boolean = true) extends ShuffleExchangeLike {
+
+  // If users specify the num partitions via APIs like `repartition`, we shouldn't change it.
+  // For `SinglePartition`, it requires exactly one partition and we can't change it either.
+  def canChangeNumPartitions: Boolean =
+    noUserSpecifiedNumPartition && outputPartitioning != SinglePartition
 
-  // NOTE: coordinator can be null after serialization/deserialization,
-  //       e.g. it can be null on the Executor side
   private lazy val writeMetrics =
     SQLShuffleWriteMetricsReporter.createShuffleWriteMetrics(sparkContext)
-  private lazy val readMetrics =
+  private[sql] lazy val readMetrics =
     SQLShuffleReadMetricsReporter.createShuffleReadMetrics(sparkContext)
   override lazy val metrics = Map(
     "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size")
@@ -67,7 +106,7 @@ case class ShuffleExchangeExec(
   @transient lazy val inputRDD: RDD[InternalRow] = child.execute()
 
   // 'mapOutputStatisticsFuture' is only needed when enable AQE.
-  @transient lazy val mapOutputStatisticsFuture: Future[MapOutputStatistics] = {
+  @transient override lazy val mapOutputStatisticsFuture: Future[MapOutputStatistics] = {
     if (inputRDD.getNumPartitions == 0) {
       Future.successful(null)
     } else {
@@ -75,6 +114,20 @@ case class ShuffleExchangeExec(
     }
   }
 
+  override def numMappers: Int = shuffleDependency.rdd.getNumPartitions
+
+  override def numPartitions: Int = shuffleDependency.partitioner.numPartitions
+
+  override def getShuffleRDD(partitionSpecs: Array[ShufflePartitionSpec]): RDD[InternalRow] = {
+    new ShuffledRowRDD(shuffleDependency, readMetrics, partitionSpecs)
+  }
+
+  override def runtimeStatistics: Statistics = {
+    val dataSize = metrics("dataSize").value
+    val rowCount = metrics(SQLShuffleWriteMetricsReporter.SHUFFLE_RECORDS_WRITTEN).value
+    Statistics(dataSize, Some(rowCount))
+  }
+
   /**
    * A [[ShuffleDependency]] that will partition rows of its child based on
    * the partitioning scheme defined in `newPartitioning`. Those partitions of
@@ -90,24 +143,6 @@ case class ShuffleExchangeExec(
       writeMetrics)
   }
 
-  def createShuffledRDD(
-      partitionRanges: Option[Array[(Int, Int)]]): ShuffledRowRDD = {
-    new ShuffledRowRDD(shuffleDependency, readMetrics, partitionRanges)
-  }
-
-  def createLocalShuffleRDD(
-      partitionStartIndicesPerMapper: Array[Array[Int]]): LocalShuffledRowRDD = {
-    new LocalShuffledRowRDD(shuffleDependency, readMetrics, partitionStartIndicesPerMapper)
-  }
-
-  def createSkewedShuffleRDD(
-      partitionIndex: Int,
-      startMapIndex: Int,
-      endMapIndex: Int): SkewedShuffledRowRDD = {
-    new SkewedShuffledRowRDD(shuffleDependency,
-      partitionIndex, startMapIndex, endMapIndex, readMetrics)
-  }
-
   /**
    * Caches the created ShuffleRowRDD so we can reuse that.
    */
@@ -116,7 +151,7 @@ case class ShuffleExchangeExec(
   protected override def doExecute(): RDD[InternalRow] = attachTree(this, "execute") {
     // Returns the same ShuffleRowRDD if this plan is used by multiple plans.
     if (cachedShuffleRDD == null) {
-      cachedShuffleRDD = createShuffledRDD(None)
+      cachedShuffleRDD = new ShuffledRowRDD(shuffleDependency, readMetrics)
     }
     cachedShuffleRDD
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
index 29645a736548c..91016a9060303 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
@@ -72,8 +72,8 @@ case class CartesianProductExec(
     } else "None"
 
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Join condition: ${joinCondStr}
+       |$formattedNodeName
+       |${ExplainUtils.generateFieldString("Join condition", joinCondStr)}
      """.stripMargin
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
index 137f0b87a2f3d..0275741844bca 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
@@ -18,12 +18,14 @@
 package org.apache.spark.sql.execution.joins
 
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.CastSupport
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution.{ExplainUtils, RowIterator, SparkPlan}
 import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{IntegralType, LongType}
 
 trait HashJoin {
@@ -48,11 +50,11 @@ trait HashJoin {
     } else "None"
 
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Left keys: ${leftKeys}
-       |Right keys: ${rightKeys}
-       |Join condition: ${joinCondStr}
-     """.stripMargin
+       |$formattedNodeName
+       |${ExplainUtils.generateFieldString("Left keys", leftKeys)}
+       |${ExplainUtils.generateFieldString("Right keys", rightKeys)}
+       |${ExplainUtils.generateFieldString("Join condition", joinCondStr)}
+       |""".stripMargin
   }
 
   override def output: Seq[Attribute] = {
@@ -80,8 +82,11 @@ trait HashJoin {
   }
 
   protected lazy val (buildKeys, streamedKeys) = {
-    require(leftKeys.map(_.dataType) == rightKeys.map(_.dataType),
-      "Join keys from two sides should have same types")
+    require(leftKeys.length == rightKeys.length &&
+      leftKeys.map(_.dataType)
+        .zip(rightKeys.map(_.dataType))
+        .forall(types => types._1.sameType(types._2)),
+      "Join keys from two sides should have same length and types")
     val lkeys = bindReferences(HashJoin.rewriteKeyExpr(leftKeys), left.output)
     val rkeys = bindReferences(HashJoin.rewriteKeyExpr(rightKeys), right.output)
     buildSide match {
@@ -237,7 +242,10 @@ trait HashJoin {
   }
 }
 
-object HashJoin {
+object HashJoin extends CastSupport {
+
+  override def conf: SQLConf = SQLConf.get
+
   /**
    * Try to rewrite the key as LongType so we can use getLong(), if they key can fit with a long.
    *
@@ -252,14 +260,14 @@ object HashJoin {
     }
 
     var keyExpr: Expression = if (keys.head.dataType != LongType) {
-      Cast(keys.head, LongType)
+      cast(keys.head, LongType)
     } else {
       keys.head
     }
     keys.tail.foreach { e =>
       val bits = e.dataType.defaultSize * 8
       keyExpr = BitwiseOr(ShiftLeft(keyExpr, Literal(bits)),
-        BitwiseAnd(Cast(e, LongType), Literal((1L << bits) - 1)))
+        BitwiseAnd(cast(e, LongType), Literal((1L << bits) - 1)))
     }
     keyExpr :: Nil
   }
@@ -272,13 +280,13 @@ object HashJoin {
     // jump over keys that have a higher index value than the required key
     if (keys.size == 1) {
       assert(index == 0)
-      Cast(BoundReference(0, LongType, nullable = false), keys(index).dataType)
+      cast(BoundReference(0, LongType, nullable = false), keys(index).dataType)
     } else {
       val shiftedBits =
         keys.slice(index + 1, keys.size).map(_.dataType.defaultSize * 8).sum
       val mask = (1L << (keys(index).dataType.defaultSize * 8)) - 1
       // build the schema for unpacking the required key
-      Cast(BitwiseAnd(
+      cast(BitwiseAnd(
         ShiftRightUnsigned(BoundReference(0, LongType, nullable = false), Literal(shiftedBits)),
         Literal(mask)), keys(index).dataType)
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
index 4001338662d53..13180d6b20902 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
@@ -177,7 +177,7 @@ private[joins] class UnsafeHashedRelation(
   }
 
   override def keys(): Iterator[InternalRow] = {
-    val iter = binaryMap.safeIterator()
+    val iter = binaryMap.iterator()
 
     new Iterator[InternalRow] {
       val unsafeRow = new UnsafeRow(numKeys)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
index 6384aed6a78e0..5b5904f1575fd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
@@ -28,7 +28,6 @@ import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.{PartialShuffleReaderExec, SkewedPartitionReaderExec}
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.util.collection.BitSet
 
@@ -42,11 +41,17 @@ case class SortMergeJoinExec(
     condition: Option[Expression],
     left: SparkPlan,
     right: SparkPlan,
-    isPartial: Boolean = false) extends BinaryExecNode with CodegenSupport {
+    isSkewJoin: Boolean = false) extends BinaryExecNode with CodegenSupport {
 
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
 
+  override def nodeName: String = {
+    if (isSkewJoin) super.nodeName + "(skew=true)" else super.nodeName
+  }
+
+  override def stringArgs: Iterator[Any] = super.stringArgs.toSeq.dropRight(1).iterator
+
   override def simpleStringWithNodeId(): String = {
     val opId = ExplainUtils.getOpId(this)
     s"$nodeName $joinType ($opId)".trim
@@ -57,11 +62,11 @@ case class SortMergeJoinExec(
       s"${condition.get}"
     } else "None"
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Left keys : ${leftKeys}
-       |Right keys: ${rightKeys}
-       |Join condition : ${joinCondStr}
-     """.stripMargin
+       |$formattedNodeName
+       |${ExplainUtils.generateFieldString("Left keys", leftKeys)}
+       |${ExplainUtils.generateFieldString("Right keys", rightKeys)}
+       |${ExplainUtils.generateFieldString("Join condition", joinCondStr)}
+       |""".stripMargin
   }
 
   override def output: Seq[Attribute] = {
@@ -98,7 +103,9 @@ case class SortMergeJoinExec(
   }
 
   override def requiredChildDistribution: Seq[Distribution] = {
-    if (isPartial) {
+    if (isSkewJoin) {
+      // We re-arrange the shuffle partitions to deal with skew join, and the new children
+      // partitioning doesn't satisfy `HashClusteredDistribution`.
       UnspecifiedDistribution :: UnspecifiedDistribution :: Nil
     } else {
       HashClusteredDistribution(leftKeys) :: HashClusteredDistribution(rightKeys) :: Nil
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
index 65aabe004d75b..92d217983f4c5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
@@ -116,26 +116,23 @@ object SQLMetrics {
     // data size total (min, med, max):
     // 100GB (100MB, 1GB, 10GB)
     val acc = new SQLMetric(SIZE_METRIC, -1)
-    acc.register(sc, name = Some(s"$name total (min, med, max (stageId (attemptId): taskId))"),
-      countFailedValues = false)
+    acc.register(sc, name = Some(name), countFailedValues = false)
     acc
   }
 
   def createTimingMetric(sc: SparkContext, name: String): SQLMetric = {
     // The final result of this metric in physical operator UI may looks like:
-    // duration(min, med, max):
+    // duration total (min, med, max):
     // 5s (800ms, 1s, 2s)
     val acc = new SQLMetric(TIMING_METRIC, -1)
-    acc.register(sc, name = Some(s"$name total (min, med, max (stageId (attemptId): taskId))"),
-      countFailedValues = false)
+    acc.register(sc, name = Some(name), countFailedValues = false)
     acc
   }
 
   def createNanoTimingMetric(sc: SparkContext, name: String): SQLMetric = {
     // Same with createTimingMetric, just normalize the unit of time to millisecond.
     val acc = new SQLMetric(NS_TIMING_METRIC, -1)
-    acc.register(sc, name = Some(s"$name total (min, med, max (stageId (attemptId): taskId))"),
-      countFailedValues = false)
+    acc.register(sc, name = Some(name), countFailedValues = false)
     acc
   }
 
@@ -150,8 +147,7 @@ object SQLMetrics {
     // probe avg (min, med, max):
     // (1.2, 2.2, 6.3)
     val acc = new SQLMetric(AVERAGE_METRIC)
-    acc.register(sc, name = Some(s"$name (min, med, max (stageId (attemptId): taskId))"),
-      countFailedValues = false)
+    acc.register(sc, name = Some(name), countFailedValues = false)
     acc
   }
 
@@ -164,34 +160,38 @@ object SQLMetrics {
     metricsType != SUM_METRIC
   }
 
+  private val METRICS_NAME_SUFFIX = "(min, med, max (stageId: taskId))"
+
   /**
    * A function that defines how we aggregate the final accumulator results among all tasks,
    * and represent it in string for a SQL physical operator.
     */
   def stringValue(metricsType: String, values: Array[Long], maxMetrics: Array[Long]): String = {
-    // stringMetric = "(driver)" OR (stage $stageId (attempt $attemptId): task $taskId))
-    val stringMetric = if (maxMetrics.isEmpty) {
+    // taskInfo = "(driver)" OR (stage ${stageId}.${attemptId}: task $taskId)
+    val taskInfo = if (maxMetrics.isEmpty) {
       "(driver)"
     } else {
-      s"(stage ${maxMetrics(1)} (attempt ${maxMetrics(2)}): task ${maxMetrics(3)})"
+      s"(stage ${maxMetrics(1)}.${maxMetrics(2)}: task ${maxMetrics(3)})"
     }
     if (metricsType == SUM_METRIC) {
       val numberFormat = NumberFormat.getIntegerInstance(Locale.US)
       numberFormat.format(values.sum)
     } else if (metricsType == AVERAGE_METRIC) {
       val validValues = values.filter(_ > 0)
-      val Seq(min, med, max) = {
-        val metric = if (validValues.isEmpty) {
-          val zeros = Seq.fill(3)(0L)
-          zeros.map(v => toNumberFormat(v))
-        } else {
+      // When there are only 1 metrics value (or None), no need to display max/min/median. This is
+      // common for driver-side SQL metrics.
+      if (validValues.length <= 1) {
+        toNumberFormat(validValues.headOption.getOrElse(0))
+      } else {
+        val Seq(min, med, max) = {
           Arrays.sort(validValues)
-          Seq(toNumberFormat(validValues(0)), toNumberFormat(validValues(validValues.length / 2)),
-            s"${toNumberFormat(validValues(validValues.length - 1))} $stringMetric")
+          Seq(
+            toNumberFormat(validValues(0)),
+            toNumberFormat(validValues(validValues.length / 2)),
+            toNumberFormat(validValues(validValues.length - 1)))
         }
-        metric
+        s"$METRICS_NAME_SUFFIX:\n($min, $med, $max $taskInfo)"
       }
-      s"\n($min, $med, $max)"
     } else {
       val strFormat: Long => String = if (metricsType == SIZE_METRIC) {
         Utils.bytesToString
@@ -204,19 +204,21 @@ object SQLMetrics {
       }
 
       val validValues = values.filter(_ >= 0)
-      val Seq(sum, min, med, max) = {
-        val metric = if (validValues.isEmpty) {
-          val zeros = Seq.fill(4)(0L)
-          zeros.map(v => strFormat(v))
-        } else {
+      // When there are only 1 metrics value (or None), no need to display max/min/median. This is
+      // common for driver-side SQL metrics.
+      if (validValues.length <= 1) {
+        strFormat(validValues.headOption.getOrElse(0))
+      } else {
+        val Seq(sum, min, med, max) = {
           Arrays.sort(validValues)
-          Seq(strFormat(validValues.sum), strFormat(validValues(0)),
+          Seq(
+            strFormat(validValues.sum),
+            strFormat(validValues(0)),
             strFormat(validValues(validValues.length / 2)),
-            s"${strFormat(validValues(validValues.length - 1))} $stringMetric")
+            strFormat(validValues(validValues.length - 1)))
         }
-        metric
+        s"total $METRICS_NAME_SUFFIX\n$sum ($min, $med, $max $taskInfo)"
       }
-      s"\n$sum ($min, $med, $max)"
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
index d05113431df41..c08db132c946f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
@@ -276,12 +276,12 @@ case class MapElementsExec(
   }
 
   override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
-    val (funcClass, methodName) = func match {
+    val (funcClass, funcName) = func match {
       case m: MapFunction[_, _] => classOf[MapFunction[_, _]] -> "call"
       case _ => FunctionUtils.getFunctionOneName(outputObjectType, child.output(0).dataType)
     }
     val funcObj = Literal.create(func, ObjectType(funcClass))
-    val callFunc = Invoke(funcObj, methodName, outputObjectType, child.output)
+    val callFunc = Invoke(funcObj, funcName, outputObjectType, child.output, propagateNull = false)
 
     val result = BindReferences.bindReference(callFunc, child.output).genCode(ctx)
 
@@ -567,7 +567,14 @@ case class FlatMapGroupsInRWithArrowExec(
       // binary in a batch due to the limitation of R API. See also ARROW-4512.
       val columnarBatchIter = runner.compute(groupedByRKey, -1)
       val outputProject = UnsafeProjection.create(output, output)
-      columnarBatchIter.flatMap(_.rowIterator().asScala).map(outputProject)
+      val outputTypes = StructType.fromAttributes(output).map(_.dataType)
+
+      columnarBatchIter.flatMap { batch =>
+        val actualDataTypes = (0 until batch.numCols()).map(i => batch.column(i).dataType())
+        assert(outputTypes == actualDataTypes, "Invalid schema from gapply(): " +
+          s"expected ${outputTypes.mkString(", ")}, got ${actualDataTypes.mkString(", ")}")
+        batch.rowIterator().asScala
+      }.map(outputProject)
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
index a0f23e925d237..57ccdb194b1f5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
@@ -21,7 +21,7 @@ import java.io.File
 
 import scala.collection.mutable.ArrayBuffer
 
-import org.apache.spark.{SparkEnv, TaskContext}
+import org.apache.spark.{ContextAwareIterator, SparkEnv, TaskContext}
 import org.apache.spark.api.python.ChainedPythonFunctions
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
@@ -88,6 +88,7 @@ abstract class EvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute],
 
     inputRDD.mapPartitions { iter =>
       val context = TaskContext.get()
+      val contextAwareIterator = new ContextAwareIterator(context, iter)
 
       // The queue used to buffer input rows so we can drain it to
       // combine input with output from Python.
@@ -119,7 +120,7 @@ abstract class EvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute],
       })
 
       // Add rows to queue to join later with the result.
-      val projectedRowIter = iter.map { inputRow =>
+      val projectedRowIter = contextAwareIterator.map { inputRow =>
         queue.add(inputRow.asInstanceOf[UnsafeRow])
         projection(inputRow)
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala
index 520afad287648..7fe3263630820 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala
@@ -198,7 +198,7 @@ object EvaluatePython {
 
     case udt: UserDefinedType[_] => makeFromJava(udt.sqlType)
 
-    case other => (obj: Any) => nullSafeConvert(other)(PartialFunction.empty)
+    case other => (obj: Any) => nullSafeConvert(obj)(PartialFunction.empty)
   }
 
   private def nullSafeConvert(input: Any)(f: PartialFunction[Any, Any]): Any = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
index 2bb808119c0ae..71f51f1abc6f5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.python
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.TaskContext
+import org.apache.spark.{ContextAwareIterator, TaskContext}
 import org.apache.spark.api.python.{ChainedPythonFunctions, PythonEvalType}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
@@ -61,16 +61,17 @@ case class MapInPandasExec(
       val pythonRunnerConf = ArrowUtils.getPythonRunnerConfMap(conf)
       val outputTypes = child.schema
 
+      val context = TaskContext.get()
+      val contextAwareIterator = new ContextAwareIterator(context, inputIter)
+
       // Here we wrap it via another row so that Python sides understand it
       // as a DataFrame.
-      val wrappedIter = inputIter.map(InternalRow(_))
+      val wrappedIter = contextAwareIterator.map(InternalRow(_))
 
       // DO NOT use iter.grouped(). See BatchIterator.
       val batchIter =
         if (batchSize > 0) new BatchIterator(wrappedIter, batchSize) else Iterator(wrappedIter)
 
-      val context = TaskContext.get()
-
       val columnarBatchIter = new ArrowPythonRunner(
         chainedFunc,
         PythonEvalType.SQL_MAP_PANDAS_ITER_UDF,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala
index 6f1b67801bb80..bcd226f95f822 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala
@@ -113,10 +113,8 @@ object FrequentItems extends Logging {
     val justItems = freqItems.map(m => m.baseMap.keys.toArray)
     val resultRow = Row(justItems : _*)
 
-    val originalSchema = df.schema
     val outputCols = cols.map { name =>
-      val index = originalSchema.fieldIndex(name)
-      val originalField = originalSchema.fields(index)
+      val originalField = df.resolve(name)
 
       // append frequent Items to the column name for easy debugging
       StructField(name + "_freqItems", ArrayType(originalField.dataType, originalField.nullable))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
index fffd8805a6525..5094e5eab5955 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
@@ -70,7 +70,7 @@ object StatFunctions extends Logging {
     require(relativeError >= 0,
       s"Relative Error must be non-negative but got $relativeError")
     val columns: Seq[Column] = cols.map { colName =>
-      val field = df.schema(colName)
+      val field = df.resolve(colName)
       require(field.dataType.isInstanceOf[NumericType],
         s"Quantile calculation for column $colName with data type ${field.dataType}" +
         " is not supported.")
@@ -154,10 +154,9 @@ object StatFunctions extends Logging {
               functionName: String): CovarianceCounter = {
     require(cols.length == 2, s"Currently $functionName calculation is supported " +
       "between two columns.")
-    cols.map(name => (name, df.schema.fields.find(_.name == name))).foreach { case (name, data) =>
-      require(data.nonEmpty, s"Couldn't find column with name $name")
-      require(data.get.dataType.isInstanceOf[NumericType], s"Currently $functionName calculation " +
-        s"for columns with dataType ${data.get.dataType.catalogString} not supported.")
+    cols.map(name => (name, df.resolve(name))).foreach { case (name, data) =>
+      require(data.dataType.isInstanceOf[NumericType], s"Currently $functionName calculation " +
+        s"for columns with dataType ${data.dataType.catalogString} not supported.")
     }
     val columns = cols.map(n => Column(Cast(Column(n).expr, DoubleType)))
     df.select(columns: _*).queryExecution.toRdd.treeAggregate(new CovarianceCounter)(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
index 905bce4d614e6..10bcfe6649802 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
@@ -28,6 +28,7 @@ import org.json4s.NoTypeHints
 import org.json4s.jackson.Serialization
 
 import org.apache.spark.sql.SparkSession
+import org.apache.spark.util.{SizeEstimator, Utils}
 
 /**
  * An abstract class for compactible metadata logs. It will write one log file for each batch.
@@ -177,16 +178,35 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
    * corresponding `batchId` file. It will delete expired files as well if enabled.
    */
   private def compact(batchId: Long, logs: Array[T]): Boolean = {
-    val validBatches = getValidBatchesBeforeCompactionBatch(batchId, compactInterval)
-    val allLogs = validBatches.flatMap { id =>
-      super.get(id).getOrElse {
-        throw new IllegalStateException(
-          s"${batchIdToPath(id)} doesn't exist when compacting batch $batchId " +
-            s"(compactInterval: $compactInterval)")
-      }
-    } ++ logs
+    val (allLogs, loadElapsedMs) = Utils.timeTakenMs {
+      val validBatches = getValidBatchesBeforeCompactionBatch(batchId, compactInterval)
+      validBatches.flatMap { id =>
+        super.get(id).getOrElse {
+          throw new IllegalStateException(
+            s"${batchIdToPath(id)} doesn't exist when compacting batch $batchId " +
+              s"(compactInterval: $compactInterval)")
+        }
+      } ++ logs
+    }
+    val compactedLogs = compactLogs(allLogs)
+
     // Return false as there is another writer.
-    super.add(batchId, compactLogs(allLogs).toArray)
+    val (writeSucceed, writeElapsedMs) = Utils.timeTakenMs {
+      super.add(batchId, compactedLogs.toArray)
+    }
+
+    val elapsedMs = loadElapsedMs + writeElapsedMs
+    if (elapsedMs >= COMPACT_LATENCY_WARN_THRESHOLD_MS) {
+      logWarning(s"Compacting took $elapsedMs ms (load: $loadElapsedMs ms," +
+        s" write: $writeElapsedMs ms) for compact batch $batchId")
+      logWarning(s"Loaded ${allLogs.size} entries (estimated ${SizeEstimator.estimate(allLogs)} " +
+        s"bytes in memory), and wrote ${compactedLogs.size} entries for compact batch $batchId")
+    } else {
+      logDebug(s"Compacting took $elapsedMs ms (load: $loadElapsedMs ms," +
+        s" write: $writeElapsedMs ms) for compact batch $batchId")
+    }
+
+    writeSucceed
   }
 
   /**
@@ -268,6 +288,7 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
 
 object CompactibleFileStreamLog {
   val COMPACT_FILE_SUFFIX = ".compact"
+  val COMPACT_LATENCY_WARN_THRESHOLD_MS = 2000
 
   def getBatchIdFromFileName(fileName: String): Long = {
     fileName.stripSuffix(COMPACT_FILE_SUFFIX).toLong
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
index eac5246904ffd..7dd52c1feabf2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
@@ -21,7 +21,7 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, UnsafeProjection}
 import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_MILLIS
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.toMillis
 import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
 import org.apache.spark.sql.types.MetadataBuilder
 import org.apache.spark.unsafe.types.CalendarInterval
@@ -100,7 +100,7 @@ case class EventTimeWatermarkExec(
     child.execute().mapPartitions { iter =>
       val getEventTime = UnsafeProjection.create(eventTime :: Nil, child.output)
       iter.map { row =>
-        eventTimeStats.add(getEventTime(row).getLong(0) / MICROS_PER_MILLIS)
+        eventTimeStats.add(toMillis(getEventTime(row).getLong(0)))
         row
       }
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala
index b679f163fc561..86a31942f3f94 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala
@@ -45,8 +45,7 @@ object FileStreamSink extends Logging {
         val hdfsPath = new Path(singlePath)
         val fs = hdfsPath.getFileSystem(hadoopConf)
         if (fs.isDirectory(hdfsPath)) {
-          val metadataPath = new Path(hdfsPath, metadataDir)
-          checkEscapedMetadataPath(fs, metadataPath, sqlConf)
+          val metadataPath = getMetadataLogPath(fs, hdfsPath, sqlConf)
           fs.exists(metadataPath)
         } else {
           false
@@ -55,6 +54,12 @@ object FileStreamSink extends Logging {
     }
   }
 
+  def getMetadataLogPath(fs: FileSystem, path: Path, sqlConf: SQLConf): Path = {
+    val metadataDir = new Path(path, FileStreamSink.metadataDir)
+    FileStreamSink.checkEscapedMetadataPath(fs, metadataDir, sqlConf)
+    metadataDir
+  }
+
   def checkEscapedMetadataPath(fs: FileSystem, metadataPath: Path, sqlConf: SQLConf): Unit = {
     if (sqlConf.getConf(SQLConf.STREAMING_CHECKPOINT_ESCAPED_PATH_CHECK_ENABLED)
         && StreamExecution.containsSpecialCharsInPath(metadataPath)) {
@@ -125,14 +130,12 @@ class FileStreamSink(
     partitionColumnNames: Seq[String],
     options: Map[String, String]) extends Sink with Logging {
 
+  import FileStreamSink._
+
   private val hadoopConf = sparkSession.sessionState.newHadoopConf()
   private val basePath = new Path(path)
-  private val logPath = {
-    val metadataDir = new Path(basePath, FileStreamSink.metadataDir)
-    val fs = metadataDir.getFileSystem(hadoopConf)
-    FileStreamSink.checkEscapedMetadataPath(fs, metadataDir, sparkSession.sessionState.conf)
-    metadataDir
-  }
+  private val logPath = getMetadataLogPath(basePath.getFileSystem(hadoopConf), basePath,
+    sparkSession.sessionState.conf)
   private val fileLog =
     new FileStreamSinkLog(FileStreamSinkLog.VERSION, sparkSession, logPath.toString)
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
index 09ae7692ec518..bfa60cf7dfd78 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.plans.physical.{AllTuples, ClusteredDistribution, HashPartitioning, SinglePartition}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.{LeafExecNode, LocalLimitExec, QueryExecution, SparkPlan, SparkPlanner, UnaryExecNode}
-import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
+import org.apache.spark.sql.execution.exchange.{ShuffleExchangeExec, ShuffleExchangeLike}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.OutputMode
 import org.apache.spark.util.Utils
@@ -76,7 +76,7 @@ class IncrementalExecution(
    * with the desired literal
    */
   override
-  lazy val optimizedPlan: LogicalPlan = tracker.measurePhase(QueryPlanningTracker.OPTIMIZATION) {
+  lazy val optimizedPlan: LogicalPlan = executePhase(QueryPlanningTracker.OPTIMIZATION) {
     sparkSession.sessionState.optimizer.executeAndTrack(withCachedData,
       tracker) transformAllExpressions {
       case ts @ CurrentBatchTimestamp(timestamp, _, _) =>
@@ -118,7 +118,7 @@ class IncrementalExecution(
           case s: StatefulOperator =>
             statefulOpFound = true
 
-          case e: ShuffleExchangeExec =>
+          case e: ShuffleExchangeLike =>
             // Don't search recursively any further as any child stateful operator as we
             // are only looking for stateful subplans that this plan has narrow dependencies on.
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
index 83bc347e23ed4..f85cfbcb33d83 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
@@ -226,7 +226,8 @@ class MicroBatchExecution(
           }
         }
 
-        finishTrigger(currentBatchHasNewData)  // Must be outside reportTimeTaken so it is recorded
+        // Must be outside reportTimeTaken so it is recorded
+        finishTrigger(currentBatchHasNewData, isCurrentBatchConstructed)
 
         // Signal waiting threads. Note this must be after finishTrigger() to ensure all
         // activities (progress generation, etc.) have completed before signaling.
@@ -317,6 +318,17 @@ class MicroBatchExecution(
               committedOffsets ++= availableOffsets
               watermarkTracker.setWatermark(
                 math.max(watermarkTracker.currentWatermark, commitMetadata.nextBatchWatermarkMs))
+            } else if (latestCommittedBatchId == latestBatchId - 1) {
+              availableOffsets.foreach {
+                case (source: Source, end: Offset) =>
+                  val start = committedOffsets.get(source).map(_.asInstanceOf[Offset])
+                  if (start.map(_ == end).getOrElse(true)) {
+                    source.getBatch(start, end)
+                  }
+                case nonV1Tuple =>
+                  // The V2 API does not have the same edge case requiring getBatch to be called
+                  // here, so we do nothing here.
+              }
             } else if (latestCommittedBatchId < latestBatchId - 1) {
               logWarning(s"Batch completion log latest batch id is " +
                 s"${latestCommittedBatchId}, which is not trailing " +
@@ -563,11 +575,10 @@ class MicroBatchExecution(
     }
 
     val nextBatch =
-      new Dataset(sparkSessionToRunBatch, lastExecution, RowEncoder(lastExecution.analyzed.schema))
+      new Dataset(lastExecution, RowEncoder(lastExecution.analyzed.schema))
 
-    val batchSinkProgress: Option[StreamWriterCommitProgress] =
-      reportTimeTaken("addBatch") {
-      SQLExecution.withNewExecutionId(sparkSessionToRunBatch, lastExecution) {
+    val batchSinkProgress: Option[StreamWriterCommitProgress] = reportTimeTaken("addBatch") {
+      SQLExecution.withNewExecutionId(lastExecution) {
         sink match {
           case s: Sink => s.addBatch(currentBatchId, nextBatch)
           case _: SupportsWrite =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
index f20291e11fd70..ea1f2ce3943b1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
@@ -85,8 +85,8 @@ trait ProgressReporter extends Logging {
   private val noDataProgressEventInterval =
     sparkSession.sessionState.conf.streamingNoDataProgressEventInterval
 
-  // The timestamp we report an event that has no input data
-  private var lastNoDataProgressEventTime = Long.MinValue
+  // The timestamp we report an event that has not executed anything
+  private var lastNoExecutionProgressEventTime = Long.MinValue
 
   private val timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
   timestampFormat.setTimeZone(DateTimeUtils.getTimeZone("UTC"))
@@ -142,12 +142,19 @@ trait ProgressReporter extends Logging {
     logInfo(s"Streaming query made progress: $newProgress")
   }
 
-  /** Finalizes the query progress and adds it to list of recent status updates. */
-  protected def finishTrigger(hasNewData: Boolean): Unit = {
+  /**
+   * Finalizes the query progress and adds it to list of recent status updates.
+   *
+   * @param hasNewData Whether the sources of this stream had new data for this trigger.
+   * @param hasExecuted Whether any batch was executed during this trigger. Streaming queries that
+   *                    perform stateful aggregations with timeouts can still run batches even
+   *                    though the sources don't have any new data.
+   */
+  protected def finishTrigger(hasNewData: Boolean, hasExecuted: Boolean): Unit = {
     assert(currentTriggerStartOffsets != null && currentTriggerEndOffsets != null)
     currentTriggerEndTimestamp = triggerClock.getTimeMillis()
 
-    val executionStats = extractExecutionStats(hasNewData)
+    val executionStats = extractExecutionStats(hasNewData, hasExecuted)
     val processingTimeMills = currentTriggerEndTimestamp - currentTriggerStartTimestamp
     val processingTimeSec = Math.max(1L, processingTimeMills).toDouble / MILLIS_PER_SECOND
 
@@ -170,9 +177,12 @@ trait ProgressReporter extends Logging {
       )
     }
 
-    val sinkProgress = SinkProgress(
-      sink.toString,
-      sinkCommitProgress.map(_.numOutputRows))
+    val sinkOutput = if (hasExecuted) {
+      sinkCommitProgress.map(_.numOutputRows)
+    } else {
+      sinkCommitProgress.map(_ => 0L)
+    }
+    val sinkProgress = SinkProgress(sink.toString, sinkOutput)
     val observedMetrics = extractObservedMetrics(hasNewData, lastExecution)
 
     val newProgress = new StreamingQueryProgress(
@@ -189,14 +199,14 @@ trait ProgressReporter extends Logging {
       sink = sinkProgress,
       observedMetrics = new java.util.HashMap(observedMetrics.asJava))
 
-    if (hasNewData) {
+    if (hasExecuted) {
       // Reset noDataEventTimestamp if we processed any data
-      lastNoDataProgressEventTime = Long.MinValue
+      lastNoExecutionProgressEventTime = triggerClock.getTimeMillis()
       updateProgress(newProgress)
     } else {
       val now = triggerClock.getTimeMillis()
-      if (now - noDataProgressEventInterval >= lastNoDataProgressEventTime) {
-        lastNoDataProgressEventTime = now
+      if (now - noDataProgressEventInterval >= lastNoExecutionProgressEventTime) {
+        lastNoExecutionProgressEventTime = now
         updateProgress(newProgress)
       }
     }
@@ -205,26 +215,26 @@ trait ProgressReporter extends Logging {
   }
 
   /** Extract statistics about stateful operators from the executed query plan. */
-  private def extractStateOperatorMetrics(hasNewData: Boolean): Seq[StateOperatorProgress] = {
+  private def extractStateOperatorMetrics(hasExecuted: Boolean): Seq[StateOperatorProgress] = {
     if (lastExecution == null) return Nil
-    // lastExecution could belong to one of the previous triggers if `!hasNewData`.
+    // lastExecution could belong to one of the previous triggers if `!hasExecuted`.
     // Walking the plan again should be inexpensive.
     lastExecution.executedPlan.collect {
       case p if p.isInstanceOf[StateStoreWriter] =>
         val progress = p.asInstanceOf[StateStoreWriter].getProgress()
-        if (hasNewData) progress else progress.copy(newNumRowsUpdated = 0)
+        if (hasExecuted) progress else progress.copy(newNumRowsUpdated = 0)
     }
   }
 
   /** Extracts statistics from the most recent query execution. */
-  private def extractExecutionStats(hasNewData: Boolean): ExecutionStats = {
+  private def extractExecutionStats(hasNewData: Boolean, hasExecuted: Boolean): ExecutionStats = {
     val hasEventTime = logicalPlan.collect { case e: EventTimeWatermark => e }.nonEmpty
     val watermarkTimestamp =
       if (hasEventTime) Map("watermark" -> formatTimestamp(offsetSeqMetadata.batchWatermarkMs))
       else Map.empty[String, String]
 
     // SPARK-19378: Still report metrics even though no data was processed while reporting progress.
-    val stateOperators = extractStateOperatorMetrics(hasNewData)
+    val stateOperators = extractStateOperatorMetrics(hasExecuted)
 
     if (!hasNewData) {
       return ExecutionStats(Map.empty, stateOperators, watermarkTimestamp)
@@ -349,7 +359,7 @@ trait ProgressReporter extends Logging {
     result
   }
 
-  private def formatTimestamp(millis: Long): String = {
+  protected def formatTimestamp(millis: Long): String = {
     timestampFormat.format(new Date(millis))
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
index 8b3534bc0837a..18fe38caa5e65 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
@@ -44,6 +44,7 @@ import org.apache.spark.sql.execution.QueryExecution
 import org.apache.spark.sql.execution.command.StreamingExplainCommand
 import org.apache.spark.sql.execution.datasources.v2.StreamWriterCommitProgress
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.connector.SupportsStreamingUpdate
 import org.apache.spark.sql.streaming._
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.{Clock, UninterruptibleThread, Utils}
@@ -307,8 +308,8 @@ abstract class StreamExecution(
       }
 
       // `postEvent` does not throw non fatal exception.
-      val submissionTime = triggerClock.getTimeMillis()
-      postEvent(new QueryStartedEvent(id, runId, name, submissionTime))
+      val startTimestamp = triggerClock.getTimeMillis()
+      postEvent(new QueryStartedEvent(id, runId, name, formatTimestamp(startTimestamp)))
 
       // Unblock starting thread
       startLatch.countDown()
@@ -451,9 +452,9 @@ abstract class StreamExecution(
       val stackTraceException = new SparkException("The stream thread was last executing:")
       stackTraceException.setStackTrace(queryExecutionThread.getStackTrace)
       val timeoutException = new TimeoutException(
-        s"Stream Execution thread failed to stop within $timeout milliseconds (specified by " +
-        s"${SQLConf.STREAMING_STOP_TIMEOUT.key}). See the cause on what was " +
-        "being executed in the streaming query thread.")
+        s"Stream Execution thread for stream $prettyIdString failed to stop within $timeout " +
+        s"milliseconds (specified by ${SQLConf.STREAMING_STOP_TIMEOUT.key}). See the cause on " +
+        s"what was being executed in the streaming query thread.")
       timeoutException.initCause(stackTraceException)
       throw timeoutException
     }
@@ -629,14 +630,9 @@ abstract class StreamExecution(
         writeBuilder.asInstanceOf[SupportsTruncate].truncate().buildForStreaming()
 
       case Update =>
-        // Although no v2 sinks really support Update mode now, but during tests we do want them
-        // to pretend to support Update mode, and treat Update mode same as Append mode.
-        if (Utils.isTesting) {
-          writeBuilder.buildForStreaming()
-        } else {
-          throw new IllegalArgumentException(
-            "Data source v2 streaming sinks does not support Update mode.")
-        }
+        require(writeBuilder.isInstanceOf[SupportsStreamingUpdate],
+          table.name + " does not support Update mode.")
+        writeBuilder.asInstanceOf[SupportsStreamingUpdate].update().buildForStreaming()
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala
index 198e17db419a7..57e62dcf513ea 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala
@@ -295,6 +295,10 @@ case class StreamingSymmetricHashJoinExec(
             postJoinFilter(joinedRow.withLeft(leftKeyValue.value).withRight(rightValue))
           }
         }
+
+        // NOTE: we need to make sure `outerOutputIter` is evaluated "after" exhausting all of
+        // elements in `innerOutputIter`, because evaluation of `innerOutputIter` may update
+        // the match flag which the logic for outer join is relying on.
         val removedRowIter = leftSideJoiner.removeOldState()
         val outerOutputIter = removedRowIter.filterNot { kv =>
           stateFormatVersion match {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
index 1a27fe61d9602..28171f4ca645e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit
 import scala.concurrent.duration.Duration
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_DAY
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.toMillis
 import org.apache.spark.sql.catalyst.util.IntervalUtils
 import org.apache.spark.sql.streaming.Trigger
 import org.apache.spark.unsafe.types.UTF8String
@@ -36,7 +37,8 @@ private object Triggers {
     if (cal.months != 0) {
       throw new IllegalArgumentException(s"Doesn't support month or year interval: $interval")
     }
-    TimeUnit.MICROSECONDS.toMillis(cal.microseconds + cal.days * MICROS_PER_DAY)
+    val microsInDays = Math.multiplyExact(cal.days, MICROS_PER_DAY)
+    toMillis(Math.addExact(cal.microseconds, microsInDays))
   }
 
   def convert(interval: Duration): Long = interval.toMillis
@@ -48,17 +50,17 @@ private object Triggers {
  * A [[Trigger]] that processes only one batch of data in a streaming query then terminates
  * the query.
  */
-private[sql] case object OneTimeTrigger extends Trigger
+case object OneTimeTrigger extends Trigger
 
 /**
  * A [[Trigger]] that runs a query periodically based on the processing time. If `interval` is 0,
  * the query will run as fast as possible.
  */
-private[sql] case class ProcessingTimeTrigger(intervalMs: Long) extends Trigger {
+case class ProcessingTimeTrigger(intervalMs: Long) extends Trigger {
   Triggers.validate(intervalMs)
 }
 
-private[sql] object ProcessingTimeTrigger {
+object ProcessingTimeTrigger {
   import Triggers._
 
   def apply(interval: String): ProcessingTimeTrigger = {
@@ -82,11 +84,11 @@ private[sql] object ProcessingTimeTrigger {
  * A [[Trigger]] that continuously processes streaming data, asynchronously checkpointing at
  * the specified interval.
  */
-private[sql] case class ContinuousTrigger(intervalMs: Long) extends Trigger {
+case class ContinuousTrigger(intervalMs: Long) extends Trigger {
   Triggers.validate(intervalMs)
 }
 
-private[sql] object ContinuousTrigger {
+object ContinuousTrigger {
   import Triggers._
 
   def apply(interval: String): ContinuousTrigger = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
index e471e6c601d16..1e64021c8105e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapabi
 import org.apache.spark.sql.connector.write.{LogicalWriteInfo, SupportsTruncate, WriteBuilder}
 import org.apache.spark.sql.connector.write.streaming.StreamingWrite
 import org.apache.spark.sql.execution.streaming.sources.ConsoleWrite
-import org.apache.spark.sql.internal.connector.SimpleTableProvider
+import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStreamingUpdate}
 import org.apache.spark.sql.sources.{BaseRelation, CreatableRelationProvider, DataSourceRegister}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -73,11 +73,12 @@ object ConsoleTable extends Table with SupportsWrite {
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    new WriteBuilder with SupportsTruncate {
+    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdate {
       private val inputSchema: StructType = info.schema()
 
-      // Do nothing for truncate. Console sink is special that it just prints all the records.
+      // Do nothing for truncate/update. Console sink is special and it just prints all the records.
       override def truncate(): WriteBuilder = this
+      override def update(): WriteBuilder = this
 
       override def buildForStreaming(): StreamingWrite = {
         assert(inputSchema != null)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
index a9b724a73a18e..a109c2171f3d2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
@@ -252,7 +252,7 @@ class ContinuousExecution(
 
       updateStatusMessage("Running")
       reportTimeTaken("runContinuous") {
-        SQLExecution.withNewExecutionId(sparkSessionForQuery, lastExecution) {
+        SQLExecution.withNewExecutionId(lastExecution) {
           lastExecution.executedPlan.execute()
         }
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala
index fc47c5ed3ac00..368dfae0cc95e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala
@@ -173,6 +173,7 @@ class TextSocketContinuousStream(
       setDaemon(true)
 
       override def run(): Unit = {
+        val toRow = encoder.createSerializer()
         try {
           while (true) {
             val line = reader.readLine()
@@ -187,7 +188,7 @@ class TextSocketContinuousStream(
                 Timestamp.valueOf(
                   TextSocketReader.DATE_FORMAT.format(Calendar.getInstance().getTime()))
               )
-              buckets(currentOffset % numPartitions) += encoder.toRow(newData)
+              buckets(currentOffset % numPartitions) += toRow(newData)
                 .copy().asInstanceOf[UnsafeRow]
             }
           }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
index ea39c549bd072..e5b9e68d71026 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
@@ -27,7 +27,7 @@ import scala.collection.mutable.ListBuffer
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.encoderFor
+import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder}
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.util.truncatedString
@@ -57,6 +57,8 @@ abstract class MemoryStreamBase[A : Encoder](sqlContext: SQLContext) extends Spa
   val encoder = encoderFor[A]
   protected val attributes = encoder.schema.toAttributes
 
+  protected lazy val toRow: ExpressionEncoder.Serializer[A] = encoder.createSerializer()
+
   def toDS(): Dataset[A] = {
     Dataset[A](sqlContext.sparkSession, logicalPlan)
   }
@@ -176,7 +178,7 @@ case class MemoryStream[A : Encoder](
 
   def addData(data: TraversableOnce[A]): Offset = {
     val objects = data.toSeq
-    val rows = objects.iterator.map(d => encoder.toRow(d).copy().asInstanceOf[UnsafeRow]).toArray
+    val rows = objects.iterator.map(d => toRow(d).copy().asInstanceOf[UnsafeRow]).toArray
     logDebug(s"Adding: $objects")
     this.synchronized {
       currentOffset = currentOffset + 1
@@ -243,7 +245,7 @@ case class MemoryStream[A : Encoder](
       rows: Seq[UnsafeRow],
       startOrdinal: Int,
       endOrdinal: Int): String = {
-    val fromRow = encoder.resolveAndBind().fromRow _
+    val fromRow = encoder.resolveAndBind().createDeserializer()
     s"MemoryBatch [$startOrdinal, $endOrdinal]: " +
         s"${rows.map(row => fromRow(row)).mkString(", ")}"
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala
index f94469385b281..d0cf602c7cca2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala
@@ -60,7 +60,7 @@ class ContinuousMemoryStream[A : Encoder](id: Int, sqlContext: SQLContext, numPa
     // Distribute data evenly among partition lists.
     data.toSeq.zipWithIndex.map {
       case (item, index) =>
-        records(index % numPartitions) += encoder.toRow(item).copy().asInstanceOf[UnsafeRow]
+        records(index % numPartitions) += toRow(item).copy().asInstanceOf[UnsafeRow]
     }
 
     // The new target offset is the offset where all records in all partitions have been processed.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala
index 03c567c58d46a..6d5e7fd5c5cf3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala
@@ -30,7 +30,8 @@ class ForeachBatchSink[T](batchWriter: (Dataset[T], Long) => Unit, encoder: Expr
     val resolvedEncoder = encoder.resolveAndBind(
       data.logicalPlan.output,
       data.sparkSession.sessionState.analyzer)
-    val rdd = data.queryExecution.toRdd.map[T](resolvedEncoder.fromRow)(encoder.clsTag)
+    val fromRow = resolvedEncoder.createDeserializer()
+    val rdd = data.queryExecution.toRdd.map[T](fromRow)(encoder.clsTag)
     val ds = data.sparkSession.createDataset(rdd)(encoder)
     batchWriter(ds, batchId)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
index 6e4f40ad080d4..57a73c740310e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapabi
 import org.apache.spark.sql.connector.write.{DataWriter, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
 import org.apache.spark.sql.execution.python.PythonForeachWriter
+import org.apache.spark.sql.internal.connector.SupportsStreamingUpdate
 import org.apache.spark.sql.types.StructType
 
 /**
@@ -54,12 +55,13 @@ case class ForeachWriterTable[T](
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    new WriteBuilder with SupportsTruncate {
+    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdate {
       private var inputSchema: StructType = info.schema()
 
-      // Do nothing for truncate. Foreach sink is special that it just forwards all the records to
-      // ForeachWriter.
+      // Do nothing for truncate/update. Foreach sink is special and it just forwards all the
+      // records to ForeachWriter.
       override def truncate(): WriteBuilder = this
+      override def update(): WriteBuilder = this
 
       override def buildForStreaming(): StreamingWrite = {
         new StreamingWrite {
@@ -73,7 +75,7 @@ case class ForeachWriterTable[T](
                 val boundEnc = enc.resolveAndBind(
                   inputSchema.toAttributes,
                   SparkSession.getActiveSession.get.sessionState.analyzer)
-                boundEnc.fromRow
+                boundEnc.createDeserializer()
               case Right(func) =>
                 func
             }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
index 2b674070a70ad..03ebbb9f1b376 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapabi
 import org.apache.spark.sql.connector.write.{DataWriter, DataWriterFactory, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
 import org.apache.spark.sql.execution.streaming.Sink
+import org.apache.spark.sql.internal.connector.SupportsStreamingUpdate
 import org.apache.spark.sql.types.StructType
 
 /**
@@ -53,7 +54,7 @@ class MemorySink extends Table with SupportsWrite with Logging {
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    new WriteBuilder with SupportsTruncate {
+    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdate {
       private var needTruncate: Boolean = false
       private val inputSchema: StructType = info.schema()
 
@@ -62,6 +63,9 @@ class MemorySink extends Table with SupportsWrite with Logging {
         this
       }
 
+      // The in-memory sink treats update as append.
+      override def update(): WriteBuilder = this
+
       override def buildForStreaming(): StreamingWrite = {
         new MemoryStreamingWrite(MemorySink.this, inputSchema, needTruncate)
       }
@@ -172,10 +176,10 @@ class MemoryDataWriter(partition: Int, schema: StructType)
 
   private val data = mutable.Buffer[Row]()
 
-  private val encoder = RowEncoder(schema).resolveAndBind()
+  private val fromRow = RowEncoder(schema).resolveAndBind().createDeserializer()
 
   override def write(row: InternalRow): Unit = {
-    data.append(encoder.fromRow(row))
+    data.append(fromRow(row))
   }
 
   override def commit(): MemoryWriterCommitMessage = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala
index 1a0a43c083879..1a5b50dcc7901 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala
@@ -451,10 +451,25 @@ class SymmetricHashJoinStateManager(
   }
 
   private trait KeyWithIndexToValueRowConverter {
+    /** Defines the schema of the value row (the value side of K-V in state store). */
     def valueAttributes: Seq[Attribute]
 
+    /**
+     * Convert the value row to (actual value, match) pair.
+     *
+     * NOTE: implementations should ensure the result row is NOT reused during execution, so
+     * that caller can safely read the value in any time.
+     */
     def convertValue(value: UnsafeRow): ValueAndMatchPair
 
+    /**
+     * Build the value row from (actual value, match) pair. This is expected to be called just
+     * before storing to the state store.
+     *
+     * NOTE: depending on the implementation, the result row "may" be reused during execution
+     * (to avoid initialization of object), so the caller should ensure that the logic doesn't
+     * affect by such behavior. Call copy() against the result row if needed.
+     */
     def convertToValueRow(value: UnsafeRow, matched: Boolean): UnsafeRow
   }
 
@@ -493,7 +508,7 @@ class SymmetricHashJoinStateManager(
 
     override def convertValue(value: UnsafeRow): ValueAndMatchPair = {
       if (value != null) {
-        ValueAndMatchPair(valueRowGenerator(value),
+        ValueAndMatchPair(valueRowGenerator(value).copy(),
           value.getBoolean(indexOrdinalInValueWithMatchedRow))
       } else {
         null
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
index c2270c57eb941..48d6210c83868 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
@@ -23,7 +23,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.{expressions, InternalRow}
-import org.apache.spark.sql.catalyst.expressions.{AttributeSeq, CreateNamedStruct, Expression, ExprId, InSet, ListQuery, Literal, PlanExpression}
+import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, ExprId, InSet, ListQuery, Literal, PlanExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.internal.SQLConf
@@ -117,6 +117,7 @@ case class InSubqueryExec(
     private var resultBroadcast: Broadcast[Array[Any]] = null) extends ExecSubqueryExpression {
 
   @transient private var result: Array[Any] = _
+  @transient private lazy val inSet = InSet(child, result.toSet)
 
   override def dataType: DataType = BooleanType
   override def children: Seq[Expression] = child :: Nil
@@ -131,10 +132,7 @@ case class InSubqueryExec(
 
   def updateResult(): Unit = {
     val rows = plan.executeCollect()
-    result = child.dataType match {
-      case _: StructType => rows.toArray
-      case _ => rows.map(_.get(0, child.dataType))
-    }
+    result = rows.map(_.get(0, child.dataType))
     resultBroadcast = plan.sqlContext.sparkContext.broadcast(result)
   }
 
@@ -149,17 +147,12 @@ case class InSubqueryExec(
 
   override def eval(input: InternalRow): Any = {
     prepareResult()
-    val v = child.eval(input)
-    if (v == null) {
-      null
-    } else {
-      result.contains(v)
-    }
+    inSet.eval(input)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     prepareResult()
-    InSet(child, result.toSet).doGenCode(ctx, ev)
+    inSet.doGenCode(ctx, ev)
   }
 
   override lazy val canonicalized: InSubqueryExec = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
index 91360e0e50314..e3e22783e18ef 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
@@ -71,6 +71,10 @@ class ExecutionPage(parent: SQLTab) extends WebUIPage("execution") with Logging
             {jobLinks(JobExecutionStatus.FAILED, "Failed Jobs:")}
           </ul>
         </div>
+        <div>
+          <input type="checkbox" id="stageId-and-taskId-checkbox"></input>
+          <span>Show the Stage ID and Task ID that corresponds to the max metric</span>
+        </div>
 
       val metrics = sqlStore.executionMetrics(executionId)
       val graph = sqlStore.planGraph(executionId)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusStore.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusStore.scala
index a90f37a80d525..c6e7f3978469d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusStore.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusStore.scala
@@ -42,10 +42,6 @@ class SQLAppStatusStore(
     store.view(classOf[SQLExecutionUIData]).asScala.toSeq
   }
 
-  def executionsList(offset: Int, length: Int): Seq[SQLExecutionUIData] = {
-    store.view(classOf[SQLExecutionUIData]).skip(offset).max(length).asScala.toSeq
-  }
-
   def execution(executionId: Long): Option[SQLExecutionUIData] = {
     try {
       Some(store.read(classOf[SQLExecutionUIData], executionId))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
index 1e767c3c043c3..274a5a414ffa2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
@@ -160,29 +160,33 @@ private[ui] class SparkPlanGraphNode(
     val metrics: Seq[SQLPlanMetric]) {
 
   def makeDotNode(metricsValue: Map[Long, String]): String = {
-    val builder = new mutable.StringBuilder(name)
+    val builder = new mutable.StringBuilder("<b>" + name + "</b>")
 
     val values = for {
       metric <- metrics
       value <- metricsValue.get(metric.accumulatorId)
     } yield {
-      metric.name + ": " + value
+      // The value may contain ":" to extend the name, like `total (min, med, max): ...`
+      if (value.contains(":")) {
+        metric.name + " " + value
+      } else {
+        metric.name + ": " + value
+      }
     }
 
-    // If there are metrics, display each entry in a separate line.
-    // Note: whitespace between two "\n"s is to create an empty line between the name of
-    // SparkPlan and metrics. If removing it, it won't display the empty line in UI.
-    builder ++= "\n \n"
-
     if (values.nonEmpty) {
-      builder ++= values.mkString("\n")
+      // If there are metrics, display each entry in a separate line.
+      // Note: whitespace between two "\n"s is to create an empty line between the name of
+      // SparkPlan and metrics. If removing it, it won't display the empty line in UI.
+      builder ++= "<br><br>"
+      builder ++= values.mkString("<br>")
+      val labelStr = StringEscapeUtils.escapeJava(builder.toString().replaceAll("\n", "<br>"))
+      s"""  $id [labelType="html" label="${labelStr}"];"""
     } else {
-      // A certain level of height is needed for a rect as a node in a sub-graph
-      // to avoid layout collapse for sub-graphs.
-      builder ++= " "
+      // SPARK-30684: when there is no metrics, add empty lines to increase the height of the node,
+      // so that there won't be gaps between an edge and a small node.
+      s"""  $id [labelType="html" label="<br><b>$name</b><br><br>"];"""
     }
-
-    s"""  $id [label="${StringEscapeUtils.escapeJava(builder.toString())}"];"""
   }
 }
 
@@ -212,6 +216,7 @@ private[ui] class SparkPlanGraphCluster(
     }
     s"""
        |  subgraph cluster${id} {
+       |    isCluster="true";
        |    label="${StringEscapeUtils.escapeJava(labelStr)}";
        |    ${nodes.map(_.makeDotNode(metricsValue)).mkString("    \n")}
        |  }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala
index d191f3790ffa8..42fa07f4a6aee 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala
@@ -38,15 +38,25 @@ import org.apache.spark.sql.types.{CalendarIntervalType, DateType, IntegerType,
  * - Entire partition: The frame is the entire partition, i.e.
  *   UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING. For this case, window function will take all
  *   rows as inputs and be evaluated once.
- * - Growing frame: We only add new rows into the frame, i.e. UNBOUNDED PRECEDING AND ....
+ * - Growing frame: We only add new rows into the frame, Examples are:
+ *     1. UNBOUNDED PRECEDING AND 1 PRECEDING
+ *     2. UNBOUNDED PRECEDING AND CURRENT ROW
+ *     3. UNBOUNDED PRECEDING AND 1 FOLLOWING
  *   Every time we move to a new row to process, we add some rows to the frame. We do not remove
  *   rows from this frame.
- * - Shrinking frame: We only remove rows from the frame, i.e. ... AND UNBOUNDED FOLLOWING.
+ * - Shrinking frame: We only remove rows from the frame, Examples are:
+ *     1. 1 PRECEDING AND UNBOUNDED FOLLOWING
+ *     2. CURRENT ROW AND UNBOUNDED FOLLOWING
+ *     3. 1 FOLLOWING AND UNBOUNDED FOLLOWING
  *   Every time we move to a new row to process, we remove some rows from the frame. We do not add
  *   rows to this frame.
  * - Moving frame: Every time we move to a new row to process, we remove some rows from the frame
  *   and we add some rows to the frame. Examples are:
- *     1 PRECEDING AND CURRENT ROW and 1 FOLLOWING AND 2 FOLLOWING.
+ *     1. 2 PRECEDING AND 1 PRECEDING
+ *     2. 1 PRECEDING AND CURRENT ROW
+ *     3. CURRENT ROW AND 1 FOLLOWING
+ *     4. 1 PRECEDING AND 1 FOLLOWING
+ *     5. 1 FOLLOWING AND 2 FOLLOWING
  * - Offset frame: The frame consist of one row, which is an offset number of rows away from the
  *   current row. Only [[OffsetWindowFunction]]s can be processed in an offset frame.
  *
@@ -105,7 +115,7 @@ case class WindowExec(
   override def outputPartitioning: Partitioning = child.outputPartitioning
 
   protected override def doExecute(): RDD[InternalRow] = {
-    // Unwrap the expressions and factories from the map.
+    // Unwrap the window expressions and window frame factories from the map.
     val expressions = windowFrameExpressionFactoryPairs.flatMap(_._1)
     val factories = windowFrameExpressionFactoryPairs.map(_._2).toArray
     val inMemoryThreshold = sqlContext.conf.windowExecBufferInMemoryThreshold
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala
index d5d11c45f8535..99c8917478865 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala
@@ -114,7 +114,7 @@ abstract class WindowExecBase(
 
   /**
    * Collection containing an entry for each window frame to process. Each entry contains a frame's
-   * [[WindowExpression]]s and factory function for the WindowFrameFunction.
+   * [[WindowExpression]]s and factory function for the [[WindowFrameFunction]].
    */
   protected lazy val windowFrameExpressionFactoryPairs = {
     type FrameKey = (String, FrameType, Expression, Expression)
@@ -170,7 +170,7 @@ abstract class WindowExecBase(
               MutableProjection.create(expressions, schema))
         }
 
-        // Create the factory
+        // Create the factory to produce WindowFunctionFrame.
         val factory = key match {
           // Offset Frame
           case ("OFFSET", _, IntegerLiteral(offset), _) =>
@@ -223,7 +223,7 @@ abstract class WindowExecBase(
         // Keep track of the number of expressions. This is a side-effect in a map...
         numExpressions += expressions.size
 
-        // Create the Frame Expression - Factory pair.
+        // Create the Window Expression - Frame Factory pair.
         (expressions, factory)
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
index d5f2ffa5573a9..dc1b919feefe4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
@@ -47,14 +47,14 @@ abstract class WindowFunctionFrame {
   /**
    * The current lower window bound in the row array (inclusive).
    *
-   * This should be called after the current row is updated via [[write]]
+   * This should be called after the current row is updated via `write`.
    */
   def currentLowerBound(): Int
 
   /**
    * The current row index of the upper window bound in the row array (exclusive)
    *
-   * This should be called after the current row is updated via [[write]]
+   * This should be called after the current row is updated via `write`.
    */
   def currentUpperBound(): Int
 }
@@ -277,6 +277,8 @@ final class UnboundedWindowFunctionFrame(
       while (iterator.hasNext) {
         processor.update(iterator.next())
       }
+
+      processor.evaluate(target)
     }
 
     upperBound = rows.length
@@ -284,11 +286,8 @@ final class UnboundedWindowFunctionFrame(
 
   /** Write the frame columns for the current row to the given target row. */
   override def write(index: Int, current: InternalRow): Unit = {
-    // Unfortunately we cannot assume that evaluation is deterministic. So we need to re-evaluate
-    // for each row.
-    if (processor != null) {
-      processor.evaluate(target)
-    }
+    // The results are the same for each row in the partition, and have been evaluated in prepare.
+    // Don't need to recalculate here.
   }
 
   override def currentLowerBound(): Int = lowerBound
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
index 878dc0d83f45a..62d04cf7f7cac 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
@@ -35,6 +35,8 @@ import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
  *     def reduce(b: Int, a: Data): Int = b + a.i
  *     def merge(b1: Int, b2: Int): Int = b1 + b2
  *     def finish(r: Int): Int = r
+ *     def bufferEncoder: Encoder[Int] = Encoders.scalaInt
+ *     def outputEncoder: Encoder[Int] = Encoders.scalaInt
  *   }.toColumn()
  *
  *   val ds: Dataset[Data] = ...
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
index 85b2cd379ba24..6a20a46756f85 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
@@ -90,10 +90,10 @@ sealed abstract class UserDefinedFunction {
   def asNondeterministic(): UserDefinedFunction
 }
 
-private[sql] case class SparkUserDefinedFunction(
+private[spark] case class SparkUserDefinedFunction(
     f: AnyRef,
     dataType: DataType,
-    inputSchemas: Seq[Option[ScalaReflection.Schema]],
+    inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil,
     name: Option[String] = None,
     nullable: Boolean = true,
     deterministic: Boolean = true) extends UserDefinedFunction {
@@ -104,18 +104,11 @@ private[sql] case class SparkUserDefinedFunction(
   }
 
   private[sql] def createScalaUDF(exprs: Seq[Expression]): ScalaUDF = {
-    // It's possible that some of the inputs don't have a specific type(e.g. `Any`),  skip type
-    // check.
-    val inputTypes = inputSchemas.map(_.map(_.dataType).getOrElse(AnyDataType))
-    // `ScalaReflection.Schema.nullable` is false iff the type is primitive. Also `Any` is not
-    // primitive.
-    val inputsPrimitive = inputSchemas.map(_.map(!_.nullable).getOrElse(false))
     ScalaUDF(
       f,
       dataType,
       exprs,
-      inputsPrimitive,
-      inputTypes,
+      inputEncoders,
       udfName = name,
       nullable = nullable,
       udfDeterministic = deterministic)
@@ -157,7 +150,8 @@ private[sql] case class UserDefinedAggregator[IN, BUF, OUT](
   // This is also used by udf.register(...) when it detects a UserDefinedAggregator
   def scalaAggregator(exprs: Seq[Expression]): ScalaAggregator[IN, BUF, OUT] = {
     val iEncoder = inputEncoder.asInstanceOf[ExpressionEncoder[IN]]
-    ScalaAggregator(exprs, aggregator, iEncoder, nullable, deterministic)
+    val bEncoder = aggregator.bufferEncoder.asInstanceOf[ExpressionEncoder[BUF]]
+    ScalaAggregator(exprs, aggregator, iEncoder, bEncoder, nullable, deterministic)
   }
 
   override def withName(name: String): UserDefinedAggregator[IN, BUF, OUT] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index da26c5a2f4625..ff8ee2555276a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, HintInfo, ResolvedHint}
+import org.apache.spark.sql.catalyst.util.TimestampFormatter
 import org.apache.spark.sql.execution.SparkSqlParser
 import org.apache.spark.sql.expressions.{Aggregator, SparkUserDefinedFunction, UserDefinedAggregator, UserDefinedFunction}
 import org.apache.spark.sql.internal.SQLConf
@@ -210,6 +211,36 @@ object functions {
   // Aggregate functions
   //////////////////////////////////////////////////////////////////////////////////////////////
 
+  /**
+   * @group agg_funcs
+   * @since 1.3.0
+   */
+  @deprecated("Use approx_count_distinct", "2.1.0")
+  def approxCountDistinct(e: Column): Column = approx_count_distinct(e)
+
+  /**
+   * @group agg_funcs
+   * @since 1.3.0
+   */
+  @deprecated("Use approx_count_distinct", "2.1.0")
+  def approxCountDistinct(columnName: String): Column = approx_count_distinct(columnName)
+
+  /**
+   * @group agg_funcs
+   * @since 1.3.0
+   */
+  @deprecated("Use approx_count_distinct", "2.1.0")
+  def approxCountDistinct(e: Column, rsd: Double): Column = approx_count_distinct(e, rsd)
+
+  /**
+   * @group agg_funcs
+   * @since 1.3.0
+   */
+  @deprecated("Use approx_count_distinct", "2.1.0")
+  def approxCountDistinct(columnName: String, rsd: Double): Column = {
+    approx_count_distinct(Column(columnName), rsd)
+  }
+
   /**
    * Aggregate function: returns the approximate number of distinct items in a group.
    *
@@ -231,7 +262,7 @@ object functions {
   /**
    * Aggregate function: returns the approximate number of distinct items in a group.
    *
-   * @param rsd maximum estimation error allowed (default = 0.05)
+   * @param rsd maximum relative standard deviation allowed (default = 0.05)
    *
    * @group agg_funcs
    * @since 2.1.0
@@ -243,7 +274,7 @@ object functions {
   /**
    * Aggregate function: returns the approximate number of distinct items in a group.
    *
-   * @param rsd maximum estimation error allowed (default = 0.05)
+   * @param rsd maximum relative standard deviation allowed (default = 0.05)
    *
    * @group agg_funcs
    * @since 2.1.0
@@ -430,7 +461,7 @@ object functions {
    * @since 2.0.0
    */
   def first(e: Column, ignoreNulls: Boolean): Column = withAggregateFunction {
-    new First(e.expr, Literal(ignoreNulls))
+    First(e.expr, ignoreNulls)
   }
 
   /**
@@ -555,7 +586,7 @@ object functions {
    * @since 2.0.0
    */
   def last(e: Column, ignoreNulls: Boolean): Column = withAggregateFunction {
-    new Last(e.expr, Literal(ignoreNulls))
+    new Last(e.expr, ignoreNulls)
   }
 
   /**
@@ -1089,6 +1120,27 @@ object functions {
    */
   def isnull(e: Column): Column = withExpr { IsNull(e.expr) }
 
+  /**
+   * A column expression that generates monotonically increasing 64-bit integers.
+   *
+   * The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive.
+   * The current implementation puts the partition ID in the upper 31 bits, and the record number
+   * within each partition in the lower 33 bits. The assumption is that the data frame has
+   * less than 1 billion partitions, and each partition has less than 8 billion records.
+   *
+   * As an example, consider a `DataFrame` with two partitions, each with 3 records.
+   * This expression would return the following IDs:
+   *
+   * {{{
+   * 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.
+   * }}}
+   *
+   * @group normal_funcs
+   * @since 1.4.0
+   */
+  @deprecated("Use monotonically_increasing_id()", "2.0.0")
+  def monotonicallyIncreasingId(): Column = monotonically_increasing_id()
+
   /**
    * A column expression that generates monotonically increasing 64-bit integers.
    *
@@ -1152,7 +1204,7 @@ object functions {
 
   /**
    * Generate a random column with independent and identically distributed (i.i.d.) samples
-   * from U[0.0, 1.0].
+   * uniformly distributed in [0.0, 1.0).
    *
    * @note The function is non-deterministic in general case.
    *
@@ -1163,7 +1215,7 @@ object functions {
 
   /**
    * Generate a random column with independent and identically distributed (i.i.d.) samples
-   * from U[0.0, 1.0].
+   * uniformly distributed in [0.0, 1.0).
    *
    * @note The function is non-deterministic in general case.
    *
@@ -1231,7 +1283,7 @@ object functions {
    * @since 1.4.0
    */
   @scala.annotation.varargs
-  def struct(cols: Column*): Column = withExpr { CreateStruct(cols.map(_.expr)) }
+  def struct(cols: Column*): Column = withExpr { CreateStruct.create(cols.map(_.expr)) }
 
   /**
    * Creates a new struct column that composes multiple input columns.
@@ -2070,6 +2122,20 @@ object functions {
    */
   def tanh(columnName: String): Column = tanh(Column(columnName))
 
+  /**
+   * @group math_funcs
+   * @since 1.4.0
+   */
+  @deprecated("Use degrees", "2.1.0")
+  def toDegrees(e: Column): Column = degrees(e)
+
+  /**
+   * @group math_funcs
+   * @since 1.4.0
+   */
+  @deprecated("Use degrees", "2.1.0")
+  def toDegrees(columnName: String): Column = degrees(Column(columnName))
+
   /**
    * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
    *
@@ -2092,6 +2158,20 @@ object functions {
    */
   def degrees(columnName: String): Column = degrees(Column(columnName))
 
+  /**
+   * @group math_funcs
+   * @since 1.4.0
+   */
+  @deprecated("Use radians", "2.1.0")
+  def toRadians(e: Column): Column = radians(e)
+
+  /**
+   * @group math_funcs
+   * @since 1.4.0
+   */
+  @deprecated("Use radians", "2.1.0")
+  def toRadians(columnName: String): Column = radians(Column(columnName))
+
   /**
    * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
    *
@@ -2459,25 +2539,25 @@ object functions {
   def soundex(e: Column): Column = withExpr { SoundEx(e.expr) }
 
   /**
-   * Splits str around matches of the given regex.
+   * Splits str around matches of the given pattern.
    *
    * @param str a string expression to split
-   * @param regex a string representing a regular expression. The regex string should be
-   *              a Java regular expression.
+   * @param pattern a string representing a regular expression. The regex string should be
+   *                a Java regular expression.
    *
    * @group string_funcs
    * @since 1.5.0
    */
-  def split(str: Column, regex: String): Column = withExpr {
-    StringSplit(str.expr, Literal(regex), Literal(-1))
+  def split(str: Column, pattern: String): Column = withExpr {
+    StringSplit(str.expr, Literal(pattern), Literal(-1))
   }
 
   /**
-   * Splits str around matches of the given regex.
+   * Splits str around matches of the given pattern.
    *
    * @param str a string expression to split
-   * @param regex a string representing a regular expression. The regex string should be
-   *              a Java regular expression.
+   * @param pattern a string representing a regular expression. The regex string should be
+   *                a Java regular expression.
    * @param limit an integer expression which controls the number of times the regex is applied.
    *        <ul>
    *          <li>limit greater than 0: The resulting array's length will not be more than limit,
@@ -2490,8 +2570,8 @@ object functions {
    * @group string_funcs
    * @since 3.0.0
    */
-  def split(str: Column, regex: String, limit: Int): Column = withExpr {
-    StringSplit(str.expr, Literal(regex), Literal(limit))
+  def split(str: Column, pattern: String, limit: Int): Column = withExpr {
+    StringSplit(str.expr, Literal(pattern), Literal(limit))
   }
 
   /**
@@ -2612,7 +2692,8 @@ object functions {
   }
 
   /**
-   * Returns the current date as a date column.
+   * Returns the current date at the start of query evaluation as a date column.
+   * All calls of current_date within the same query return the same value.
    *
    * @group datetime_funcs
    * @since 1.5.0
@@ -2620,7 +2701,8 @@ object functions {
   def current_date(): Column = withExpr { CurrentDate() }
 
   /**
-   * Returns the current timestamp as a timestamp column.
+   * Returns the current timestamp at the start of query evaluation as a timestamp column.
+   * All calls of current_timestamp within the same query return the same value.
    *
    * @group datetime_funcs
    * @since 1.5.0
@@ -2631,11 +2713,13 @@ object functions {
    * Converts a date/timestamp/string to a value of string in the format specified by the date
    * format given by the second argument.
    *
-   * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
+   * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>
+   * for valid date and time format patterns
    *
    * @param dateExpr A date, timestamp or string. If a string, the data must be in a format that
-   *                 can be cast to a timestamp, such as `uuuu-MM-dd` or `uuuu-MM-dd HH:mm:ss.SSSS`
-   * @param format A pattern `dd.MM.uuuu` would return a string like `18.03.1993`
+   *                 can be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+   * @param format A pattern `dd.MM.yyyy` would return a string like `18.03.1993`
    * @return A string, or null if `dateExpr` was a string that could not be cast to a timestamp
    * @note Use specialized functions like [[year]] whenever possible as they benefit from a
    * specialized implementation.
@@ -2872,7 +2956,7 @@ object functions {
   /**
    * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string
    * representing the timestamp of that moment in the current system time zone in the
-   * uuuu-MM-dd HH:mm:ss format.
+   * yyyy-MM-dd HH:mm:ss format.
    *
    * @param ut A number of a type that is castable to a long, such as string or integer. Can be
    *           negative for timestamps before the unix epoch
@@ -2881,7 +2965,7 @@ object functions {
    * @since 1.5.0
    */
   def from_unixtime(ut: Column): Column = withExpr {
-    FromUnixTime(ut.expr, Literal("uuuu-MM-dd HH:mm:ss"))
+    FromUnixTime(ut.expr, Literal(TimestampFormatter.defaultPattern))
   }
 
   /**
@@ -2889,7 +2973,9 @@ object functions {
    * representing the timestamp of that moment in the current system time zone in the given
    * format.
    *
-   * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
+   * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>
+   * for valid date and time format patterns
    *
    * @param ut A number of a type that is castable to a long, such as string or integer. Can be
    *           negative for timestamps before the unix epoch
@@ -2913,30 +2999,32 @@ object functions {
    * @since 1.5.0
    */
   def unix_timestamp(): Column = withExpr {
-    UnixTimestamp(CurrentTimestamp(), Literal("uuuu-MM-dd HH:mm:ss"))
+    UnixTimestamp(CurrentTimestamp(), Literal(TimestampFormatter.defaultPattern))
   }
 
   /**
-   * Converts time string in format uuuu-MM-dd HH:mm:ss to Unix timestamp (in seconds),
+   * Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds),
    * using the default timezone and the default locale.
    *
    * @param s A date, timestamp or string. If a string, the data must be in the
-   *          `uuuu-MM-dd HH:mm:ss` format
+   *          `yyyy-MM-dd HH:mm:ss` format
    * @return A long, or null if the input was a string not of the correct format
    * @group datetime_funcs
    * @since 1.5.0
    */
   def unix_timestamp(s: Column): Column = withExpr {
-    UnixTimestamp(s.expr, Literal("uuuu-MM-dd HH:mm:ss"))
+    UnixTimestamp(s.expr, Literal(TimestampFormatter.defaultPattern))
   }
 
   /**
    * Converts time string with given pattern to Unix timestamp (in seconds).
    *
-   * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
+   * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>
+   * for valid date and time format patterns
    *
    * @param s A date, timestamp or string. If a string, the data must be in a format that can be
-   *          cast to a date, such as `uuuu-MM-dd` or `uuuu-MM-dd HH:mm:ss.SSSS`
+   *          cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @param p A date time pattern detailing the format of `s` when `s` is a string
    * @return A long, or null if `s` was a string that could not be cast to a date or `p` was
    *         an invalid format
@@ -2949,7 +3037,7 @@ object functions {
    * Converts to a timestamp by casting rules to `TimestampType`.
    *
    * @param s A date, timestamp or string. If a string, the data must be in a format that can be
-   *          cast to a timestamp, such as `uuuu-MM-dd` or `uuuu-MM-dd HH:mm:ss.SSSS`
+   *          cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @return A timestamp, or null if the input was a string that could not be cast to a timestamp
    * @group datetime_funcs
    * @since 2.2.0
@@ -2961,10 +3049,12 @@ object functions {
   /**
    * Converts time string with the given pattern to timestamp.
    *
-   * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
+   * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>
+   * for valid date and time format patterns
    *
    * @param s   A date, timestamp or string. If a string, the data must be in a format that can be
-   *            cast to a timestamp, such as `uuuu-MM-dd` or `uuuu-MM-dd HH:mm:ss.SSSS`
+   *            cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @param fmt A date time pattern detailing the format of `s` when `s` is a string
    * @return A timestamp, or null if `s` was a string that could not be cast to a timestamp or
    *         `fmt` was an invalid format
@@ -2986,10 +3076,12 @@ object functions {
   /**
    * Converts the column into a `DateType` with a specified format
    *
-   * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
+   * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>
+   * for valid date and time format patterns
    *
    * @param e   A date, timestamp or string. If a string, the data must be in a format that can be
-   *            cast to a date, such as `uuuu-MM-dd` or `uuuu-MM-dd HH:mm:ss.SSSS`
+   *            cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @param fmt A date time pattern detailing the format of `e` when `e`is a string
    * @return A date, or null if `e` was a string that could not be cast to a date or `fmt` was an
    *         invalid format
@@ -3009,6 +3101,7 @@ object functions {
    *             cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @param format: 'year', 'yyyy', 'yy' to truncate by year,
    *               or 'month', 'mon', 'mm' to truncate by month
+   *               Other options are: 'week', 'quarter'
    *
    * @return A date, or null if `date` was a string that could not be cast to a date or `format`
    *         was an invalid value
@@ -3027,7 +3120,8 @@ object functions {
    * @param format: 'year', 'yyyy', 'yy' to truncate by year,
    *                'month', 'mon', 'mm' to truncate by month,
    *                'day', 'dd' to truncate by day,
-   *                Other options are: 'second', 'minute', 'hour', 'week', 'month', 'quarter'
+   *                Other options are:
+   *                'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter'
    * @param timestamp A date, timestamp or string. If a string, the data must be in a format that
    *                  can be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @return A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp
@@ -3046,8 +3140,12 @@ object functions {
    *
    * @param ts A date, timestamp or string. If a string, the data must be in a format that can be
    *           cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
-   * @param tz A string detailing the time zone that the input should be adjusted to, such as
-   *           `Europe/London`, `PST` or `GMT+5`
+   * @param tz A string detailing the time zone ID that the input should be adjusted to. It should
+   *           be in the format of either region-based zone IDs or zone offsets. Region IDs must
+   *           have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
+   *           the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
+   *           supported as aliases of '+00:00'. Other short names are not recommended to use
+   *           because they can be ambiguous.
    * @return A timestamp, or null if `ts` was a string that could not be cast to a timestamp or
    *         `tz` was an invalid value
    * @group datetime_funcs
@@ -3075,8 +3173,12 @@ object functions {
    *
    * @param ts A date, timestamp or string. If a string, the data must be in a format that can be
    *           cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
-   * @param tz A string detailing the time zone that the input belongs to, such as `Europe/London`,
-   *           `PST` or `GMT+5`
+   * @param tz A string detailing the time zone ID that the input should be adjusted to. It should
+   *           be in the format of either region-based zone IDs or zone offsets. Region IDs must
+   *           have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
+   *           the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
+   *           supported as aliases of '+00:00'. Other short names are not recommended to use
+   *           because they can be ambiguous.
    * @return A timestamp, or null if `ts` was a string that could not be cast to a timestamp or
    *         `tz` was an invalid value
    * @group datetime_funcs
@@ -3410,6 +3512,12 @@ object functions {
   /**
    * Returns an array of elements after applying a transformation to each element
    * in the input array.
+   * {{{
+   *   df.select(transform(col("i"), x => x + 1))
+   * }}}
+   *
+   * @param column the input array column
+   * @param f col => transformed_col, the lambda function to transform the input column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3421,6 +3529,13 @@ object functions {
   /**
    * Returns an array of elements after applying a transformation to each element
    * in the input array.
+   * {{{
+   *   df.select(transform(col("i"), (x, i) => x + i))
+   * }}}
+   *
+   * @param column the input array column
+   * @param f (col, index) => transformed_col, the lambda function to filter the input column
+   *           given the index. Indices start at 0.
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3431,6 +3546,12 @@ object functions {
 
   /**
    * Returns whether a predicate holds for one or more elements in the array.
+   * {{{
+   *   df.select(exists(col("i"), _ % 2 === 0))
+   * }}}
+   *
+   * @param column the input array column
+   * @param f col => predicate, the Boolean predicate to check the input column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3441,6 +3562,12 @@ object functions {
 
   /**
    * Returns whether a predicate holds for every element in the array.
+   * {{{
+   *   df.select(forall(col("i"), x => x % 2 === 0))
+   * }}}
+   *
+   * @param column the input array column
+   * @param f col => predicate, the Boolean predicate to check the input column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3453,11 +3580,10 @@ object functions {
    * Returns an array of elements for which a predicate holds in a given array.
    * {{{
    *   df.select(filter(col("s"), x => x % 2 === 0))
-   *   df.selectExpr("filter(col, x -> x % 2 == 0)")
    * }}}
    *
-   * @param column: the input array column
-   * @param f: col => predicate, the Boolean predicate to filter the input column
+   * @param column the input array column
+   * @param f col => predicate, the Boolean predicate to filter the input column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3470,11 +3596,10 @@ object functions {
    * Returns an array of elements for which a predicate holds in a given array.
    * {{{
    *   df.select(filter(col("s"), (x, i) => i % 2 === 0))
-   *   df.selectExpr("filter(col, (x, i) -> i % 2 == 0)")
    * }}}
    *
-   * @param column: the input array column
-   * @param f: (col, index) => predicate, the Boolean predicate to filter the input column
+   * @param column the input array column
+   * @param f (col, index) => predicate, the Boolean predicate to filter the input column
    *           given the index. Indices start at 0.
    *
    * @group collection_funcs
@@ -3488,18 +3613,28 @@ object functions {
    * Applies a binary operator to an initial state and all elements in the array,
    * and reduces this to a single state. The final state is converted into the final result
    * by applying a finish function.
+   * {{{
+   *   df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10))
+   * }}}
+   *
+   * @param expr the input array column
+   * @param initialValue the initial value
+   * @param merge (combined_value, input_value) => combined_value, the merge function to merge
+   *              an input value to the combined_value
+   * @param finish combined_value => final_value, the lambda function to convert the combined value
+   *               of all inputs to final result
    *
    * @group collection_funcs
    * @since 3.0.0
    */
   def aggregate(
       expr: Column,
-      zero: Column,
+      initialValue: Column,
       merge: (Column, Column) => Column,
       finish: Column => Column): Column = withExpr {
     ArrayAggregate(
       expr.expr,
-      zero.expr,
+      initialValue.expr,
       createLambda(merge),
       createLambda(finish)
     )
@@ -3508,17 +3643,31 @@ object functions {
   /**
    * Applies a binary operator to an initial state and all elements in the array,
    * and reduces this to a single state.
+   * {{{
+   *   df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x))
+   * }}}
    *
+   * @param expr the input array column
+   * @param initialValue the initial value
+   * @param merge (combined_value, input_value) => combined_value, the merge function to merge
+   *              an input value to the combined_value
    * @group collection_funcs
    * @since 3.0.0
    */
-  def aggregate(expr: Column, zero: Column, merge: (Column, Column) => Column): Column =
-    aggregate(expr, zero, merge, c => c)
+  def aggregate(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column =
+    aggregate(expr, initialValue, merge, c => c)
 
   /**
    * Merge two given arrays, element-wise, into a single array using a function.
    * If one array is shorter, nulls are appended at the end to match the length of the longer
    * array, before applying the function.
+   * {{{
+   *   df.select(zip_with(df1("val1"), df1("val2"), (x, y) => x + y))
+   * }}}
+   *
+   * @param left the left input array column
+   * @param right the right input array column
+   * @param f (lCol, rCol) => col, the lambda function to merge two input columns into one column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3530,6 +3679,12 @@ object functions {
   /**
    * Applies a function to every key-value pair in a map and returns
    * a map with the results of those applications as the new keys for the pairs.
+   * {{{
+   *   df.select(transform_keys(col("i"), (k, v) => k + v))
+   * }}}
+   *
+   * @param expr the input map column
+   * @param f (key, value) => new_key, the lambda function to transform the key of input map column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3541,6 +3696,13 @@ object functions {
   /**
    * Applies a function to every key-value pair in a map and returns
    * a map with the results of those applications as the new values for the pairs.
+   * {{{
+   *   df.select(transform_values(col("i"), (k, v) => k + v))
+   * }}}
+   *
+   * @param expr the input map column
+   * @param f (key, value) => new_value, the lambda function to transform the value of input map
+   *          column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3551,6 +3713,12 @@ object functions {
 
   /**
    * Returns a map whose key-value pairs satisfy a predicate.
+   * {{{
+   *   df.select(map_filter(col("m"), (k, v) => k * 10 === v))
+   * }}}
+   *
+   * @param expr the input map column
+   * @param f (key, value) => predicate, the Boolean predicate to filter the input map column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3561,6 +3729,13 @@ object functions {
 
   /**
    * Merge two given maps, key-wise into a single map using a function.
+   * {{{
+   *   df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2))
+   * }}}
+   *
+   * @param left the left input map column
+   * @param right the right input map column
+   * @param f (key, value1, value2) => new_value, the lambda function to merge the map values
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3883,6 +4058,10 @@ object functions {
   /**
    * Returns length of array or map.
    *
+   * The function returns null for null input if spark.sql.legacy.sizeOfNull is set to false or
+   * spark.sql.ansi.enabled is set to true. Otherwise, the function returns -1 for null input.
+   * With the default settings, the function returns -1 for null input.
+   *
    * @group collection_funcs
    * @since 1.5.0
    */
@@ -4194,7 +4373,7 @@ object functions {
   (0 to 10).foreach { x =>
     val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"})
     val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _)
-    val inputSchemas = (1 to x).foldRight("Nil")((i, s) => {s"Try(ScalaReflection.schemaFor(typeTag[A$i])).toOption :: $s"})
+    val inputEncoders = (1 to x).foldRight("Nil")((i, s) => {s"Try(ExpressionEncoder[A$i]()).toOption :: $s"})
     println(s"""
       |/**
       | * Defines a Scala closure of $x arguments as user-defined function (UDF).
@@ -4207,8 +4386,8 @@ object functions {
       | */
       |def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = {
       |  val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-      |  val inputSchemas = $inputSchemas
-      |  val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+      |  val inputEncoders = $inputEncoders
+      |  val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
       |  if (nullable) udf else udf.asNonNullable()
       |}""".stripMargin)
   }
@@ -4231,7 +4410,7 @@ object functions {
       | */
       |def udf(f: UDF$i[$extTypeArgs], returnType: DataType): UserDefinedFunction = {
       |  val func = $funcCall
-      |  SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill($i)(None))
+      |  SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill($i)(None))
       |}""".stripMargin)
   }
 
@@ -4313,8 +4492,8 @@ object functions {
    */
   def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4329,8 +4508,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4345,8 +4524,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4361,8 +4540,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4377,8 +4556,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4393,8 +4572,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4409,8 +4588,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4425,8 +4604,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4441,8 +4620,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A8])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4457,8 +4636,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A8])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A9])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4473,8 +4652,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A8])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A9])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A10])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4493,7 +4672,7 @@ object functions {
    */
   def udf(f: UDF0[_], returnType: DataType): UserDefinedFunction = {
     val func = () => f.asInstanceOf[UDF0[Any]].call()
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(0)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(0)(None))
   }
 
   /**
@@ -4507,7 +4686,7 @@ object functions {
    */
   def udf(f: UDF1[_, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF1[Any, Any]].call(_: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(1)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(1)(None))
   }
 
   /**
@@ -4521,7 +4700,7 @@ object functions {
    */
   def udf(f: UDF2[_, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(2)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(2)(None))
   }
 
   /**
@@ -4535,7 +4714,7 @@ object functions {
    */
   def udf(f: UDF3[_, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(3)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(3)(None))
   }
 
   /**
@@ -4549,7 +4728,7 @@ object functions {
    */
   def udf(f: UDF4[_, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(4)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(4)(None))
   }
 
   /**
@@ -4563,7 +4742,7 @@ object functions {
    */
   def udf(f: UDF5[_, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(5)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(5)(None))
   }
 
   /**
@@ -4577,7 +4756,7 @@ object functions {
    */
   def udf(f: UDF6[_, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(6)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(6)(None))
   }
 
   /**
@@ -4591,7 +4770,7 @@ object functions {
    */
   def udf(f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(7)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(7)(None))
   }
 
   /**
@@ -4605,7 +4784,7 @@ object functions {
    */
   def udf(f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(8)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(8)(None))
   }
 
   /**
@@ -4619,7 +4798,7 @@ object functions {
    */
   def udf(f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(9)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(9)(None))
   }
 
   /**
@@ -4633,7 +4812,7 @@ object functions {
    */
   def udf(f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(10)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(10)(None))
   }
 
   // scalastyle:on parameter.number
@@ -4658,8 +4837,24 @@ object functions {
    * @group udf_funcs
    * @since 2.0.0
    */
+  @deprecated("Scala `udf` method with return type parameter is deprecated. " +
+    "Please use Scala `udf` method without return type parameter.", "3.0.0")
   def udf(f: AnyRef, dataType: DataType): UserDefinedFunction = {
-    SparkUserDefinedFunction(f, dataType, inputSchemas = Nil)
+    if (!SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF)) {
+      val errorMsg = "You're using untyped Scala UDF, which does not have the input type " +
+        "information. Spark may blindly pass null to the Scala closure with primitive-type " +
+        "argument, and the closure will see the default value of the Java type for the null " +
+        "argument, e.g. `udf((x: Int) => x, IntegerType)`, the result is 0 for null input. " +
+        "To get rid of this error, you could:\n" +
+        "1. use typed Scala UDF APIs(without return type parameter), e.g. `udf((x: Int) => x)`\n" +
+        "2. use Java UDF APIs, e.g. `udf(new UDF1[String, Integer] { " +
+        "override def call(s: String): Integer = s.length() }, IntegerType)`, " +
+        "if input types are all non primitive\n" +
+        s"3. set ${SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF.key} to true and " +
+        s"use this API with caution"
+      throw new AnalysisException(errorMsg)
+    }
+    SparkUserDefinedFunction(f, dataType, inputEncoders = Nil)
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index eb658e2d8850e..83a7a557305e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -26,8 +26,10 @@ import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
-import org.apache.spark.sql.execution.{ColumnarRule, QueryExecution, SparkOptimizer, SparkPlanner, SparkSqlParser}
+import org.apache.spark.sql.execution.{ColumnarRule, QueryExecution, SparkOptimizer, SparkPlan, SparkPlanner, SparkSqlParser}
+import org.apache.spark.sql.execution.aggregate.ResolveEncodersInScalaAgg
 import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin
+import org.apache.spark.sql.execution.command.CommandCheck
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.execution.datasources.v2.{TableCapabilityCheck, V2SessionCatalog}
 import org.apache.spark.sql.streaming.StreamingQueryManager
@@ -174,7 +176,9 @@ abstract class BaseSessionStateBuilder(
       new FindDataSourceTable(session) +:
         new ResolveSQLOnFile(session) +:
         new FallBackFileSourceV2(session) +:
-        new ResolveSessionCatalog(catalogManager, conf, catalog.isView) +:
+        ResolveEncodersInScalaAgg +:
+        new ResolveSessionCatalog(
+          catalogManager, conf, catalog.isTempView, catalog.isTempFunction) +:
         customResolutionRules
 
     override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =
@@ -189,6 +193,7 @@ abstract class BaseSessionStateBuilder(
         PreReadCheck +:
         HiveOnlyCheck +:
         TableCapabilityCheck +:
+        CommandCheck(conf) +:
         customCheckRules
   }
 
@@ -229,6 +234,9 @@ abstract class BaseSessionStateBuilder(
    */
   protected def optimizer: Optimizer = {
     new SparkOptimizer(catalogManager, catalog, experimentalMethods) {
+      override def earlyScanPushDownRules: Seq[Rule[LogicalPlan]] =
+        super.earlyScanPushDownRules ++ customEarlyScanPushDownRules
+
       override def extendedOperatorOptimizationRules: Seq[Rule[LogicalPlan]] =
         super.extendedOperatorOptimizationRules ++ customOperatorOptimizationRules
     }
@@ -244,6 +252,14 @@ abstract class BaseSessionStateBuilder(
     extensions.buildOptimizerRules(session)
   }
 
+  /**
+   * Custom early scan push down rules to add to the Optimizer. Prefer overriding this instead
+   * of creating your own Optimizer.
+   *
+   * Note that this may NOT depend on the `optimizer` function.
+   */
+  protected def customEarlyScanPushDownRules: Seq[Rule[LogicalPlan]] = Nil
+
   /**
    * Planner that converts optimized logical plans to physical plans.
    *
@@ -270,6 +286,10 @@ abstract class BaseSessionStateBuilder(
     extensions.buildColumnarRules(session)
   }
 
+  protected def queryStagePrepRules: Seq[Rule[SparkPlan]] = {
+    extensions.buildQueryStagePrepRules(session)
+  }
+
   /**
    * Create a query execution object.
    */
@@ -321,7 +341,8 @@ abstract class BaseSessionStateBuilder(
       () => resourceLoader,
       createQueryExecution,
       createClone,
-      columnarRules)
+      columnarRules,
+      queryStagePrepRules)
   }
 }
 
@@ -333,8 +354,14 @@ private[sql] trait WithTestConf { self: BaseSessionStateBuilder =>
 
   override protected lazy val conf: SQLConf = {
     val overrideConfigurations = overrideConfs
-    val conf = parentState.map(_.conf.clone()).getOrElse {
-      new SQLConf {
+    parentState.map { s =>
+      val cloned = s.conf.clone()
+      if (session.sparkContext.conf.get(StaticSQLConf.SQL_LEGACY_SESSION_INIT_WITH_DEFAULTS)) {
+        mergeSparkConf(conf, session.sparkContext.conf)
+      }
+      cloned
+    }.getOrElse {
+      val conf = new SQLConf {
         clear()
         override def clear(): Unit = {
           super.clear()
@@ -342,8 +369,8 @@ private[sql] trait WithTestConf { self: BaseSessionStateBuilder =>
           overrideConfigurations.foreach { case (key, value) => setConfString(key, value) }
         }
       }
+      mergeSparkConf(conf, session.sparkContext.conf)
+      conf
     }
-    mergeSparkConf(conf, session.sparkContext.conf)
-    conf
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
index 3740b56cb9cbb..7c76168b0dd9b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
@@ -464,6 +464,9 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
    * If this table is cached as an InMemoryRelation, drop the original cached version and make the
    * new version cached lazily.
    *
+   * In addition, refreshing a table also invalidate all caches that have reference to the table
+   * in a cascading manner. This is to prevent incorrect result from the otherwise staled caches.
+   *
    * @group cachemgmt
    * @since 2.0.0
    */
@@ -484,14 +487,17 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
     // If this table is cached as an InMemoryRelation, drop the original
     // cached version and make the new version cached lazily.
     val cache = sparkSession.sharedState.cacheManager.lookupCachedData(table)
+
+    // uncache the logical plan.
+    // note this is a no-op for the table itself if it's not cached, but will invalidate all
+    // caches referencing this table.
+    sparkSession.sharedState.cacheManager.uncacheQuery(table, cascade = true)
+
     if (cache.nonEmpty) {
       // save the cache name and cache level for recreation
       val cacheName = cache.get.cachedRepresentation.cacheBuilder.tableName
       val cacheLevel = cache.get.cachedRepresentation.cacheBuilder.storageLevel
 
-      // uncache the logical plan.
-      sparkSession.sharedState.cacheManager.uncacheQuery(table, cascade = true)
-
       // recache with the same name and cache level.
       sparkSession.sharedState.cacheManager.cacheQuery(table, cacheName, cacheLevel)
     }
@@ -517,10 +523,11 @@ private[sql] object CatalogImpl {
       data: Seq[T],
       sparkSession: SparkSession): Dataset[T] = {
     val enc = ExpressionEncoder[T]()
-    val encoded = data.map(d => enc.toRow(d).copy())
+    val toRow = enc.createSerializer()
+    val encoded = data.map(d => toRow(d).copy())
     val plan = new LocalRelation(enc.schema.toAttributes, encoded)
     val queryExecution = sparkSession.sessionState.executePlan(plan)
-    new Dataset[T](sparkSession, queryExecution, enc)
+    new Dataset[T](queryExecution, enc)
   }
 
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
index abd1250628539..cd425b04ef311 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.optimizer.Optimizer
 import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.streaming.StreamingQueryManager
@@ -73,7 +74,8 @@ private[sql] class SessionState(
     resourceLoaderBuilder: () => SessionResourceLoader,
     createQueryExecution: LogicalPlan => QueryExecution,
     createClone: (SparkSession, SessionState) => SessionState,
-    val columnarRules: Seq[ColumnarRule]) {
+    val columnarRules: Seq[ColumnarRule],
+    val queryStagePrepRules: Seq[Rule[SparkPlan]]) {
 
   // The following fields are lazy to avoid creating the Hive client when creating SessionState.
   lazy val catalog: SessionCatalog = catalogBuilder()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index fefd72dcf1752..1922a58ce16af 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -22,6 +22,7 @@ import java.util.UUID
 import java.util.concurrent.ConcurrentHashMap
 import javax.annotation.concurrent.GuardedBy
 
+import scala.collection.JavaConverters._
 import scala.reflect.ClassTag
 import scala.util.control.NonFatal
 
@@ -41,7 +42,6 @@ import org.apache.spark.sql.streaming.ui.{StreamingQueryStatusListener, Streamin
 import org.apache.spark.status.ElementTrackingStore
 import org.apache.spark.util.Utils
 
-
 /**
  * A class that holds all state shared across sessions in a given [[SQLContext]].
  *
@@ -53,47 +53,12 @@ private[sql] class SharedState(
     initialConfigs: scala.collection.Map[String, String])
   extends Logging {
 
-  SharedState.setFsUrlStreamHandlerFactory(sparkContext.conf)
-
-  // Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on
-  // the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf.
-  val warehousePath: String = {
-    val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
-    if (configFile != null) {
-      logInfo(s"loading hive config file: $configFile")
-      sparkContext.hadoopConfiguration.addResource(configFile)
-    }
-
-    // hive.metastore.warehouse.dir only stay in hadoopConf
-    sparkContext.conf.remove("hive.metastore.warehouse.dir")
-    // Set the Hive metastore warehouse path to the one we use
-    val hiveWarehouseDir = sparkContext.hadoopConfiguration.get("hive.metastore.warehouse.dir")
-    if (hiveWarehouseDir != null && !sparkContext.conf.contains(WAREHOUSE_PATH.key)) {
-      // If hive.metastore.warehouse.dir is set and spark.sql.warehouse.dir is not set,
-      // we will respect the value of hive.metastore.warehouse.dir.
-      sparkContext.conf.set(WAREHOUSE_PATH.key, hiveWarehouseDir)
-      logInfo(s"${WAREHOUSE_PATH.key} is not set, but hive.metastore.warehouse.dir " +
-        s"is set. Setting ${WAREHOUSE_PATH.key} to the value of " +
-        s"hive.metastore.warehouse.dir ('$hiveWarehouseDir').")
-      hiveWarehouseDir
-    } else {
-      // If spark.sql.warehouse.dir is set, we will override hive.metastore.warehouse.dir using
-      // the value of spark.sql.warehouse.dir.
-      // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set,
-      // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir.
-      val sparkWarehouseDir = sparkContext.conf.get(WAREHOUSE_PATH)
-      logInfo(s"Setting hive.metastore.warehouse.dir ('$hiveWarehouseDir') to the value of " +
-        s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').")
-      sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir)
-      sparkWarehouseDir
-    }
-  }
-  logInfo(s"Warehouse path is '$warehousePath'.")
+  SharedState.setFsUrlStreamHandlerFactory(sparkContext.conf, sparkContext.hadoopConfiguration)
 
-  // These 2 variables should be initiated after `warehousePath`, because in the first place we need
-  // to load hive-site.xml into hadoopConf and determine the warehouse path which will be set into
-  // both spark conf and hadoop conf avoiding be affected by any SparkSession level options
   private val (conf, hadoopConf) = {
+    // Load hive-site.xml into hadoopConf and determine the warehouse path which will be set into
+    // both spark conf and hadoop conf avoiding be affected by any SparkSession level options
+    SharedState.loadHiveConfFile(sparkContext.conf, sparkContext.hadoopConfiguration)
     val confClone = sparkContext.conf.clone()
     val hadoopConfClone = new Configuration(sparkContext.hadoopConfiguration)
     // If `SparkSession` is instantiated using an existing `SparkContext` instance and no existing
@@ -145,13 +110,14 @@ private[sql] class SharedState(
    * data to show.
    */
   lazy val streamingQueryStatusListener: Option[StreamingQueryStatusListener] = {
-    val sqlConf = SQLConf.get
-    if (sqlConf.isStreamingUIEnabled) {
-      val statusListener = new StreamingQueryStatusListener(sqlConf)
-      sparkContext.ui.foreach(new StreamingQueryTab(statusListener, _))
-      Some(statusListener)
-    } else {
-      None
+    sparkContext.ui.flatMap { ui =>
+      if (conf.get(STREAMING_UI_ENABLED)) {
+        val statusListener = new StreamingQueryStatusListener(conf)
+        new StreamingQueryTab(statusListener, ui)
+        Some(statusListener)
+      } else {
+        None
+      }
     }
   }
 
@@ -165,7 +131,7 @@ private[sql] class SharedState(
     val defaultDbDefinition = CatalogDatabase(
       SessionCatalog.DEFAULT_DATABASE,
       "default database",
-      CatalogUtils.stringToURI(warehousePath),
+      CatalogUtils.stringToURI(conf.get(WAREHOUSE_PATH)),
       Map())
     // Create default database if it doesn't exist
     if (!externalCatalog.databaseExists(SessionCatalog.DEFAULT_DATABASE)) {
@@ -187,9 +153,6 @@ private[sql] class SharedState(
    * A manager for global temporary views.
    */
   lazy val globalTempViewManager: GlobalTempViewManager = {
-    // System preserved database should not exists in metastore. However it's hard to guarantee it
-    // for every session, because case-sensitivity differs. Here we always lowercase it to make our
-    // life easier.
     val globalTempDB = conf.get(GLOBAL_TEMP_DATABASE)
     if (externalCatalog.databaseExists(globalTempDB)) {
       throw new SparkException(
@@ -211,13 +174,13 @@ private[sql] class SharedState(
 object SharedState extends Logging {
   @volatile private var fsUrlStreamHandlerFactoryInitialized = false
 
-  private def setFsUrlStreamHandlerFactory(conf: SparkConf): Unit = {
+  private def setFsUrlStreamHandlerFactory(conf: SparkConf, hadoopConf: Configuration): Unit = {
     if (!fsUrlStreamHandlerFactoryInitialized &&
         conf.get(DEFAULT_URL_STREAM_HANDLER_FACTORY_ENABLED)) {
       synchronized {
         if (!fsUrlStreamHandlerFactoryInitialized) {
           try {
-            URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory())
+            URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(hadoopConf))
             fsUrlStreamHandlerFactoryInitialized = true
           } catch {
             case NonFatal(_) =>
@@ -257,4 +220,52 @@ object SharedState extends Logging {
         throw new IllegalArgumentException(s"Error while instantiating '$className':", e)
     }
   }
+
+  /**
+   * Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on
+   * the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf.
+   */
+  def loadHiveConfFile(
+      sparkConf: SparkConf,
+      hadoopConf: Configuration): Unit = {
+    def containsInSparkConf(key: String): Boolean = {
+      sparkConf.contains(key) || sparkConf.contains("spark.hadoop." + key) ||
+        (key.startsWith("hive") && sparkConf.contains("spark." + key))
+    }
+
+    val hiveWarehouseKey = "hive.metastore.warehouse.dir"
+    val configFile = Utils.getContextOrSparkClassLoader.getResourceAsStream("hive-site.xml")
+    if (configFile != null) {
+      logInfo(s"loading hive config file: $configFile")
+      val hadoopConfTemp = new Configuration()
+      hadoopConfTemp.clear()
+      hadoopConfTemp.addResource(configFile)
+      for (entry <- hadoopConfTemp.asScala if !containsInSparkConf(entry.getKey)) {
+        hadoopConf.set(entry.getKey, entry.getValue)
+      }
+    }
+    // hive.metastore.warehouse.dir only stay in hadoopConf
+    sparkConf.remove(hiveWarehouseKey)
+    // Set the Hive metastore warehouse path to the one we use
+    val hiveWarehouseDir = hadoopConf.get(hiveWarehouseKey)
+    val warehousePath = if (hiveWarehouseDir != null && !sparkConf.contains(WAREHOUSE_PATH.key)) {
+      // If hive.metastore.warehouse.dir is set and spark.sql.warehouse.dir is not set,
+      // we will respect the value of hive.metastore.warehouse.dir.
+      sparkConf.set(WAREHOUSE_PATH.key, hiveWarehouseDir)
+      logInfo(s"${WAREHOUSE_PATH.key} is not set, but $hiveWarehouseKey is set. Setting" +
+        s" ${WAREHOUSE_PATH.key} to the value of $hiveWarehouseKey ('$hiveWarehouseDir').")
+      hiveWarehouseDir
+    } else {
+      // If spark.sql.warehouse.dir is set, we will override hive.metastore.warehouse.dir using
+      // the value of spark.sql.warehouse.dir.
+      // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set
+      // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir.
+      val sparkWarehouseDir = sparkConf.get(WAREHOUSE_PATH)
+      logInfo(s"Setting $hiveWarehouseKey ('$hiveWarehouseDir') to the value of " +
+        s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').")
+      hadoopConf.set(hiveWarehouseKey, sparkWarehouseDir)
+      sparkWarehouseDir
+    }
+    logInfo(s"Warehouse path is '$warehousePath'.")
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
index 72284b5996201..72cdc68d56116 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
@@ -25,6 +25,13 @@ import org.apache.spark.sql.types._
 
 private object MsSqlServerDialect extends JdbcDialect {
 
+  // Special JDBC types in Microsoft SQL Server.
+  // https://github.com/microsoft/mssql-jdbc/blob/v7.2.1/src/main/java/microsoft/sql/Types.java
+  private object SpecificTypes {
+    val GEOMETRY = -157
+    val GEOGRAPHY = -158
+  }
+
   override def canHandle(url: String): Boolean =
     url.toLowerCase(Locale.ROOT).startsWith("jdbc:sqlserver")
 
@@ -40,6 +47,7 @@ private object MsSqlServerDialect extends JdbcDialect {
         sqlType match {
           case java.sql.Types.SMALLINT => Some(ShortType)
           case java.sql.Types.REAL => Some(FloatType)
+          case SpecificTypes.GEOMETRY | SpecificTypes.GEOGRAPHY => Some(BinaryType)
           case _ => None
         }
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
index c8d8a3392128e..a1ce25a0464c3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
@@ -57,7 +57,7 @@ private object PostgresDialect extends JdbcDialect {
     case "int8" | "oid" => Some(LongType)
     case "float4" => Some(FloatType)
     case "money" | "float8" => Some(DoubleType)
-    case "text" | "varchar" | "char" | "cidr" | "inet" | "json" | "jsonb" | "uuid" =>
+    case "text" | "varchar" | "char" | "bpchar" | "cidr" | "inet" | "json" | "jsonb" | "uuid" =>
       Some(StringType)
     case "bytea" => Some(BinaryType)
     case "timestamp" | "timestamptz" | "time" | "timetz" => Some(TimestampType)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
index 61875931d226e..c0397010acba3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
@@ -49,8 +49,15 @@ package object sql {
    * Metadata key which is used to write Spark version in the followings:
    * - Parquet file metadata
    * - ORC file metadata
+   * - Avro file metadata
    *
    * Note that Hive table property `spark.sql.create.version` also has Spark version.
    */
   private[sql] val SPARK_VERSION_METADATA_KEY = "org.apache.spark.version"
+
+  /**
+   * Parquet/Avro file metadata key to indicate that the file was written with legacy datetime
+   * values.
+   */
+  private[sql] val SPARK_LEGACY_DATETIME = "org.apache.spark.legacyDateTime"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index 0eb4776988d9f..6b30949a4dc80 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -24,6 +24,7 @@ import scala.collection.JavaConverters._
 import org.apache.spark.annotation.Evolving
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.connector.catalog.{SupportsRead, TableProvider}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.execution.command.DDLUtils
@@ -81,8 +82,19 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -118,8 +130,19 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV data sources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -134,8 +157,19 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV data sources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -177,7 +211,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
       case provider: TableProvider if !provider.isInstanceOf[FileDataSourceV2] =>
         val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
           source = provider, conf = sparkSession.sessionState.conf)
-        val options = sessionOptions ++ extraOptions
+        val options = sessionOptions ++ extraOptions.toMap
         val dsOptions = new CaseInsensitiveStringMap(options.asJava)
         val table = DataSourceV2Utils.getTableFromProvider(provider, dsOptions, userSpecifiedSchema)
         import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._
@@ -252,12 +286,16 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * <li>`columnNameOfCorruptRecord` (default is the value specified in
    * `spark.sql.columnNameOfCorruptRecord`): allows renaming the new field having malformed string
    * created by `PERMISSIVE` mode. This overrides `spark.sql.columnNameOfCorruptRecord`.</li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`multiLine` (default `false`): parse one record, which may span multiple lines,
    * per file</li>
    * <li>`lineSep` (default covers all `\r`, `\r\n` and `\n`): defines the line separator
@@ -318,12 +356,16 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * value.</li>
    * <li>`negativeInf` (default `-Inf`): sets the string representation of a negative infinity
    * value.</li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`maxColumns` (default `20480`): defines a hard limit of how many columns
    * a record can have.</li>
    * <li>`maxCharsPerColumn` (default `-1`): defines the maximum number of characters allowed
@@ -479,5 +521,5 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
 
   private var userSpecifiedSchema: Option[StructType] = None
 
-  private var extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
index 1c21a30dd5bd6..07ab4008ca63f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
@@ -26,6 +26,7 @@ import org.apache.spark.annotation.Evolving
 import org.apache.spark.api.java.function.VoidFunction2
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.connector.catalog.{SupportsWrite, TableProvider}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.execution.command.DDLUtils
@@ -161,8 +162,19 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -198,8 +210,19 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -214,8 +237,19 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -316,7 +350,7 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
         val provider = cls.getConstructor().newInstance().asInstanceOf[TableProvider]
         val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
           source = provider, conf = df.sparkSession.sessionState.conf)
-        val options = sessionOptions ++ extraOptions
+        val options = sessionOptions ++ extraOptions.toMap
         val dsOptions = new CaseInsensitiveStringMap(options.asJava)
         val table = DataSourceV2Utils.getTableFromProvider(
           provider, dsOptions, userSpecifiedSchema = None)
@@ -439,7 +473,7 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
 
   private var trigger: Trigger = Trigger.ProcessingTime(0L)
 
-  private var extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 
   private var foreachWriter: ForeachWriter[T] = null
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
index dd842cd1a3e99..7ae38c71a005f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
@@ -82,7 +82,7 @@ object StreamingQueryListener {
    * @param id A unique query id that persists across restarts. See `StreamingQuery.id()`.
    * @param runId A query id that is unique for every start/restart. See `StreamingQuery.runId()`.
    * @param name User-specified name of the query, null if not specified.
-   * @param submissionTime The timestamp to start a query.
+   * @param timestamp The timestamp to start a query.
    * @since 2.1.0
    */
   @Evolving
@@ -90,7 +90,7 @@ object StreamingQueryListener {
       val id: UUID,
       val runId: UUID,
       val name: String,
-      val submissionTime: Long) extends Event
+      val timestamp: String) extends Event
 
   /**
    * Event representing any progress updates in a query.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
index 4d0d8ffd959c6..0fe2d0be966d0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
@@ -58,8 +58,17 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
   private val activeQueriesSharedLock = sparkSession.sharedState.activeQueriesLock
   private val awaitTerminationLock = new Object
 
+  /**
+   * Track the last terminated query and remember the last failure since the creation of the
+   * context, or since `resetTerminated()` was called. There are three possible values:
+   *
+   * - null: no query has been been terminated.
+   * - None: some queries have been terminated and no one has failed.
+   * - Some(StreamingQueryException): Some queries have been terminated and at least one query has
+   *   failed. The exception is the exception of the last failed query.
+   */
   @GuardedBy("awaitTerminationLock")
-  private var lastTerminatedQuery: StreamingQuery = null
+  private var lastTerminatedQueryException: Option[StreamingQueryException] = null
 
   try {
     sparkSession.sparkContext.conf.get(STREAMING_QUERY_LISTENERS).foreach { classNames =>
@@ -125,11 +134,11 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
   @throws[StreamingQueryException]
   def awaitAnyTermination(): Unit = {
     awaitTerminationLock.synchronized {
-      while (lastTerminatedQuery == null) {
+      while (lastTerminatedQueryException == null) {
         awaitTerminationLock.wait(10)
       }
-      if (lastTerminatedQuery != null && lastTerminatedQuery.exception.nonEmpty) {
-        throw lastTerminatedQuery.exception.get
+      if (lastTerminatedQueryException != null && lastTerminatedQueryException.nonEmpty) {
+        throw lastTerminatedQueryException.get
       }
     }
   }
@@ -164,13 +173,13 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
     }
 
     awaitTerminationLock.synchronized {
-      while (!isTimedout && lastTerminatedQuery == null) {
+      while (!isTimedout && lastTerminatedQueryException == null) {
         awaitTerminationLock.wait(10)
       }
-      if (lastTerminatedQuery != null && lastTerminatedQuery.exception.nonEmpty) {
-        throw lastTerminatedQuery.exception.get
+      if (lastTerminatedQueryException != null && lastTerminatedQueryException.nonEmpty) {
+        throw lastTerminatedQueryException.get
       }
-      lastTerminatedQuery != null
+      lastTerminatedQueryException != null
     }
   }
 
@@ -182,7 +191,7 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
    */
   def resetTerminated(): Unit = {
     awaitTerminationLock.synchronized {
-      lastTerminatedQuery = null
+      lastTerminatedQueryException = null
     }
   }
 
@@ -422,8 +431,8 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
   private[sql] def notifyQueryTermination(terminatedQuery: StreamingQuery): Unit = {
     unregisterTerminatedStream(terminatedQuery)
     awaitTerminationLock.synchronized {
-      if (lastTerminatedQuery == null || terminatedQuery.exception.nonEmpty) {
-        lastTerminatedQuery = terminatedQuery
+      if (lastTerminatedQueryException == null || terminatedQuery.exception.nonEmpty) {
+        lastTerminatedQueryException = terminatedQuery.exception
       }
       awaitTerminationLock.notifyAll()
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
index 650f64fe1688c..43b93a3654f4c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
@@ -17,22 +17,18 @@
 
 package org.apache.spark.sql.streaming.ui
 
-import java.text.SimpleDateFormat
 import javax.servlet.http.HttpServletRequest
 
 import scala.xml.Node
 
-import org.apache.commons.lang3.StringEscapeUtils
+import org.apache.commons.text.StringEscapeUtils
 
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.getTimeZone
 import org.apache.spark.sql.streaming.ui.UIUtils._
 import org.apache.spark.ui.{UIUtils => SparkUIUtils, WebUIPage}
 
 private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
     extends WebUIPage("") with Logging {
-  val df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
-  df.setTimeZone(getTimeZone("UTC"))
 
   override def render(request: HttpServletRequest): Seq[Node] = {
     val content = generateStreamingQueryTable(request)
@@ -61,12 +57,12 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
     val name = UIUtils.getQueryName(query)
     val status = UIUtils.getQueryStatus(query)
     val duration = if (queryActive) {
-      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.submissionTime)
+      System.currentTimeMillis() - query.startTimestamp
     } else {
       withNoProgress(query, {
         val endTimeMs = query.lastProgress.timestamp
-        SparkUIUtils.formatDurationVerbose(df.parse(endTimeMs).getTime - query.submissionTime)
-      }, "-")
+        parseProgressTimestamp(endTimeMs) - query.startTimestamp
+      }, 0)
     }
 
     <tr>
@@ -74,8 +70,10 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
       <td> {status} </td>
       <td> {query.id} </td>
       <td> <a href={statisticsLink}> {query.runId} </a> </td>
-      <td> {SparkUIUtils.formatDate(query.submissionTime)} </td>
-      <td> {duration} </td>
+      <td> {SparkUIUtils.formatDate(query.startTimestamp)} </td>
+      <td sorttable_customkey={duration.toString}>
+        {SparkUIUtils.formatDurationVerbose(duration)}
+      </td>
       <td> {withNoProgress(query, {
         (query.recentProgress.map(p => withNumberInvalid(p.inputRowsPerSecond)).sum /
           query.recentProgress.length).formatted("%.2f") }, "NaN")}
@@ -94,29 +92,38 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
       .partition(_.isActive)
     val activeQueryTables = if (activeQueries.nonEmpty) {
       val headerRow = Seq(
-        "Name", "Status", "Id", "Run ID", "Submitted Time", "Duration", "Avg Input /sec",
+        "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
         "Avg Process /sec", "Lastest Batch")
 
+      val headerCss = Seq("", "", "", "", "", "sorttable_numeric", "sorttable_numeric",
+        "sorttable_numeric", "")
+      // header classes size must be equal to header row size
+      assert(headerRow.size == headerCss.size)
+
       Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = true),
-        activeQueries, true, None, Seq(null), false))
+        activeQueries, true, Some("activeQueries-table"), headerCss, false))
     } else {
       None
     }
 
     val inactiveQueryTables = if (inactiveQueries.nonEmpty) {
       val headerRow = Seq(
-        "Name", "Status", "Id", "Run ID", "Submitted Time", "Duration", "Avg Input /sec",
+        "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
         "Avg Process /sec", "Lastest Batch", "Error")
 
+      val headerCss = Seq("", "", "", "", "", "sorttable_numeric", "sorttable_numeric",
+        "sorttable_numeric", "", "")
+      assert(headerRow.size == headerCss.size)
+
       Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = false),
-        inactiveQueries, true, None, Seq(null), false))
+        inactiveQueries, true, Some("completedQueries-table"), headerCss, false))
     } else {
       None
     }
 
     // scalastyle:off
     val content =
-      <span id="completed" class="collapse-aggregated-activeQueries collapse-table"
+      <span id="active" class="collapse-aggregated-activeQueries collapse-table"
             onClick="collapseTable('collapse-aggregated-activeQueries','aggregated-activeQueries')">
         <h5 id="activequeries">
           <span class="collapse-table-arrow arrow-open"></span>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
index 56672ce328bff..227e5e5af3983 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
@@ -19,21 +19,17 @@ package org.apache.spark.sql.streaming.ui
 
 import java.{util => ju}
 import java.lang.{Long => JLong}
-import java.text.SimpleDateFormat
 import java.util.UUID
 import javax.servlet.http.HttpServletRequest
 
 import scala.xml.{Node, Unparsed}
 
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.getTimeZone
 import org.apache.spark.sql.streaming.ui.UIUtils._
 import org.apache.spark.ui.{GraphUIData, JsCollector, UIUtils => SparkUIUtils, WebUIPage}
 
 private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
   extends WebUIPage("statistics") with Logging {
-  val df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
-  df.setTimeZone(getTimeZone("UTC"))
 
   def generateLoadResources(request: HttpServletRequest): Seq[Node] = {
     // scalastyle:off
@@ -70,11 +66,30 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
     <script>{Unparsed(js)}</script>
   }
 
-  def generateVar(values: Array[(Long, ju.Map[String, JLong])]): Seq[Node] = {
+  def generateTimeTipStrings(values: Array[(Long, Long)]): Seq[Node] = {
+    val js = "var timeTipStrings = {};\n" + values.map { case (batchId, time) =>
+      val formattedTime = SparkUIUtils.formatBatchTime(time, 1, showYYYYMMSS = false)
+      s"timeTipStrings[$time] = 'batch $batchId ($formattedTime)';"
+    }.mkString("\n")
+
+    <script>{Unparsed(js)}</script>
+  }
+
+  def generateFormattedTimeTipStrings(values: Array[(Long, Long)]): Seq[Node] = {
+    val js = "var formattedTimeTipStrings = {};\n" + values.map { case (batchId, time) =>
+      val formattedTime = SparkUIUtils.formatBatchTime(time, 1, showYYYYMMSS = false)
+      s"""formattedTimeTipStrings["$formattedTime"] = 'batch $batchId ($formattedTime)';"""
+    }.mkString("\n")
+
+    <script>{Unparsed(js)}</script>
+  }
+
+  def generateTimeToValues(values: Array[(Long, ju.Map[String, JLong])]): Seq[Node] = {
     val durationDataPadding = SparkUIUtils.durationDataPadding(values)
-    val js = "var timeToValues = {};\n" + durationDataPadding.map { case (x, y) =>
+    val js = "var formattedTimeToValues = {};\n" + durationDataPadding.map { case (x, y) =>
       val s = y.toSeq.sortBy(_._1).map(e => s""""${e._2}"""").mkString("[", ",", "]")
-      s"""timeToValues["${SparkUIUtils.formatBatchTime(x, 1, showYYYYMMSS = false)}"] = $s;"""
+      val formattedTime = SparkUIUtils.formatBatchTime(x, 1, showYYYYMMSS = false)
+      s"""formattedTimeToValues["$formattedTime"] = $s;"""
     }.mkString("\n")
 
     <script>{Unparsed(js)}</script>
@@ -82,13 +97,13 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
 
   def generateBasicInfo(query: StreamingQueryUIData): Seq[Node] = {
     val duration = if (query.isActive) {
-      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.submissionTime)
+      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.startTimestamp)
     } else {
       withNoProgress(query, {
         val end = query.lastProgress.timestamp
         val start = query.recentProgress.head.timestamp
         SparkUIUtils.formatDurationVerbose(
-          df.parse(end).getTime - df.parse(start).getTime)
+          parseProgressTimestamp(end) - parseProgressTimestamp(start))
       }, "-")
     }
 
@@ -100,7 +115,7 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
       </strong>
       since
       <strong>
-        {SparkUIUtils.formatDate(query.submissionTime)}
+        {SparkUIUtils.formatDate(query.startTimestamp)}
       </strong>
       (<strong>{numBatches}</strong> completed batches)
     </div>
@@ -112,12 +127,14 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
   }
 
   def generateStatTable(query: StreamingQueryUIData): Seq[Node] = {
-    val batchTimes = withNoProgress(query,
-      query.recentProgress.map(p => df.parse(p.timestamp).getTime), Array.empty[Long])
+    val batchToTimestamps = withNoProgress(query,
+      query.recentProgress.map(p => (p.batchId, parseProgressTimestamp(p.timestamp))),
+      Array.empty[(Long, Long)])
+    val batchTimes = batchToTimestamps.map(_._2)
     val minBatchTime =
-      withNoProgress(query, df.parse(query.recentProgress.head.timestamp).getTime, 0L)
+      withNoProgress(query, parseProgressTimestamp(query.recentProgress.head.timestamp), 0L)
     val maxBatchTime =
-      withNoProgress(query, df.parse(query.lastProgress.timestamp).getTime, 0L)
+      withNoProgress(query, parseProgressTimestamp(query.lastProgress.timestamp), 0L)
     val maxRecordRate =
       withNoProgress(query, query.recentProgress.map(_.inputRowsPerSecond).max, 0L)
     val minRecordRate = 0L
@@ -131,22 +148,26 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
     val minBatchDuration = 0L
 
     val inputRateData = withNoProgress(query,
-      query.recentProgress.map(p => (df.parse(p.timestamp).getTime,
+      query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
         withNumberInvalid { p.inputRowsPerSecond })), Array.empty[(Long, Double)])
     val processRateData = withNoProgress(query,
-      query.recentProgress.map(p => (df.parse(p.timestamp).getTime,
+      query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
         withNumberInvalid { p.processedRowsPerSecond })), Array.empty[(Long, Double)])
     val inputRowsData = withNoProgress(query,
-      query.recentProgress.map(p => (df.parse(p.timestamp).getTime,
+      query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
         withNumberInvalid { p.numInputRows })), Array.empty[(Long, Double)])
     val batchDurations = withNoProgress(query,
-      query.recentProgress.map(p => (df.parse(p.timestamp).getTime,
+      query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
         withNumberInvalid { p.batchDuration })), Array.empty[(Long, Double)])
-    val operationDurationData = withNoProgress(query, query.recentProgress.map { p =>
-      val durationMs = p.durationMs
-      // remove "triggerExecution" as it count the other operation duration.
-      durationMs.remove("triggerExecution")
-      (df.parse(p.timestamp).getTime, durationMs)}, Array.empty[(Long, ju.Map[String, JLong])])
+    val operationDurationData = withNoProgress(
+      query,
+      query.recentProgress.map { p =>
+        val durationMs = p.durationMs
+        // remove "triggerExecution" as it count the other operation duration.
+        durationMs.remove("triggerExecution")
+        (parseProgressTimestamp(p.timestamp), durationMs)
+      },
+      Array.empty[(Long, ju.Map[String, JLong])])
 
     val jsCollector = new JsCollector
     val graphUIDataForInputRate =
@@ -208,14 +229,15 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
         0L,
         "ms")
 
-    val table =
-    // scalastyle:off
+    val table = if (query.lastProgress != null) {
+      // scalastyle:off
       <table id="stat-table" class="table table-bordered" style="width: auto">
         <thead>
           <tr>
             <th style="width: 160px;"></th>
             <th style="width: 492px;">Timelines</th>
-            <th style="width: 350px;">Histograms</th></tr>
+            <th style="width: 350px;">Histograms</th>
+          </tr>
         </thead>
         <tbody>
           <tr>
@@ -264,8 +286,16 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
           </tr>
         </tbody>
       </table>
-    // scalastyle:on
+    } else {
+      <div id="empty-streaming-query-message">
+        <b>No visualization information available.</b>
+      </div>
+      // scalastyle:on
+    }
 
-    generateVar(operationDurationData) ++ generateTimeMap(batchTimes) ++ table ++ jsCollector.toHtml
+    generateTimeToValues(operationDurationData) ++
+      generateFormattedTimeTipStrings(batchToTimestamps) ++
+      generateTimeMap(batchTimes) ++ generateTimeTipStrings(batchToTimestamps) ++
+      table ++ jsCollector.toHtml
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
index db085dbe87ec4..e331083b30024 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
@@ -17,26 +17,23 @@
 
 package org.apache.spark.sql.streaming.ui
 
-import java.text.SimpleDateFormat
 import java.util.UUID
 import java.util.concurrent.ConcurrentHashMap
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.streaming.{StreamingQueryListener, StreamingQueryProgress}
+import org.apache.spark.sql.streaming.ui.UIUtils.parseProgressTimestamp
 
 /**
  * A customized StreamingQueryListener used in structured streaming UI, which contains all
  * UI data for both active and inactive query.
  * TODO: Add support for history server.
  */
-private[sql] class StreamingQueryStatusListener(sqlConf: SQLConf) extends StreamingQueryListener {
-
-  private val timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
-  timestampFormat.setTimeZone(DateTimeUtils.getTimeZone("UTC"))
+private[sql] class StreamingQueryStatusListener(conf: SparkConf) extends StreamingQueryListener {
 
   /**
    * We use runId as the key here instead of id in active query status map,
@@ -45,16 +42,18 @@ private[sql] class StreamingQueryStatusListener(sqlConf: SQLConf) extends Stream
   private[ui] val activeQueryStatus = new ConcurrentHashMap[UUID, StreamingQueryUIData]()
   private[ui] val inactiveQueryStatus = new mutable.Queue[StreamingQueryUIData]()
 
-  private val streamingProgressRetention = sqlConf.streamingProgressRetention
-  private val inactiveQueryStatusRetention = sqlConf.streamingUIInactiveQueryRetention
+  private val streamingProgressRetention =
+    conf.get(StaticSQLConf.STREAMING_UI_RETAINED_PROGRESS_UPDATES)
+  private val inactiveQueryStatusRetention = conf.get(StaticSQLConf.STREAMING_UI_RETAINED_QUERIES)
 
   override def onQueryStarted(event: StreamingQueryListener.QueryStartedEvent): Unit = {
+    val startTimestamp = parseProgressTimestamp(event.timestamp)
     activeQueryStatus.putIfAbsent(event.runId,
-      new StreamingQueryUIData(event.name, event.id, event.runId, event.submissionTime))
+      new StreamingQueryUIData(event.name, event.id, event.runId, startTimestamp))
   }
 
   override def onQueryProgress(event: StreamingQueryListener.QueryProgressEvent): Unit = {
-    val batchTimestamp = timestampFormat.parse(event.progress.timestamp).getTime
+    val batchTimestamp = parseProgressTimestamp(event.progress.timestamp)
     val queryStatus = activeQueryStatus.getOrDefault(
       event.progress.runId,
       new StreamingQueryUIData(event.progress.name, event.progress.id, event.progress.runId,
@@ -87,7 +86,7 @@ private[ui] class StreamingQueryUIData(
     val name: String,
     val id: UUID,
     val runId: UUID,
-    val submissionTime: Long) {
+    val startTimestamp: Long) {
 
   /** Holds the most recent query progress updates. */
   private val progressBuffer = new mutable.Queue[StreamingQueryProgress]()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala
index f909cfd97514e..bb097ffc06912 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala
@@ -34,6 +34,6 @@ private[sql] class StreamingQueryTab(
   parent.addStaticHandler(StreamingQueryTab.STATIC_RESOURCE_DIR, "/static/sql")
 }
 
-object StreamingQueryTab {
+private[sql] object StreamingQueryTab {
   private val STATIC_RESOURCE_DIR = "org/apache/spark/sql/execution/ui/static"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala
index 57b9dec81f28a..cdad5ed9942b5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala
@@ -17,6 +17,11 @@
 
 package org.apache.spark.sql.streaming.ui
 
+import java.text.SimpleDateFormat
+import java.util.Locale
+
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.getTimeZone
+
 private[ui] object UIUtils {
 
   /**
@@ -57,4 +62,16 @@ private[ui] object UIUtils {
       query.exception.map(_ => "FAILED").getOrElse("FINISHED")
     }
   }
+
+  private val progressTimestampFormat = new ThreadLocal[SimpleDateFormat]() {
+    override def initialValue(): SimpleDateFormat = {
+      val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
+      format.setTimeZone(getTimeZone("UTC"))
+      format
+    }
+  }
+
+  def parseProgressTimestamp(timestamp: String): Long = {
+    progressTimestampFormat.get.parse(timestamp).getTime
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
index 01f81825f6bfd..0b5951ec2ac97 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
@@ -23,7 +23,7 @@ import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.internal.Logging
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.execution.QueryExecution
+import org.apache.spark.sql.execution.{QueryExecution, QueryExecutionException}
 import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.util.{ListenerBus, Utils}
@@ -55,12 +55,13 @@ trait QueryExecutionListener {
    * @param funcName the name of the action that triggered this query.
    * @param qe the QueryExecution object that carries detail information like logical plan,
    *           physical plan, etc.
-   * @param error the error that failed this query.
-   *
+   * @param exception the exception that failed this query. If `java.lang.Error` is thrown during
+   *                  execution, it will be wrapped with an `Exception` and it can be accessed by
+   *                  `exception.getCause`.
    * @note This can be invoked by multiple different threads.
    */
   @DeveloperApi
-  def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit
+  def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit
 }
 
 
@@ -140,7 +141,14 @@ private[sql] class ExecutionListenerBus(session: SparkSession)
       val funcName = event.executionName.get
       event.executionFailure match {
         case Some(ex) =>
-          listener.onFailure(funcName, event.qe, ex)
+          val exception = ex match {
+            case e: Exception => e
+            case other: Throwable =>
+              val message = "Hit an error when executing a query" +
+                (if (other.getMessage == null) "" else s": ${other.getMessage}")
+              new QueryExecutionException(message, other)
+          }
+          listener.onFailure(funcName, event.qe, exception)
         case _ =>
           listener.onSuccess(funcName, event.qe, event.duration)
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
deleted file mode 100644
index 346e07f2bef15..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.status.api.v1.sql
-
-import java.util.Date
-import javax.ws.rs._
-import javax.ws.rs.core.MediaType
-
-import org.apache.spark.JobExecutionStatus
-import org.apache.spark.sql.execution.ui.{SQLAppStatusStore, SQLExecutionUIData, SQLPlanMetric}
-import org.apache.spark.status.api.v1.{BaseAppResource, NotFoundException}
-
-@Produces(Array(MediaType.APPLICATION_JSON))
-private[v1] class SqlResource extends BaseAppResource {
-
-  @GET
-  def sqlList(
-      @DefaultValue("false") @QueryParam("details") details: Boolean,
-      @DefaultValue("0") @QueryParam("offset") offset: Int,
-      @DefaultValue("20") @QueryParam("length") length: Int): Seq[ExecutionData] = {
-    withUI { ui =>
-      val sqlStore = new SQLAppStatusStore(ui.store.store)
-      sqlStore.executionsList(offset, length).map(prepareExecutionData(_, details))
-    }
-  }
-
-  @GET
-  @Path("{executionId:\\d+}")
-  def sql(
-      @PathParam("executionId") execId: Long,
-      @DefaultValue("false") @QueryParam("details") details: Boolean): ExecutionData = {
-    withUI { ui =>
-      val sqlStore = new SQLAppStatusStore(ui.store.store)
-      sqlStore
-        .execution(execId)
-        .map(prepareExecutionData(_, details))
-        .getOrElse(throw new NotFoundException("unknown id: " + execId))
-    }
-  }
-
-  private def printableMetrics(
-      metrics: Seq[SQLPlanMetric],
-      metricValues: Map[Long, String]): Seq[Metrics] = {
-    metrics.map(metric =>
-      Metrics(metric.name, metricValues.get(metric.accumulatorId).getOrElse("")))
-  }
-
-  private def prepareExecutionData(exec: SQLExecutionUIData, details: Boolean): ExecutionData = {
-    var running = Seq[Int]()
-    var completed = Seq[Int]()
-    var failed = Seq[Int]()
-
-    exec.jobs.foreach {
-      case (id, JobExecutionStatus.RUNNING) =>
-        running = running :+ id
-      case (id, JobExecutionStatus.SUCCEEDED) =>
-        completed = completed :+ id
-      case (id, JobExecutionStatus.FAILED) =>
-        failed = failed :+ id
-      case _ =>
-    }
-
-    val status = if (exec.jobs.size == completed.size) {
-      "COMPLETED"
-    } else if (failed.nonEmpty) {
-      "FAILED"
-    } else {
-      "RUNNING"
-    }
-
-    val duration = exec.completionTime.getOrElse(new Date()).getTime - exec.submissionTime
-    val planDetails = if (details) exec.physicalPlanDescription else ""
-    val metrics = if (details) printableMetrics(exec.metrics, exec.metricValues) else Seq.empty
-    new ExecutionData(
-      exec.executionId,
-      status,
-      exec.description,
-      planDetails,
-      metrics,
-      new Date(exec.submissionTime),
-      duration,
-      running,
-      completed,
-      failed)
-  }
-}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java
index 5603cb988b8e7..af0a22b036030 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java
@@ -18,6 +18,8 @@
 package test.org.apache.spark.sql;
 
 import java.io.Serializable;
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.util.*;
@@ -210,6 +212,17 @@ private static Row createRecordSpark22000Row(Long index) {
     return new GenericRow(values);
   }
 
+  private static String timestampToString(Timestamp ts) {
+    String timestampString = String.valueOf(ts);
+    String formatted = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(ts);
+
+    if (timestampString.length() > 19 && !timestampString.substring(19).equals(".0")) {
+      return formatted + timestampString.substring(19);
+    } else {
+      return formatted;
+    }
+  }
+
   private static RecordSpark22000 createRecordSpark22000(Row recordRow) {
     RecordSpark22000 record = new RecordSpark22000();
     record.setShortField(String.valueOf(recordRow.getShort(0)));
@@ -219,7 +232,7 @@ private static RecordSpark22000 createRecordSpark22000(Row recordRow) {
     record.setDoubleField(String.valueOf(recordRow.getDouble(4)));
     record.setStringField(recordRow.getString(5));
     record.setBooleanField(String.valueOf(recordRow.getBoolean(6)));
-    record.setTimestampField(String.valueOf(recordRow.getTimestamp(7)));
+    record.setTimestampField(timestampToString(recordRow.getTimestamp(7)));
     // This would figure out that null value will not become "null".
     record.setNullIntField(null);
     return record;
diff --git a/sql/core/src/test/resources/hive-site.xml b/sql/core/src/test/resources/hive-site.xml
index 17297b3e22a7e..4bf6189b73ca9 100644
--- a/sql/core/src/test/resources/hive-site.xml
+++ b/sql/core/src/test/resources/hive-site.xml
@@ -23,4 +23,9 @@
       <value>true</value>
       <description>Internal marker for test.</description>
   </property>
+  <property>
+    <name>hadoop.tmp.dir</name>
+    <value>/tmp/hive_one</value>
+    <description>default is /tmp/hadoop-${user.name} and will be overridden </description>
+  </property>
 </configuration>
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
new file mode 100644
index 0000000000000..b84abe5afc1ef
--- /dev/null
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -0,0 +1,336 @@
+<!-- Automatically generated by ExpressionsSchemaSuite -->
+## Summary
+  - Number of queries: 328
+  - Number of expressions that missing example: 34
+  - Expressions missing examples: and,string,tinyint,double,smallint,date,decimal,boolean,float,binary,bigint,int,timestamp,struct,cume_dist,dense_rank,input_file_block_length,input_file_block_start,input_file_name,lag,lead,monotonically_increasing_id,ntile,!,not,or,percent_rank,rank,row_number,spark_partition_id,version,window,positive,count_min_sketch
+## Schema of Built-in Functions
+| Class name | Function name or alias | Query example | Output schema |
+| ---------- | ---------------------- | ------------- | ------------- |
+| org.apache.spark.sql.catalyst.expressions.Abs | abs | SELECT abs(-1) | struct<abs(-1):int> |
+| org.apache.spark.sql.catalyst.expressions.Acos | acos | SELECT acos(1) | struct<ACOS(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Acosh | acosh | SELECT acosh(1) | struct<ACOSH(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Add | + | SELECT 1 + 2 | struct<(1 + 2):int> |
+| org.apache.spark.sql.catalyst.expressions.AddMonths | add_months | SELECT add_months('2016-08-31', 1) | struct<add_months(CAST(2016-08-31 AS DATE), 1):date> |
+| org.apache.spark.sql.catalyst.expressions.And | and | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.ArrayAggregate | aggregate | SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x) | struct<aggregate(array(1, 2, 3), 0, lambdafunction((namedlambdavariable() + namedlambdavariable()), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):int> |
+| org.apache.spark.sql.catalyst.expressions.ArrayContains | array_contains | SELECT array_contains(array(1, 2, 3), 2) | struct<array_contains(array(1, 2, 3), 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.ArrayDistinct | array_distinct | SELECT array_distinct(array(1, 2, 3, null, 3)) | struct<array_distinct(array(1, 2, 3, CAST(NULL AS INT), 3)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayExcept | array_except | SELECT array_except(array(1, 2, 3), array(1, 3, 5)) | struct<array_except(array(1, 2, 3), array(1, 3, 5)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayExists | exists | SELECT exists(array(1, 2, 3), x -> x % 2 == 0) | struct<exists(array(1, 2, 3), lambdafunction(((namedlambdavariable() % 2) = 0), namedlambdavariable())):boolean> |
+| org.apache.spark.sql.catalyst.expressions.ArrayFilter | filter | SELECT filter(array(1, 2, 3), x -> x % 2 == 1) | struct<filter(array(1, 2, 3), lambdafunction(((namedlambdavariable() % 2) = 1), namedlambdavariable())):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayForAll | forall | SELECT forall(array(1, 2, 3), x -> x % 2 == 0) | struct<forall(array(1, 2, 3), lambdafunction(((namedlambdavariable() % 2) = 0), namedlambdavariable())):boolean> |
+| org.apache.spark.sql.catalyst.expressions.ArrayIntersect | array_intersect | SELECT array_intersect(array(1, 2, 3), array(1, 3, 5)) | struct<array_intersect(array(1, 2, 3), array(1, 3, 5)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayJoin | array_join | SELECT array_join(array('hello', 'world'), ' ') | struct<array_join(array(hello, world),  ):string> |
+| org.apache.spark.sql.catalyst.expressions.ArrayMax | array_max | SELECT array_max(array(1, 20, null, 3)) | struct<array_max(array(1, 20, CAST(NULL AS INT), 3)):int> |
+| org.apache.spark.sql.catalyst.expressions.ArrayMin | array_min | SELECT array_min(array(1, 20, null, 3)) | struct<array_min(array(1, 20, CAST(NULL AS INT), 3)):int> |
+| org.apache.spark.sql.catalyst.expressions.ArrayPosition | array_position | SELECT array_position(array(3, 2, 1), 1) | struct<array_position(array(3, 2, 1), 1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.ArrayRemove | array_remove | SELECT array_remove(array(1, 2, 3, null, 3), 3) | struct<array_remove(array(1, 2, 3, CAST(NULL AS INT), 3), 3):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayRepeat | array_repeat | SELECT array_repeat('123', 2) | struct<array_repeat(123, 2):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.ArraySort | array_sort | SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end) | struct<array_sort(array(5, 6, 1), lambdafunction(CASE WHEN (namedlambdavariable() < namedlambdavariable()) THEN -1 WHEN (namedlambdavariable() > namedlambdavariable()) THEN 1 ELSE 0 END, namedlambdavariable(), namedlambdavariable())):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayTransform | transform | SELECT transform(array(1, 2, 3), x -> x + 1) | struct<transform(array(1, 2, 3), lambdafunction((namedlambdavariable() + 1), namedlambdavariable())):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayUnion | array_union | SELECT array_union(array(1, 2, 3), array(1, 3, 5)) | struct<array_union(array(1, 2, 3), array(1, 3, 5)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArraysOverlap | arrays_overlap | SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)) | struct<arrays_overlap(array(1, 2, 3), array(3, 4, 5)):boolean> |
+| org.apache.spark.sql.catalyst.expressions.ArraysZip | arrays_zip | SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4)) | struct<arrays_zip(array(1, 2, 3), array(2, 3, 4)):array<struct<0:int,1:int>>> |
+| org.apache.spark.sql.catalyst.expressions.Ascii | ascii | SELECT ascii('222') | struct<ascii(222):int> |
+| org.apache.spark.sql.catalyst.expressions.Asin | asin | SELECT asin(0) | struct<ASIN(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Asinh | asinh | SELECT asinh(0) | struct<ASINH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.AssertTrue | assert_true | SELECT assert_true(0 < 1) | struct<assert_true((0 < 1)):null> |
+| org.apache.spark.sql.catalyst.expressions.Atan | atan | SELECT atan(0) | struct<ATAN(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Atan2 | atan2 | SELECT atan2(0, 0) | struct<ATAN2(CAST(0 AS DOUBLE), CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Atanh | atanh | SELECT atanh(0) | struct<ATANH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.BRound | bround | SELECT bround(2.5, 0) | struct<bround(2.5, 0):decimal(2,0)> |
+| org.apache.spark.sql.catalyst.expressions.Base64 | base64 | SELECT base64('Spark SQL') | struct<base64(CAST(Spark SQL AS BINARY)):string> |
+| org.apache.spark.sql.catalyst.expressions.Bin | bin | SELECT bin(13) | struct<bin(CAST(13 AS BIGINT)):string> |
+| org.apache.spark.sql.catalyst.expressions.BitLength | bit_length | SELECT bit_length('Spark SQL') | struct<bit_length(Spark SQL):int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseAnd | & | SELECT 3 & 5 | struct<(3 & 5):int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseCount | bit_count | SELECT bit_count(0) | struct<bit_count(0):int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseNot | ~ | SELECT ~ 0 | struct<~0:int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseOr | &#124; | SELECT 3 &#124; 5 | struct<(3 &#124; 5):int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseXor | ^ | SELECT 3 ^ 5 | struct<(3 ^ 5):int> |
+| org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection | java_method | SELECT java_method('java.util.UUID', 'randomUUID') | struct<java_method(java.util.UUID, randomUUID):string> |
+| org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection | reflect | SELECT reflect('java.util.UUID', 'randomUUID') | struct<reflect(java.util.UUID, randomUUID):string> |
+| org.apache.spark.sql.catalyst.expressions.CaseWhen | when | SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END | struct<CASE WHEN (1 > 0) THEN CAST(1 AS DECIMAL(11,1)) WHEN (2 > 0) THEN CAST(2.0 AS DECIMAL(11,1)) ELSE CAST(1.2 AS DECIMAL(11,1)) END:decimal(11,1)> |
+| org.apache.spark.sql.catalyst.expressions.Cast | string | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | cast | SELECT cast('10' as int) | struct<CAST(10 AS INT):int> |
+| org.apache.spark.sql.catalyst.expressions.Cast | tinyint | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | double | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | smallint | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | date | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | decimal | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | boolean | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | float | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | binary | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | bigint | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | int | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | timestamp | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cbrt | cbrt | SELECT cbrt(27.0) | struct<CBRT(CAST(27.0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Ceil | ceil | SELECT ceil(-0.1) | struct<CEIL(-0.1):decimal(1,0)> |
+| org.apache.spark.sql.catalyst.expressions.Ceil | ceiling | SELECT ceiling(-0.1) | struct<ceiling(-0.1):decimal(1,0)> |
+| org.apache.spark.sql.catalyst.expressions.Chr | char | SELECT char(65) | struct<char(CAST(65 AS BIGINT)):string> |
+| org.apache.spark.sql.catalyst.expressions.Chr | chr | SELECT chr(65) | struct<chr(CAST(65 AS BIGINT)):string> |
+| org.apache.spark.sql.catalyst.expressions.Coalesce | coalesce | SELECT coalesce(NULL, 1, NULL) | struct<coalesce(CAST(NULL AS INT), 1, CAST(NULL AS INT)):int> |
+| org.apache.spark.sql.catalyst.expressions.Concat | concat | SELECT concat('Spark', 'SQL') | struct<concat(Spark, SQL):string> |
+| org.apache.spark.sql.catalyst.expressions.ConcatWs | concat_ws | SELECT concat_ws(' ', 'Spark', 'SQL') | struct<concat_ws( , Spark, SQL):string> |
+| org.apache.spark.sql.catalyst.expressions.Conv | conv | SELECT conv('100', 2, 10) | struct<conv(100, 2, 10):string> |
+| org.apache.spark.sql.catalyst.expressions.Cos | cos | SELECT cos(0) | struct<COS(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Cosh | cosh | SELECT cosh(0) | struct<COSH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Cot | cot | SELECT cot(1) | struct<COT(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Crc32 | crc32 | SELECT crc32('Spark') | struct<crc32(CAST(Spark AS BINARY)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.CreateArray | array | SELECT array(1, 2, 3) | struct<array(1, 2, 3):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.CreateMap | map | SELECT map(1.0, '2', 3.0, '4') | struct<map(1.0, 2, 3.0, 4):map<decimal(2,1),string>> |
+| org.apache.spark.sql.catalyst.expressions.CreateNamedStruct | named_struct | SELECT named_struct("a", 1, "b", 2, "c", 3) | struct<named_struct(a, 1, b, 2, c, 3):struct<a:int,b:int,c:int>> |
+| org.apache.spark.sql.catalyst.expressions.CreateNamedStruct | struct | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.CsvToStructs | from_csv | SELECT from_csv('1, 0.8', 'a INT, b DOUBLE') | struct<from_csv(1, 0.8):struct<a:int,b:double>> |
+| org.apache.spark.sql.catalyst.expressions.Cube | cube | SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name, age) | struct<name:string,age:int,count(1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.CumeDist | cume_dist | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.CurrentDatabase | current_database | SELECT current_database() | struct<current_database():string> |
+| org.apache.spark.sql.catalyst.expressions.CurrentDate | current_date | SELECT current_date() | struct<current_date():date> |
+| org.apache.spark.sql.catalyst.expressions.CurrentTimestamp | current_timestamp | SELECT current_timestamp() | struct<current_timestamp():timestamp> |
+| org.apache.spark.sql.catalyst.expressions.DateAdd | date_add | SELECT date_add('2016-07-30', 1) | struct<date_add(CAST(2016-07-30 AS DATE), 1):date> |
+| org.apache.spark.sql.catalyst.expressions.DateDiff | datediff | SELECT datediff('2009-07-31', '2009-07-30') | struct<datediff(CAST(2009-07-31 AS DATE), CAST(2009-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.DateFormatClass | date_format | SELECT date_format('2016-04-08', 'y') | struct<date_format(CAST(2016-04-08 AS TIMESTAMP), y):string> |
+| org.apache.spark.sql.catalyst.expressions.DatePart | date_part | SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456') | struct<date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'):int> |
+| org.apache.spark.sql.catalyst.expressions.DateSub | date_sub | SELECT date_sub('2016-07-30', 1) | struct<date_sub(CAST(2016-07-30 AS DATE), 1):date> |
+| org.apache.spark.sql.catalyst.expressions.DayOfMonth | day | SELECT day('2009-07-30') | struct<day(CAST(2009-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.DayOfMonth | dayofmonth | SELECT dayofmonth('2009-07-30') | struct<dayofmonth(CAST(2009-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.DayOfWeek | dayofweek | SELECT dayofweek('2009-07-30') | struct<dayofweek(CAST(2009-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.DayOfYear | dayofyear | SELECT dayofyear('2016-04-09') | struct<dayofyear(CAST(2016-04-09 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.Decode | decode | SELECT decode(encode('abc', 'utf-8'), 'utf-8') | struct<decode(encode(abc, utf-8), utf-8):string> |
+| org.apache.spark.sql.catalyst.expressions.DenseRank | dense_rank | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Divide | / | SELECT 3 / 2 | struct<(CAST(3 AS DOUBLE) / CAST(2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.ElementAt | element_at | SELECT element_at(array(1, 2, 3), 2) | struct<element_at(array(1, 2, 3), 2):int> |
+| org.apache.spark.sql.catalyst.expressions.Elt | elt | SELECT elt(1, 'scala', 'java') | struct<elt(1, scala, java):string> |
+| org.apache.spark.sql.catalyst.expressions.Encode | encode | SELECT encode('abc', 'utf-8') | struct<encode(abc, utf-8):binary> |
+| org.apache.spark.sql.catalyst.expressions.EqualNullSafe | <=> | SELECT 2 <=> 2 | struct<(2 <=> 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.EqualTo | = | SELECT 2 = 2 | struct<(2 = 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.EqualTo | == | SELECT 2 == 2 | struct<(2 = 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.EulerNumber | e | SELECT e() | struct<E():double> |
+| org.apache.spark.sql.catalyst.expressions.Exp | exp | SELECT exp(0) | struct<EXP(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Explode | explode | SELECT explode(array(10, 20)) | struct<col:int> |
+| org.apache.spark.sql.catalyst.expressions.Explode | explode_outer | SELECT explode_outer(array(10, 20)) | struct<col:int> |
+| org.apache.spark.sql.catalyst.expressions.Expm1 | expm1 | SELECT expm1(0) | struct<EXPM1(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Extract | extract | SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456') | struct<extract('YEAR' FROM TIMESTAMP '2019-08-12 01:00:00.123456'):int> |
+| org.apache.spark.sql.catalyst.expressions.Factorial | factorial | SELECT factorial(5) | struct<factorial(5):bigint> |
+| org.apache.spark.sql.catalyst.expressions.FindInSet | find_in_set | SELECT find_in_set('ab','abc,b,ab,c,def') | struct<find_in_set(ab, abc,b,ab,c,def):int> |
+| org.apache.spark.sql.catalyst.expressions.Flatten | flatten | SELECT flatten(array(array(1, 2), array(3, 4))) | struct<flatten(array(array(1, 2), array(3, 4))):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.Floor | floor | SELECT floor(-0.1) | struct<FLOOR(-0.1):decimal(1,0)> |
+| org.apache.spark.sql.catalyst.expressions.FormatNumber | format_number | SELECT format_number(12332.123456, 4) | struct<format_number(12332.123456, 4):string> |
+| org.apache.spark.sql.catalyst.expressions.FormatString | printf | SELECT printf("Hello World %d %s", 100, "days") | struct<printf(Hello World %d %s, 100, days):string> |
+| org.apache.spark.sql.catalyst.expressions.FormatString | format_string | SELECT format_string("Hello World %d %s", 100, "days") | struct<format_string(Hello World %d %s, 100, days):string> |
+| org.apache.spark.sql.catalyst.expressions.FromUTCTimestamp | from_utc_timestamp | SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul') | struct<from_utc_timestamp(CAST(2016-08-31 AS TIMESTAMP), Asia/Seoul):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.FromUnixTime | from_unixtime | SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss') | struct<from_unixtime(CAST(0 AS BIGINT), yyyy-MM-dd HH:mm:ss):string> |
+| org.apache.spark.sql.catalyst.expressions.GetJsonObject | get_json_object | SELECT get_json_object('{"a":"b"}', '$.a') | struct<get_json_object({"a":"b"}, $.a):string> |
+| org.apache.spark.sql.catalyst.expressions.GreaterThan | > | SELECT 2 > 1 | struct<(2 > 1):boolean> |
+| org.apache.spark.sql.catalyst.expressions.GreaterThanOrEqual | >= | SELECT 2 >= 1 | struct<(2 >= 1):boolean> |
+| org.apache.spark.sql.catalyst.expressions.Greatest | greatest | SELECT greatest(10, 9, 2, 4, 3) | struct<greatest(10, 9, 2, 4, 3):int> |
+| org.apache.spark.sql.catalyst.expressions.Grouping | grouping | SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name) | struct<name:string,grouping(name):tinyint,sum(age):bigint> |
+| org.apache.spark.sql.catalyst.expressions.GroupingID | grouping_id | SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height) | struct<name:string,grouping_id():int,sum(age):bigint,avg(height):double> |
+| org.apache.spark.sql.catalyst.expressions.Hex | hex | SELECT hex(17) | struct<hex(CAST(17 AS BIGINT)):string> |
+| org.apache.spark.sql.catalyst.expressions.Hour | hour | SELECT hour('2009-07-30 12:58:59') | struct<hour(CAST(2009-07-30 12:58:59 AS TIMESTAMP)):int> |
+| org.apache.spark.sql.catalyst.expressions.Hypot | hypot | SELECT hypot(3, 4) | struct<HYPOT(CAST(3 AS DOUBLE), CAST(4 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.If | if | SELECT if(1 < 2, 'a', 'b') | struct<(IF((1 < 2), a, b)):string> |
+| org.apache.spark.sql.catalyst.expressions.IfNull | ifnull | SELECT ifnull(NULL, array('2')) | struct<ifnull(NULL, array('2')):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.In | in | SELECT 1 in(1, 2, 3) | struct<(1 IN (1, 2, 3)):boolean> |
+| org.apache.spark.sql.catalyst.expressions.InitCap | initcap | SELECT initcap('sPark sql') | struct<initcap(sPark sql):string> |
+| org.apache.spark.sql.catalyst.expressions.Inline | inline | SELECT inline(array(struct(1, 'a'), struct(2, 'b'))) | struct<col1:int,col2:string> |
+| org.apache.spark.sql.catalyst.expressions.Inline | inline_outer | SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b'))) | struct<col1:int,col2:string> |
+| org.apache.spark.sql.catalyst.expressions.InputFileBlockLength | input_file_block_length | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.InputFileBlockStart | input_file_block_start | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.InputFileName | input_file_name | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.IntegralDivide | div | SELECT 3 div 2 | struct<(CAST(3 AS BIGINT) div CAST(2 AS BIGINT)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.IsNaN | isnan | SELECT isnan(cast('NaN' as double)) | struct<isnan(CAST(NaN AS DOUBLE)):boolean> |
+| org.apache.spark.sql.catalyst.expressions.IsNotNull | isnotnull | SELECT isnotnull(1) | struct<(1 IS NOT NULL):boolean> |
+| org.apache.spark.sql.catalyst.expressions.IsNull | isnull | SELECT isnull(1) | struct<(1 IS NULL):boolean> |
+| org.apache.spark.sql.catalyst.expressions.JsonToStructs | from_json | SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE') | struct<from_json({"a":1, "b":0.8}):struct<a:int,b:double>> |
+| org.apache.spark.sql.catalyst.expressions.JsonTuple | json_tuple | SELECT json_tuple('{"a":1, "b":2}', 'a', 'b') | struct<c0:string,c1:string> |
+| org.apache.spark.sql.catalyst.expressions.Lag | lag | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.LastDay | last_day | SELECT last_day('2009-01-12') | struct<last_day(CAST(2009-01-12 AS DATE)):date> |
+| org.apache.spark.sql.catalyst.expressions.Lead | lead | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Least | least | SELECT least(10, 9, 2, 4, 3) | struct<least(10, 9, 2, 4, 3):int> |
+| org.apache.spark.sql.catalyst.expressions.Left | left | SELECT left('Spark SQL', 3) | struct<left('Spark SQL', 3):string> |
+| org.apache.spark.sql.catalyst.expressions.Length | character_length | SELECT character_length('Spark SQL ') | struct<character_length(Spark SQL ):int> |
+| org.apache.spark.sql.catalyst.expressions.Length | char_length | SELECT char_length('Spark SQL ') | struct<char_length(Spark SQL ):int> |
+| org.apache.spark.sql.catalyst.expressions.Length | length | SELECT length('Spark SQL ') | struct<length(Spark SQL ):int> |
+| org.apache.spark.sql.catalyst.expressions.LessThan | < | SELECT 1 < 2 | struct<(1 < 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.LessThanOrEqual | <= | SELECT 2 <= 2 | struct<(2 <= 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.Levenshtein | levenshtein | SELECT levenshtein('kitten', 'sitting') | struct<levenshtein(kitten, sitting):int> |
+| org.apache.spark.sql.catalyst.expressions.Like | like | SELECT like('Spark', '_park') | struct<Spark LIKE _park:boolean> |
+| org.apache.spark.sql.catalyst.expressions.Log | ln | SELECT ln(1) | struct<ln(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Log10 | log10 | SELECT log10(10) | struct<LOG10(CAST(10 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Log1p | log1p | SELECT log1p(0) | struct<LOG1P(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Log2 | log2 | SELECT log2(2) | struct<LOG2(CAST(2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Logarithm | log | SELECT log(10, 100) | struct<LOG(CAST(10 AS DOUBLE), CAST(100 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Lower | lcase | SELECT lcase('SparkSql') | struct<lcase(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.Lower | lower | SELECT lower('SparkSql') | struct<lower(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.MakeDate | make_date | SELECT make_date(2013, 7, 15) | struct<make_date(2013, 7, 15):date> |
+| org.apache.spark.sql.catalyst.expressions.MakeInterval | make_interval | SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001) | struct<make_interval(100, 11, 1, 1, 12, 30, CAST(1.001001 AS DECIMAL(18,6))):interval> |
+| org.apache.spark.sql.catalyst.expressions.MakeTimestamp | make_timestamp | SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887) | struct<make_timestamp(2014, 12, 28, 6, 30, CAST(45.887 AS DECIMAL(8,6))):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.MapConcat | map_concat | SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c')) | struct<map_concat(map(1, a, 2, b), map(3, c)):map<int,string>> |
+| org.apache.spark.sql.catalyst.expressions.MapEntries | map_entries | SELECT map_entries(map(1, 'a', 2, 'b')) | struct<map_entries(map(1, a, 2, b)):array<struct<key:int,value:string>>> |
+| org.apache.spark.sql.catalyst.expressions.MapFilter | map_filter | SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v) | struct<map_filter(map(1, 0, 2, 2, 3, -1), lambdafunction((namedlambdavariable() > namedlambdavariable()), namedlambdavariable(), namedlambdavariable())):map<int,int>> |
+| org.apache.spark.sql.catalyst.expressions.MapFromArrays | map_from_arrays | SELECT map_from_arrays(array(1.0, 3.0), array('2', '4')) | struct<map_from_arrays(array(1.0, 3.0), array(2, 4)):map<decimal(2,1),string>> |
+| org.apache.spark.sql.catalyst.expressions.MapFromEntries | map_from_entries | SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))) | struct<map_from_entries(array(struct(1, a), struct(2, b))):map<int,string>> |
+| org.apache.spark.sql.catalyst.expressions.MapKeys | map_keys | SELECT map_keys(map(1, 'a', 2, 'b')) | struct<map_keys(map(1, a, 2, b)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.MapValues | map_values | SELECT map_values(map(1, 'a', 2, 'b')) | struct<map_values(map(1, a, 2, b)):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.MapZipWith | map_zip_with | SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)) | struct<map_zip_with(map(1, a, 2, b), map(1, x, 2, y), lambdafunction(concat(namedlambdavariable(), namedlambdavariable()), namedlambdavariable(), namedlambdavariable(), namedlambdavariable())):map<int,string>> |
+| org.apache.spark.sql.catalyst.expressions.Md5 | md5 | SELECT md5('Spark') | struct<md5(CAST(Spark AS BINARY)):string> |
+| org.apache.spark.sql.catalyst.expressions.Minute | minute | SELECT minute('2009-07-30 12:58:59') | struct<minute(CAST(2009-07-30 12:58:59 AS TIMESTAMP)):int> |
+| org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID | monotonically_increasing_id | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Month | month | SELECT month('2016-07-30') | struct<month(CAST(2016-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.MonthsBetween | months_between | SELECT months_between('1997-02-28 10:30:00', '1996-10-30') | struct<months_between(CAST(1997-02-28 10:30:00 AS TIMESTAMP), CAST(1996-10-30 AS TIMESTAMP), true):double> |
+| org.apache.spark.sql.catalyst.expressions.Multiply | * | SELECT 2 * 3 | struct<(2 * 3):int> |
+| org.apache.spark.sql.catalyst.expressions.Murmur3Hash | hash | SELECT hash('Spark', array(123), 2) | struct<hash(Spark, array(123), 2):int> |
+| org.apache.spark.sql.catalyst.expressions.NTile | ntile | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.NaNvl | nanvl | SELECT nanvl(cast('NaN' as double), 123) | struct<nanvl(CAST(NaN AS DOUBLE), CAST(123 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.NextDay | next_day | SELECT next_day('2015-01-14', 'TU') | struct<next_day(CAST(2015-01-14 AS DATE), TU):date> |
+| org.apache.spark.sql.catalyst.expressions.Not | ! | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Not | not | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Now | now | SELECT now() | struct<now():timestamp> |
+| org.apache.spark.sql.catalyst.expressions.NullIf | nullif | SELECT nullif(2, 2) | struct<nullif(2, 2):int> |
+| org.apache.spark.sql.catalyst.expressions.Nvl | nvl | SELECT nvl(NULL, array('2')) | struct<nvl(NULL, array('2')):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.Nvl2 | nvl2 | SELECT nvl2(NULL, 2, 1) | struct<nvl2(NULL, 2, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.OctetLength | octet_length | SELECT octet_length('Spark SQL') | struct<octet_length(Spark SQL):int> |
+| org.apache.spark.sql.catalyst.expressions.Or | or | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Overlay | overlay | SELECT overlay('Spark SQL' PLACING '_' FROM 6) | struct<overlay(Spark SQL, _, 6, -1):string> |
+| org.apache.spark.sql.catalyst.expressions.ParseToDate | to_date | SELECT to_date('2009-07-30 04:17:52') | struct<to_date('2009-07-30 04:17:52'):date> |
+| org.apache.spark.sql.catalyst.expressions.ParseToTimestamp | to_timestamp | SELECT to_timestamp('2016-12-31 00:12:00') | struct<to_timestamp('2016-12-31 00:12:00'):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.ParseUrl | parse_url | SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST') | struct<parse_url(http://spark.apache.org/path?query=1, HOST):string> |
+| org.apache.spark.sql.catalyst.expressions.PercentRank | percent_rank | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Pi | pi | SELECT pi() | struct<PI():double> |
+| org.apache.spark.sql.catalyst.expressions.Pmod | pmod | SELECT pmod(10, 3) | struct<pmod(10, 3):int> |
+| org.apache.spark.sql.catalyst.expressions.PosExplode | posexplode_outer | SELECT posexplode_outer(array(10,20)) | struct<pos:int,col:int> |
+| org.apache.spark.sql.catalyst.expressions.PosExplode | posexplode | SELECT posexplode(array(10,20)) | struct<pos:int,col:int> |
+| org.apache.spark.sql.catalyst.expressions.Pow | pow | SELECT pow(2, 3) | struct<pow(CAST(2 AS DOUBLE), CAST(3 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Pow | power | SELECT power(2, 3) | struct<POWER(CAST(2 AS DOUBLE), CAST(3 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Quarter | quarter | SELECT quarter('2016-08-31') | struct<quarter(CAST(2016-08-31 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.RLike | rlike | SELECT '%SystemDrive%\Users\John' rlike '%SystemDrive%\\Users.*' | struct<%SystemDrive%UsersJohn RLIKE %SystemDrive%\Users.*:boolean> |
+| org.apache.spark.sql.catalyst.expressions.Rank | rank | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.RegExpExtract | regexp_extract | SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1) | struct<regexp_extract(100-200, (\d+)-(\d+), 1):string> |
+| org.apache.spark.sql.catalyst.expressions.RegExpReplace | regexp_replace | SELECT regexp_replace('100-200', '(\\d+)', 'num') | struct<regexp_replace(100-200, (\d+), num):string> |
+| org.apache.spark.sql.catalyst.expressions.Remainder | % | SELECT 2 % 1.8 | struct<(CAST(CAST(2 AS DECIMAL(1,0)) AS DECIMAL(2,1)) % CAST(1.8 AS DECIMAL(2,1))):decimal(2,1)> |
+| org.apache.spark.sql.catalyst.expressions.Remainder | mod | SELECT 2 % 1.8 | struct<(CAST(CAST(2 AS DECIMAL(1,0)) AS DECIMAL(2,1)) % CAST(1.8 AS DECIMAL(2,1))):decimal(2,1)> |
+| org.apache.spark.sql.catalyst.expressions.Reverse | reverse | SELECT reverse('Spark SQL') | struct<reverse(Spark SQL):string> |
+| org.apache.spark.sql.catalyst.expressions.Right | right | SELECT right('Spark SQL', 3) | struct<right('Spark SQL', 3):string> |
+| org.apache.spark.sql.catalyst.expressions.Rint | rint | SELECT rint(12.3456) | struct<rint(CAST(12.3456 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Rollup | rollup | SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY rollup(name, age) | struct<name:string,age:int,count(1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.Round | round | SELECT round(2.5, 0) | struct<round(2.5, 0):decimal(2,0)> |
+| org.apache.spark.sql.catalyst.expressions.RowNumber | row_number | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.SchemaOfCsv | schema_of_csv | SELECT schema_of_csv('1,abc') | struct<schema_of_csv(1,abc):string> |
+| org.apache.spark.sql.catalyst.expressions.SchemaOfJson | schema_of_json | SELECT schema_of_json('[{"col":0}]') | struct<schema_of_json([{"col":0}]):string> |
+| org.apache.spark.sql.catalyst.expressions.Second | second | SELECT second('2009-07-30 12:58:59') | struct<second(CAST(2009-07-30 12:58:59 AS TIMESTAMP)):int> |
+| org.apache.spark.sql.catalyst.expressions.Sentences | sentences | SELECT sentences('Hi there! Good morning.') | struct<sentences(Hi there! Good morning., , ):array<array<string>>> |
+| org.apache.spark.sql.catalyst.expressions.Sequence | sequence | SELECT sequence(1, 5) | struct<sequence(1, 5):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.Sha1 | sha1 | SELECT sha1('Spark') | struct<sha1(CAST(Spark AS BINARY)):string> |
+| org.apache.spark.sql.catalyst.expressions.Sha1 | sha | SELECT sha('Spark') | struct<sha(CAST(Spark AS BINARY)):string> |
+| org.apache.spark.sql.catalyst.expressions.Sha2 | sha2 | SELECT sha2('Spark', 256) | struct<sha2(CAST(Spark AS BINARY), 256):string> |
+| org.apache.spark.sql.catalyst.expressions.ShiftLeft | shiftleft | SELECT shiftleft(2, 1) | struct<shiftleft(2, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.ShiftRight | shiftright | SELECT shiftright(4, 1) | struct<shiftright(4, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.ShiftRightUnsigned | shiftrightunsigned | SELECT shiftrightunsigned(4, 1) | struct<shiftrightunsigned(4, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.Shuffle | shuffle | SELECT shuffle(array(1, 20, 3, 5)) | struct<shuffle(array(1, 20, 3, 5)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.Signum | signum | SELECT signum(40) | struct<SIGNUM(CAST(40 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Signum | sign | SELECT sign(40) | struct<sign(CAST(40 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Sin | sin | SELECT sin(0) | struct<SIN(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Sinh | sinh | SELECT sinh(0) | struct<SINH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Size | size | SELECT size(array('b', 'd', 'c', 'a')) | struct<size(array(b, d, c, a)):int> |
+| org.apache.spark.sql.catalyst.expressions.Size | cardinality | SELECT cardinality(array('b', 'd', 'c', 'a')) | struct<cardinality(array(b, d, c, a)):int> |
+| org.apache.spark.sql.catalyst.expressions.Slice | slice | SELECT slice(array(1, 2, 3, 4), 2, 2) | struct<slice(array(1, 2, 3, 4), 2, 2):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.SortArray | sort_array | SELECT sort_array(array('b', 'd', null, 'c', 'a'), true) | struct<sort_array(array(b, d, CAST(NULL AS STRING), c, a), true):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.SoundEx | soundex | SELECT soundex('Miller') | struct<soundex(Miller):string> |
+| org.apache.spark.sql.catalyst.expressions.SparkPartitionID | spark_partition_id | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.SparkVersion | version | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Sqrt | sqrt | SELECT sqrt(4) | struct<SQRT(CAST(4 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Stack | stack | SELECT stack(2, 1, 2, 3) | struct<col0:int,col1:int> |
+| org.apache.spark.sql.catalyst.expressions.StringInstr | instr | SELECT instr('SparkSQL', 'SQL') | struct<instr(SparkSQL, SQL):int> |
+| org.apache.spark.sql.catalyst.expressions.StringLPad | lpad | SELECT lpad('hi', 5, '??') | struct<lpad(hi, 5, ??):string> |
+| org.apache.spark.sql.catalyst.expressions.StringLocate | position | SELECT position('bar', 'foobarbar') | struct<position(bar, foobarbar, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.StringLocate | locate | SELECT locate('bar', 'foobarbar') | struct<locate(bar, foobarbar, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.StringRPad | rpad | SELECT rpad('hi', 5, '??') | struct<rpad(hi, 5, ??):string> |
+| org.apache.spark.sql.catalyst.expressions.StringRepeat | repeat | SELECT repeat('123', 2) | struct<repeat(123, 2):string> |
+| org.apache.spark.sql.catalyst.expressions.StringReplace | replace | SELECT replace('ABCabc', 'abc', 'DEF') | struct<replace(ABCabc, abc, DEF):string> |
+| org.apache.spark.sql.catalyst.expressions.StringSpace | space | SELECT concat(space(2), '1') | struct<concat(space(2), 1):string> |
+| org.apache.spark.sql.catalyst.expressions.StringSplit | split | SELECT split('oneAtwoBthreeC', '[ABC]') | struct<split(oneAtwoBthreeC, [ABC], -1):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.StringToMap | str_to_map | SELECT str_to_map('a:1,b:2,c:3', ',', ':') | struct<str_to_map(a:1,b:2,c:3, ,, :):map<string,string>> |
+| org.apache.spark.sql.catalyst.expressions.StringTranslate | translate | SELECT translate('AaBbCc', 'abc', '123') | struct<translate(AaBbCc, abc, 123):string> |
+| org.apache.spark.sql.catalyst.expressions.StringTrim | trim | SELECT trim('    SparkSQL   ') | struct<trim(    SparkSQL   ):string> |
+| org.apache.spark.sql.catalyst.expressions.StringTrimLeft | ltrim | SELECT ltrim('    SparkSQL   ') | struct<ltrim(    SparkSQL   ):string> |
+| org.apache.spark.sql.catalyst.expressions.StringTrimRight | rtrim | SELECT rtrim('    SparkSQL   ') | struct<rtrim(    SparkSQL   ):string> |
+| org.apache.spark.sql.catalyst.expressions.StructsToCsv | to_csv | SELECT to_csv(named_struct('a', 1, 'b', 2)) | struct<to_csv(named_struct(a, 1, b, 2)):string> |
+| org.apache.spark.sql.catalyst.expressions.StructsToJson | to_json | SELECT to_json(named_struct('a', 1, 'b', 2)) | struct<to_json(named_struct(a, 1, b, 2)):string> |
+| org.apache.spark.sql.catalyst.expressions.Substring | substr | SELECT substr('Spark SQL', 5) | struct<substr(Spark SQL, 5, 2147483647):string> |
+| org.apache.spark.sql.catalyst.expressions.Substring | substring | SELECT substring('Spark SQL', 5) | struct<substring(Spark SQL, 5, 2147483647):string> |
+| org.apache.spark.sql.catalyst.expressions.SubstringIndex | substring_index | SELECT substring_index('www.apache.org', '.', 2) | struct<substring_index(www.apache.org, ., 2):string> |
+| org.apache.spark.sql.catalyst.expressions.Subtract | - | SELECT 2 - 1 | struct<(2 - 1):int> |
+| org.apache.spark.sql.catalyst.expressions.Tan | tan | SELECT tan(0) | struct<TAN(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Tanh | tanh | SELECT tanh(0) | struct<TANH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.TimeWindow | window | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.ToDegrees | degrees | SELECT degrees(3.141592653589793) | struct<DEGREES(CAST(3.141592653589793 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.ToRadians | radians | SELECT radians(180) | struct<RADIANS(CAST(180 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.ToUTCTimestamp | to_utc_timestamp | SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul') | struct<to_utc_timestamp(CAST(2016-08-31 AS TIMESTAMP), Asia/Seoul):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.ToUnixTimestamp | to_unix_timestamp | SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd') | struct<to_unix_timestamp(2016-04-08, yyyy-MM-dd):bigint> |
+| org.apache.spark.sql.catalyst.expressions.TransformKeys | transform_keys | SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1) | struct<transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), lambdafunction((namedlambdavariable() + 1), namedlambdavariable(), namedlambdavariable())):map<int,int>> |
+| org.apache.spark.sql.catalyst.expressions.TransformValues | transform_values | SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1) | struct<transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), lambdafunction((namedlambdavariable() + 1), namedlambdavariable(), namedlambdavariable())):map<int,int>> |
+| org.apache.spark.sql.catalyst.expressions.TruncDate | trunc | SELECT trunc('2019-08-04', 'week') | struct<trunc(CAST(2019-08-04 AS DATE), week):date> |
+| org.apache.spark.sql.catalyst.expressions.TruncTimestamp | date_trunc | SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359') | struct<date_trunc(YEAR, CAST(2015-03-05T09:32:05.359 AS TIMESTAMP)):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.TypeOf | typeof | SELECT typeof(1) | struct<typeof(1):string> |
+| org.apache.spark.sql.catalyst.expressions.UnBase64 | unbase64 | SELECT unbase64('U3BhcmsgU1FM') | struct<unbase64(U3BhcmsgU1FM):binary> |
+| org.apache.spark.sql.catalyst.expressions.UnaryMinus | negative | SELECT negative(1) | struct<negative(1):int> |
+| org.apache.spark.sql.catalyst.expressions.UnaryPositive | positive | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Unhex | unhex | SELECT decode(unhex('537061726B2053514C'), 'UTF-8') | struct<decode(unhex(537061726B2053514C), UTF-8):string> |
+| org.apache.spark.sql.catalyst.expressions.UnixTimestamp | unix_timestamp | SELECT unix_timestamp() | struct<unix_timestamp(current_timestamp(), yyyy-MM-dd HH:mm:ss):bigint> |
+| org.apache.spark.sql.catalyst.expressions.Upper | ucase | SELECT ucase('SparkSql') | struct<ucase(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.Upper | upper | SELECT upper('SparkSql') | struct<upper(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.Uuid | uuid | SELECT uuid() | struct<uuid():string> |
+| org.apache.spark.sql.catalyst.expressions.WeekDay | weekday | SELECT weekday('2009-07-30') | struct<weekday(CAST(2009-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.WeekOfYear | weekofyear | SELECT weekofyear('2008-02-20') | struct<weekofyear(CAST(2008-02-20 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.XxHash64 | xxhash64 | SELECT xxhash64('Spark', array(123), 2) | struct<xxhash64(Spark, array(123), 2):bigint> |
+| org.apache.spark.sql.catalyst.expressions.Year | year | SELECT year('2016-07-30') | struct<year(CAST(2016-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.ZipWith | zip_with | SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)) | struct<zip_with(array(1, 2, 3), array(a, b, c), lambdafunction(named_struct(y, namedlambdavariable(), x, namedlambdavariable()), namedlambdavariable(), namedlambdavariable())):array<struct<y:string,x:int>>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | approx_percentile | SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col) | struct<approx_percentile(col, array(0.5, 0.4, 0.1), 100):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | percentile_approx | SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col) | struct<percentile_approx(col, array(0.5, 0.4, 0.1), 100):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Average | avg | SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<avg(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Average | mean | SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<mean(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BitAndAgg | bit_and | SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col) | struct<bit_and(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BitOrAgg | bit_or | SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col) | struct<bit_or(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BitXorAgg | bit_xor | SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col) | struct<bit_xor(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolAnd | every | SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col) | struct<every(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolAnd | bool_and | SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col) | struct<bool_and(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr | bool_or | SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col) | struct<bool_or(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr | some | SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col) | struct<some(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr | any | SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col) | struct<any(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CollectList | collect_list | SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col) | struct<collect_list(col):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CollectSet | collect_set | SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col) | struct<collect_set(col):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Corr | corr | SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2) | struct<corr(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Count | count | SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col) | struct<count(1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CountIf | count_if | SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col) | struct<count_if(((col % 2) = 0)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CountMinSketchAgg | count_min_sketch | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CovPopulation | covar_pop | SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2) | struct<covar_pop(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CovSample | covar_samp | SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2) | struct<covar_samp(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.First | first_value | SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first_value(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.First | first | SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.HyperLogLogPlusPlus | approx_count_distinct | SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1) | struct<approx_count_distinct(col1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Kurtosis | kurtosis | SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<kurtosis(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last_value | SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last_value(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last | SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Max | max | SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col) | struct<max(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.MaxBy | max_by | SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<max_by(x, y):string> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Min | min | SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col) | struct<min(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.MinBy | min_by | SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<min_by(x, y):string> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Percentile | percentile | SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col) | struct<percentile(col, CAST(0.3 AS DOUBLE), 1):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Skewness | skewness | SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<skewness(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevPop | stddev_pop | SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev_pop(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | stddev_samp | SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev_samp(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | stddev | SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | std | SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<std(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Sum | sum | SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col) | struct<sum(col):bigint> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.VariancePop | var_pop | SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<var_pop(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.VarianceSamp | var_samp | SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<var_samp(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.VarianceSamp | variance | SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<variance(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathBoolean | xpath_boolean | SELECT xpath_boolean('<a><b>1</b></a>','a/b') | struct<xpath_boolean(<a><b>1</b></a>, a/b):boolean> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathDouble | xpath_number | SELECT xpath_number('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_number(<a><b>1</b><b>2</b></a>, sum(a/b)):double> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathDouble | xpath_double | SELECT xpath_double('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_double(<a><b>1</b><b>2</b></a>, sum(a/b)):double> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathFloat | xpath_float | SELECT xpath_float('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_float(<a><b>1</b><b>2</b></a>, sum(a/b)):float> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathInt | xpath_int | SELECT xpath_int('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_int(<a><b>1</b><b>2</b></a>, sum(a/b)):int> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathList | xpath | SELECT xpath('<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>','a/b/text()') | struct<xpath(<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>, a/b/text()):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathLong | xpath_long | SELECT xpath_long('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_long(<a><b>1</b><b>2</b></a>, sum(a/b)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathShort | xpath_short | SELECT xpath_short('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_short(<a><b>1</b><b>2</b></a>, sum(a/b)):smallint> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathString | xpath_string | SELECT xpath_string('<a><b>b</b><c>cc</c></a>','a/c') | struct<xpath_string(<a><b>b</b><c>cc</c></a>, a/c):string> |
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/datetime.sql
new file mode 100644
index 0000000000000..58ecf80637883
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/datetime.sql
@@ -0,0 +1 @@
+--IMPORT datetime.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/string-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/string-functions.sql
new file mode 100644
index 0000000000000..dd28e9b97fb20
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/string-functions.sql
@@ -0,0 +1 @@
+--IMPORT string-functions.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cast.sql b/sql/core/src/test/resources/sql-tests/inputs/cast.sql
index 972ebdd01f61e..81c741a5ca8e9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cast.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cast.sql
@@ -70,6 +70,11 @@ select cast(' 1' as bigint);
 select cast(' 1' as float);
 select cast(' 1 ' as DOUBLE);
 select cast('1.0 ' as DEC);
+select cast('1中文' as tinyint);
+select cast('1中文' as smallint);
+select cast('1中文' as INT);
+select cast('中文1' as bigint);
+select cast('1中文' as bigint);
 
 -- trim string before cast to boolean
 select cast('\t\t true \n\r ' as boolean);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/change-column.sql b/sql/core/src/test/resources/sql-tests/inputs/change-column.sql
index dd2fc660b53e3..2b57891cfcbc5 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/change-column.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/change-column.sql
@@ -15,29 +15,34 @@ ALTER TABLE test_change CHANGE a TYPE STRING;
 DESC test_change;
 
 -- Change column position (not supported yet)
-ALTER TABLE test_change CHANGE a TYPE INT AFTER b;
-ALTER TABLE test_change CHANGE b TYPE STRING FIRST;
+ALTER TABLE test_change CHANGE a AFTER b;
+ALTER TABLE test_change CHANGE b FIRST;
 DESC test_change;
 
 -- Change column comment
-ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a';
-ALTER TABLE test_change CHANGE b TYPE STRING COMMENT '#*02?`';
-ALTER TABLE test_change CHANGE c TYPE INT COMMENT '';
+ALTER TABLE test_change CHANGE a COMMENT 'this is column a';
+ALTER TABLE test_change CHANGE b COMMENT '#*02?`';
+ALTER TABLE test_change CHANGE c COMMENT '';
 DESC test_change;
 
 -- Don't change anything.
-ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a';
+ALTER TABLE test_change CHANGE a TYPE INT;
+ALTER TABLE test_change CHANGE a COMMENT 'this is column a';
 DESC test_change;
 
 -- Change a invalid column
 ALTER TABLE test_change CHANGE invalid_col TYPE INT;
 DESC test_change;
 
+-- Check case insensitivity.
+ALTER TABLE test_change CHANGE A COMMENT 'case insensitivity';
+DESC test_change;
+
 -- Change column can't apply to a temporary/global_temporary view
 CREATE TEMPORARY VIEW temp_view(a, b) AS SELECT 1, "one";
-ALTER TABLE temp_view CHANGE a TYPE INT COMMENT 'this is column a';
+ALTER TABLE temp_view CHANGE a TYPE INT;
 CREATE GLOBAL TEMPORARY VIEW global_temp_view(a, b) AS SELECT 1, "one";
-ALTER TABLE global_temp.global_temp_view CHANGE a TYPE INT COMMENT 'this is column a';
+ALTER TABLE global_temp.global_temp_view CHANGE a TYPE INT;
 
 -- DROP TEST TABLE
 DROP TABLE test_change;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql
index 2f2606d44d910..29dee1a3afd38 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql
@@ -1,115 +1,2 @@
-create temporary view t as select * from values 0, 1, 2 as t(id);
-create temporary view t2 as select * from values 0, 1 as t(id);
-
--- CTE legacy substitution
-SET spark.sql.legacy.ctePrecedence.enabled=true;
-
--- CTE in CTE definition
-WITH t as (
-  WITH t2 AS (SELECT 1)
-  SELECT * FROM t2
-)
-SELECT * FROM t;
-
--- CTE in subquery
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 1)
-  SELECT * FROM t
-);
-
--- CTE in subquery expression
-SELECT (
-  WITH t AS (SELECT 1)
-  SELECT * FROM t
-);
-
--- CTE in CTE definition shadows outer
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-SELECT * FROM t2;
-
--- CTE in CTE definition shadows outer 2
-WITH
-  t(c) AS (SELECT 1),
-  t2 AS (
-    SELECT (
-      SELECT max(c) FROM (
-        WITH t(c) AS (SELECT 2)
-        SELECT * FROM t
-      )
-    )
-  )
-SELECT * FROM t2;
-
--- CTE in CTE definition shadows outer 3
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2),
-    t2 AS (
-      WITH t AS (SELECT 3)
-      SELECT * FROM t
-    )
-    SELECT * FROM t2
-  )
-SELECT * FROM t2;
-
--- CTE in subquery shadows outer
-WITH t(c) AS (SELECT 1)
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT * FROM t
-);
-
--- CTE in subquery shadows outer 2
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 2)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery shadows outer 3
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 3)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery expression shadows outer
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT * FROM t
-);
-
--- CTE in subquery expression shadows outer 2
-WITH t AS (SELECT 1)
-SELECT (
-  SELECT (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery expression shadows outer 3
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT (
-    WITH t AS (SELECT 3)
-    SELECT * FROM t
-  )
-);
-
--- Clean up
-DROP VIEW IF EXISTS t;
-DROP VIEW IF EXISTS t2;
+--SET spark.sql.legacy.ctePrecedencePolicy = legacy
+--IMPORT cte-nested.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
new file mode 100644
index 0000000000000..3b64b5daa82db
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
@@ -0,0 +1,138 @@
+-- CTE in CTE definition
+WITH t as (
+  WITH t2 AS (SELECT 1)
+  SELECT * FROM t2
+)
+SELECT * FROM t;
+
+-- CTE in subquery
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 1)
+  SELECT * FROM t
+);
+
+-- CTE in subquery expression
+SELECT (
+  WITH t AS (SELECT 1)
+  SELECT * FROM t
+);
+
+-- CTE in CTE definition shadows outer
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+SELECT * FROM t2;
+
+-- CTE in CTE definition shadows outer 2
+WITH
+  t(c) AS (SELECT 1),
+  t2 AS (
+    SELECT (
+      SELECT max(c) FROM (
+        WITH t(c) AS (SELECT 2)
+        SELECT * FROM t
+      )
+    )
+  )
+SELECT * FROM t2;
+
+-- CTE in CTE definition shadows outer 3
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2),
+    t2 AS (
+      WITH t AS (SELECT 3)
+      SELECT * FROM t
+    )
+    SELECT * FROM t2
+  )
+SELECT * FROM t2;
+
+-- CTE in subquery shadows outer
+WITH t(c) AS (SELECT 1)
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+);
+
+-- CTE in subquery shadows outer 2
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 2)
+    SELECT * FROM t
+  )
+);
+
+-- CTE in subquery shadows outer 3
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 3)
+    SELECT * FROM t
+  )
+);
+
+-- CTE in subquery expression shadows outer
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT * FROM t
+);
+
+-- CTE in subquery expression shadows outer 2
+WITH t AS (SELECT 1)
+SELECT (
+  SELECT (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+);
+
+-- CTE in subquery expression shadows outer 3
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT (
+    WITH t AS (SELECT 3)
+    SELECT * FROM t
+  )
+);
+
+-- CTE in subquery expression shadows outer 4
+WITH t(c) AS (SELECT 1)
+SELECT * FROM t
+WHERE c IN (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+);
+
+-- forward name conflict is not a real conflict
+WITH
+  t AS (
+    WITH t2 AS (SELECT 1)
+    SELECT * FROM t2
+  ),
+  t2 AS (SELECT 2)
+SELECT * FROM t;
+
+-- case insensitive name conflicts: in other CTE relations
+WITH
+  abc AS (SELECT 1),
+  t AS (
+    WITH aBc AS (SELECT 2)
+    SELECT * FROM aBC
+  )
+SELECT * FROM t;
+
+-- case insensitive name conflicts: in subquery expressions
+WITH abc AS (SELECT 1)
+SELECT (
+  WITH aBc AS (SELECT 2)
+  SELECT * FROM aBC
+);
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
new file mode 100644
index 0000000000000..d3b4539da2f3e
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.ctePrecedencePolicy = corrected
+--IMPORT cte-nested.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte.sql b/sql/core/src/test/resources/sql-tests/inputs/cte.sql
index d0e145c35a9fe..ec2f4808fcfc9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cte.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte.sql
@@ -49,112 +49,6 @@ WITH
   t(x) AS (SELECT 2)
 SELECT * FROM t;
 
--- CTE in CTE definition
-WITH t as (
-  WITH t2 AS (SELECT 1)
-  SELECT * FROM t2
-)
-SELECT * FROM t;
-
--- CTE in subquery
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 1)
-  SELECT * FROM t
-);
-
--- CTE in subquery expression
-SELECT (
-  WITH t AS (SELECT 1)
-  SELECT * FROM t
-);
-
--- CTE in CTE definition shadows outer
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-SELECT * FROM t2;
-
--- CTE in CTE definition shadows outer 2
-WITH
-  t(c) AS (SELECT 1),
-  t2 AS (
-    SELECT (
-      SELECT max(c) FROM (
-        WITH t(c) AS (SELECT 2)
-        SELECT * FROM t
-      )
-    )
-  )
-SELECT * FROM t2;
-
--- CTE in CTE definition shadows outer 3
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2),
-    t2 AS (
-      WITH t AS (SELECT 3)
-      SELECT * FROM t
-    )
-    SELECT * FROM t2
-  )
-SELECT * FROM t2;
-
--- CTE in subquery shadows outer
-WITH t(c) AS (SELECT 1)
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT * FROM t
-);
-
--- CTE in subquery shadows outer 2
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 2)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery shadows outer 3
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 3)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery expression shadows outer
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT * FROM t
-);
-
--- CTE in subquery expression shadows outer 2
-WITH t AS (SELECT 1)
-SELECT (
-  SELECT (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery expression shadows outer 3
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT (
-    WITH t AS (SELECT 3)
-    SELECT * FROM t
-  )
-);
-
 -- Clean up
 DROP VIEW IF EXISTS t;
 DROP VIEW IF EXISTS t2;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql
deleted file mode 100644
index a63cdafb745a0..0000000000000
--- a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql
+++ /dev/null
@@ -1,145 +0,0 @@
-CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c;
-
-select date_part('millennium', c) from t;
-select date_part('millennia', c) from t;
-select date_part('mil', c) from t;
-select date_part('mils', c) from t;
-
-select date_part('century', c) from t;
-select date_part('centuries', c) from t;
-select date_part('c', c) from t;
-select date_part('cent', c) from t;
-
-select date_part('decade', c) from t;
-select date_part('decades', c) from t;
-select date_part('dec', c) from t;
-select date_part('decs', c) from t;
-
-select date_part('year', c) from t;
-select date_part('y', c) from t;
-select date_part('years', c) from t;
-select date_part('yr', c) from t;
-select date_part('yrs', c) from t;
-
-select date_part('quarter', c) from t;
-select date_part('qtr', c) from t;
-
-select date_part('month', c) from t;
-select date_part('mon', c) from t;
-select date_part('mons', c) from t;
-select date_part('months', c) from t;
-
-select date_part('week', c) from t;
-select date_part('w', c) from t;
-select date_part('weeks', c) from t;
-
-select date_part('day', c) from t;
-select date_part('d', c) from t;
-select date_part('days', c) from t;
-
-select date_part('dayofweek', c) from t;
-
-select date_part('dow', c) from t;
-
-select date_part('isodow', c) from t;
-
-select date_part('doy', c) from t;
-
-select date_part('hour', c) from t;
-select date_part('h', c) from t;
-select date_part('hours', c) from t;
-select date_part('hr', c) from t;
-select date_part('hrs', c) from t;
-
-select date_part('minute', c) from t;
-select date_part('m', c) from t;
-select date_part('min', c) from t;
-select date_part('mins', c) from t;
-select date_part('minutes', c) from t;
-
-select date_part('second', c) from t;
-select date_part('s', c) from t;
-select date_part('sec', c) from t;
-select date_part('seconds', c) from t;
-select date_part('secs', c) from t;
-
-select date_part('not_supported', c) from t;
-
-select date_part(c, c) from t;
-
-select date_part(null, c) from t;
-
-CREATE TEMPORARY VIEW t2 AS select interval 1010 year 9 month 8 day 7 hour 6 minute 5 second 4 millisecond 3 microsecond as c;
-
-select date_part('millennium', c) from t2;
-select date_part('millennia', c) from t2;
-select date_part('mil', c) from t2;
-select date_part('mils', c) from t2;
-
-select date_part('century', c) from t2;
-select date_part('centuries', c) from t2;
-select date_part('c', c) from t2;
-select date_part('cent', c) from t2;
-
-select date_part('decade', c) from t2;
-select date_part('decades', c) from t2;
-select date_part('dec', c) from t2;
-select date_part('decs', c) from t2;
-
-select date_part('year', c) from t2;
-select date_part('y', c) from t2;
-select date_part('years', c) from t2;
-select date_part('yr', c) from t2;
-select date_part('yrs', c) from t2;
-
-select date_part('quarter', c) from t2;
-select date_part('qtr', c) from t2;
-
-select date_part('month', c) from t2;
-select date_part('mon', c) from t2;
-select date_part('mons', c) from t2;
-select date_part('months', c) from t2;
-
-select date_part('day', c) from t2;
-select date_part('d', c) from t2;
-select date_part('days', c) from t2;
-
-select date_part('hour', c) from t2;
-select date_part('h', c) from t2;
-select date_part('hours', c) from t2;
-select date_part('hr', c) from t2;
-select date_part('hrs', c) from t2;
-
-select date_part('minute', c) from t2;
-select date_part('m', c) from t2;
-select date_part('min', c) from t2;
-select date_part('mins', c) from t2;
-select date_part('minutes', c) from t2;
-
-select date_part('second', c) from t2;
-select date_part('s', c) from t2;
-select date_part('sec', c) from t2;
-select date_part('seconds', c) from t2;
-select date_part('secs', c) from t2;
-
-select date_part('milliseconds', c) from t2;
-select date_part('msec', c) from t2;
-select date_part('msecs', c) from t2;
-select date_part('millisecon', c) from t2;
-select date_part('mseconds', c) from t2;
-select date_part('ms', c) from t2;
-
-select date_part('microseconds', c) from t2;
-select date_part('usec', c) from t2;
-select date_part('usecs', c) from t2;
-select date_part('useconds', c) from t2;
-select date_part('microsecon', c) from t2;
-select date_part('us', c) from t2;
-
-select date_part('epoch', c) from t2;
-
-select date_part('not_supported', c) from t2;
-
-select date_part(c, c) from t2;
-
-select date_part(null, c) from t2;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql
new file mode 100644
index 0000000000000..11bba00e91abf
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql
@@ -0,0 +1,53 @@
+--- TESTS FOR DATETIME FORMATTING FUNCTIONS WITH INVALID PATTERNS ---
+
+-- separating this from datetime-formatting.sql, because the text form
+-- for patterns with 5 letters in SimpleDateFormat varies from different JDKs
+select date_format('2018-11-17 13:33:33.333', 'GGGGG');
+-- pattern letter count can not be greater than 6
+select date_format('2018-11-17 13:33:33.333', 'yyyyyyy');
+-- q/L in JDK 8 will fail when the count is more than 2
+select date_format('2018-11-17 13:33:33.333', 'qqqqq');
+select date_format('2018-11-17 13:33:33.333', 'QQQQQ');
+select date_format('2018-11-17 13:33:33.333', 'MMMMM');
+select date_format('2018-11-17 13:33:33.333', 'LLLLL');
+
+select date_format('2018-11-17 13:33:33.333', 'EEEEE');
+select date_format('2018-11-17 13:33:33.333', 'FF');
+select date_format('2018-11-17 13:33:33.333', 'ddd');
+-- DD is invalid if the day-of-year exceeds 100, but it becomes valid in Java 11
+-- select date_format('2018-11-17 13:33:33.333', 'DD');
+select date_format('2018-11-17 13:33:33.333', 'DDDD');
+select date_format('2018-11-17 13:33:33.333', 'HHH');
+select date_format('2018-11-17 13:33:33.333', 'hhh');
+select date_format('2018-11-17 13:33:33.333', 'kkk');
+select date_format('2018-11-17 13:33:33.333', 'KKK');
+select date_format('2018-11-17 13:33:33.333', 'mmm');
+select date_format('2018-11-17 13:33:33.333', 'sss');
+select date_format('2018-11-17 13:33:33.333', 'SSSSSSSSSS');
+select date_format('2018-11-17 13:33:33.333', 'aa');
+select date_format('2018-11-17 13:33:33.333', 'V');
+select date_format('2018-11-17 13:33:33.333', 'zzzzz');
+select date_format('2018-11-17 13:33:33.333', 'XXXXXX');
+select date_format('2018-11-17 13:33:33.333', 'ZZZZZZ');
+select date_format('2018-11-17 13:33:33.333', 'OO');
+select date_format('2018-11-17 13:33:33.333', 'xxxxxx');
+
+select date_format('2018-11-17 13:33:33.333', 'A');
+select date_format('2018-11-17 13:33:33.333', 'n');
+select date_format('2018-11-17 13:33:33.333', 'N');
+select date_format('2018-11-17 13:33:33.333', 'p');
+
+-- disabled week-based patterns
+select date_format('2018-11-17 13:33:33.333', 'Y');
+select date_format('2018-11-17 13:33:33.333', 'w');
+select date_format('2018-11-17 13:33:33.333', 'W');
+select date_format('2018-11-17 13:33:33.333', 'u');
+select date_format('2018-11-17 13:33:33.333', 'e');
+select date_format('2018-11-17 13:33:33.333', 'c');
+
+-- others
+select date_format('2018-11-17 13:33:33.333', 'B');
+select date_format('2018-11-17 13:33:33.333', 'C');
+select date_format('2018-11-17 13:33:33.333', 'I');
+
+
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-legacy.sql
new file mode 100644
index 0000000000000..19cab61a7ee56
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-legacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.timeParserPolicy=LEGACY
+--IMPORT datetime-formatting.sql
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql
new file mode 100644
index 0000000000000..2d70326f4f3c0
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql
@@ -0,0 +1,68 @@
+--- TESTS FOR DATETIME FORMATTING FUNCTIONS ---
+
+create temporary view v as select col from values
+ (timestamp '1582-06-01 11:33:33.123UTC+080000'),
+ (timestamp '1970-01-01 00:00:00.000Europe/Paris'),
+ (timestamp '1970-12-31 23:59:59.999Asia/Srednekolymsk'),
+ (timestamp '1996-04-01 00:33:33.123Australia/Darwin'),
+ (timestamp '2018-11-17 13:33:33.123Z'),
+ (timestamp '2020-01-01 01:33:33.123Asia/Shanghai'),
+ (timestamp '2100-01-01 01:33:33.123America/Los_Angeles') t(col);
+
+select col, date_format(col, 'G GG GGG GGGG') from v;
+
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy') from v;
+
+select col, date_format(col, 'q qq') from v;
+
+select col, date_format(col, 'Q QQ QQQ QQQQ') from v;
+
+select col, date_format(col, 'M MM MMM MMMM') from v;
+
+select col, date_format(col, 'L LL') from v;
+
+select col, date_format(col, 'E EE EEE EEEE') from v;
+
+select col, date_format(col, 'F') from v;
+
+select col, date_format(col, 'd dd') from v;
+
+select col, date_format(col, 'DD') from v where col = timestamp '2100-01-01 01:33:33.123America/Los_Angeles';
+select col, date_format(col, 'D DDD') from v;
+
+select col, date_format(col, 'H HH') from v;
+
+select col, date_format(col, 'h hh') from v;
+
+select col, date_format(col, 'k kk') from v;
+
+select col, date_format(col, 'K KK') from v;
+
+select col, date_format(col, 'm mm') from v;
+
+select col, date_format(col, 's ss') from v;
+
+select col, date_format(col, 'S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS') from v;
+
+select col, date_format(col, 'a') from v;
+
+select col, date_format(col, 'VV') from v;
+
+select col, date_format(col, 'z zz zzz zzzz') from v;
+
+select col, date_format(col, 'X XX XXX') from v;
+select col, date_format(col, 'XXXX XXXXX') from v;
+
+select col, date_format(col, 'Z ZZ ZZZ ZZZZ ZZZZZ') from v;
+
+select col, date_format(col, 'O OOOO') from v;
+
+select col, date_format(col, 'x xx xxx xxxx xxxx xxxxx') from v;
+
+-- optional pattern, but the results won't be optional for formatting
+select col, date_format(col, '[yyyy-MM-dd HH:mm:ss]') from v;
+
+-- literals
+select col, date_format(col, "姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'") from v;
+select col, date_format(col, "''") from v;
+select col, date_format(col, '') from v;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-legacy.sql
new file mode 100644
index 0000000000000..e573f8a6b4be9
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-legacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.timeParserPolicy=LEGACY
+--IMPORT datetime.sql
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
new file mode 100644
index 0000000000000..a1c02eaa3b0a0
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
@@ -0,0 +1,29 @@
+--- TESTS FOR DATETIME PARSING FUNCTIONS WITH INVALID VALUES ---
+
+-- parsing invalid value with pattern 'y'
+select to_timestamp('294248', 'y'); -- out of year value range [0, 294247]
+select to_timestamp('1', 'yy'); -- the number of digits must be 2 for 'yy'.
+select to_timestamp('-12', 'yy'); -- out of year value range [0, 99] for reduced two digit form
+select to_timestamp('123', 'yy'); -- the number of digits must be 2 for 'yy'.
+select to_timestamp('1', 'yyy'); -- the number of digits must be in [3, 6] for 'yyy'
+
+select to_timestamp('1234567', 'yyyyyyy'); -- the length of 'y' pattern must be less than 7
+
+-- parsing invalid values with pattern 'D'
+select to_timestamp('366', 'D');
+select to_timestamp('9', 'DD');
+-- in java 8 this case is invalid, but valid in java 11, disabled for jenkins
+-- select to_timestamp('100', 'DD');
+select to_timestamp('366', 'DD');
+select to_timestamp('9', 'DDD');
+select to_timestamp('99', 'DDD');
+select to_timestamp('30-365', 'dd-DDD');
+select to_timestamp('11-365', 'MM-DDD');
+select to_timestamp('2019-366', 'yyyy-DDD');
+select to_timestamp('12-30-365', 'MM-dd-DDD');
+select to_timestamp('2020-01-365', 'yyyy-dd-DDD');
+select to_timestamp('2020-10-350', 'yyyy-MM-DDD');
+select to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD');
+-- add a special case to test csv, because the legacy formatter it uses is lenient then Spark should
+-- throw SparkUpgradeException
+select from_csv('2018-366', 'date Date', map('dateFormat', 'yyyy-DDD'))
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
new file mode 100644
index 0000000000000..ee1afe502ab79
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.timeParserPolicy=LEGACY
+--IMPORT datetime-parsing.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
new file mode 100644
index 0000000000000..e058bd675c375
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
@@ -0,0 +1,44 @@
+--- TESTS FOR DATETIME PARSING FUNCTIONS ---
+
+-- parsing with pattern 'y'.
+-- the range of valid year is [-290307, 294247],
+-- but particularly, some thrift client use java.sql.Timestamp to parse timestamp, which allows
+-- only positive year values less or equal than 9999. So the cases bellow only use [1, 9999] to pass
+-- ThriftServerQueryTestSuite
+select to_timestamp('1', 'y');
+select to_timestamp('009999', 'y');
+
+-- reduced two digit form is used, the range of valid year is 20-[01, 99]
+select to_timestamp('00', 'yy');
+select to_timestamp('99', 'yy');
+
+-- the range of valid year is [-290307, 294247], the number of digits must be in [3, 6] for 'yyy'
+select to_timestamp('001', 'yyy');
+select to_timestamp('009999', 'yyy');
+
+-- the range of valid year is [-9999, 9999], the number of digits must be 4 for 'yyyy'.
+select to_timestamp('0001', 'yyyy');
+select to_timestamp('9999', 'yyyy');
+
+-- the range of valid year is [-99999, 99999], the number of digits must be 5 for 'yyyyy'.
+select to_timestamp('00001', 'yyyyy');
+select to_timestamp('09999', 'yyyyy');
+
+-- the range of valid year is [-290307, 294247], the number of digits must be 6 for 'yyyyyy'.
+select to_timestamp('000001', 'yyyyyy');
+select to_timestamp('009999', 'yyyyyy');
+
+-- parsing with pattern 'D'
+select to_timestamp('9', 'D');
+select to_timestamp('300', 'D');
+select to_timestamp('09', 'DD');
+select to_timestamp('99', 'DD');
+select to_timestamp('009', 'DDD');
+select to_timestamp('365', 'DDD');
+select to_timestamp('31-365', 'dd-DDD');
+select to_timestamp('12-365', 'MM-DDD');
+select to_timestamp('2020-365', 'yyyy-DDD');
+select to_timestamp('12-31-365', 'MM-dd-DDD');
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD');
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD');
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index b14778b91510e..ae5831c61913c 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -33,6 +33,8 @@ select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01');
 
 select date '2019-01-01\t';
 select timestamp '2019-01-01\t';
+select date '2020-01-01中文';
+select timestamp '2019-01-01中文';
 
 -- time add/sub
 select timestamp'2011-11-11 11:11:11' + interval '2' day;
@@ -58,20 +60,91 @@ select date_add('2011-11-11', 1L);
 select date_add('2011-11-11', 1.0);
 select date_add('2011-11-11', 1E1);
 select date_add('2011-11-11', '1');
+select date_add('2011-11-11', '1.2');
 select date_add(date'2011-11-11', 1);
 select date_add(timestamp'2011-11-11', 1);
 select date_sub(date'2011-11-11', 1);
+select date_sub(date'2011-11-11', '1');
+select date_sub(date'2011-11-11', '1.2');
 select date_sub(timestamp'2011-11-11', 1);
 select date_sub(null, 1);
 select date_sub(date'2011-11-11', null);
 select date'2011-11-11' + 1E1;
+select date'2011-11-11' + '1';
 select null + date '2001-09-28';
 select date '2001-09-28' + 7Y;
 select 7S + date '2001-09-28';
 select date '2001-10-01' - 7;
+select date '2001-10-01' - '7';
 select date '2001-09-28' + null;
 select date '2001-09-28' - null;
 
+-- date add/sub with non-literal string column
+create temp view v as select '1' str;
+select date_add('2011-11-11', str) from v;
+select date_sub('2011-11-11', str) from v;
+
 -- subtract dates
 select null - date '2019-10-06';
 select date '2001-10-01' - date '2001-09-28';
+
+-- variable-length second fraction tests
+select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+-- second fraction exceeded max variable length
+select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+-- special cases
+select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]');
+select to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]');
+select to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]');
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]');
+select to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]');
+select to_timestamp("2019-10-06S10:11:12.12345", "yyyy-MM-dd'S'HH:mm:ss.SSSSSS");
+select to_timestamp("12.12342019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm");
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm");
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm");
+select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm");
+
+select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'");
+select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd");
+
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS"); -- middle
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''"); -- tail
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss"); -- head
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss"); -- head but as single quote
+
+-- missing fields
+select to_timestamp("16", "dd");
+select to_timestamp("02-29", "MM-dd");
+select to_date("16", "dd");
+select to_date("02-29", "MM-dd");
+select to_timestamp("2019 40", "yyyy mm");
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss");
+
+-- Unsupported narrow text style
+select date_format(date '2020-05-23', 'GGGGG');
+select date_format(date '2020-05-23', 'MMMMM');
+select date_format(date '2020-05-23', 'LLLLL');
+select date_format(timestamp '2020-05-23', 'EEEEE');
+select date_format(timestamp '2020-05-23', 'uuuuu');
+select date_format('2020-05-23', 'QQQQQ');
+select date_format('2020-05-23', 'qqqqq');
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG');
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE');
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE');
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE');
+select from_unixtime(12345, 'MMMMM');
+select from_unixtime(54321, 'QQQQQ');
+select from_unixtime(23456, 'aaaaa');
+select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'));
+select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'));
+select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'));
+select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'));
diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql b/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql
index 821cb473751eb..d55e398329b76 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql
@@ -52,7 +52,7 @@ DROP TABLE desc_complex_col_table;
 
 --Test case insensitive
 
-CREATE TABLE customer(CName STRING);
+CREATE TABLE customer(CName STRING) USING PARQUET;
 
 INSERT INTO customer VALUES('Maria');
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/except.sql b/sql/core/src/test/resources/sql-tests/inputs/except.sql
index 1d579e65f3473..ffdf1f4f3d24d 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/except.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/except.sql
@@ -55,3 +55,22 @@ FROM   t1
 WHERE  t1.v >= (SELECT   min(t2.v)
                 FROM     t2
                 WHERE    t2.k = t1.k);
+
+-- SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v);
+SELECT t.v FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t;
+
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t;
+
+-- Clean-up
+DROP VIEW IF EXISTS t1;
+DROP VIEW IF EXISTS t2;
+DROP VIEW IF EXISTS t3;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
new file mode 100644
index 0000000000000..f4afa2b77a9d7
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
@@ -0,0 +1,3 @@
+--IMPORT explain.sql
+
+--SET spark.sql.adaptive.enabled=true
diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain.sql b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
index d5253e3daddb0..80bf258704c70 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/explain.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
@@ -5,6 +5,7 @@
 CREATE table  explain_temp1 (key int, val int) USING PARQUET;
 CREATE table  explain_temp2 (key int, val int) USING PARQUET;
 CREATE table  explain_temp3 (key int, val int) USING PARQUET;
+CREATE table  explain_temp4 (key int, val string) USING PARQUET;
 
 SET spark.sql.codegen.wholeStage = true;
 
@@ -61,7 +62,7 @@ EXPLAIN FORMATTED
                 FROM   explain_temp2 
                 WHERE  val > 0) 
          OR
-         key = (SELECT max(key) 
+         key = (SELECT avg(key)
                 FROM   explain_temp3
                 WHERE  val > 0);
 
@@ -93,7 +94,27 @@ EXPLAIN FORMATTED
   CREATE VIEW explain_view AS
     SELECT key, val FROM explain_temp1;
 
+-- HashAggregate
+EXPLAIN FORMATTED
+  SELECT
+    COUNT(val) + SUM(key) as TOTAL,
+    COUNT(key) FILTER (WHERE val > 1)
+  FROM explain_temp1;
+
+-- ObjectHashAggregate
+EXPLAIN FORMATTED
+  SELECT key, sort_array(collect_set(val))[0]
+  FROM explain_temp4
+  GROUP BY key;
+
+-- SortAggregate
+EXPLAIN FORMATTED
+  SELECT key, MIN(val)
+  FROM explain_temp4
+  GROUP BY key;
+
 -- cleanup
 DROP TABLE explain_temp1;
 DROP TABLE explain_temp2;
 DROP TABLE explain_temp3;
+DROP TABLE explain_temp4;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/extract.sql b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
index 2d6cad8838704..abb9e82c9ef2e 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/extract.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
@@ -1,84 +1,125 @@
-CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c;
+CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c, interval 10 year 20 month 30 day 40 hour 50 minute 6.7890 second as i;
 
-select extract(millennium from c) from t;
-select extract(millennia from c) from t;
-select extract(mil from c) from t;
-select extract(mils from c) from t;
+select extract(year from c), extract(year from i) from t;
+select extract(y from c), extract(y from i) from t;
+select extract(years from c), extract(years from i) from t;
+select extract(yr from c), extract(yr from i) from t;
+select extract(yrs from c), extract(yrs from i) from t;
 
-select extract(century from c) from t;
-select extract(centuries from c) from t;
-select extract(c from c) from t;
-select extract(cent from c) from t;
-
-select extract(decade from c) from t;
-select extract(decades from c) from t;
-select extract(dec from c) from t;
-select extract(decs from c) from t;
-
-select extract(year from c) from t;
-select extract(y from c) from t;
-select extract(years from c) from t;
-select extract(yr from c) from t;
-select extract(yrs from c) from t;
-
-select extract(isoyear from c) from t;
+select extract(yearofweek from c) from t;
 
 select extract(quarter from c) from t;
 select extract(qtr from c) from t;
 
-select extract(month from c) from t;
-select extract(mon from c) from t;
-select extract(mons from c) from t;
-select extract(months from c) from t;
+select extract(month from c), extract(month from i) from t;
+select extract(mon from c), extract(mon from i) from t;
+select extract(mons from c), extract(mons from i) from t;
+select extract(months from c), extract(months from i) from t;
 
 select extract(week from c) from t;
 select extract(w from c) from t;
 select extract(weeks from c) from t;
 
-select extract(day from c) from t;
-select extract(d from c) from t;
-select extract(days from c) from t;
+select extract(day from c), extract(day from i) from t;
+select extract(d from c), extract(d from i) from t;
+select extract(days from c), extract(days from i) from t;
 
 select extract(dayofweek from c) from t;
-
 select extract(dow from c) from t;
 
-select extract(isodow from c) from t;
+select extract(dayofweek_iso from c) from t;
+select extract(dow_iso from c) from t;
 
 select extract(doy from c) from t;
 
-select extract(hour from c) from t;
-select extract(h from c) from t;
-select extract(hours from c) from t;
-select extract(hr from c) from t;
-select extract(hrs from c) from t;
-
-select extract(minute from c) from t;
-select extract(m from c) from t;
-select extract(min from c) from t;
-select extract(mins from c) from t;
-select extract(minutes from c) from t;
-
-select extract(second from c) from t;
-select extract(s from c) from t;
-select extract(sec from c) from t;
-select extract(seconds from c) from t;
-select extract(secs from c) from t;
-
-select extract(milliseconds from c) from t;
-select extract(msec from c) from t;
-select extract(msecs from c) from t;
-select extract(millisecon from c) from t;
-select extract(mseconds from c) from t;
-select extract(ms from c) from t;
-
-select extract(microseconds from c) from t;
-select extract(usec from c) from t;
-select extract(usecs from c) from t;
-select extract(useconds from c) from t;
-select extract(microsecon from c) from t;
-select extract(us from c) from t;
-
-select extract(epoch from c) from t;
+select extract(hour from c), extract(hour from i) from t;
+select extract(h from c), extract(h from i) from t;
+select extract(hours from c), extract(hours from i) from t;
+select extract(hr from c), extract(hr from i) from t;
+select extract(hrs from c), extract(hrs from i) from t;
+
+select extract(minute from c), extract(minute from i) from t;
+select extract(m from c), extract(m from i) from t;
+select extract(min from c), extract(min from i) from t;
+select extract(mins from c), extract(mins from i) from t;
+select extract(minutes from c), extract(minutes from i) from t;
+
+select extract(second from c), extract(second from i) from t;
+select extract(s from c), extract(s from i) from t;
+select extract(sec from c), extract(sec from i) from t;
+select extract(seconds from c), extract(seconds from i) from t;
+select extract(secs from c), extract(secs from i) from t;
 
 select extract(not_supported from c) from t;
+select extract(not_supported from i) from t;
+
+select date_part('year', c), date_part('year', i) from t;
+select date_part('y', c), date_part('y', i) from t;
+select date_part('years', c), date_part('years', i) from t;
+select date_part('yr', c), date_part('yr', i) from t;
+select date_part('yrs', c), date_part('yrs', i) from t;
+
+select date_part('yearofweek', c) from t;
+
+select date_part('quarter', c) from t;
+select date_part('qtr', c) from t;
+
+select date_part('month', c), date_part('month', i) from t;
+select date_part('mon', c), date_part('mon', i) from t;
+select date_part('mons', c), date_part('mons', i) from t;
+select date_part('months', c), date_part('months', i) from t;
+
+select date_part('week', c) from t;
+select date_part('w', c) from t;
+select date_part('weeks', c) from t;
+
+select date_part('day', c), date_part('day', i) from t;
+select date_part('d', c), date_part('d', i) from t;
+select date_part('days', c), date_part('days', i) from t;
+
+select date_part('dayofweek', c) from t;
+select date_part('dow', c) from t;
+
+select date_part('dayofweek_iso', c) from t;
+select date_part('dow_iso', c) from t;
+
+select date_part('doy', c) from t;
+
+select date_part('hour', c), date_part('hour', i) from t;
+select date_part('h', c), date_part('h', i) from t;
+select date_part('hours', c), date_part('hours', i) from t;
+select date_part('hr', c), date_part('hr', i) from t;
+select date_part('hrs', c), date_part('hrs', i) from t;
+
+select date_part('minute', c), date_part('minute', i) from t;
+select date_part('m', c), date_part('m', i) from t;
+select date_part('min', c), date_part('min', i) from t;
+select date_part('mins', c), date_part('mins', i) from t;
+select date_part('minutes', c), date_part('minutes', i) from t;
+
+select date_part('second', c), date_part('second', i) from t;
+select date_part('s', c), date_part('s', i) from t;
+select date_part('sec', c), date_part('sec', i) from t;
+select date_part('seconds', c), date_part('seconds', i) from t;
+select date_part('secs', c), date_part('secs', i) from t;
+
+select date_part('not_supported', c) from t;
+select date_part(c, c) from t;
+select date_part(null, c) from t;
+
+select date_part(i, i) from t;
+select date_part(null, i) from t;
+
+-- In SPARK-31476, we've supported extract('field', source), too
+select extract('year', c) from t;
+select extract('quarter', c) from t;
+select extract('month', c) from t;
+select extract('week', c) from t;
+select extract('day', c) from t;
+select extract('days', c) from t;
+select extract('dayofweek', c) from t;
+select extract('dow', c) from t;
+select extract('doy', c) from t;
+select extract('hour', c) from t;
+select extract('minute', c) from t;
+select extract('second', c) from t;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/group-by.sql b/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
index fedf03d774e42..3f5f5568b0720 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
@@ -86,6 +86,16 @@ SELECT 1 FROM range(10) HAVING MAX(id) > 0;
 
 SELECT id FROM range(10) HAVING id > 0;
 
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=true;
+
+SELECT 1 FROM range(10) HAVING true;
+
+SELECT 1 FROM range(10) HAVING MAX(id) > 0;
+
+SELECT id FROM range(10) HAVING id > 0;
+
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=false;
+
 -- Test data
 CREATE OR REPLACE TEMPORARY VIEW test_agg AS SELECT * FROM VALUES
   (1, true), (1, false),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/having.sql b/sql/core/src/test/resources/sql-tests/inputs/having.sql
index 868a911e787f6..2799b1a94d085 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/having.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/having.sql
@@ -16,3 +16,17 @@ SELECT MIN(t.v) FROM (SELECT * FROM hav WHERE v > 0) t HAVING(COUNT(1) > 0);
 
 -- SPARK-20329: make sure we handle timezones correctly
 SELECT a + b FROM VALUES (1L, 2), (3L, 4) AS T(a, b) GROUP BY a + b HAVING a + b > 1;
+
+-- SPARK-31519: Cast in having aggregate expressions returns the wrong result
+SELECT SUM(a) AS b, CAST('2020-01-01' AS DATE) AS fake FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY b HAVING b > 10;
+
+-- SPARK-31663: Grouping sets with having clause returns the wrong result
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY GROUPING SETS ((b), (a, b)) HAVING b > 10;
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY CUBE(a, b) HAVING b > 10;
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY ROLLUP(a, b) HAVING b > 10;
+
+-- SPARK-33131: Grouping sets with having clause can not resolve qualified col name.
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY GROUPING SETS(t.c1) HAVING t.c1 = 1;
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY CUBE(t.c1) HAVING t.c1 = 1;
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY ROLLUP(t.c1) HAVING t.c1 = 1;
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY t.c1 HAVING t.c1 = 1;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
index cfa06aea82b04..73dfa91827b08 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
@@ -92,3 +92,6 @@ select transform_values(ys, (k, v) -> k + v) as v from nested;
 -- use non reversed keywords: all is non reversed only if !ansi
 select transform(ys, all -> all * all) as v from values (array(32, 97)) as t(ys);
 select transform(ys, (all, i) -> all + i) as v from values (array(32, 97)) as t(ys);
+
+-- SPARK-32819: Aggregate on nested string arrays
+select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)));
diff --git a/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql b/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
index b0b2244048caa..077caa5dd44a0 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
@@ -155,6 +155,21 @@ SELECT * FROM tab2;
 -- Restore the property
 SET spark.sql.legacy.setopsPrecedence.enabled = false;
 
+-- SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes
+CREATE OR REPLACE TEMPORARY VIEW tab3 AS VALUES (decimal(1)), (decimal(2)) tbl3(v);
+SELECT t.v FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t;
+
+SELECT SUM(t.v) FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t;
+
 -- Clean-up 
 DROP VIEW IF EXISTS tab1;
 DROP VIEW IF EXISTS tab2;
+DROP VIEW IF EXISTS tab3;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index fb6c485f619ae..8f6cf0504baf2 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -4,6 +4,10 @@
 select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15');
 select interval 4 month 2 weeks 3 microseconds * 1.5;
 select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5;
+select interval 2147483647 month * 2;
+select interval 2147483647 month / 0.5;
+select interval 2147483647 day * 2;
+select interval 2147483647 day / 0.5;
 
 -- interval operation with null and zero case
 select interval '2 seconds' / 0;
@@ -25,6 +29,8 @@ select make_interval(1, 2, 3, 4);
 select make_interval(1, 2, 3, 4, 5);
 select make_interval(1, 2, 3, 4, 5, 6);
 select make_interval(1, 2, 3, 4, 5, 6, 7.008009);
+select make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456);
+select make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789);
 
 -- cast string to intervals
 select cast('1 second' as interval);
@@ -49,6 +55,7 @@ select interval '15:40:32.99899999' hour to second;
 select interval '40:32.99899999' minute to second;
 select interval '40:32' minute to second;
 select interval 30 day day;
+select interval 30 days days;
 
 -- invalid day-time string intervals
 select interval '20 15:40:32.99899999' day to hour;
@@ -67,6 +74,7 @@ select map(1, interval 1 day, 2, interval 3 week);
 -- typed interval expression
 select interval 'interval 3 year 1 hour';
 select interval '3 year 1 hour';
+SELECT interval '1 year 3 months 2 weeks 2 days 1 hour 3 minutes 2 seconds 100 millisecond 200 microseconds';
 
 -- malformed interval literal
 select interval;
@@ -83,70 +91,9 @@ select interval '12:11:10' hour to second '1' year;
 select interval (-30) day;
 select interval (a + 1) day;
 select interval 30 day day day;
-
--- sum interval values
--- null
-select sum(cast(null as interval));
-
--- empty set
-select sum(cast(v as interval)) from VALUES ('1 seconds') t(v) where 1=0;
-
--- basic interval sum
-select sum(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v);
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v);
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v);
-select sum(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v);
-
--- group by
-select
-    i,
-    sum(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i;
-
--- having
-select
-    sum(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null;
-
--- window
-SELECT
-    i,
-    sum(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES(1, '1 seconds'), (1, '2 seconds'), (2, NULL), (2, NULL) t(i,v);
-
--- average with interval type
--- null
-select avg(cast(v as interval)) from VALUES (null) t(v);
-
--- empty set
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v) where 1=0;
-
--- basic interval avg
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v);
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v);
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v);
-select avg(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v);
-
--- group by
-select
-    i,
-    avg(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i;
-
--- having
-select
-    avg(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null;
-
--- window
-SELECT
-    i,
-    avg(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES (1,'1 seconds'), (1,'2 seconds'), (2,NULL), (2,NULL) t(i,v);
+select interval (-30) days;
+select interval (a + 1) days;
+select interval 30 days days days;
 
 -- Interval year-month arithmetic
 
@@ -213,6 +160,11 @@ select interval 'interval \t 1\tday';
 select interval 'interval\t1\tday';
 select interval '1\t' day;
 select interval '1 ' day;
+select interval '2-2\t' year to month;
+select interval '-\t2-2\t' year to month;
+select interval '\n0 12:34:46.789\t' day to second;
+select interval '\n-\t10\t 12:34:46.789\t' day to second;
+select interval '中文 interval 1 day';
 
 -- interval overflow if (ansi) exception else NULL
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b);
@@ -222,7 +174,25 @@ select a * 1.1 from values (interval '-2147483648 months', interval '2147483647
 select a / 0.5 from values (interval '-2147483648 months', interval '2147483647 months') t(a, b);
 
 -- interval support for csv and json functions
-SELECT from_csv('1, 1 day', 'a INT, b interval');
-SELECT to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute));
-SELECT from_json('{"a":"1 days"}', 'a interval');
-SELECT to_json(map('a', interval 25 month 100 day 130 minute));
+SELECT
+  from_csv('1, 1 day', 'a INT, b interval'),
+  to_csv(from_csv('1, 1 day', 'a INT, b interval')),
+  to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)),
+  from_csv(to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)), 'a interval, b interval');
+SELECT
+  from_json('{"a":"1 days"}', 'a interval'),
+  to_json(from_json('{"a":"1 days"}', 'a interval')),
+  to_json(map('a', interval 25 month 100 day 130 minute)),
+  from_json(to_json(map('a', interval 25 month 100 day 130 minute)), 'a interval');
+
+select interval '+';
+select interval '+.';
+select interval '1';
+select interval '1.2';
+select interval '- 2';
+select interval '1 day -';
+select interval '1 day 1';
+
+select interval '1 day 2' day;
+select interval 'interval 1' day;
+select interval '-\t 1' day;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
index 6c14eee2e4e61..5bd78f5f6af3a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
@@ -48,6 +48,21 @@ select from_json('[null, {"a":2}]', 'array<struct<a:int>>');
 select from_json('[{"a": 1}, {"b":2}]', 'array<map<string,int>>');
 select from_json('[{"a": 1}, 2]', 'array<map<string,int>>');
 
+-- from_json - datetime type
+select from_json('{"d": "2012-12-15", "t": "2012-12-15 15:15:15"}', 'd date, t timestamp');
+select from_json(
+  '{"d": "12/15 2012", "t": "12/15 2012 15:15:15"}',
+  'd date, t timestamp',
+  map('dateFormat', 'MM/dd yyyy', 'timestampFormat', 'MM/dd yyyy HH:mm:ss'));
+select from_json(
+  '{"d": "02-29"}',
+  'd date',
+  map('dateFormat', 'MM-dd'));
+select from_json(
+  '{"t": "02-29"}',
+  't timestamp',
+  map('timestampFormat', 'MM-dd'));
+
 -- to_json - array type
 select to_json(array('1', '2', '3'));
 select to_json(array(array(1, 2, 3), array(4)));
diff --git a/sql/core/src/test/resources/sql-tests/inputs/operator-div.sql b/sql/core/src/test/resources/sql-tests/inputs/operator-div.sql
deleted file mode 100644
index 67b2d39aacd18..0000000000000
--- a/sql/core/src/test/resources/sql-tests/inputs/operator-div.sql
+++ /dev/null
@@ -1,21 +0,0 @@
-set spark.sql.legacy.integralDivide.returnBigint=true;
-
-select 5 div 2;
-select 5 div 0;
-select 5 div null;
-select null div 5;
-select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0));
-select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0));
-select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0));
-select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0));
-
-set spark.sql.legacy.integralDivide.returnBigint=false;
-
-select 5 div 2;
-select 5 div 0;
-select 5 div null;
-select null div 5;
-select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0));
-select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0));
-select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0));
-select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0));
diff --git a/sql/core/src/test/resources/sql-tests/inputs/operators.sql b/sql/core/src/test/resources/sql-tests/inputs/operators.sql
index ba14789d48db6..20bf0eb15c5b2 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/operators.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/operators.sql
@@ -22,6 +22,16 @@ select 5 / 0;
 select 5 / null;
 select null / 5;
 
+-- integral div
+select 5 div 2;
+select 5 div 0;
+select 5 div null;
+select null div 5;
+select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0));
+select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0));
+select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0));
+select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0));
+
 -- other arithmetics
 select 1 + 2;
 select 1 - 2;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/comments.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/comments.sql
index 6725ce45e72a5..1a454179ef79f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/comments.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/comments.sql
@@ -11,17 +11,19 @@ SELECT /* embedded single line */ 'embedded' AS `second`;
 SELECT /* both embedded and trailing single line */ 'both' AS third; -- trailing single line
 
 SELECT 'before multi-line' AS fourth;
+--QUERY-DELIMITER-START
 -- [SPARK-28880] ANSI SQL: Bracketed comments
 /* This is an example of SQL which should not execute:
  * select 'multi-line';
  */
 SELECT 'after multi-line' AS fifth;
+--QUERY-DELIMITER-END
 
 -- [SPARK-28880] ANSI SQL: Bracketed comments
 --
 -- Nested comments
 --
-
+--QUERY-DELIMITER-START
 /*
 SELECT 'trailing' as x1; -- inside block comment
 */
@@ -44,5 +46,5 @@ Hoo boy. Still two deep...
 Now just one deep...
 */
 'deeply nested example' AS sixth;
-
+--QUERY-DELIMITER-END
 /* and this is the end of the file */
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql
index 39e708478e298..21ffd85f7d01f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql
@@ -41,7 +41,7 @@ DROP TABLE emp;
 -- These views are left around mainly to exercise special cases in pg_dump.
 
 -- [SPARK-19842] Informational Referential Integrity Constraints Support in Spark
-CREATE TABLE view_base_table (key int /* PRIMARY KEY */, data varchar(20));
+CREATE TABLE view_base_table (key int /* PRIMARY KEY */, data varchar(20)) USING PARQUET;
 --
 CREATE VIEW key_dependent_view AS
    SELECT * FROM view_base_table GROUP BY key;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/date.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/date.sql
index 0bab2f884d976..69851080847b9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/date.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/date.sql
@@ -230,49 +230,49 @@ SELECT date 'tomorrow' - date 'yesterday' AS `Two days`;
 --
 -- epoch
 --
-SELECT EXTRACT(EPOCH FROM DATE        '1970-01-01');     --  0
-SELECT EXTRACT(EPOCH FROM TIMESTAMP   '1970-01-01');     --  0
+-- SELECT EXTRACT(EPOCH FROM DATE        '1970-01-01');     --  0
+-- SELECT EXTRACT(EPOCH FROM TIMESTAMP   '1970-01-01');     --  0
 -- SELECT EXTRACT(EPOCH FROM TIMESTAMPTZ '1970-01-01+00');  --  0
 --
 -- century
 --
-SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')); -- -2
-SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')); -- -1
-SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')); -- -1
-SELECT EXTRACT(CENTURY FROM DATE '0001-01-01');    --  1
-SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD'); --  1
-SELECT EXTRACT(CENTURY FROM DATE '1900-12-31');    -- 19
-SELECT EXTRACT(CENTURY FROM DATE '1901-01-01');    -- 20
-SELECT EXTRACT(CENTURY FROM DATE '2000-12-31');    -- 20
-SELECT EXTRACT(CENTURY FROM DATE '2001-01-01');    -- 21
-SELECT EXTRACT(CENTURY FROM CURRENT_DATE)>=21 AS True;     -- true
+-- SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')); -- -2
+-- SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')); -- -1
+-- SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')); -- -1
+-- SELECT EXTRACT(CENTURY FROM DATE '0001-01-01');    --  1
+-- SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD'); --  1
+-- SELECT EXTRACT(CENTURY FROM DATE '1900-12-31');    -- 19
+-- SELECT EXTRACT(CENTURY FROM DATE '1901-01-01');    -- 20
+-- SELECT EXTRACT(CENTURY FROM DATE '2000-12-31');    -- 20
+-- SELECT EXTRACT(CENTURY FROM DATE '2001-01-01');    -- 21
+-- SELECT EXTRACT(CENTURY FROM CURRENT_DATE)>=21 AS True;     -- true
 --
 -- millennium
 --
-SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')); -- -1
-SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD'); --  1
-SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31');    --  1
-SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01');    --  2
-SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31');    --  2
-SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01');    --  3
+-- SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')); -- -1
+-- SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD'); --  1
+-- SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31');    --  1
+-- SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01');    --  2
+-- SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31');    --  2
+-- SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01');    --  3
 -- next test to be fixed on the turn of the next millennium;-)
-SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE);         --  3
+-- SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE);         --  3
 --
 -- decade
 --
-SELECT EXTRACT(DECADE FROM DATE '1994-12-25');    -- 199
-SELECT EXTRACT(DECADE FROM DATE '0010-01-01');    --   1
-SELECT EXTRACT(DECADE FROM DATE '0009-12-31');    --   0
-SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')); --   0
-SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')); --  -1
-SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')); --  -1
-SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G')); --  -2
+-- SELECT EXTRACT(DECADE FROM DATE '1994-12-25');    -- 199
+-- SELECT EXTRACT(DECADE FROM DATE '0010-01-01');    --   1
+-- SELECT EXTRACT(DECADE FROM DATE '0009-12-31');    --   0
+-- SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')); --   0
+-- SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')); --  -1
+-- SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')); --  -1
+-- SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G')); --  -2
 --
 -- some other types:
 --
 -- on a timestamp.
-SELECT EXTRACT(CENTURY FROM NOW())>=21 AS True;       -- true
-SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000'); -- 20
+-- SELECT EXTRACT(CENTURY FROM NOW())>=21 AS True;       -- true
+-- SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000'); -- 20
 -- on an interval
 -- SELECT EXTRACT(CENTURY FROM INTERVAL '100 y');  -- 1
 -- SELECT EXTRACT(CENTURY FROM INTERVAL '99 y');   -- 0
@@ -280,16 +280,16 @@ SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000'); -- 20
 -- SELECT EXTRACT(CENTURY FROM INTERVAL '-100 y'); -- -1
 --
 -- test trunc function!
-SELECT DATE_TRUNC('MILLENNIUM', TIMESTAMP '1970-03-20 04:30:00.00000'); -- 1001
-SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20'); -- 1001-01-01
-SELECT DATE_TRUNC('CENTURY', TIMESTAMP '1970-03-20 04:30:00.00000'); -- 1901
-SELECT DATE_TRUNC('CENTURY', DATE '1970-03-20'); -- 1901
-SELECT DATE_TRUNC('CENTURY', DATE '2004-08-10'); -- 2001-01-01
-SELECT DATE_TRUNC('CENTURY', DATE '0002-02-04'); -- 0001-01-01
-SELECT DATE_TRUNC('CENTURY', TO_DATE('0055-08-10 BC', 'yyyy-MM-dd G')); -- 0100-01-01 BC
-SELECT DATE_TRUNC('DECADE', DATE '1993-12-25'); -- 1990-01-01
-SELECT DATE_TRUNC('DECADE', DATE '0004-12-25'); -- 0001-01-01 BC
-SELECT DATE_TRUNC('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')); -- 0011-01-01 BC
+-- SELECT DATE_TRUNC('MILLENNIUM', TIMESTAMP '1970-03-20 04:30:00.00000'); -- 1001
+-- SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20'); -- 1001-01-01
+-- SELECT DATE_TRUNC('CENTURY', TIMESTAMP '1970-03-20 04:30:00.00000'); -- 1901
+-- SELECT DATE_TRUNC('CENTURY', DATE '1970-03-20'); -- 1901
+-- SELECT DATE_TRUNC('CENTURY', DATE '2004-08-10'); -- 2001-01-01
+-- SELECT DATE_TRUNC('CENTURY', DATE '0002-02-04'); -- 0001-01-01
+-- SELECT DATE_TRUNC('CENTURY', TO_DATE('0055-08-10 BC', 'yyyy-MM-dd G')); -- 0100-01-01 BC
+-- SELECT DATE_TRUNC('DECADE', DATE '1993-12-25'); -- 1990-01-01
+-- SELECT DATE_TRUNC('DECADE', DATE '0004-12-25'); -- 0001-01-01 BC
+-- SELECT DATE_TRUNC('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')); -- 0011-01-01 BC
 
 -- [SPARK-29006] Support special date/timestamp values `infinity`/`-infinity`
 --
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql
index 541ff0bdad745..e2a94404395bc 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql
@@ -630,7 +630,8 @@ SELECT rpad('hi', -5, 'xy');
 SELECT rpad('hello', 2);
 SELECT rpad('hi', 5, '');
 
-SELECT ltrim('zzzytrim', 'xyz');
+-- skip this test because PostgreSQL has different parameter order compares to SparkSQL
+-- SELECT ltrim('zzzytrim', 'xyz');
 
 SELECT translate('', '14', 'ax');
 SELECT translate('12345', '14', 'ax');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/timestamp.sql
index bf69da295a960..ade29cc41358e 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/timestamp.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/timestamp.sql
@@ -200,15 +200,15 @@ SELECT '' AS `54`, d1 as `timestamp`,
     date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second`
     FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
 
-SELECT '' AS `54`, d1 as `timestamp`,
-    date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec,
-    date_part( 'usec', d1) AS usec
-    FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
-
-SELECT '' AS `54`, d1 as `timestamp`,
-    date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week,
-    date_part( 'dow', d1) AS dow
-    FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
+-- SELECT '' AS `54`, d1 as `timestamp`,
+--     date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec,
+--     date_part( 'usec', d1) AS usec
+--     FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
+
+-- SELECT '' AS `54`, d1 as `timestamp`,
+--     date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week,
+--     date_part( 'dow', d1) AS dow
+--     FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
 
 -- [SPARK-28137] Data Type Formatting Functions
 -- TO_CHAR()
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql
index 087d7a5befd19..6e95aca7aff62 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql
@@ -146,7 +146,7 @@ SELECT count(*) OVER (PARTITION BY four) FROM (SELECT * FROM tenk1 WHERE FALSE)s
 -- mixture of agg/wfunc in the same window
 -- SELECT sum(salary) OVER w, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary DESC);
 
--- Cannot safely cast 'enroll_date': StringType to DateType;
+-- Cannot safely cast 'enroll_date': string to date;
 -- SELECT empno, depname, salary, bonus, depadj, MIN(bonus) OVER (ORDER BY empno), MAX(depadj) OVER () FROM(
 -- SELECT *,
 --   CASE WHEN enroll_date < '2008-01-01' THEN 2008 - extract(year FROM enroll_date) END * 500 AS bonus,
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql
index 395149e48d5c8..ba1acc9f56b4a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql
@@ -15,6 +15,18 @@ CREATE TABLE empsalary (
     enroll_date date
 ) USING parquet;
 
+INSERT INTO empsalary VALUES
+  ('develop', 10, 5200, date '2007-08-01'),
+  ('sales', 1, 5000, date '2006-10-01'),
+  ('personnel', 5, 3500, date '2007-12-10'),
+  ('sales', 4, 4800, date '2007-08-08'),
+  ('personnel', 2, 3900, date '2006-12-23'),
+  ('develop', 7, 4200, date '2008-01-01'),
+  ('develop', 9, 4500, date '2008-01-01'),
+  ('sales', 3, 4800, date '2007-08-01'),
+  ('develop', 8, 6000, date '2006-10-01'),
+  ('develop', 11, 5200, date '2007-08-15');
+
 -- [SPARK-28429] SQL Datetime util function being casted to double instead of timestamp
 -- CREATE TEMP VIEW v_window AS
 -- SELECT i, min(i) over (order by i range between '1 day' preceding and '10 days' following) as min_i
@@ -99,9 +111,10 @@ FROM tenk1 WHERE unique1 < 10;
 -- nth_value(salary, 1) over(order by salary range between 1000 preceding and 1000 following),
 -- salary from empsalary;
 
-select last(salary) over(order by salary range between 1000 preceding and 1000 following),
-lag(salary) over(order by salary range between 1000 preceding and 1000 following),
-salary from empsalary;
+-- [SPARK-30734] AnalysisException that window RangeFrame not match RowFrame
+-- select last(salary) over(order by salary range between 1000 preceding and 1000 following),
+-- lag(salary) over(order by salary range between 1000 preceding and 1000 following),
+-- salary from empsalary;
 
 -- [SPARK-27951] ANSI SQL: NTH_VALUE function
 -- select first_value(salary) over(order by salary range between 1000 following and 3000 following
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql
index cd3b74b3aa03f..f4b8454da0d82 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql
@@ -42,7 +42,7 @@ create table datetimes (
     f_timestamp timestamp
 ) using parquet;
 
--- Spark cannot safely cast StringType to TimestampType
+-- Spark cannot safely cast string to timestamp
 -- [SPARK-29636] Spark can't parse '11:00 BST' or '2000-10-19 10:23:54+01' signatures to timestamp
 insert into datetimes values
 (1, timestamp '11:00', cast ('11:00 BST' as timestamp), cast ('1 year' as timestamp), cast ('2000-10-19 10:23:54+01' as timestamp), timestamp '2000-10-19 10:23:54'),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
new file mode 100644
index 0000000000000..8a531be30d896
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
@@ -0,0 +1,11 @@
+-- regexp_extract
+SELECT regexp_extract('1a 2b 14m', '\\d+');
+SELECT regexp_extract('1a 2b 14m', '\\d+', 0);
+SELECT regexp_extract('1a 2b 14m', '\\d+', 1);
+SELECT regexp_extract('1a 2b 14m', '\\d+', 2);
+SELECT regexp_extract('1a 2b 14m', '\\d+', -1);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)');
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 0);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 1);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 2);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', -1);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql b/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql
index dc77f87d9743a..00b46d1951fcf 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql
@@ -74,6 +74,9 @@ CREATE VIEW view_SPARK_30302 (aaa, bbb)
 AS SELECT a, b FROM tbl;
 
 SHOW CREATE TABLE view_SPARK_30302 AS SERDE;
+
+SHOW CREATE TABLE view_SPARK_30302;
+
 DROP VIEW view_SPARK_30302;
 
 
@@ -83,6 +86,9 @@ COMMENT 'This is a comment with \'quoted text\' for view'
 AS SELECT a, b FROM tbl;
 
 SHOW CREATE TABLE view_SPARK_30302 AS SERDE;
+
+SHOW CREATE TABLE view_SPARK_30302;
+
 DROP VIEW view_SPARK_30302;
 
 
@@ -92,13 +98,9 @@ TBLPROPERTIES ('a' = '1', 'b' = '2')
 AS SELECT a, b FROM tbl;
 
 SHOW CREATE TABLE view_SPARK_30302 AS SERDE;
-DROP VIEW view_SPARK_30302;
-
--- SHOW CREATE TABLE does not support view
-CREATE VIEW view_SPARK_30302 (aaa, bbb)
-AS SELECT a, b FROM tbl;
 
 SHOW CREATE TABLE view_SPARK_30302;
+
 DROP VIEW view_SPARK_30302;
 
 DROP TABLE tbl;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-tables.sql b/sql/core/src/test/resources/sql-tests/inputs/show-tables.sql
index 3c77c9977d80f..8f46c93ee3233 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/show-tables.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/show-tables.sql
@@ -15,6 +15,7 @@ SHOW TABLES IN showdb;
 SHOW TABLES 'show_t*';
 SHOW TABLES LIKE 'show_t1*|show_t2*';
 SHOW TABLES IN showdb 'show_t*';
+SHOW TABLES IN showdb LIKE 'show_t*';
 
 -- SHOW TABLE EXTENDED
 SHOW TABLE EXTENDED LIKE 'show_t*';
diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql b/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql
new file mode 100644
index 0000000000000..2861b2b43a113
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql
@@ -0,0 +1,26 @@
+-- create a table with properties
+CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
+TBLPROPERTIES('p1'='v1', 'p2'='v2');
+
+SHOW TBLPROPERTIES tbl;
+SHOW TBLPROPERTIES tbl("p1");
+SHOW TBLPROPERTIES tbl("p3");
+
+DROP TABLE tbl;
+
+-- create a view with properties
+CREATE VIEW view TBLPROPERTIES('p1'='v1', 'p2'='v2') AS SELECT 1 AS c1;
+
+SHOW TBLPROPERTIES view;
+SHOW TBLPROPERTIES view("p1");
+SHOW TBLPROPERTIES view("p3");
+
+DROP VIEW view;
+
+-- create a temporary view with properties
+CREATE TEMPORARY VIEW tv TBLPROPERTIES('p1'='v1') AS SELECT 1 AS c1;
+
+-- Properties for a temporary view should be empty
+SHOW TBLPROPERTIES tv;
+
+DROP VIEW tv;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-views.sql b/sql/core/src/test/resources/sql-tests/inputs/show-views.sql
new file mode 100644
index 0000000000000..bdf9ef4aa6780
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/show-views.sql
@@ -0,0 +1,28 @@
+-- Test data.
+CREATE DATABASE showdb;
+USE showdb;
+CREATE TABLE tbl(a STRING, b INT, c STRING, d STRING) USING parquet;
+CREATE VIEW view_1 AS SELECT * FROM tbl;
+CREATE VIEW view_2 AS SELECT * FROM tbl WHERE c='a';
+CREATE GLOBAL TEMP VIEW view_3 AS SELECT 1 as col1;
+CREATE TEMPORARY VIEW view_4(e INT) USING parquet;
+
+-- SHOW VIEWS
+SHOW VIEWS;
+SHOW VIEWS FROM showdb;
+SHOW VIEWS IN showdb;
+SHOW VIEWS IN global_temp;
+
+-- SHOW VIEWS WITH wildcard match
+SHOW VIEWS 'view_*';
+SHOW VIEWS LIKE 'view_1*|view_2*';
+SHOW VIEWS IN showdb 'view_*';
+SHOW VIEWS IN showdb LIKE 'view_*';
+-- Error when database not exists
+SHOW VIEWS IN wrongdb LIKE 'view_*';
+
+-- Clean Up
+DROP VIEW global_temp.view_3;
+DROP VIEW view_4;
+USE default;
+DROP DATABASE showdb CASCADE;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
index 59987b9792e25..f5ed2036dc8ac 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
@@ -39,12 +39,17 @@ SELECT substring('Spark SQL' from 5);
 SELECT substring('Spark SQL' from -3);
 SELECT substring('Spark SQL' from 5 for 1);
 
--- trim/ltrim/rtrim
-SELECT trim('yxTomxx', 'xyz'), trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx');
-SELECT trim('xxxbarxxx', 'x'), trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx');
-SELECT ltrim('zzzytest', 'xyz'), trim(LEADING 'xyz' FROM 'zzzytest');
-SELECT ltrim('zzzytestxyz', 'xyz'), trim(LEADING 'xyz' FROM 'zzzytestxyz');
-SELECT ltrim('xyxXxyLAST WORD', 'xy'), trim(LEADING 'xy' FROM 'xyxXxyLAST WORD');
-SELECT rtrim('testxxzx', 'xyz'), trim(TRAILING 'xyz' FROM 'testxxzx');
-SELECT rtrim('xyztestxxzx', 'xyz'), trim(TRAILING 'xyz' FROM 'xyztestxxzx');
-SELECT rtrim('TURNERyxXxy', 'xy'), trim(TRAILING 'xy' FROM 'TURNERyxXxy');
+-- trim
+SELECT trim(" xyz "), ltrim(" xyz "), rtrim(" xyz ");
+SELECT trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx');
+SELECT trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx');
+SELECT trim(LEADING 'xyz' FROM 'zzzytest');
+SELECT trim(LEADING 'xyz' FROM 'zzzytestxyz');
+SELECT trim(LEADING 'xy' FROM 'xyxXxyLAST WORD');
+SELECT trim(TRAILING 'xyz' FROM 'testxxzx');
+SELECT trim(TRAILING 'xyz' FROM 'xyztestxxzx');
+SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy');
+
+-- Check lpad/rpad with invalid length parameter
+SELECT lpad('hi', 'invalid_length');
+SELECT rpad('hi', 'invalid_length');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
index 481b5e8cc7700..0a16f118f0455 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
@@ -72,7 +72,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d IN (SELECT t2d
                FROM   t2
-               ORDER  BY t2c
+               ORDER  BY t2c, t2d
                LIMIT 2)
 GROUP  BY t1b
 ORDER  BY t1b DESC NULLS FIRST
@@ -93,7 +93,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d NOT IN (SELECT t2d
                    FROM   t2
-                   ORDER  BY t2b DESC nulls first
+                   ORDER  BY t2b DESC nulls first, t2d
                    LIMIT 1)
 GROUP  BY t1b
 ORDER BY t1b NULLS last
diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql
index e6fe1078b0d24..77bcfe7e10cbc 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql
@@ -8,6 +8,9 @@
 --
 -- This test file was converted from postgreSQL/join.sql.
 
+-- Disable BroadcastHashJoin optimization to avoid changing result order when we enable AQE
+--SET spark.sql.autoBroadcastJoinThreshold = -1
+
 CREATE OR REPLACE TEMPORARY VIEW INT4_TBL AS SELECT * FROM
   (VALUES (0), (123456), (-123456), (2147483647), (-2147483647))
   AS v(f1);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/union.sql b/sql/core/src/test/resources/sql-tests/inputs/union.sql
index 6da1b9b49b226..8a5b6c50fc1e3 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/union.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/union.sql
@@ -45,10 +45,24 @@ SELECT array(1, 2), 'str'
 UNION ALL
 SELECT array(1, 2, 3, NULL), 1;
 
+-- SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v);
+SELECT t.v FROM (
+  SELECT v FROM t3
+  UNION ALL
+  SELECT v + v AS v FROM t3
+) t;
+
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  UNION
+  SELECT v + v AS v FROM t3
+) t;
 
 -- Clean-up
 DROP VIEW IF EXISTS t1;
 DROP VIEW IF EXISTS t2;
+DROP VIEW IF EXISTS t3;
 DROP VIEW IF EXISTS p1;
 DROP VIEW IF EXISTS p2;
 DROP VIEW IF EXISTS p3;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/window.sql b/sql/core/src/test/resources/sql-tests/inputs/window.sql
index e25a252418301..3d05dfda6c3fa 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/window.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/window.sql
@@ -120,3 +120,8 @@ SELECT cate, sum(val) OVER (w)
 FROM testData
 WHERE val is not null
 WINDOW w AS (PARTITION BY cate ORDER BY val);
+
+-- with filter predicate
+SELECT val, cate,
+count(val) FILTER (WHERE val > 1) OVER(PARTITION BY cate)
+FROM testData ORDER BY cate, val;
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
new file mode 100644
index 0000000000000..7e5568dc5cb36
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -0,0 +1,937 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 108
+
+
+-- !query
+select current_date = current_date(), current_timestamp = current_timestamp()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'current_date'(line 1, pos 22)
+
+== SQL ==
+select current_date = current_date(), current_timestamp = current_timestamp()
+----------------------^^^
+
+
+-- !query
+select to_date(null), to_date('2016-12-31'), to_date('2016-12-31', 'yyyy-MM-dd')
+-- !query schema
+struct<to_date(NULL):date,to_date('2016-12-31'):date,to_date('2016-12-31', 'yyyy-MM-dd'):date>
+-- !query output
+NULL	2016-12-31	2016-12-31
+
+
+-- !query
+select to_timestamp(null), to_timestamp('2016-12-31 00:12:00'), to_timestamp('2016-12-31', 'yyyy-MM-dd')
+-- !query schema
+struct<to_timestamp(NULL):timestamp,to_timestamp('2016-12-31 00:12:00'):timestamp,to_timestamp('2016-12-31', 'yyyy-MM-dd'):timestamp>
+-- !query output
+NULL	2016-12-31 00:12:00	2016-12-31 00:00:00
+
+
+-- !query
+select dayofweek('2007-02-03'), dayofweek('2009-07-30'), dayofweek('2017-05-27'), dayofweek(null), dayofweek('1582-10-15 13:10:15')
+-- !query schema
+struct<dayofweek(CAST(2007-02-03 AS DATE)):int,dayofweek(CAST(2009-07-30 AS DATE)):int,dayofweek(CAST(2017-05-27 AS DATE)):int,dayofweek(CAST(NULL AS DATE)):int,dayofweek(CAST(1582-10-15 13:10:15 AS DATE)):int>
+-- !query output
+7	5	7	NULL	6
+
+
+-- !query
+create temporary view ttf1 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf1(current_date, current_timestamp)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'current_date'(line 4, pos 10)
+
+== SQL ==
+create temporary view ttf1 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf1(current_date, current_timestamp)
+----------^^^
+
+
+-- !query
+select current_date, current_timestamp from ttf1
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Table or view not found: ttf1; line 1 pos 44
+
+
+-- !query
+create temporary view ttf2 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf2(a, b)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select current_date = current_date(), current_timestamp = current_timestamp(), a, b from ttf2
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'current_date'(line 1, pos 22)
+
+== SQL ==
+select current_date = current_date(), current_timestamp = current_timestamp(), a, b from ttf2
+----------------------^^^
+
+
+-- !query
+select a, b from ttf2 order by a, current_date
+-- !query schema
+struct<a:int,b:int>
+-- !query output
+1	2
+2	3
+
+
+-- !query
+select weekday('2007-02-03'), weekday('2009-07-30'), weekday('2017-05-27'), weekday(null), weekday('1582-10-15 13:10:15')
+-- !query schema
+struct<weekday(CAST(2007-02-03 AS DATE)):int,weekday(CAST(2009-07-30 AS DATE)):int,weekday(CAST(2017-05-27 AS DATE)):int,weekday(CAST(NULL AS DATE)):int,weekday(CAST(1582-10-15 13:10:15 AS DATE)):int>
+-- !query output
+5	3	5	NULL	4
+
+
+-- !query
+select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
+-- !query schema
+struct<year(CAST(1500-01-01 AS DATE)):int,month(CAST(1500-01-01 AS DATE)):int,dayofyear(CAST(1500-01-01 AS DATE)):int>
+-- !query output
+1500	1	1
+
+
+-- !query
+select date '2019-01-01\t'
+-- !query schema
+struct<DATE '2019-01-01':date>
+-- !query output
+2019-01-01
+
+
+-- !query
+select timestamp '2019-01-01\t'
+-- !query schema
+struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
+-- !query output
+2019-01-01 00:00:00
+
+
+-- !query
+select date '2020-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)
+
+== SQL ==
+select date '2020-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp '2019-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)
+
+== SQL ==
+select timestamp '2019-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp'2011-11-11 11:11:11' + interval '2' day
+-- !query schema
+struct<CAST(TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days' AS TIMESTAMP):timestamp>
+-- !query output
+2011-11-13 11:11:11
+
+
+-- !query
+select timestamp'2011-11-11 11:11:11' - interval '2' day
+-- !query schema
+struct<CAST(TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days' AS TIMESTAMP):timestamp>
+-- !query output
+2011-11-09 11:11:11
+
+
+-- !query
+select date'2011-11-11 11:11:11' + interval '2' second
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+requirement failed: Cannot add hours, minutes or seconds, milliseconds, microseconds to a date
+
+
+-- !query
+select date'2011-11-11 11:11:11' - interval '2' second
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+requirement failed: Cannot add hours, minutes or seconds, milliseconds, microseconds to a date
+
+
+-- !query
+select '2011-11-11' - interval '2' day
+-- !query schema
+struct<CAST(CAST(2011-11-11 AS TIMESTAMP) - INTERVAL '2 days' AS STRING):string>
+-- !query output
+2011-11-09 00:00:00
+
+
+-- !query
+select '2011-11-11 11:11:11' - interval '2' second
+-- !query schema
+struct<CAST(CAST(2011-11-11 11:11:11 AS TIMESTAMP) - INTERVAL '2 seconds' AS STRING):string>
+-- !query output
+2011-11-11 11:11:09
+
+
+-- !query
+select '1' - interval '2' second
+-- !query schema
+struct<CAST(CAST(1 AS TIMESTAMP) - INTERVAL '2 seconds' AS STRING):string>
+-- !query output
+NULL
+
+
+-- !query
+select 1 - interval '2' second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve '1 - INTERVAL '2 seconds'' due to data type mismatch: argument 1 requires timestamp type, however, '1' is of int type.; line 1 pos 7
+
+
+-- !query
+select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'
+-- !query schema
+struct<subtracttimestamps(CAST(DATE '2020-01-01' AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+-- !query output
+2078 hours 48 minutes 47.654322 seconds
+
+
+-- !query
+select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'
+-- !query schema
+struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(DATE '2020-01-01' AS TIMESTAMP)):interval>
+-- !query output
+-2078 hours -48 minutes -47.654322 seconds
+
+
+-- !query
+select timestamp'2019-10-06 10:11:12.345678' - null
+-- !query schema
+struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(NULL AS TIMESTAMP)):interval>
+-- !query output
+NULL
+
+
+-- !query
+select null - timestamp'2019-10-06 10:11:12.345678'
+-- !query schema
+struct<subtracttimestamps(CAST(NULL AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+-- !query output
+NULL
+
+
+-- !query
+select date_add('2011-11-11', 1Y)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1S)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1L)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 1L)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '1L' is of bigint type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', 1.0)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 1.0BD)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '1.0BD' is of decimal(2,1) type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', 1E1)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', '1')
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_add' function needs to be an integer.;
+
+
+-- !query
+select date_add(date'2011-11-11', 1)
+-- !query schema
+struct<date_add(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add(timestamp'2011-11-11', 1)
+-- !query schema
+struct<date_add(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_sub(date'2011-11-11', 1)
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1')
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_sub' function needs to be an integer.;
+
+
+-- !query
+select date_sub(timestamp'2011-11-11', 1)
+-- !query schema
+struct<date_sub(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(null, 1)
+-- !query schema
+struct<date_sub(CAST(NULL AS DATE), 1):date>
+-- !query output
+NULL
+
+
+-- !query
+select date_sub(date'2011-11-11', null)
+-- !query schema
+struct<date_sub(DATE '2011-11-11', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date'2011-11-11' + 1E1
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
+
+
+-- !query
+select date'2011-11-11' + '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
+-- !query
+select null + date '2001-09-28'
+-- !query schema
+struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-09-28' + 7Y
+-- !query schema
+struct<date_add(DATE '2001-09-28', 7):date>
+-- !query output
+2001-10-05
+
+
+-- !query
+select 7S + date '2001-09-28'
+-- !query schema
+struct<date_add(DATE '2001-09-28', 7):date>
+-- !query output
+2001-10-05
+
+
+-- !query
+select date '2001-10-01' - 7
+-- !query schema
+struct<date_sub(DATE '2001-10-01', 7):date>
+-- !query output
+2001-09-24
+
+
+-- !query
+select date '2001-10-01' - '7'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(DATE '2001-10-01', CAST('7' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('7' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
+-- !query
+select date '2001-09-28' + null
+-- !query schema
+struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-09-28' - null
+-- !query schema
+struct<date_sub(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+create temp view v as select '1' str
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select date_add('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select date_sub('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select null - date '2019-10-06'
+-- !query schema
+struct<subtractdates(CAST(NULL AS DATE), DATE '2019-10-06'):interval>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-10-01' - date '2001-09-28'
+-- !query schema
+struct<subtractdates(DATE '2001-10-01', DATE '2001-09-28'):interval>
+-- !query output
+3 days
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 03:11:12.123
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 08:11:12.12345
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123456
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123456
+
+
+-- !query
+select to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:00
+
+
+-- !query
+select to_timestamp("2019-10-06S10:11:12.12345", "yyyy-MM-dd'S'HH:mm:ss.SSSSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06S10:11:12.12345', 'yyyy-MM-dd\'S\'HH:mm:ss.SSSSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12345
+
+
+-- !query
+select to_timestamp("12.12342019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.12342019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1234019-10-06S10:11', 'ss.SSSSy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+0019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'")
+-- !query schema
+struct<to_timestamp('2019-10-06S', 'yyyy-MM-dd\'S\''):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd")
+-- !query schema
+struct<to_timestamp('S2019-10-06', '\'S\'yyyy-MM-dd'):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'12', 'yyyy-MM-dd\'T\'HH:mm:ss\'\'SSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'', 'yyyy-MM-dd\'T\'HH:mm:ss\'\''):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('\'2019-10-06T10:11:12', '\'\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('P2019-10-06T10:11:12', '\'P\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("16", "dd")
+-- !query schema
+struct<to_timestamp('16', 'dd'):timestamp>
+-- !query output
+1970-01-16 00:00:00
+
+
+-- !query
+select to_timestamp("02-29", "MM-dd")
+-- !query schema
+struct<to_timestamp('02-29', 'MM-dd'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_date("16", "dd")
+-- !query schema
+struct<to_date('16', 'dd'):date>
+-- !query output
+1970-01-16
+
+
+-- !query
+select to_date("02-29", "MM-dd")
+-- !query schema
+struct<to_date('02-29', 'MM-dd'):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019 40", "yyyy mm")
+-- !query schema
+struct<to_timestamp('2019 40', 'yyyy mm'):timestamp>
+-- !query output
+2019-01-01 00:40:00
+
+
+-- !query
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
+-- !query schema
+struct<to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss'):timestamp>
+-- !query output
+2019-01-01 10:10:10
+
+
+-- !query
+select date_format(date '2020-05-23', 'GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(date '2020-05-23', 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(date '2020-05-23', 'LLLLL')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'LLLLL' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'uuuuu')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'uuuuu' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2020-05-23', 'QQQQQ')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: Q
+
+
+-- !query
+select date_format('2020-05-23', 'qqqqq')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: q
+
+
+-- !query
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyy-MM-dd GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_unixtime(12345, 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_unixtime(54321, 'QQQQQ')
+-- !query schema
+struct<from_unixtime(CAST(54321 AS BIGINT), QQQQQ):string>
+-- !query output
+NULL
+
+
+-- !query
+select from_unixtime(23456, 'aaaaa')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aaaaa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out
index 7bef1bad4507e..6d26fae73b11c 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 29
+-- Number of queries: 30
 
 
 -- !query
@@ -282,3 +282,11 @@ no viable alternative at input 'all'(line 1, pos 22)
 == SQL ==
 select transform(ys, (all, i) -> all + i) as v from values (array(32, 97)) as t(ys)
 ----------------------^^^
+
+
+-- !query
+select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)))
+-- !query schema
+struct<aggregate(split(abcdefgh, , -1), array(array()), lambdafunction(array(array(namedlambdavariable())), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):array<array<string>>>
+-- !query output
+[[""]]
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index ab6130da869c4..2cee0c8886935 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 101
+-- Number of queries: 107
 
 
 -- !query
@@ -26,6 +26,42 @@ struct<divide_interval(subtracttimestamps(TIMESTAMP '2019-10-15 00:00:00', TIMES
 16 hours
 
 
+-- !query
+select interval 2147483647 month * 2
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+integer overflow
+
+
+-- !query
+select interval 2147483647 month / 0.5
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+integer overflow
+
+
+-- !query
+select interval 2147483647 day * 2
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+integer overflow
+
+
+-- !query
+select interval 2147483647 day / 0.5
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+integer overflow
+
+
 -- !query
 select interval '2 seconds' / 0
 -- !query schema
@@ -142,11 +178,28 @@ struct<make_interval(1, 2, 3, 4, 5, 6, 0.000000):interval>
 -- !query
 select make_interval(1, 2, 3, 4, 5, 6, 7.008009)
 -- !query schema
-struct<make_interval(1, 2, 3, 4, 5, 6, CAST(7.008009 AS DECIMAL(8,6))):interval>
+struct<make_interval(1, 2, 3, 4, 5, 6, CAST(7.008009 AS DECIMAL(18,6))):interval>
 -- !query output
 1 years 2 months 25 days 5 hours 6 minutes 7.008009 seconds
 
 
+-- !query
+select make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456)
+-- !query schema
+struct<make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456):interval>
+-- !query output
+1 years 2 months 25 days 34293552 hours 30 minutes 12.123456 seconds
+
+
+-- !query
+select make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Decimal(expanded,1234567890123456789,20,0}) cannot be represented as Decimal(18, 6).
+
+
 -- !query
 select cast('1 second' as interval)
 -- !query schema
@@ -302,15 +355,17 @@ struct<INTERVAL '40 minutes 32 seconds':interval>
 -- !query
 select interval 30 day day
 -- !query schema
-struct<>
+struct<day:interval>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
+30 days
 
-no viable alternative at input 'day'(line 1, pos 23)
 
-== SQL ==
-select interval 30 day day
------------------------^^^
+-- !query
+select interval 30 days days
+-- !query schema
+struct<days:interval>
+-- !query output
+30 days
 
 
 -- !query
@@ -320,7 +375,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 15:40:32.99899999' day to hour
@@ -334,7 +389,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 15:40:32.99899999' day to minute
@@ -348,7 +403,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40:32.99899999' hour to minute
@@ -362,7 +417,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40.99899999' hour to second
@@ -376,7 +431,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40' hour to second
@@ -390,7 +445,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 40:32.99899999' minute to second
@@ -435,6 +490,14 @@ struct<INTERVAL '3 years 1 hours':interval>
 3 years 1 hours
 
 
+-- !query
+SELECT interval '1 year 3 months 2 weeks 2 days 1 hour 3 minutes 2 seconds 100 millisecond 200 microseconds'
+-- !query schema
+struct<INTERVAL '1 years 3 months 16 days 1 hours 3 minutes 2.1002 seconds':interval>
+-- !query output
+1 years 3 months 16 days 1 hours 3 minutes 2.1002 seconds
+
+
 -- !query
 select interval
 -- !query schema
@@ -594,13 +657,8 @@ select interval (-30) day
 -- !query schema
 struct<>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-no viable alternative at input 'day'(line 1, pos 22)
-
-== SQL ==
-select interval (-30) day
-----------------------^^^
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
 
 
 -- !query
@@ -608,13 +666,8 @@ select interval (a + 1) day
 -- !query schema
 struct<>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-no viable alternative at input 'day'(line 1, pos 24)
-
-== SQL ==
-select interval (a + 1) day
-------------------------^^^
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
 
 
 -- !query
@@ -624,185 +677,43 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-no viable alternative at input 'day'(line 1, pos 23)
+extraneous input 'day' expecting {<EOF>, ';'}(line 1, pos 27)
 
 == SQL ==
 select interval 30 day day day
------------------------^^^
+---------------------------^^^
 
 
 -- !query
-select sum(cast(null as interval))
+select interval (-30) days
 -- !query schema
-struct<sum(CAST(NULL AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('1 seconds') t(v) where 1=0
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-3 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-1 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
--3 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
--7 days 2 seconds
-
-
--- !query
-select
-    i,
-    sum(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i
--- !query schema
-struct<i:int,sum(CAST(v AS INTERVAL)):interval>
--- !query output
-1	-2 days
-2	2 seconds
-3	NULL
-
-
--- !query
-select
-    sum(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null
--- !query schema
-struct<sv:interval>
--- !query output
--2 days 2 seconds
-
-
--- !query
-SELECT
-    i,
-    sum(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES(1, '1 seconds'), (1, '2 seconds'), (2, NULL), (2, NULL) t(i,v)
--- !query schema
-struct<i:int,sum(CAST(v AS INTERVAL)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):interval>
--- !query output
-1	2 seconds
-1	3 seconds
-2	NULL
-2	NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v) where 1=0
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-1.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-0.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
+struct<>
 -- !query output
--1.5 seconds
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
 
 
 -- !query
-select avg(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v)
+select interval (a + 1) days
 -- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
+struct<>
 -- !query output
--3 days -11 hours -59 minutes -59 seconds
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
 
 
 -- !query
-select
-    i,
-    avg(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i
+select interval 30 days days days
 -- !query schema
-struct<i:int,avg(CAST(v AS INTERVAL)):interval>
--- !query output
-1	-1 days
-2	2 seconds
-3	NULL
-
-
--- !query
-select
-    avg(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null
--- !query schema
-struct<sv:interval>
+struct<>
 -- !query output
--15 hours -59 minutes -59.333333 seconds
+org.apache.spark.sql.catalyst.parser.ParseException
 
+extraneous input 'days' expecting {<EOF>, ';'}(line 1, pos 29)
 
--- !query
-SELECT
-    i,
-    avg(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES (1,'1 seconds'), (1,'2 seconds'), (2,NULL), (2,NULL) t(i,v)
--- !query schema
-struct<i:int,avg(CAST(v AS INTERVAL)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):interval>
--- !query output
-1	1.5 seconds
-1	2 seconds
-2	NULL
-2	NULL
+== SQL ==
+select interval 30 days days days
+-----------------------------^^^
 
 
 -- !query
@@ -827,7 +738,7 @@ select
   interval '2-2' year to month + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '2 years 2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '-2 years -2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '2 years 2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '-2 years -2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + (- INTERVAL '2 years 2 months') AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '2 years 2 months' AS DATE):date>
+struct<dateval:date,dateval + (- INTERVAL '2 years 2 months'):date,dateval + (- INTERVAL '-2 years -2 months'):date,dateval + INTERVAL '2 years 2 months':date,dateval + INTERVAL '-2 years -2 months':date,dateval + (- INTERVAL '2 years 2 months'):date,dateval + INTERVAL '2 years 2 months':date>
 -- !query output
 2012-01-01	2009-11-01	2014-03-01	2014-03-01	2009-11-01	2009-11-01	2014-03-01
 
@@ -870,9 +781,10 @@ select
   interval '99 11:22:33.123456789' day to second + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds') AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date>
+struct<>
 -- !query output
-2012-01-01	2011-09-23	2012-04-09	2012-04-09	2011-09-23	2011-09-23	2012-04-09
+java.lang.IllegalArgumentException
+requirement failed: Cannot add hours, minutes or seconds, milliseconds, microseconds to a date
 
 
 -- !query
@@ -942,6 +854,65 @@ struct<INTERVAL '1 days':interval>
 1 days
 
 
+-- !query
+select interval '2-2\t' year to month
+-- !query schema
+struct<INTERVAL '2 years 2 months':interval>
+-- !query output
+2 years 2 months
+
+
+-- !query
+select interval '-\t2-2\t' year to month
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Interval string does not match year-month format of 'y-m': -	2-2	(line 1, pos 16)
+
+== SQL ==
+select interval '-\t2-2\t' year to month
+----------------^^^
+
+
+-- !query
+select interval '\n0 12:34:46.789\t' day to second
+-- !query schema
+struct<INTERVAL '12 hours 34 minutes 46.789 seconds':interval>
+-- !query output
+12 hours 34 minutes 46.789 seconds
+
+
+-- !query
+select interval '\n-\t10\t 12:34:46.789\t' day to second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 
+-	10	 12:34:46.789	, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
+
+== SQL ==
+select interval '\n-\t10\t 12:34:46.789\t' day to second
+----------------^^^
+
+
+-- !query
+select interval '中文 interval 1 day'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 中文 interval 1 day(line 1, pos 7)
+
+== SQL ==
+select interval '中文 interval 1 day'
+-------^^^
+
+
 -- !query
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
@@ -988,32 +959,158 @@ integer overflow
 
 
 -- !query
-SELECT from_csv('1, 1 day', 'a INT, b interval')
+SELECT
+  from_csv('1, 1 day', 'a INT, b interval'),
+  to_csv(from_csv('1, 1 day', 'a INT, b interval')),
+  to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)),
+  from_csv(to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)), 'a interval, b interval')
+-- !query schema
+struct<from_csv(1, 1 day):struct<a:int,b:interval>,to_csv(from_csv(1, 1 day)):string,to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes')):string,from_csv(to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes'))):struct<a:interval,b:interval>>
+-- !query output
+{"a":1,"b":1 days}	1,1 days	2 years 8 months,1 hours 10 minutes	{"a":2 years 8 months,"b":1 hours 10 minutes}
+
+
+-- !query
+SELECT
+  from_json('{"a":"1 days"}', 'a interval'),
+  to_json(from_json('{"a":"1 days"}', 'a interval')),
+  to_json(map('a', interval 25 month 100 day 130 minute)),
+  from_json(to_json(map('a', interval 25 month 100 day 130 minute)), 'a interval')
+-- !query schema
+struct<from_json({"a":"1 days"}):struct<a:interval>,to_json(from_json({"a":"1 days"})):string,to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string,from_json(to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes'))):struct<a:interval>>
+-- !query output
+{"a":1 days}	{"a":"1 days"}	{"a":"2 years 1 months 100 days 2 hours 10 minutes"}	{"a":2 years 1 months 100 days 2 hours 10 minutes}
+
+
+-- !query
+select interval '+'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +(line 1, pos 7)
+
+== SQL ==
+select interval '+'
+-------^^^
+
+
+-- !query
+select interval '+.'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +.(line 1, pos 7)
+
+== SQL ==
+select interval '+.'
+-------^^^
+
+
+-- !query
+select interval '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1(line 1, pos 7)
+
+== SQL ==
+select interval '1'
+-------^^^
+
+
+-- !query
+select interval '1.2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1.2(line 1, pos 7)
+
+== SQL ==
+select interval '1.2'
+-------^^^
+
+
+-- !query
+select interval '- 2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: - 2(line 1, pos 7)
+
+== SQL ==
+select interval '- 2'
+-------^^^
+
+
+-- !query
+select interval '1 day -'
 -- !query schema
-struct<from_csv(1, 1 day):struct<a:int,b:interval>>
+struct<>
 -- !query output
-{"a":1,"b":1 days}
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day -(line 1, pos 7)
+
+== SQL ==
+select interval '1 day -'
+-------^^^
 
 
 -- !query
-SELECT to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute))
+select interval '1 day 1'
 -- !query schema
-struct<to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes')):string>
+struct<>
 -- !query output
-2 years 8 months,1 hours 10 minutes
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day 1(line 1, pos 7)
+
+== SQL ==
+select interval '1 day 1'
+-------^^^
+
+
+-- !query
+select interval '1 day 2' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: 1 day 2(line 1, pos 16)
+
+== SQL ==
+select interval '1 day 2' day
+----------------^^^
 
 
 -- !query
-SELECT from_json('{"a":"1 days"}', 'a interval')
+select interval 'interval 1' day
 -- !query schema
-struct<from_json({"a":"1 days"}):struct<a:interval>>
+struct<>
 -- !query output
-{"a":1 days}
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: interval 1(line 1, pos 16)
+
+== SQL ==
+select interval 'interval 1' day
+----------------^^^
 
 
 -- !query
-SELECT to_json(map('a', interval 25 month 100 day 130 minute))
+select interval '-\t 1' day
 -- !query schema
-struct<to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string>
+struct<INTERVAL '-1 days':interval>
 -- !query output
-{"a":"2 years 1 months 100 days 2 hours 10 minutes"}
+-1 days
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
new file mode 100644
index 0000000000000..dfdf183233b42
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
@@ -0,0 +1,296 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 36
+
+
+-- !query
+select concat_ws()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+requirement failed: concat_ws requires at least one argument.; line 1 pos 7
+
+
+-- !query
+select format_string()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+requirement failed: format_string() should take at least 1 argument; line 1 pos 7
+
+
+-- !query
+select 'a' || 'b' || 'c'
+-- !query schema
+struct<concat(concat(a, b), c):string>
+-- !query output
+abc
+
+
+-- !query
+select replace('abc', 'b', '123')
+-- !query schema
+struct<replace(abc, b, 123):string>
+-- !query output
+a123c
+
+
+-- !query
+select replace('abc', 'b')
+-- !query schema
+struct<replace(abc, b, ):string>
+-- !query output
+ac
+
+
+-- !query
+select length(uuid()), (uuid() <> uuid())
+-- !query schema
+struct<length(uuid()):int,(NOT (uuid() = uuid())):boolean>
+-- !query output
+36	true
+
+
+-- !query
+select position('bar' in 'foobarbar'), position(null, 'foobarbar'), position('aaads', null)
+-- !query schema
+struct<locate(bar, foobarbar, 1):int,position(CAST(NULL AS STRING), foobarbar, 1):int,position(aaads, CAST(NULL AS STRING), 1):int>
+-- !query output
+4	NULL	NULL
+
+
+-- !query
+select left("abcd", 2), left("abcd", 5), left("abcd", '2'), left("abcd", null)
+-- !query schema
+struct<left('abcd', 2):string,left('abcd', 5):string,left('abcd', '2'):string,left('abcd', NULL):string>
+-- !query output
+ab	abcd	ab	NULL
+
+
+-- !query
+select left(null, -2), left("abcd", -2), left("abcd", 0), left("abcd", 'a')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: a
+
+
+-- !query
+select right("abcd", 2), right("abcd", 5), right("abcd", '2'), right("abcd", null)
+-- !query schema
+struct<right('abcd', 2):string,right('abcd', 5):string,right('abcd', '2'):string,right('abcd', NULL):string>
+-- !query output
+cd	abcd	cd	NULL
+
+
+-- !query
+select right(null, -2), right("abcd", -2), right("abcd", 0), right("abcd", 'a')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: a
+
+
+-- !query
+SELECT split('aa1cc2ee3', '[1-9]+')
+-- !query schema
+struct<split(aa1cc2ee3, [1-9]+, -1):array<string>>
+-- !query output
+["aa","cc","ee",""]
+
+
+-- !query
+SELECT split('aa1cc2ee3', '[1-9]+', 2)
+-- !query schema
+struct<split(aa1cc2ee3, [1-9]+, 2):array<string>>
+-- !query output
+["aa","cc2ee3"]
+
+
+-- !query
+SELECT substr('Spark SQL', 5)
+-- !query schema
+struct<substr(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substr('Spark SQL', -3)
+-- !query schema
+struct<substr(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substr('Spark SQL', 5, 1)
+-- !query schema
+struct<substr(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT substr('Spark SQL' from 5)
+-- !query schema
+struct<substring(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substr('Spark SQL' from -3)
+-- !query schema
+struct<substring(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substr('Spark SQL' from 5 for 1)
+-- !query schema
+struct<substring(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT substring('Spark SQL', 5)
+-- !query schema
+struct<substring(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substring('Spark SQL', -3)
+-- !query schema
+struct<substring(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substring('Spark SQL', 5, 1)
+-- !query schema
+struct<substring(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT substring('Spark SQL' from 5)
+-- !query schema
+struct<substring(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substring('Spark SQL' from -3)
+-- !query schema
+struct<substring(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substring('Spark SQL' from 5 for 1)
+-- !query schema
+struct<substring(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT trim(" xyz "), ltrim(" xyz "), rtrim(" xyz ")
+-- !query schema
+struct<trim( xyz ):string,ltrim( xyz ):string,rtrim( xyz ):string>
+-- !query output
+xyz	xyz 	 xyz
+
+
+-- !query
+SELECT trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx')
+-- !query schema
+struct<TRIM(BOTH xyz FROM yxTomxx):string,TRIM(BOTH xyz FROM yxTomxx):string>
+-- !query output
+Tom	Tom
+
+
+-- !query
+SELECT trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx')
+-- !query schema
+struct<TRIM(BOTH x FROM xxxbarxxx):string,TRIM(BOTH x FROM xxxbarxxx):string>
+-- !query output
+bar	bar
+
+
+-- !query
+SELECT trim(LEADING 'xyz' FROM 'zzzytest')
+-- !query schema
+struct<TRIM(LEADING xyz FROM zzzytest):string>
+-- !query output
+test
+
+
+-- !query
+SELECT trim(LEADING 'xyz' FROM 'zzzytestxyz')
+-- !query schema
+struct<TRIM(LEADING xyz FROM zzzytestxyz):string>
+-- !query output
+testxyz
+
+
+-- !query
+SELECT trim(LEADING 'xy' FROM 'xyxXxyLAST WORD')
+-- !query schema
+struct<TRIM(LEADING xy FROM xyxXxyLAST WORD):string>
+-- !query output
+XxyLAST WORD
+
+
+-- !query
+SELECT trim(TRAILING 'xyz' FROM 'testxxzx')
+-- !query schema
+struct<TRIM(TRAILING xyz FROM testxxzx):string>
+-- !query output
+test
+
+
+-- !query
+SELECT trim(TRAILING 'xyz' FROM 'xyztestxxzx')
+-- !query schema
+struct<TRIM(TRAILING xyz FROM xyztestxxzx):string>
+-- !query output
+xyztest
+
+
+-- !query
+SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy')
+-- !query schema
+struct<TRIM(TRAILING xy FROM TURNERyxXxy):string>
+-- !query output
+TURNERyxX
+
+
+-- !query
+SELECT lpad('hi', 'invalid_length')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: invalid_length
+
+
+-- !query
+SELECT rpad('hi', 'invalid_length')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: invalid_length
diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
index 35b4c0e79720b..d4872ca03199b 100644
--- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 46
+-- Number of queries: 51
 
 
 -- !query
@@ -353,6 +353,46 @@ struct<CAST(1.0  AS DECIMAL(10,0)):decimal(10,0)>
 1
 
 
+-- !query
+select cast('1中文' as tinyint)
+-- !query schema
+struct<CAST(1中文 AS TINYINT):tinyint>
+-- !query output
+NULL
+
+
+-- !query
+select cast('1中文' as smallint)
+-- !query schema
+struct<CAST(1中文 AS SMALLINT):smallint>
+-- !query output
+NULL
+
+
+-- !query
+select cast('1中文' as INT)
+-- !query schema
+struct<CAST(1中文 AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+select cast('中文1' as bigint)
+-- !query schema
+struct<CAST(中文1 AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast('1中文' as bigint)
+-- !query schema
+struct<CAST(1中文 AS BIGINT):bigint>
+-- !query output
+NULL
+
+
 -- !query
 select cast('\t\t true \n\r ' as boolean)
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
index 5bb00e028c4b7..b1a32ad1f63e9 100644
--- a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 25
+-- Number of queries: 28
 
 
 -- !query
@@ -27,7 +27,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-Operation not allowed: ALTER TABLE table CHANGE COLUMN requires a TYPE or a COMMENT or a FIRST/AFTER(line 1, pos 0)
+Operation not allowed: ALTER TABLE table CHANGE COLUMN requires a TYPE, a SET/DROP, a COMMENT, or a FIRST/AFTER(line 1, pos 0)
 
 == SQL ==
 ALTER TABLE test_change CHANGE a
@@ -83,7 +83,7 @@ c                   	int
 
 
 -- !query
-ALTER TABLE test_change CHANGE a TYPE INT AFTER b
+ALTER TABLE test_change CHANGE a AFTER b
 -- !query schema
 struct<>
 -- !query output
@@ -92,7 +92,7 @@ ALTER COLUMN ... FIRST | ALTER is only supported with v2 tables.;
 
 
 -- !query
-ALTER TABLE test_change CHANGE b TYPE STRING FIRST
+ALTER TABLE test_change CHANGE b FIRST
 -- !query schema
 struct<>
 -- !query output
@@ -111,7 +111,7 @@ c                   	int
 
 
 -- !query
-ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a'
+ALTER TABLE test_change CHANGE a COMMENT 'this is column a'
 -- !query schema
 struct<>
 -- !query output
@@ -119,7 +119,7 @@ struct<>
 
 
 -- !query
-ALTER TABLE test_change CHANGE b TYPE STRING COMMENT '#*02?`'
+ALTER TABLE test_change CHANGE b COMMENT '#*02?`'
 -- !query schema
 struct<>
 -- !query output
@@ -127,7 +127,7 @@ struct<>
 
 
 -- !query
-ALTER TABLE test_change CHANGE c TYPE INT COMMENT ''
+ALTER TABLE test_change CHANGE c COMMENT ''
 -- !query schema
 struct<>
 -- !query output
@@ -145,7 +145,15 @@ c                   	int
 
 
 -- !query
-ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a'
+ALTER TABLE test_change CHANGE a TYPE INT
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+ALTER TABLE test_change CHANGE a COMMENT 'this is column a'
 -- !query schema
 struct<>
 -- !query output
@@ -181,6 +189,24 @@ b                   	string              	#*02?`
 c                   	int
 
 
+-- !query
+ALTER TABLE test_change CHANGE A COMMENT 'case insensitivity'
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DESC test_change
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+a                   	int                 	case insensitivity  
+b                   	string              	#*02?`              
+c                   	int
+
+
 -- !query
 CREATE TEMPORARY VIEW temp_view(a, b) AS SELECT 1, "one"
 -- !query schema
@@ -190,7 +216,7 @@ struct<>
 
 
 -- !query
-ALTER TABLE temp_view CHANGE a TYPE INT COMMENT 'this is column a'
+ALTER TABLE temp_view CHANGE a TYPE INT
 -- !query schema
 struct<>
 -- !query output
@@ -207,7 +233,7 @@ struct<>
 
 
 -- !query
-ALTER TABLE global_temp.global_temp_view CHANGE a TYPE INT COMMENT 'this is column a'
+ALTER TABLE global_temp.global_temp_view CHANGE a TYPE INT
 -- !query schema
 struct<>
 -- !query output
diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
index f34b75a379aae..04ddfe0ac128c 100644
--- a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
@@ -72,7 +72,7 @@ SELECT i1 FROM t1, mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -81,7 +81,7 @@ SELECT t1.i1 FROM t1, mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -90,7 +90,7 @@ SELECT mydb1.t1.i1 FROM t1, mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'mydb1.t1.i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 'mydb1.t1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -99,7 +99,7 @@ SELECT i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb1.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -108,7 +108,7 @@ SELECT t1.i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb1.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -125,7 +125,7 @@ SELECT i1 FROM t1, mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb2.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -134,7 +134,7 @@ SELECT t1.i1 FROM t1, mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb2.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -143,7 +143,7 @@ SELECT i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb2.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -152,7 +152,7 @@ SELECT t1.i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb2.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -161,7 +161,7 @@ SELECT db1.t1.i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`db1.t1.i1`' given input columns: [mydb2.t1.i1, mydb2.t1.i1]; line 1 pos 7
+cannot resolve '`db1.t1.i1`' given input columns: [spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1]; line 1 pos 7
 
 
 -- !query
@@ -186,7 +186,7 @@ SELECT mydb1.t1 FROM t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`mydb1.t1`' given input columns: [mydb1.t1.i1]; line 1 pos 7
+cannot resolve '`mydb1.t1`' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7
 
 
 -- !query
@@ -204,7 +204,7 @@ SELECT t1 FROM mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`t1`' given input columns: [mydb1.t1.i1]; line 1 pos 7
+cannot resolve '`t1`' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7
 
 
 -- !query
@@ -221,7 +221,7 @@ SELECT mydb1.t1.i1 FROM t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`mydb1.t1.i1`' given input columns: [mydb2.t1.i1]; line 1 pos 7
+cannot resolve '`mydb1.t1.i1`' given input columns: [spark_catalog.mydb2.t1.i1]; line 1 pos 7
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
index a9709c4a79793..4d0e5ea829d3f 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
@@ -1,29 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 17
-
-
--- !query
-create temporary view t as select * from values 0, 1, 2 as t(id)
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-create temporary view t2 as select * from values 0, 1 as t(id)
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-SET spark.sql.legacy.ctePrecedence.enabled=true
--- !query schema
-struct<key:string,value:string>
--- !query output
-spark.sql.legacy.ctePrecedence.enabled	true
+-- Number of queries: 16
 
 
 -- !query
@@ -193,16 +169,53 @@ struct<scalarsubquery():int>
 
 
 -- !query
-DROP VIEW IF EXISTS t
+WITH t(c) AS (SELECT 1)
+SELECT * FROM t
+WHERE c IN (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+)
 -- !query schema
-struct<>
+struct<c:int>
 -- !query output
+1
 
 
+-- !query
+WITH
+  t AS (
+    WITH t2 AS (SELECT 1)
+    SELECT * FROM t2
+  ),
+  t2 AS (SELECT 2)
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
 
 -- !query
-DROP VIEW IF EXISTS t2
+WITH
+  abc AS (SELECT 1),
+  t AS (
+    WITH aBc AS (SELECT 2)
+    SELECT * FROM aBC
+  )
+SELECT * FROM t
 -- !query schema
-struct<>
+struct<1:int>
 -- !query output
+1
 
+
+-- !query
+WITH abc AS (SELECT 1)
+SELECT (
+  WITH aBc AS (SELECT 2)
+  SELECT * FROM aBC
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+1
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
new file mode 100644
index 0000000000000..2f736c7b4978f
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
@@ -0,0 +1,229 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 16
+
+
+-- !query
+WITH t as (
+  WITH t2 AS (SELECT 1)
+  SELECT * FROM t2
+)
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 1)
+  SELECT * FROM t
+)
+-- !query schema
+struct<max(c):int>
+-- !query output
+1
+
+
+-- !query
+SELECT (
+  WITH t AS (SELECT 1)
+  SELECT * FROM t
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+1
+
+
+-- !query
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+SELECT * FROM t2
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH
+  t(c) AS (SELECT 1),
+  t2 AS (
+    SELECT (
+      SELECT max(c) FROM (
+        WITH t(c) AS (SELECT 2)
+        SELECT * FROM t
+      )
+    )
+  )
+SELECT * FROM t2
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+2
+
+
+-- !query
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2),
+    t2 AS (
+      WITH t AS (SELECT 3)
+      SELECT * FROM t
+    )
+    SELECT * FROM t2
+  )
+SELECT * FROM t2
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<max(c):int>
+-- !query output
+2
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 2)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<sum(c):bigint>
+-- !query output
+2
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 3)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<sum(c):bigint>
+-- !query output
+3
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  SELECT (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT (
+    WITH t AS (SELECT 3)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT * FROM t
+WHERE c IN (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH
+  t AS (
+    WITH t2 AS (SELECT 1)
+    SELECT * FROM t2
+  ),
+  t2 AS (SELECT 2)
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
+
+-- !query
+WITH
+  abc AS (SELECT 1),
+  t AS (
+    WITH aBc AS (SELECT 2)
+    SELECT * FROM aBC
+  )
+SELECT * FROM t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name aBc is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH abc AS (SELECT 1)
+SELECT (
+  WITH aBc AS (SELECT 2)
+  SELECT * FROM aBC
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name aBc is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
new file mode 100644
index 0000000000000..74394ee3ffc89
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
@@ -0,0 +1,221 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 16
+
+
+-- !query
+WITH t as (
+  WITH t2 AS (SELECT 1)
+  SELECT * FROM t2
+)
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 1)
+  SELECT * FROM t
+)
+-- !query schema
+struct<max(c):int>
+-- !query output
+1
+
+
+-- !query
+SELECT (
+  WITH t AS (SELECT 1)
+  SELECT * FROM t
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+1
+
+
+-- !query
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+SELECT * FROM t2
+-- !query schema
+struct<2:int>
+-- !query output
+2
+
+
+-- !query
+WITH
+  t(c) AS (SELECT 1),
+  t2 AS (
+    SELECT (
+      SELECT max(c) FROM (
+        WITH t(c) AS (SELECT 2)
+        SELECT * FROM t
+      )
+    )
+  )
+SELECT * FROM t2
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+2
+
+
+-- !query
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2),
+    t2 AS (
+      WITH t AS (SELECT 3)
+      SELECT * FROM t
+    )
+    SELECT * FROM t2
+  )
+SELECT * FROM t2
+-- !query schema
+struct<3:int>
+-- !query output
+3
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<max(c):int>
+-- !query output
+2
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 2)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<sum(c):bigint>
+-- !query output
+2
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 3)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<sum(c):bigint>
+-- !query output
+3
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+2
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  SELECT (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+2
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT (
+    WITH t AS (SELECT 3)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+3
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT * FROM t
+WHERE c IN (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<c:int>
+-- !query output
+
+
+
+-- !query
+WITH
+  t AS (
+    WITH t2 AS (SELECT 1)
+    SELECT * FROM t2
+  ),
+  t2 AS (SELECT 2)
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
+
+-- !query
+WITH
+  abc AS (SELECT 1),
+  t AS (
+    WITH aBc AS (SELECT 2)
+    SELECT * FROM aBC
+  )
+SELECT * FROM t
+-- !query schema
+struct<2:int>
+-- !query output
+2
+
+
+-- !query
+WITH abc AS (SELECT 1)
+SELECT (
+  WITH aBc AS (SELECT 2)
+  SELECT * FROM aBC
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+2
diff --git a/sql/core/src/test/resources/sql-tests/results/cte.sql.out b/sql/core/src/test/resources/sql-tests/results/cte.sql.out
index 2d87781193c25..b8f666586ce45 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 27
+-- Number of queries: 15
 
 
 -- !query
@@ -161,172 +161,6 @@ WITH
 SELECT * FROM t
 
 
--- !query
-WITH t as (
-  WITH t2 AS (SELECT 1)
-  SELECT * FROM t2
-)
-SELECT * FROM t
--- !query schema
-struct<1:int>
--- !query output
-1
-
-
--- !query
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 1)
-  SELECT * FROM t
-)
--- !query schema
-struct<max(c):int>
--- !query output
-1
-
-
--- !query
-SELECT (
-  WITH t AS (SELECT 1)
-  SELECT * FROM t
-)
--- !query schema
-struct<scalarsubquery():int>
--- !query output
-1
-
-
--- !query
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-SELECT * FROM t2
--- !query schema
-struct<2:int>
--- !query output
-2
-
-
--- !query
-WITH
-  t(c) AS (SELECT 1),
-  t2 AS (
-    SELECT (
-      SELECT max(c) FROM (
-        WITH t(c) AS (SELECT 2)
-        SELECT * FROM t
-      )
-    )
-  )
-SELECT * FROM t2
--- !query schema
-struct<scalarsubquery():int>
--- !query output
-2
-
-
--- !query
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2),
-    t2 AS (
-      WITH t AS (SELECT 3)
-      SELECT * FROM t
-    )
-    SELECT * FROM t2
-  )
-SELECT * FROM t2
--- !query schema
-struct<3:int>
--- !query output
-3
-
-
--- !query
-WITH t(c) AS (SELECT 1)
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT * FROM t
-)
--- !query schema
-struct<max(c):int>
--- !query output
-2
-
-
--- !query
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 2)
-    SELECT * FROM t
-  )
-)
--- !query schema
-struct<sum(c):bigint>
--- !query output
-2
-
-
--- !query
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 3)
-    SELECT * FROM t
-  )
-)
--- !query schema
-struct<sum(c):bigint>
--- !query output
-3
-
-
--- !query
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT * FROM t
-)
--- !query schema
-struct<scalarsubquery():int>
--- !query output
-2
-
-
--- !query
-WITH t AS (SELECT 1)
-SELECT (
-  SELECT (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-)
--- !query schema
-struct<scalarsubquery():int>
--- !query output
-2
-
-
--- !query
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT (
-    WITH t AS (SELECT 3)
-    SELECT * FROM t
-  )
-)
--- !query schema
-struct<scalarsubquery():int>
--- !query output
-3
-
-
 -- !query
 DROP VIEW IF EXISTS t
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out
deleted file mode 100644
index b4cceedffd98b..0000000000000
--- a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out
+++ /dev/null
@@ -1,886 +0,0 @@
--- Automatically generated by SQLQueryTestSuite
--- Number of queries: 110
-
-
--- !query
-CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-select date_part('millennium', c) from t
--- !query schema
-struct<date_part('millennium', t.`c`):int>
--- !query output
-3
-
-
--- !query
-select date_part('millennia', c) from t
--- !query schema
-struct<date_part('millennia', t.`c`):int>
--- !query output
-3
-
-
--- !query
-select date_part('mil', c) from t
--- !query schema
-struct<date_part('mil', t.`c`):int>
--- !query output
-3
-
-
--- !query
-select date_part('mils', c) from t
--- !query schema
-struct<date_part('mils', t.`c`):int>
--- !query output
-3
-
-
--- !query
-select date_part('century', c) from t
--- !query schema
-struct<date_part('century', t.`c`):int>
--- !query output
-21
-
-
--- !query
-select date_part('centuries', c) from t
--- !query schema
-struct<date_part('centuries', t.`c`):int>
--- !query output
-21
-
-
--- !query
-select date_part('c', c) from t
--- !query schema
-struct<date_part('c', t.`c`):int>
--- !query output
-21
-
-
--- !query
-select date_part('cent', c) from t
--- !query schema
-struct<date_part('cent', t.`c`):int>
--- !query output
-21
-
-
--- !query
-select date_part('decade', c) from t
--- !query schema
-struct<date_part('decade', t.`c`):int>
--- !query output
-201
-
-
--- !query
-select date_part('decades', c) from t
--- !query schema
-struct<date_part('decades', t.`c`):int>
--- !query output
-201
-
-
--- !query
-select date_part('dec', c) from t
--- !query schema
-struct<date_part('dec', t.`c`):int>
--- !query output
-201
-
-
--- !query
-select date_part('decs', c) from t
--- !query schema
-struct<date_part('decs', t.`c`):int>
--- !query output
-201
-
-
--- !query
-select date_part('year', c) from t
--- !query schema
-struct<date_part('year', t.`c`):int>
--- !query output
-2011
-
-
--- !query
-select date_part('y', c) from t
--- !query schema
-struct<date_part('y', t.`c`):int>
--- !query output
-2011
-
-
--- !query
-select date_part('years', c) from t
--- !query schema
-struct<date_part('years', t.`c`):int>
--- !query output
-2011
-
-
--- !query
-select date_part('yr', c) from t
--- !query schema
-struct<date_part('yr', t.`c`):int>
--- !query output
-2011
-
-
--- !query
-select date_part('yrs', c) from t
--- !query schema
-struct<date_part('yrs', t.`c`):int>
--- !query output
-2011
-
-
--- !query
-select date_part('quarter', c) from t
--- !query schema
-struct<date_part('quarter', t.`c`):int>
--- !query output
-2
-
-
--- !query
-select date_part('qtr', c) from t
--- !query schema
-struct<date_part('qtr', t.`c`):int>
--- !query output
-2
-
-
--- !query
-select date_part('month', c) from t
--- !query schema
-struct<date_part('month', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('mon', c) from t
--- !query schema
-struct<date_part('mon', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('mons', c) from t
--- !query schema
-struct<date_part('mons', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('months', c) from t
--- !query schema
-struct<date_part('months', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('week', c) from t
--- !query schema
-struct<date_part('week', t.`c`):int>
--- !query output
-18
-
-
--- !query
-select date_part('w', c) from t
--- !query schema
-struct<date_part('w', t.`c`):int>
--- !query output
-18
-
-
--- !query
-select date_part('weeks', c) from t
--- !query schema
-struct<date_part('weeks', t.`c`):int>
--- !query output
-18
-
-
--- !query
-select date_part('day', c) from t
--- !query schema
-struct<date_part('day', t.`c`):int>
--- !query output
-6
-
-
--- !query
-select date_part('d', c) from t
--- !query schema
-struct<date_part('d', t.`c`):int>
--- !query output
-6
-
-
--- !query
-select date_part('days', c) from t
--- !query schema
-struct<date_part('days', t.`c`):int>
--- !query output
-6
-
-
--- !query
-select date_part('dayofweek', c) from t
--- !query schema
-struct<date_part('dayofweek', t.`c`):int>
--- !query output
-6
-
-
--- !query
-select date_part('dow', c) from t
--- !query schema
-struct<date_part('dow', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('isodow', c) from t
--- !query schema
-struct<date_part('isodow', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('doy', c) from t
--- !query schema
-struct<date_part('doy', t.`c`):int>
--- !query output
-126
-
-
--- !query
-select date_part('hour', c) from t
--- !query schema
-struct<date_part('hour', t.`c`):int>
--- !query output
-7
-
-
--- !query
-select date_part('h', c) from t
--- !query schema
-struct<date_part('h', t.`c`):int>
--- !query output
-7
-
-
--- !query
-select date_part('hours', c) from t
--- !query schema
-struct<date_part('hours', t.`c`):int>
--- !query output
-7
-
-
--- !query
-select date_part('hr', c) from t
--- !query schema
-struct<date_part('hr', t.`c`):int>
--- !query output
-7
-
-
--- !query
-select date_part('hrs', c) from t
--- !query schema
-struct<date_part('hrs', t.`c`):int>
--- !query output
-7
-
-
--- !query
-select date_part('minute', c) from t
--- !query schema
-struct<date_part('minute', t.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('m', c) from t
--- !query schema
-struct<date_part('m', t.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('min', c) from t
--- !query schema
-struct<date_part('min', t.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('mins', c) from t
--- !query schema
-struct<date_part('mins', t.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('minutes', c) from t
--- !query schema
-struct<date_part('minutes', t.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('second', c) from t
--- !query schema
-struct<date_part('second', t.`c`):decimal(8,6)>
--- !query output
-9.123456
-
-
--- !query
-select date_part('s', c) from t
--- !query schema
-struct<date_part('s', t.`c`):decimal(8,6)>
--- !query output
-9.123456
-
-
--- !query
-select date_part('sec', c) from t
--- !query schema
-struct<date_part('sec', t.`c`):decimal(8,6)>
--- !query output
-9.123456
-
-
--- !query
-select date_part('seconds', c) from t
--- !query schema
-struct<date_part('seconds', t.`c`):decimal(8,6)>
--- !query output
-9.123456
-
-
--- !query
-select date_part('secs', c) from t
--- !query schema
-struct<date_part('secs', t.`c`):decimal(8,6)>
--- !query output
-9.123456
-
-
--- !query
-select date_part('not_supported', c) from t
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Literals of type 'not_supported' are currently not supported for the string type.;; line 1 pos 7
-
-
--- !query
-select date_part(c, c) from t
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-The field parameter needs to be a foldable string value.;; line 1 pos 7
-
-
--- !query
-select date_part(null, c) from t
--- !query schema
-struct<date_part(NULL, t.`c`):double>
--- !query output
-NULL
-
-
--- !query
-CREATE TEMPORARY VIEW t2 AS select interval 1010 year 9 month 8 day 7 hour 6 minute 5 second 4 millisecond 3 microsecond as c
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-select date_part('millennium', c) from t2
--- !query schema
-struct<date_part('millennium', t2.`c`):int>
--- !query output
-1
-
-
--- !query
-select date_part('millennia', c) from t2
--- !query schema
-struct<date_part('millennia', t2.`c`):int>
--- !query output
-1
-
-
--- !query
-select date_part('mil', c) from t2
--- !query schema
-struct<date_part('mil', t2.`c`):int>
--- !query output
-1
-
-
--- !query
-select date_part('mils', c) from t2
--- !query schema
-struct<date_part('mils', t2.`c`):int>
--- !query output
-1
-
-
--- !query
-select date_part('century', c) from t2
--- !query schema
-struct<date_part('century', t2.`c`):int>
--- !query output
-10
-
-
--- !query
-select date_part('centuries', c) from t2
--- !query schema
-struct<date_part('centuries', t2.`c`):int>
--- !query output
-10
-
-
--- !query
-select date_part('c', c) from t2
--- !query schema
-struct<date_part('c', t2.`c`):int>
--- !query output
-10
-
-
--- !query
-select date_part('cent', c) from t2
--- !query schema
-struct<date_part('cent', t2.`c`):int>
--- !query output
-10
-
-
--- !query
-select date_part('decade', c) from t2
--- !query schema
-struct<date_part('decade', t2.`c`):int>
--- !query output
-101
-
-
--- !query
-select date_part('decades', c) from t2
--- !query schema
-struct<date_part('decades', t2.`c`):int>
--- !query output
-101
-
-
--- !query
-select date_part('dec', c) from t2
--- !query schema
-struct<date_part('dec', t2.`c`):int>
--- !query output
-101
-
-
--- !query
-select date_part('decs', c) from t2
--- !query schema
-struct<date_part('decs', t2.`c`):int>
--- !query output
-101
-
-
--- !query
-select date_part('year', c) from t2
--- !query schema
-struct<date_part('year', t2.`c`):int>
--- !query output
-1010
-
-
--- !query
-select date_part('y', c) from t2
--- !query schema
-struct<date_part('y', t2.`c`):int>
--- !query output
-1010
-
-
--- !query
-select date_part('years', c) from t2
--- !query schema
-struct<date_part('years', t2.`c`):int>
--- !query output
-1010
-
-
--- !query
-select date_part('yr', c) from t2
--- !query schema
-struct<date_part('yr', t2.`c`):int>
--- !query output
-1010
-
-
--- !query
-select date_part('yrs', c) from t2
--- !query schema
-struct<date_part('yrs', t2.`c`):int>
--- !query output
-1010
-
-
--- !query
-select date_part('quarter', c) from t2
--- !query schema
-struct<date_part('quarter', t2.`c`):tinyint>
--- !query output
-4
-
-
--- !query
-select date_part('qtr', c) from t2
--- !query schema
-struct<date_part('qtr', t2.`c`):tinyint>
--- !query output
-4
-
-
--- !query
-select date_part('month', c) from t2
--- !query schema
-struct<date_part('month', t2.`c`):tinyint>
--- !query output
-9
-
-
--- !query
-select date_part('mon', c) from t2
--- !query schema
-struct<date_part('mon', t2.`c`):tinyint>
--- !query output
-9
-
-
--- !query
-select date_part('mons', c) from t2
--- !query schema
-struct<date_part('mons', t2.`c`):tinyint>
--- !query output
-9
-
-
--- !query
-select date_part('months', c) from t2
--- !query schema
-struct<date_part('months', t2.`c`):tinyint>
--- !query output
-9
-
-
--- !query
-select date_part('day', c) from t2
--- !query schema
-struct<date_part('day', t2.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('d', c) from t2
--- !query schema
-struct<date_part('d', t2.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('days', c) from t2
--- !query schema
-struct<date_part('days', t2.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('hour', c) from t2
--- !query schema
-struct<date_part('hour', t2.`c`):bigint>
--- !query output
-7
-
-
--- !query
-select date_part('h', c) from t2
--- !query schema
-struct<date_part('h', t2.`c`):bigint>
--- !query output
-7
-
-
--- !query
-select date_part('hours', c) from t2
--- !query schema
-struct<date_part('hours', t2.`c`):bigint>
--- !query output
-7
-
-
--- !query
-select date_part('hr', c) from t2
--- !query schema
-struct<date_part('hr', t2.`c`):bigint>
--- !query output
-7
-
-
--- !query
-select date_part('hrs', c) from t2
--- !query schema
-struct<date_part('hrs', t2.`c`):bigint>
--- !query output
-7
-
-
--- !query
-select date_part('minute', c) from t2
--- !query schema
-struct<date_part('minute', t2.`c`):tinyint>
--- !query output
-6
-
-
--- !query
-select date_part('m', c) from t2
--- !query schema
-struct<date_part('m', t2.`c`):tinyint>
--- !query output
-6
-
-
--- !query
-select date_part('min', c) from t2
--- !query schema
-struct<date_part('min', t2.`c`):tinyint>
--- !query output
-6
-
-
--- !query
-select date_part('mins', c) from t2
--- !query schema
-struct<date_part('mins', t2.`c`):tinyint>
--- !query output
-6
-
-
--- !query
-select date_part('minutes', c) from t2
--- !query schema
-struct<date_part('minutes', t2.`c`):tinyint>
--- !query output
-6
-
-
--- !query
-select date_part('second', c) from t2
--- !query schema
-struct<date_part('second', t2.`c`):decimal(8,6)>
--- !query output
-5.004003
-
-
--- !query
-select date_part('s', c) from t2
--- !query schema
-struct<date_part('s', t2.`c`):decimal(8,6)>
--- !query output
-5.004003
-
-
--- !query
-select date_part('sec', c) from t2
--- !query schema
-struct<date_part('sec', t2.`c`):decimal(8,6)>
--- !query output
-5.004003
-
-
--- !query
-select date_part('seconds', c) from t2
--- !query schema
-struct<date_part('seconds', t2.`c`):decimal(8,6)>
--- !query output
-5.004003
-
-
--- !query
-select date_part('secs', c) from t2
--- !query schema
-struct<date_part('secs', t2.`c`):decimal(8,6)>
--- !query output
-5.004003
-
-
--- !query
-select date_part('milliseconds', c) from t2
--- !query schema
-struct<date_part('milliseconds', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('msec', c) from t2
--- !query schema
-struct<date_part('msec', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('msecs', c) from t2
--- !query schema
-struct<date_part('msecs', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('millisecon', c) from t2
--- !query schema
-struct<date_part('millisecon', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('mseconds', c) from t2
--- !query schema
-struct<date_part('mseconds', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('ms', c) from t2
--- !query schema
-struct<date_part('ms', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('microseconds', c) from t2
--- !query schema
-struct<date_part('microseconds', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('usec', c) from t2
--- !query schema
-struct<date_part('usec', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('usecs', c) from t2
--- !query schema
-struct<date_part('usecs', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('useconds', c) from t2
--- !query schema
-struct<date_part('useconds', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('microsecon', c) from t2
--- !query schema
-struct<date_part('microsecon', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('us', c) from t2
--- !query schema
-struct<date_part('us', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('epoch', c) from t2
--- !query schema
-struct<date_part('epoch', t2.`c`):decimal(18,6)>
--- !query output
-31897220765.004003
-
-
--- !query
-select date_part('not_supported', c) from t2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Literals of type 'not_supported' are currently not supported for the interval type.;; line 1 pos 7
-
-
--- !query
-select date_part(c, c) from t2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-The field parameter needs to be a foldable string value.;; line 1 pos 7
-
-
--- !query
-select date_part(null, c) from t2
--- !query schema
-struct<date_part(NULL, t2.`c`):double>
--- !query output
-NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out
new file mode 100644
index 0000000000000..18d1a10068794
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out
@@ -0,0 +1,335 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 37
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'yyyyyyy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'qqqqq')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: q
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'QQQQQ')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: Q
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'LLLLL')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'LLLLL' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'FF')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'FF' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'ddd')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'ddd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'DDDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'DDDD' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'HHH')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'HHH' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'hhh')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'hhh' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'kkk')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'kkk' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'KKK')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'KKK' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'mmm')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'mmm' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'sss')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'sss' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'SSSSSSSSSS')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'SSSSSSSSSS' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'aa')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'V')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Pattern letter count must be 2: V
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'zzzzz')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'zzzzz' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'XXXXXX')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: X
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'ZZZZZZ')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'ZZZZZZ' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'OO')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Pattern letter count must be 1 or 4: O
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'xxxxxx')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: x
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'A')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: A
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'n')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: n
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'N')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: N
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'p')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: p
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'Y')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'Y' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'w')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'w' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'W')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'W' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'u')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'u' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'e')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+All week-based patterns are unsupported since Spark 3.0, detected: e, Please use the SQL function EXTRACT instead
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'c')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+All week-based patterns are unsupported since Spark 3.0, detected: c, Please use the SQL function EXTRACT instead
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'B')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Unknown pattern letter: B
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'C')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Unknown pattern letter: C
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'I')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Unknown pattern letter: I
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out
new file mode 100644
index 0000000000000..b37922b20807d
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out
@@ -0,0 +1,401 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 31
+
+
+-- !query
+create temporary view v as select col from values
+ (timestamp '1582-06-01 11:33:33.123UTC+080000'),
+ (timestamp '1970-01-01 00:00:00.000Europe/Paris'),
+ (timestamp '1970-12-31 23:59:59.999Asia/Srednekolymsk'),
+ (timestamp '1996-04-01 00:33:33.123Australia/Darwin'),
+ (timestamp '2018-11-17 13:33:33.123Z'),
+ (timestamp '2020-01-01 01:33:33.123Asia/Shanghai'),
+ (timestamp '2100-01-01 01:33:33.123America/Los_Angeles') t(col)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select col, date_format(col, 'G GG GGG GGGG') from v
+-- !query schema
+struct<col:timestamp,date_format(col, G GG GGG GGGG):string>
+-- !query output
+1582-05-31 19:40:35.123	AD AD AD AD
+1969-12-31 15:00:00	AD AD AD AD
+1970-12-31 04:59:59.999	AD AD AD AD
+1996-03-31 07:03:33.123	AD AD AD AD
+2018-11-17 05:33:33.123	AD AD AD AD
+2019-12-31 09:33:33.123	AD AD AD AD
+2100-01-01 01:33:33.123	AD AD AD AD
+
+
+-- !query
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy') from v
+-- !query schema
+struct<col:timestamp,date_format(col, y yy yyy yyyy yyyyy yyyyyy):string>
+-- !query output
+1582-05-31 19:40:35.123	1582 82 1582 1582 01582 001582
+1969-12-31 15:00:00	1969 69 1969 1969 01969 001969
+1970-12-31 04:59:59.999	1970 70 1970 1970 01970 001970
+1996-03-31 07:03:33.123	1996 96 1996 1996 01996 001996
+2018-11-17 05:33:33.123	2018 18 2018 2018 02018 002018
+2019-12-31 09:33:33.123	2019 19 2019 2019 02019 002019
+2100-01-01 01:33:33.123	2100 00 2100 2100 02100 002100
+
+
+-- !query
+select col, date_format(col, 'q qq') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'q'
+
+
+-- !query
+select col, date_format(col, 'Q QQ QQQ QQQQ') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'Q'
+
+
+-- !query
+select col, date_format(col, 'M MM MMM MMMM') from v
+-- !query schema
+struct<col:timestamp,date_format(col, M MM MMM MMMM):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05 May May
+1969-12-31 15:00:00	12 12 Dec December
+1970-12-31 04:59:59.999	12 12 Dec December
+1996-03-31 07:03:33.123	3 03 Mar March
+2018-11-17 05:33:33.123	11 11 Nov November
+2019-12-31 09:33:33.123	12 12 Dec December
+2100-01-01 01:33:33.123	1 01 Jan January
+
+
+-- !query
+select col, date_format(col, 'L LL') from v
+-- !query schema
+struct<col:timestamp,date_format(col, L LL):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05
+1969-12-31 15:00:00	12 12
+1970-12-31 04:59:59.999	12 12
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	11 11
+2019-12-31 09:33:33.123	12 12
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'E EE EEE EEEE') from v
+-- !query schema
+struct<col:timestamp,date_format(col, E EE EEE EEEE):string>
+-- !query output
+1582-05-31 19:40:35.123	Thu Thu Thu Thursday
+1969-12-31 15:00:00	Wed Wed Wed Wednesday
+1970-12-31 04:59:59.999	Thu Thu Thu Thursday
+1996-03-31 07:03:33.123	Sun Sun Sun Sunday
+2018-11-17 05:33:33.123	Sat Sat Sat Saturday
+2019-12-31 09:33:33.123	Tue Tue Tue Tuesday
+2100-01-01 01:33:33.123	Fri Fri Fri Friday
+
+
+-- !query
+select col, date_format(col, 'F') from v
+-- !query schema
+struct<col:timestamp,date_format(col, F):string>
+-- !query output
+1582-05-31 19:40:35.123	5
+1969-12-31 15:00:00	5
+1970-12-31 04:59:59.999	5
+1996-03-31 07:03:33.123	5
+2018-11-17 05:33:33.123	3
+2019-12-31 09:33:33.123	5
+2100-01-01 01:33:33.123	1
+
+
+-- !query
+select col, date_format(col, 'd dd') from v
+-- !query schema
+struct<col:timestamp,date_format(col, d dd):string>
+-- !query output
+1582-05-31 19:40:35.123	31 31
+1969-12-31 15:00:00	31 31
+1970-12-31 04:59:59.999	31 31
+1996-03-31 07:03:33.123	31 31
+2018-11-17 05:33:33.123	17 17
+2019-12-31 09:33:33.123	31 31
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'DD') from v where col = timestamp '2100-01-01 01:33:33.123America/Los_Angeles'
+-- !query schema
+struct<col:timestamp,date_format(col, DD):string>
+-- !query output
+2100-01-01 01:33:33.123	01
+
+
+-- !query
+select col, date_format(col, 'D DDD') from v
+-- !query schema
+struct<col:timestamp,date_format(col, D DDD):string>
+-- !query output
+1582-05-31 19:40:35.123	151 151
+1969-12-31 15:00:00	365 365
+1970-12-31 04:59:59.999	365 365
+1996-03-31 07:03:33.123	91 091
+2018-11-17 05:33:33.123	321 321
+2019-12-31 09:33:33.123	365 365
+2100-01-01 01:33:33.123	1 001
+
+
+-- !query
+select col, date_format(col, 'H HH') from v
+-- !query schema
+struct<col:timestamp,date_format(col, H HH):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'h hh') from v
+-- !query schema
+struct<col:timestamp,date_format(col, h hh):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'k kk') from v
+-- !query schema
+struct<col:timestamp,date_format(col, k kk):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'K KK') from v
+-- !query schema
+struct<col:timestamp,date_format(col, K KK):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'm mm') from v
+-- !query schema
+struct<col:timestamp,date_format(col, m mm):string>
+-- !query output
+1582-05-31 19:40:35.123	40 40
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 's ss') from v
+-- !query schema
+struct<col:timestamp,date_format(col, s ss):string>
+-- !query output
+1582-05-31 19:40:35.123	35 35
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	33 33
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 'S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS') from v
+-- !query schema
+struct<col:timestamp,date_format(col, S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS):string>
+-- !query output
+1582-05-31 19:40:35.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+1969-12-31 15:00:00	0 00 000 0000 00000 000000 0000000 00000000 000000000
+1970-12-31 04:59:59.999	999 999 999 0999 00999 000999 0000999 00000999 000000999
+1996-03-31 07:03:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+2018-11-17 05:33:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+2019-12-31 09:33:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+2100-01-01 01:33:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+
+
+-- !query
+select col, date_format(col, 'a') from v
+-- !query schema
+struct<col:timestamp,date_format(col, a):string>
+-- !query output
+1582-05-31 19:40:35.123	PM
+1969-12-31 15:00:00	PM
+1970-12-31 04:59:59.999	AM
+1996-03-31 07:03:33.123	AM
+2018-11-17 05:33:33.123	AM
+2019-12-31 09:33:33.123	AM
+2100-01-01 01:33:33.123	AM
+
+
+-- !query
+select col, date_format(col, 'VV') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'V'
+
+
+-- !query
+select col, date_format(col, 'z zz zzz zzzz') from v
+-- !query schema
+struct<col:timestamp,date_format(col, z zz zzz zzzz):string>
+-- !query output
+1582-05-31 19:40:35.123	PST PST PST Pacific Standard Time
+1969-12-31 15:00:00	PST PST PST Pacific Standard Time
+1970-12-31 04:59:59.999	PST PST PST Pacific Standard Time
+1996-03-31 07:03:33.123	PST PST PST Pacific Standard Time
+2018-11-17 05:33:33.123	PST PST PST Pacific Standard Time
+2019-12-31 09:33:33.123	PST PST PST Pacific Standard Time
+2100-01-01 01:33:33.123	PST PST PST Pacific Standard Time
+
+
+-- !query
+select col, date_format(col, 'X XX XXX') from v
+-- !query schema
+struct<col:timestamp,date_format(col, X XX XXX):string>
+-- !query output
+1582-05-31 19:40:35.123	-08 -0800 -08:00
+1969-12-31 15:00:00	-08 -0800 -08:00
+1970-12-31 04:59:59.999	-08 -0800 -08:00
+1996-03-31 07:03:33.123	-08 -0800 -08:00
+2018-11-17 05:33:33.123	-08 -0800 -08:00
+2019-12-31 09:33:33.123	-08 -0800 -08:00
+2100-01-01 01:33:33.123	-08 -0800 -08:00
+
+
+-- !query
+select col, date_format(col, 'XXXX XXXXX') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+invalid ISO 8601 format: length=4
+
+
+-- !query
+select col, date_format(col, 'Z ZZ ZZZ ZZZZ ZZZZZ') from v
+-- !query schema
+struct<col:timestamp,date_format(col, Z ZZ ZZZ ZZZZ ZZZZZ):string>
+-- !query output
+1582-05-31 19:40:35.123	-0800 -0800 -0800 -0800 -0800
+1969-12-31 15:00:00	-0800 -0800 -0800 -0800 -0800
+1970-12-31 04:59:59.999	-0800 -0800 -0800 -0800 -0800
+1996-03-31 07:03:33.123	-0800 -0800 -0800 -0800 -0800
+2018-11-17 05:33:33.123	-0800 -0800 -0800 -0800 -0800
+2019-12-31 09:33:33.123	-0800 -0800 -0800 -0800 -0800
+2100-01-01 01:33:33.123	-0800 -0800 -0800 -0800 -0800
+
+
+-- !query
+select col, date_format(col, 'O OOOO') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'O'
+
+
+-- !query
+select col, date_format(col, 'x xx xxx xxxx xxxx xxxxx') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'x'
+
+
+-- !query
+select col, date_format(col, '[yyyy-MM-dd HH:mm:ss]') from v
+-- !query schema
+struct<col:timestamp,date_format(col, [yyyy-MM-dd HH:mm:ss]):string>
+-- !query output
+1582-05-31 19:40:35.123	[1582-05-31 19:40:35]
+1969-12-31 15:00:00	[1969-12-31 15:00:00]
+1970-12-31 04:59:59.999	[1970-12-31 04:59:59]
+1996-03-31 07:03:33.123	[1996-03-31 07:03:33]
+2018-11-17 05:33:33.123	[2018-11-17 05:33:33]
+2019-12-31 09:33:33.123	[2019-12-31 09:33:33]
+2100-01-01 01:33:33.123	[2100-01-01 01:33:33]
+
+
+-- !query
+select col, date_format(col, "姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'") from v
+-- !query schema
+struct<col:timestamp,date_format(col, 姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'):string>
+-- !query output
+1582-05-31 19:40:35.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1969-12-31 15:00:00	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1970-12-31 04:59:59.999	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1996-03-31 07:03:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2018-11-17 05:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2019-12-31 09:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2100-01-01 01:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+
+
+-- !query
+select col, date_format(col, "''") from v
+-- !query schema
+struct<col:timestamp,date_format(col, ''):string>
+-- !query output
+1582-05-31 19:40:35.123	'
+1969-12-31 15:00:00	'
+1970-12-31 04:59:59.999	'
+1996-03-31 07:03:33.123	'
+2018-11-17 05:33:33.123	'
+2019-12-31 09:33:33.123	'
+2100-01-01 01:33:33.123	'
+
+
+-- !query
+select col, date_format(col, '') from v
+-- !query schema
+struct<col:timestamp,date_format(col, ):string>
+-- !query output
+1582-05-31 19:40:35.123	
+1969-12-31 15:00:00	
+1970-12-31 04:59:59.999	
+1996-03-31 07:03:33.123	
+2018-11-17 05:33:33.123	
+2019-12-31 09:33:33.123	
+2100-01-01 01:33:33.123
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out
new file mode 100644
index 0000000000000..5bed88e168f1e
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out
@@ -0,0 +1,431 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 31
+
+
+-- !query
+create temporary view v as select col from values
+ (timestamp '1582-06-01 11:33:33.123UTC+080000'),
+ (timestamp '1970-01-01 00:00:00.000Europe/Paris'),
+ (timestamp '1970-12-31 23:59:59.999Asia/Srednekolymsk'),
+ (timestamp '1996-04-01 00:33:33.123Australia/Darwin'),
+ (timestamp '2018-11-17 13:33:33.123Z'),
+ (timestamp '2020-01-01 01:33:33.123Asia/Shanghai'),
+ (timestamp '2100-01-01 01:33:33.123America/Los_Angeles') t(col)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select col, date_format(col, 'G GG GGG GGGG') from v
+-- !query schema
+struct<col:timestamp,date_format(col, G GG GGG GGGG):string>
+-- !query output
+1582-05-31 19:40:35.123	AD AD AD Anno Domini
+1969-12-31 15:00:00	AD AD AD Anno Domini
+1970-12-31 04:59:59.999	AD AD AD Anno Domini
+1996-03-31 07:03:33.123	AD AD AD Anno Domini
+2018-11-17 05:33:33.123	AD AD AD Anno Domini
+2019-12-31 09:33:33.123	AD AD AD Anno Domini
+2100-01-01 01:33:33.123	AD AD AD Anno Domini
+
+
+-- !query
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy') from v
+-- !query schema
+struct<col:timestamp,date_format(col, y yy yyy yyyy yyyyy yyyyyy):string>
+-- !query output
+1582-05-31 19:40:35.123	1582 82 1582 1582 01582 001582
+1969-12-31 15:00:00	1969 69 1969 1969 01969 001969
+1970-12-31 04:59:59.999	1970 70 1970 1970 01970 001970
+1996-03-31 07:03:33.123	1996 96 1996 1996 01996 001996
+2018-11-17 05:33:33.123	2018 18 2018 2018 02018 002018
+2019-12-31 09:33:33.123	2019 19 2019 2019 02019 002019
+2100-01-01 01:33:33.123	2100 00 2100 2100 02100 002100
+
+
+-- !query
+select col, date_format(col, 'q qq') from v
+-- !query schema
+struct<col:timestamp,date_format(col, q qq):string>
+-- !query output
+1582-05-31 19:40:35.123	2 02
+1969-12-31 15:00:00	4 04
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	1 01
+2018-11-17 05:33:33.123	4 04
+2019-12-31 09:33:33.123	4 04
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'Q QQ QQQ QQQQ') from v
+-- !query schema
+struct<col:timestamp,date_format(col, Q QQ QQQ QQQQ):string>
+-- !query output
+1582-05-31 19:40:35.123	2 02 Q2 2nd quarter
+1969-12-31 15:00:00	4 04 Q4 4th quarter
+1970-12-31 04:59:59.999	4 04 Q4 4th quarter
+1996-03-31 07:03:33.123	1 01 Q1 1st quarter
+2018-11-17 05:33:33.123	4 04 Q4 4th quarter
+2019-12-31 09:33:33.123	4 04 Q4 4th quarter
+2100-01-01 01:33:33.123	1 01 Q1 1st quarter
+
+
+-- !query
+select col, date_format(col, 'M MM MMM MMMM') from v
+-- !query schema
+struct<col:timestamp,date_format(col, M MM MMM MMMM):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05 May May
+1969-12-31 15:00:00	12 12 Dec December
+1970-12-31 04:59:59.999	12 12 Dec December
+1996-03-31 07:03:33.123	3 03 Mar March
+2018-11-17 05:33:33.123	11 11 Nov November
+2019-12-31 09:33:33.123	12 12 Dec December
+2100-01-01 01:33:33.123	1 01 Jan January
+
+
+-- !query
+select col, date_format(col, 'L LL') from v
+-- !query schema
+struct<col:timestamp,date_format(col, L LL):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05
+1969-12-31 15:00:00	12 12
+1970-12-31 04:59:59.999	12 12
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	11 11
+2019-12-31 09:33:33.123	12 12
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'E EE EEE EEEE') from v
+-- !query schema
+struct<col:timestamp,date_format(col, E EE EEE EEEE):string>
+-- !query output
+1582-05-31 19:40:35.123	Mon Mon Mon Monday
+1969-12-31 15:00:00	Wed Wed Wed Wednesday
+1970-12-31 04:59:59.999	Thu Thu Thu Thursday
+1996-03-31 07:03:33.123	Sun Sun Sun Sunday
+2018-11-17 05:33:33.123	Sat Sat Sat Saturday
+2019-12-31 09:33:33.123	Tue Tue Tue Tuesday
+2100-01-01 01:33:33.123	Fri Fri Fri Friday
+
+
+-- !query
+select col, date_format(col, 'F') from v
+-- !query schema
+struct<col:timestamp,date_format(col, F):string>
+-- !query output
+1582-05-31 19:40:35.123	3
+1969-12-31 15:00:00	3
+1970-12-31 04:59:59.999	3
+1996-03-31 07:03:33.123	3
+2018-11-17 05:33:33.123	3
+2019-12-31 09:33:33.123	3
+2100-01-01 01:33:33.123	1
+
+
+-- !query
+select col, date_format(col, 'd dd') from v
+-- !query schema
+struct<col:timestamp,date_format(col, d dd):string>
+-- !query output
+1582-05-31 19:40:35.123	31 31
+1969-12-31 15:00:00	31 31
+1970-12-31 04:59:59.999	31 31
+1996-03-31 07:03:33.123	31 31
+2018-11-17 05:33:33.123	17 17
+2019-12-31 09:33:33.123	31 31
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'DD') from v where col = timestamp '2100-01-01 01:33:33.123America/Los_Angeles'
+-- !query schema
+struct<col:timestamp,date_format(col, DD):string>
+-- !query output
+2100-01-01 01:33:33.123	01
+
+
+-- !query
+select col, date_format(col, 'D DDD') from v
+-- !query schema
+struct<col:timestamp,date_format(col, D DDD):string>
+-- !query output
+1582-05-31 19:40:35.123	151 151
+1969-12-31 15:00:00	365 365
+1970-12-31 04:59:59.999	365 365
+1996-03-31 07:03:33.123	91 091
+2018-11-17 05:33:33.123	321 321
+2019-12-31 09:33:33.123	365 365
+2100-01-01 01:33:33.123	1 001
+
+
+-- !query
+select col, date_format(col, 'H HH') from v
+-- !query schema
+struct<col:timestamp,date_format(col, H HH):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'h hh') from v
+-- !query schema
+struct<col:timestamp,date_format(col, h hh):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'k kk') from v
+-- !query schema
+struct<col:timestamp,date_format(col, k kk):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'K KK') from v
+-- !query schema
+struct<col:timestamp,date_format(col, K KK):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'm mm') from v
+-- !query schema
+struct<col:timestamp,date_format(col, m mm):string>
+-- !query output
+1582-05-31 19:40:35.123	40 40
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 's ss') from v
+-- !query schema
+struct<col:timestamp,date_format(col, s ss):string>
+-- !query output
+1582-05-31 19:40:35.123	35 35
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	33 33
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 'S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS') from v
+-- !query schema
+struct<col:timestamp,date_format(col, S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS):string>
+-- !query output
+1582-05-31 19:40:35.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+1969-12-31 15:00:00	0 00 000 0000 00000 000000 0000000 00000000 000000000
+1970-12-31 04:59:59.999	9 99 999 9990 99900 999000 9990000 99900000 999000000
+1996-03-31 07:03:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+2018-11-17 05:33:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+2019-12-31 09:33:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+2100-01-01 01:33:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+
+
+-- !query
+select col, date_format(col, 'a') from v
+-- !query schema
+struct<col:timestamp,date_format(col, a):string>
+-- !query output
+1582-05-31 19:40:35.123	PM
+1969-12-31 15:00:00	PM
+1970-12-31 04:59:59.999	AM
+1996-03-31 07:03:33.123	AM
+2018-11-17 05:33:33.123	AM
+2019-12-31 09:33:33.123	AM
+2100-01-01 01:33:33.123	AM
+
+
+-- !query
+select col, date_format(col, 'VV') from v
+-- !query schema
+struct<col:timestamp,date_format(col, VV):string>
+-- !query output
+1582-05-31 19:40:35.123	America/Los_Angeles
+1969-12-31 15:00:00	America/Los_Angeles
+1970-12-31 04:59:59.999	America/Los_Angeles
+1996-03-31 07:03:33.123	America/Los_Angeles
+2018-11-17 05:33:33.123	America/Los_Angeles
+2019-12-31 09:33:33.123	America/Los_Angeles
+2100-01-01 01:33:33.123	America/Los_Angeles
+
+
+-- !query
+select col, date_format(col, 'z zz zzz zzzz') from v
+-- !query schema
+struct<col:timestamp,date_format(col, z zz zzz zzzz):string>
+-- !query output
+1582-05-31 19:40:35.123	PST PST PST Pacific Standard Time
+1969-12-31 15:00:00	PST PST PST Pacific Standard Time
+1970-12-31 04:59:59.999	PST PST PST Pacific Standard Time
+1996-03-31 07:03:33.123	PST PST PST Pacific Standard Time
+2018-11-17 05:33:33.123	PST PST PST Pacific Standard Time
+2019-12-31 09:33:33.123	PST PST PST Pacific Standard Time
+2100-01-01 01:33:33.123	PST PST PST Pacific Standard Time
+
+
+-- !query
+select col, date_format(col, 'X XX XXX') from v
+-- !query schema
+struct<col:timestamp,date_format(col, X XX XXX):string>
+-- !query output
+1582-05-31 19:40:35.123	-0752 -0752 -07:52
+1969-12-31 15:00:00	-08 -0800 -08:00
+1970-12-31 04:59:59.999	-08 -0800 -08:00
+1996-03-31 07:03:33.123	-08 -0800 -08:00
+2018-11-17 05:33:33.123	-08 -0800 -08:00
+2019-12-31 09:33:33.123	-08 -0800 -08:00
+2100-01-01 01:33:33.123	-08 -0800 -08:00
+
+
+-- !query
+select col, date_format(col, 'XXXX XXXXX') from v
+-- !query schema
+struct<col:timestamp,date_format(col, XXXX XXXXX):string>
+-- !query output
+1582-05-31 19:40:35.123	-075258 -07:52:58
+1969-12-31 15:00:00	-0800 -08:00
+1970-12-31 04:59:59.999	-0800 -08:00
+1996-03-31 07:03:33.123	-0800 -08:00
+2018-11-17 05:33:33.123	-0800 -08:00
+2019-12-31 09:33:33.123	-0800 -08:00
+2100-01-01 01:33:33.123	-0800 -08:00
+
+
+-- !query
+select col, date_format(col, 'Z ZZ ZZZ ZZZZ ZZZZZ') from v
+-- !query schema
+struct<col:timestamp,date_format(col, Z ZZ ZZZ ZZZZ ZZZZZ):string>
+-- !query output
+1582-05-31 19:40:35.123	-0752 -0752 -0752 GMT-07:52:58 -07:52:58
+1969-12-31 15:00:00	-0800 -0800 -0800 GMT-08:00 -08:00
+1970-12-31 04:59:59.999	-0800 -0800 -0800 GMT-08:00 -08:00
+1996-03-31 07:03:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+2018-11-17 05:33:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+2019-12-31 09:33:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+2100-01-01 01:33:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+
+
+-- !query
+select col, date_format(col, 'O OOOO') from v
+-- !query schema
+struct<col:timestamp,date_format(col, O OOOO):string>
+-- !query output
+1582-05-31 19:40:35.123	GMT-7:52:58 GMT-07:52:58
+1969-12-31 15:00:00	GMT-8 GMT-08:00
+1970-12-31 04:59:59.999	GMT-8 GMT-08:00
+1996-03-31 07:03:33.123	GMT-8 GMT-08:00
+2018-11-17 05:33:33.123	GMT-8 GMT-08:00
+2019-12-31 09:33:33.123	GMT-8 GMT-08:00
+2100-01-01 01:33:33.123	GMT-8 GMT-08:00
+
+
+-- !query
+select col, date_format(col, 'x xx xxx xxxx xxxx xxxxx') from v
+-- !query schema
+struct<col:timestamp,date_format(col, x xx xxx xxxx xxxx xxxxx):string>
+-- !query output
+1582-05-31 19:40:35.123	-0752 -0752 -07:52 -075258 -075258 -07:52:58
+1969-12-31 15:00:00	-08 -0800 -08:00 -0800 -0800 -08:00
+1970-12-31 04:59:59.999	-08 -0800 -08:00 -0800 -0800 -08:00
+1996-03-31 07:03:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+2018-11-17 05:33:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+2019-12-31 09:33:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+2100-01-01 01:33:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+
+
+-- !query
+select col, date_format(col, '[yyyy-MM-dd HH:mm:ss]') from v
+-- !query schema
+struct<col:timestamp,date_format(col, [yyyy-MM-dd HH:mm:ss]):string>
+-- !query output
+1582-05-31 19:40:35.123	1582-05-31 19:40:35
+1969-12-31 15:00:00	1969-12-31 15:00:00
+1970-12-31 04:59:59.999	1970-12-31 04:59:59
+1996-03-31 07:03:33.123	1996-03-31 07:03:33
+2018-11-17 05:33:33.123	2018-11-17 05:33:33
+2019-12-31 09:33:33.123	2019-12-31 09:33:33
+2100-01-01 01:33:33.123	2100-01-01 01:33:33
+
+
+-- !query
+select col, date_format(col, "姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'") from v
+-- !query schema
+struct<col:timestamp,date_format(col, 姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'):string>
+-- !query output
+1582-05-31 19:40:35.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1969-12-31 15:00:00	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1970-12-31 04:59:59.999	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1996-03-31 07:03:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2018-11-17 05:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2019-12-31 09:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2100-01-01 01:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+
+
+-- !query
+select col, date_format(col, "''") from v
+-- !query schema
+struct<col:timestamp,date_format(col, ''):string>
+-- !query output
+1582-05-31 19:40:35.123	'
+1969-12-31 15:00:00	'
+1970-12-31 04:59:59.999	'
+1996-03-31 07:03:33.123	'
+2018-11-17 05:33:33.123	'
+2019-12-31 09:33:33.123	'
+2100-01-01 01:33:33.123	'
+
+
+-- !query
+select col, date_format(col, '') from v
+-- !query schema
+struct<col:timestamp,date_format(col, ):string>
+-- !query output
+1582-05-31 19:40:35.123	
+1969-12-31 15:00:00	
+1970-12-31 04:59:59.999	
+1996-03-31 07:03:33.123	
+2018-11-17 05:33:33.123	
+2019-12-31 09:33:33.123	
+2100-01-01 01:33:33.123
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
new file mode 100644
index 0000000000000..fbbdb5f3feb48
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
@@ -0,0 +1,900 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 108
+
+
+-- !query
+select current_date = current_date(), current_timestamp = current_timestamp()
+-- !query schema
+struct<(current_date() = current_date()):boolean,(current_timestamp() = current_timestamp()):boolean>
+-- !query output
+true	true
+
+
+-- !query
+select to_date(null), to_date('2016-12-31'), to_date('2016-12-31', 'yyyy-MM-dd')
+-- !query schema
+struct<to_date(NULL):date,to_date('2016-12-31'):date,to_date('2016-12-31', 'yyyy-MM-dd'):date>
+-- !query output
+NULL	2016-12-31	2016-12-31
+
+
+-- !query
+select to_timestamp(null), to_timestamp('2016-12-31 00:12:00'), to_timestamp('2016-12-31', 'yyyy-MM-dd')
+-- !query schema
+struct<to_timestamp(NULL):timestamp,to_timestamp('2016-12-31 00:12:00'):timestamp,to_timestamp('2016-12-31', 'yyyy-MM-dd'):timestamp>
+-- !query output
+NULL	2016-12-31 00:12:00	2016-12-31 00:00:00
+
+
+-- !query
+select dayofweek('2007-02-03'), dayofweek('2009-07-30'), dayofweek('2017-05-27'), dayofweek(null), dayofweek('1582-10-15 13:10:15')
+-- !query schema
+struct<dayofweek(CAST(2007-02-03 AS DATE)):int,dayofweek(CAST(2009-07-30 AS DATE)):int,dayofweek(CAST(2017-05-27 AS DATE)):int,dayofweek(CAST(NULL AS DATE)):int,dayofweek(CAST(1582-10-15 13:10:15 AS DATE)):int>
+-- !query output
+7	5	7	NULL	6
+
+
+-- !query
+create temporary view ttf1 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf1(current_date, current_timestamp)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select current_date, current_timestamp from ttf1
+-- !query schema
+struct<current_date:int,current_timestamp:int>
+-- !query output
+1	2
+2	3
+
+
+-- !query
+create temporary view ttf2 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf2(a, b)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select current_date = current_date(), current_timestamp = current_timestamp(), a, b from ttf2
+-- !query schema
+struct<(current_date() = current_date()):boolean,(current_timestamp() = current_timestamp()):boolean,a:int,b:int>
+-- !query output
+true	true	1	2
+true	true	2	3
+
+
+-- !query
+select a, b from ttf2 order by a, current_date
+-- !query schema
+struct<a:int,b:int>
+-- !query output
+1	2
+2	3
+
+
+-- !query
+select weekday('2007-02-03'), weekday('2009-07-30'), weekday('2017-05-27'), weekday(null), weekday('1582-10-15 13:10:15')
+-- !query schema
+struct<weekday(CAST(2007-02-03 AS DATE)):int,weekday(CAST(2009-07-30 AS DATE)):int,weekday(CAST(2017-05-27 AS DATE)):int,weekday(CAST(NULL AS DATE)):int,weekday(CAST(1582-10-15 13:10:15 AS DATE)):int>
+-- !query output
+5	3	5	NULL	4
+
+
+-- !query
+select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
+-- !query schema
+struct<year(CAST(1500-01-01 AS DATE)):int,month(CAST(1500-01-01 AS DATE)):int,dayofyear(CAST(1500-01-01 AS DATE)):int>
+-- !query output
+1500	1	1
+
+
+-- !query
+select date '2019-01-01\t'
+-- !query schema
+struct<DATE '2019-01-01':date>
+-- !query output
+2019-01-01
+
+
+-- !query
+select timestamp '2019-01-01\t'
+-- !query schema
+struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
+-- !query output
+2019-01-01 00:00:00
+
+
+-- !query
+select date '2020-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)
+
+== SQL ==
+select date '2020-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp '2019-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)
+
+== SQL ==
+select timestamp '2019-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp'2011-11-11 11:11:11' + interval '2' day
+-- !query schema
+struct<CAST(TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days' AS TIMESTAMP):timestamp>
+-- !query output
+2011-11-13 11:11:11
+
+
+-- !query
+select timestamp'2011-11-11 11:11:11' - interval '2' day
+-- !query schema
+struct<CAST(TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days' AS TIMESTAMP):timestamp>
+-- !query output
+2011-11-09 11:11:11
+
+
+-- !query
+select date'2011-11-11 11:11:11' + interval '2' second
+-- !query schema
+struct<DATE '2011-11-11' + INTERVAL '2 seconds':date>
+-- !query output
+2011-11-11
+
+
+-- !query
+select date'2011-11-11 11:11:11' - interval '2' second
+-- !query schema
+struct<DATE '2011-11-11' + (- INTERVAL '2 seconds'):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select '2011-11-11' - interval '2' day
+-- !query schema
+struct<CAST(CAST(2011-11-11 AS TIMESTAMP) - INTERVAL '2 days' AS STRING):string>
+-- !query output
+2011-11-09 00:00:00
+
+
+-- !query
+select '2011-11-11 11:11:11' - interval '2' second
+-- !query schema
+struct<CAST(CAST(2011-11-11 11:11:11 AS TIMESTAMP) - INTERVAL '2 seconds' AS STRING):string>
+-- !query output
+2011-11-11 11:11:09
+
+
+-- !query
+select '1' - interval '2' second
+-- !query schema
+struct<CAST(CAST(1 AS TIMESTAMP) - INTERVAL '2 seconds' AS STRING):string>
+-- !query output
+NULL
+
+
+-- !query
+select 1 - interval '2' second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve '1 - INTERVAL '2 seconds'' due to data type mismatch: argument 1 requires timestamp type, however, '1' is of int type.; line 1 pos 7
+
+
+-- !query
+select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'
+-- !query schema
+struct<subtracttimestamps(CAST(DATE '2020-01-01' AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+-- !query output
+2078 hours 48 minutes 47.654322 seconds
+
+
+-- !query
+select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'
+-- !query schema
+struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(DATE '2020-01-01' AS TIMESTAMP)):interval>
+-- !query output
+-2078 hours -48 minutes -47.654322 seconds
+
+
+-- !query
+select timestamp'2019-10-06 10:11:12.345678' - null
+-- !query schema
+struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(NULL AS TIMESTAMP)):interval>
+-- !query output
+NULL
+
+
+-- !query
+select null - timestamp'2019-10-06 10:11:12.345678'
+-- !query schema
+struct<subtracttimestamps(CAST(NULL AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+-- !query output
+NULL
+
+
+-- !query
+select date_add('2011-11-11', 1Y)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1S)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1L)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 1L)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '1L' is of bigint type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', 1.0)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 1.0BD)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '1.0BD' is of decimal(2,1) type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', 1E1)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', '1')
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_add' function needs to be an integer.;
+
+
+-- !query
+select date_add(date'2011-11-11', 1)
+-- !query schema
+struct<date_add(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add(timestamp'2011-11-11', 1)
+-- !query schema
+struct<date_add(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_sub(date'2011-11-11', 1)
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1')
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_sub' function needs to be an integer.;
+
+
+-- !query
+select date_sub(timestamp'2011-11-11', 1)
+-- !query schema
+struct<date_sub(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(null, 1)
+-- !query schema
+struct<date_sub(CAST(NULL AS DATE), 1):date>
+-- !query output
+NULL
+
+
+-- !query
+select date_sub(date'2011-11-11', null)
+-- !query schema
+struct<date_sub(DATE '2011-11-11', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date'2011-11-11' + 1E1
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
+
+
+-- !query
+select date'2011-11-11' + '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
+-- !query
+select null + date '2001-09-28'
+-- !query schema
+struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-09-28' + 7Y
+-- !query schema
+struct<date_add(DATE '2001-09-28', 7):date>
+-- !query output
+2001-10-05
+
+
+-- !query
+select 7S + date '2001-09-28'
+-- !query schema
+struct<date_add(DATE '2001-09-28', 7):date>
+-- !query output
+2001-10-05
+
+
+-- !query
+select date '2001-10-01' - 7
+-- !query schema
+struct<date_sub(DATE '2001-10-01', 7):date>
+-- !query output
+2001-09-24
+
+
+-- !query
+select date '2001-10-01' - '7'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(DATE '2001-10-01', CAST('7' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('7' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
+-- !query
+select date '2001-09-28' + null
+-- !query schema
+struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-09-28' - null
+-- !query schema
+struct<date_sub(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+create temp view v as select '1' str
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select date_add('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select date_sub('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select null - date '2019-10-06'
+-- !query schema
+struct<subtractdates(CAST(NULL AS DATE), DATE '2019-10-06'):interval>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-10-01' - date '2001-09-28'
+-- !query schema
+struct<subtractdates(DATE '2001-10-01', DATE '2001-09-28'):interval>
+-- !query output
+3 days
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019-10-06S10:11:12.12345", "yyyy-MM-dd'S'HH:mm:ss.SSSSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06S10:11:12.12345', 'yyyy-MM-dd\'S\'HH:mm:ss.SSSSSS'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.12342019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.12342019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1234019-10-06S10:11', 'ss.SSSSy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'")
+-- !query schema
+struct<to_timestamp('2019-10-06S', 'yyyy-MM-dd\'S\''):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd")
+-- !query schema
+struct<to_timestamp('S2019-10-06', '\'S\'yyyy-MM-dd'):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'12', 'yyyy-MM-dd\'T\'HH:mm:ss\'\'SSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.012
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'', 'yyyy-MM-dd\'T\'HH:mm:ss\'\''):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('\'2019-10-06T10:11:12', '\'\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('P2019-10-06T10:11:12', '\'P\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("16", "dd")
+-- !query schema
+struct<to_timestamp('16', 'dd'):timestamp>
+-- !query output
+1970-01-16 00:00:00
+
+
+-- !query
+select to_timestamp("02-29", "MM-dd")
+-- !query schema
+struct<to_timestamp('02-29', 'MM-dd'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_date("16", "dd")
+-- !query schema
+struct<to_date('16', 'dd'):date>
+-- !query output
+1970-01-16
+
+
+-- !query
+select to_date("02-29", "MM-dd")
+-- !query schema
+struct<to_date('02-29', 'MM-dd'):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019 40", "yyyy mm")
+-- !query schema
+struct<to_timestamp('2019 40', 'yyyy mm'):timestamp>
+-- !query output
+2019-01-01 00:40:00
+
+
+-- !query
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
+-- !query schema
+struct<to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss'):timestamp>
+-- !query output
+2019-01-01 10:10:10
+
+
+-- !query
+select date_format(date '2020-05-23', 'GGGGG')
+-- !query schema
+struct<date_format(CAST(DATE '2020-05-23' AS TIMESTAMP), GGGGG):string>
+-- !query output
+AD
+
+
+-- !query
+select date_format(date '2020-05-23', 'MMMMM')
+-- !query schema
+struct<date_format(CAST(DATE '2020-05-23' AS TIMESTAMP), MMMMM):string>
+-- !query output
+May
+
+
+-- !query
+select date_format(date '2020-05-23', 'LLLLL')
+-- !query schema
+struct<date_format(CAST(DATE '2020-05-23' AS TIMESTAMP), LLLLL):string>
+-- !query output
+May
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'EEEEE')
+-- !query schema
+struct<date_format(TIMESTAMP '2020-05-23 00:00:00', EEEEE):string>
+-- !query output
+Saturday
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'uuuuu')
+-- !query schema
+struct<date_format(TIMESTAMP '2020-05-23 00:00:00', uuuuu):string>
+-- !query output
+00006
+
+
+-- !query
+select date_format('2020-05-23', 'QQQQQ')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'Q'
+
+
+-- !query
+select date_format('2020-05-23', 'qqqqq')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'q'
+
+
+-- !query
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG')
+-- !query schema
+struct<to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE')
+-- !query schema
+struct<to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE'):timestamp>
+-- !query output
+2020-05-22 00:00:00
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE'):timestamp>
+-- !query output
+2020-05-22 00:00:00
+
+
+-- !query
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<unix_timestamp(22 05 2020 Friday, dd MM yyyy EEEEE):bigint>
+-- !query output
+1590130800
+
+
+-- !query
+select from_unixtime(12345, 'MMMMM')
+-- !query schema
+struct<from_unixtime(CAST(12345 AS BIGINT), MMMMM):string>
+-- !query output
+December
+
+
+-- !query
+select from_unixtime(54321, 'QQQQQ')
+-- !query schema
+struct<from_unixtime(CAST(54321 AS BIGINT), QQQQQ):string>
+-- !query output
+NULL
+
+
+-- !query
+select from_unixtime(23456, 'aaaaa')
+-- !query schema
+struct<from_unixtime(CAST(23456 AS BIGINT), aaaaa):string>
+-- !query output
+PM
+
+
+-- !query
+select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_json({"time":"26/October/2015"}):struct<time:timestamp>>
+-- !query output
+{"time":2015-10-26 00:00:00}
+
+
+-- !query
+select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_json({"date":"26/October/2015"}):struct<date:date>>
+-- !query output
+{"date":2015-10-26}
+
+
+-- !query
+select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_csv(26/October/2015):struct<time:timestamp>>
+-- !query output
+{"time":2015-10-26 00:00:00}
+
+
+-- !query
+select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_csv(26/October/2015):struct<date:date>>
+-- !query output
+{"date":2015-10-26}
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
new file mode 100644
index 0000000000000..76008d9c97dc6
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
@@ -0,0 +1,162 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 19
+
+
+-- !query
+select to_timestamp('294248', 'y')
+-- !query schema
+struct<to_timestamp('294248', 'y'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('1', 'yy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('-12', 'yy')
+-- !query schema
+struct<to_timestamp('-12', 'yy'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('123', 'yy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '123' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('1', 'yyy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('1234567', 'yyyyyyy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('366', 'D')
+-- !query schema
+struct<to_timestamp('366', 'D'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('9', 'DD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '9' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('366', 'DD')
+-- !query schema
+struct<to_timestamp('366', 'DD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('9', 'DDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '9' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('99', 'DDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '99' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('30-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp('30-365', 'dd-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('11-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp('11-365', 'MM-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-366', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp('2019-366', 'yyyy-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('12-30-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('12-30-365', 'MM-dd-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-01-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-01-365', 'yyyy-dd-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-10-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp('2020-10-350', 'yyyy-MM-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select from_csv('2018-366', 'date Date', map('dateFormat', 'yyyy-DDD'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2018-366' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
new file mode 100644
index 0000000000000..258afe43b99ee
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
@@ -0,0 +1,202 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 25
+
+
+-- !query
+select to_timestamp('1', 'y')
+-- !query schema
+struct<to_timestamp('1', 'y'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'y')
+-- !query schema
+struct<to_timestamp('009999', 'y'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00', 'yy')
+-- !query schema
+struct<to_timestamp('00', 'yy'):timestamp>
+-- !query output
+2000-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'yy')
+-- !query schema
+struct<to_timestamp('99', 'yy'):timestamp>
+-- !query output
+1999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('001', 'yyy')
+-- !query schema
+struct<to_timestamp('001', 'yyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyy')
+-- !query schema
+struct<to_timestamp('009999', 'yyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('0001', 'yyyy')
+-- !query schema
+struct<to_timestamp('0001', 'yyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('9999', 'yyyy')
+-- !query schema
+struct<to_timestamp('9999', 'yyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00001', 'yyyyy')
+-- !query schema
+struct<to_timestamp('00001', 'yyyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('09999', 'yyyyy')
+-- !query schema
+struct<to_timestamp('09999', 'yyyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('000001', 'yyyyyy')
+-- !query schema
+struct<to_timestamp('000001', 'yyyyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyyyyy')
+-- !query schema
+struct<to_timestamp('009999', 'yyyyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('9', 'D')
+-- !query schema
+struct<to_timestamp('9', 'D'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('300', 'D')
+-- !query schema
+struct<to_timestamp('300', 'D'):timestamp>
+-- !query output
+1970-10-27 00:00:00
+
+
+-- !query
+select to_timestamp('09', 'DD')
+-- !query schema
+struct<to_timestamp('09', 'DD'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'DD')
+-- !query schema
+struct<to_timestamp('99', 'DD'):timestamp>
+-- !query output
+1970-04-09 00:00:00
+
+
+-- !query
+select to_timestamp('009', 'DDD')
+-- !query schema
+struct<to_timestamp('009', 'DDD'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('365', 'DDD')
+-- !query schema
+struct<to_timestamp('365', 'DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('31-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp('31-365', 'dd-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('12-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp('12-365', 'MM-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-365', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp('2020-365', 'yyyy-DDD'):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('12-31-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('12-31-365', 'MM-dd-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-30-365', 'yyyy-dd-DDD'):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp('2020-12-350', 'yyyy-MM-DDD'):timestamp>
+-- !query output
+2020-12-15 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD'):timestamp>
+-- !query output
+2020-12-31 00:00:00
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
new file mode 100644
index 0000000000000..f030141d288ec
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
@@ -0,0 +1,202 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 25
+
+
+-- !query
+select to_timestamp('1', 'y')
+-- !query schema
+struct<to_timestamp('1', 'y'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'y')
+-- !query schema
+struct<to_timestamp('009999', 'y'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00', 'yy')
+-- !query schema
+struct<to_timestamp('00', 'yy'):timestamp>
+-- !query output
+2000-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'yy')
+-- !query schema
+struct<to_timestamp('99', 'yy'):timestamp>
+-- !query output
+2099-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('001', 'yyy')
+-- !query schema
+struct<to_timestamp('001', 'yyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyy')
+-- !query schema
+struct<to_timestamp('009999', 'yyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('0001', 'yyyy')
+-- !query schema
+struct<to_timestamp('0001', 'yyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('9999', 'yyyy')
+-- !query schema
+struct<to_timestamp('9999', 'yyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00001', 'yyyyy')
+-- !query schema
+struct<to_timestamp('00001', 'yyyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('09999', 'yyyyy')
+-- !query schema
+struct<to_timestamp('09999', 'yyyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('000001', 'yyyyyy')
+-- !query schema
+struct<to_timestamp('000001', 'yyyyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyyyyy')
+-- !query schema
+struct<to_timestamp('009999', 'yyyyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('9', 'D')
+-- !query schema
+struct<to_timestamp('9', 'D'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('300', 'D')
+-- !query schema
+struct<to_timestamp('300', 'D'):timestamp>
+-- !query output
+1970-10-27 00:00:00
+
+
+-- !query
+select to_timestamp('09', 'DD')
+-- !query schema
+struct<to_timestamp('09', 'DD'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'DD')
+-- !query schema
+struct<to_timestamp('99', 'DD'):timestamp>
+-- !query output
+1970-04-09 00:00:00
+
+
+-- !query
+select to_timestamp('009', 'DDD')
+-- !query schema
+struct<to_timestamp('009', 'DDD'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('365', 'DDD')
+-- !query schema
+struct<to_timestamp('365', 'DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('31-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp('31-365', 'dd-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('12-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp('12-365', 'MM-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-365', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp('2020-365', 'yyyy-DDD'):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('12-31-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('12-31-365', 'MM-dd-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-30-365', 'yyyy-dd-DDD'):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp('2020-12-350', 'yyyy-MM-DDD'):timestamp>
+-- !query output
+2020-12-15 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD'):timestamp>
+-- !query output
+2020-12-31 00:00:00
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index a7b098d79a706..d7e960a9a3db3 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 47
+-- Number of queries: 108
 
 
 -- !query
@@ -115,6 +115,34 @@ struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
 2019-01-01 00:00:00
 
 
+-- !query
+select date '2020-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)
+
+== SQL ==
+select date '2020-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp '2019-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)
+
+== SQL ==
+select timestamp '2019-01-01中文'
+-------^^^
+
+
 -- !query
 select timestamp'2011-11-11 11:11:11' + interval '2' day
 -- !query schema
@@ -134,7 +162,7 @@ struct<CAST(TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days' AS TIMESTAMP):ti
 -- !query
 select date'2011-11-11 11:11:11' + interval '2' second
 -- !query schema
-struct<CAST(CAST(DATE '2011-11-11' AS TIMESTAMP) + INTERVAL '2 seconds' AS DATE):date>
+struct<DATE '2011-11-11' + INTERVAL '2 seconds':date>
 -- !query output
 2011-11-11
 
@@ -142,7 +170,7 @@ struct<CAST(CAST(DATE '2011-11-11' AS TIMESTAMP) + INTERVAL '2 seconds' AS DATE)
 -- !query
 select date'2011-11-11 11:11:11' - interval '2' second
 -- !query schema
-struct<CAST(CAST(DATE '2011-11-11' AS TIMESTAMP) - INTERVAL '2 seconds' AS DATE):date>
+struct<DATE '2011-11-11' + (- INTERVAL '2 seconds'):date>
 -- !query output
 2011-11-10
 
@@ -266,10 +294,18 @@ cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 10.0D)' due to data type mi
 -- !query
 select date_add('2011-11-11', '1')
 -- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', '1.2')
+-- !query schema
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_add(CAST('2011-11-11' AS DATE), '1')' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, ''1'' is of string type.; line 1 pos 7
+The second argument of 'date_add' function needs to be an integer.;
 
 
 -- !query
@@ -296,6 +332,23 @@ struct<date_sub(DATE '2011-11-11', 1):date>
 2011-11-10
 
 
+-- !query
+select date_sub(date'2011-11-11', '1')
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_sub' function needs to be an integer.;
+
+
 -- !query
 select date_sub(timestamp'2011-11-11', 1)
 -- !query schema
@@ -329,6 +382,15 @@ org.apache.spark.sql.AnalysisException
 cannot resolve 'date_add(DATE '2011-11-11', 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
 
 
+-- !query
+select date'2011-11-11' + '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
 -- !query
 select null + date '2001-09-28'
 -- !query schema
@@ -361,6 +423,15 @@ struct<date_sub(DATE '2001-10-01', 7):date>
 2001-09-24
 
 
+-- !query
+select date '2001-10-01' - '7'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(DATE '2001-10-01', CAST('7' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('7' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
 -- !query
 select date '2001-09-28' + null
 -- !query schema
@@ -377,6 +448,32 @@ struct<date_sub(DATE '2001-09-28', CAST(NULL AS INT)):date>
 NULL
 
 
+-- !query
+create temp view v as select '1' str
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select date_add('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select date_sub('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
 -- !query
 select null - date '2019-10-06'
 -- !query schema
@@ -391,3 +488,428 @@ select date '2001-10-01' - date '2001-09-28'
 struct<subtractdates(DATE '2001-10-01', DATE '2001-09-28'):interval>
 -- !query output
 3 days
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 03:11:12.123
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 08:11:12.12345
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123456
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123456
+
+
+-- !query
+select to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:00
+
+
+-- !query
+select to_timestamp("2019-10-06S10:11:12.12345", "yyyy-MM-dd'S'HH:mm:ss.SSSSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06S10:11:12.12345', 'yyyy-MM-dd\'S\'HH:mm:ss.SSSSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12345
+
+
+-- !query
+select to_timestamp("12.12342019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.12342019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1234019-10-06S10:11', 'ss.SSSSy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+0019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'")
+-- !query schema
+struct<to_timestamp('2019-10-06S', 'yyyy-MM-dd\'S\''):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd")
+-- !query schema
+struct<to_timestamp('S2019-10-06', '\'S\'yyyy-MM-dd'):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'12', 'yyyy-MM-dd\'T\'HH:mm:ss\'\'SSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'', 'yyyy-MM-dd\'T\'HH:mm:ss\'\''):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('\'2019-10-06T10:11:12', '\'\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('P2019-10-06T10:11:12', '\'P\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("16", "dd")
+-- !query schema
+struct<to_timestamp('16', 'dd'):timestamp>
+-- !query output
+1970-01-16 00:00:00
+
+
+-- !query
+select to_timestamp("02-29", "MM-dd")
+-- !query schema
+struct<to_timestamp('02-29', 'MM-dd'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_date("16", "dd")
+-- !query schema
+struct<to_date('16', 'dd'):date>
+-- !query output
+1970-01-16
+
+
+-- !query
+select to_date("02-29", "MM-dd")
+-- !query schema
+struct<to_date('02-29', 'MM-dd'):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019 40", "yyyy mm")
+-- !query schema
+struct<to_timestamp('2019 40', 'yyyy mm'):timestamp>
+-- !query output
+2019-01-01 00:40:00
+
+
+-- !query
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
+-- !query schema
+struct<to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss'):timestamp>
+-- !query output
+2019-01-01 10:10:10
+
+
+-- !query
+select date_format(date '2020-05-23', 'GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(date '2020-05-23', 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(date '2020-05-23', 'LLLLL')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'LLLLL' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'uuuuu')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'uuuuu' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2020-05-23', 'QQQQQ')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: Q
+
+
+-- !query
+select date_format('2020-05-23', 'qqqqq')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: q
+
+
+-- !query
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyy-MM-dd GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_unixtime(12345, 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_unixtime(54321, 'QQQQQ')
+-- !query schema
+struct<from_unixtime(CAST(54321 AS BIGINT), QQQQQ):string>
+-- !query output
+NULL
+
+
+-- !query
+select from_unixtime(23456, 'aaaaa')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aaaaa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
diff --git a/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out b/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out
index 6b16aba268f50..2199fc0312d25 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out
@@ -112,7 +112,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-mismatched input 'desc_temp1' expecting {<EOF>, '.'}(line 1, pos 21)
+mismatched input 'desc_temp1' expecting {<EOF>, ';'}(line 1, pos 21)
 
 == SQL ==
 DESCRIBE INSERT INTO desc_temp1 values (1, 'val1')
@@ -126,7 +126,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-mismatched input 'desc_temp1' expecting {<EOF>, '.'}(line 1, pos 21)
+mismatched input 'desc_temp1' expecting {<EOF>, ';'}(line 1, pos 21)
 
 == SQL ==
 DESCRIBE INSERT INTO desc_temp1 SELECT * FROM desc_temp2
diff --git a/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out b/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out
index ae9240ec588da..c6d3d45879eb1 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out
@@ -267,7 +267,7 @@ struct<>
 
 
 -- !query
-CREATE TABLE customer(CName STRING)
+CREATE TABLE customer(CName STRING) USING PARQUET
 -- !query schema
 struct<>
 -- !query output
diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
index 697e006544acf..a7de033e3a1ac 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
@@ -520,7 +520,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `t`, false
+   +- DescribeTableCommand `default`.`t`, false
 
 
 -- !query
@@ -530,7 +530,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `t`, true
+   +- DescribeTableCommand `default`.`t`, true
 
 
 -- !query
@@ -544,14 +544,14 @@ struct<plan:string>
 
 == Analyzed Logical Plan ==
 col_name: string, data_type: string, comment: string
-DescribeTableCommand `t`, false
+DescribeTableCommand `default`.`t`, false
 
 == Optimized Logical Plan ==
-DescribeTableCommand `t`, false
+DescribeTableCommand `default`.`t`, false
 
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `t`, false
+   +- DescribeTableCommand `default`.`t`, false
 
 
 -- !query
@@ -561,7 +561,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeColumnCommand
-   +- DescribeColumnCommand `t`, [b], false
+   +- DescribeColumnCommand `default`.`t`, [b], false
 
 
 -- !query
@@ -571,7 +571,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `t`, Map(c -> Us, d -> 2), false
+   +- DescribeTableCommand `default`.`t`, Map(c -> Us, d -> 2), false
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/except.sql.out b/sql/core/src/test/resources/sql-tests/results/except.sql.out
index 62d695219d01d..061b122eac7cf 100644
--- a/sql/core/src/test/resources/sql-tests/results/except.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/except.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 9
+-- Number of queries: 15
 
 
 -- !query
@@ -103,3 +103,59 @@ WHERE  t1.v >= (SELECT   min(t2.v)
 struct<k:string>
 -- !query output
 two
+
+
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT t.v FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<v:decimal(11,0)>
+-- !query output
+1
+
+
+-- !query
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<sum(v):decimal(21,0)>
+-- !query output
+1
+
+
+-- !query
+DROP VIEW IF EXISTS t1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP VIEW IF EXISTS t2
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP VIEW IF EXISTS t3
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
new file mode 100644
index 0000000000000..36757863ffcb5
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
@@ -0,0 +1,860 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 23
+
+
+-- !query
+CREATE table  explain_temp1 (key int, val int) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE table  explain_temp2 (key int, val int) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE table  explain_temp3 (key int, val int) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE table  explain_temp4 (key int, val string) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SET spark.sql.codegen.wholeStage = true
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.codegen.wholeStage	true
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, max(val) 
+  FROM   explain_temp1 
+  WHERE  key > 0 
+  GROUP  BY key 
+  ORDER  BY key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (9)
++- Sort (8)
+   +- Exchange (7)
+      +- HashAggregate (6)
+         +- Exchange (5)
+            +- HashAggregate (4)
+               +- Project (3)
+                  +- Filter (2)
+                     +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
+ReadSchema: struct<key:int,val:int>
+
+(2) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 0))
+
+(3) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(4) HashAggregate
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+
+(5) Exchange
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
+(6) HashAggregate
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
+
+(7) Exchange
+Input [2]: [key#x, max(val)#x]
+Arguments: rangepartitioning(key#x ASC NULLS FIRST, 4), true, [id=#x]
+
+(8) Sort
+Input [2]: [key#x, max(val)#x]
+Arguments: [key#x ASC NULLS FIRST], true, 0
+
+(9) AdaptiveSparkPlan
+Output [2]: [key#x, max(val)#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, max(val)
+  FROM explain_temp1
+  WHERE key > 0
+  GROUP BY key
+  HAVING max(val) > 0
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (9)
++- Project (8)
+   +- Filter (7)
+      +- HashAggregate (6)
+         +- Exchange (5)
+            +- HashAggregate (4)
+               +- Project (3)
+                  +- Filter (2)
+                     +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
+ReadSchema: struct<key:int,val:int>
+
+(2) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 0))
+
+(3) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(4) HashAggregate
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+
+(5) Exchange
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
+(6) HashAggregate
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [3]: [key#x, max(val#x)#x AS max(val)#x, max(val#x)#x AS max(val#x)#x]
+
+(7) Filter
+Input [3]: [key#x, max(val)#x, max(val#x)#x]
+Condition : (isnotnull(max(val#x)#x) AND (max(val#x)#x > 0))
+
+(8) Project
+Output [2]: [key#x, max(val)#x]
+Input [3]: [key#x, max(val)#x, max(val#x)#x]
+
+(9) AdaptiveSparkPlan
+Output [2]: [key#x, max(val)#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, val FROM explain_temp1 WHERE key > 0
+  UNION 
+  SELECT key, val FROM explain_temp1 WHERE key > 0
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (11)
++- HashAggregate (10)
+   +- Exchange (9)
+      +- HashAggregate (8)
+         +- Union (7)
+            :- Project (3)
+            :  +- Filter (2)
+            :     +- Scan parquet default.explain_temp1 (1)
+            +- Project (6)
+               +- Filter (5)
+                  +- Scan parquet default.explain_temp1 (4)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
+ReadSchema: struct<key:int,val:int>
+
+(2) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 0))
+
+(3) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(4) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
+ReadSchema: struct<key:int,val:int>
+
+(5) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 0))
+
+(6) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(7) Union
+
+(8) HashAggregate
+Input [2]: [key#x, val#x]
+Keys [2]: [key#x, val#x]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [key#x, val#x]
+
+(9) Exchange
+Input [2]: [key#x, val#x]
+Arguments: hashpartitioning(key#x, val#x, 4), true, [id=#x]
+
+(10) HashAggregate
+Input [2]: [key#x, val#x]
+Keys [2]: [key#x, val#x]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [key#x, val#x]
+
+(11) AdaptiveSparkPlan
+Output [2]: [key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT * 
+  FROM   explain_temp1 a, 
+         explain_temp2 b 
+  WHERE  a.key = b.key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (9)
++- BroadcastHashJoin Inner BuildRight (8)
+   :- Project (3)
+   :  +- Filter (2)
+   :     +- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (7)
+      +- Project (6)
+         +- Filter (5)
+            +- Scan parquet default.explain_temp2 (4)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key)]
+ReadSchema: struct<key:int,val:int>
+
+(2) Filter
+Input [2]: [key#x, val#x]
+Condition : isnotnull(key#x)
+
+(3) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(4) Scan parquet default.explain_temp2
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp2]
+PushedFilters: [IsNotNull(key)]
+ReadSchema: struct<key:int,val:int>
+
+(5) Filter
+Input [2]: [key#x, val#x]
+Condition : isnotnull(key#x)
+
+(6) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(7) BroadcastExchange
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+
+(8) BroadcastHashJoin
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
+Join condition: None
+
+(9) AdaptiveSparkPlan
+Output [4]: [key#x, val#x, key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT * 
+  FROM   explain_temp1 a 
+         LEFT OUTER JOIN explain_temp2 b 
+                      ON a.key = b.key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (7)
++- BroadcastHashJoin LeftOuter BuildRight (6)
+   :- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (5)
+      +- Project (4)
+         +- Filter (3)
+            +- Scan parquet default.explain_temp2 (2)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<key:int,val:int>
+
+(2) Scan parquet default.explain_temp2
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp2]
+PushedFilters: [IsNotNull(key)]
+ReadSchema: struct<key:int,val:int>
+
+(3) Filter
+Input [2]: [key#x, val#x]
+Condition : isnotnull(key#x)
+
+(4) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(5) BroadcastExchange
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+
+(6) BroadcastHashJoin
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
+Join condition: None
+
+(7) AdaptiveSparkPlan
+Output [4]: [key#x, val#x, key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT * 
+  FROM   explain_temp1 
+  WHERE  key = (SELECT max(key) 
+                FROM   explain_temp2 
+                WHERE  key = (SELECT max(key) 
+                              FROM   explain_temp3 
+                              WHERE  val > 0) 
+                       AND val = 2) 
+         AND val > 3
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (4)
++- Project (3)
+   +- Filter (2)
+      +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), IsNotNull(val), GreaterThan(val,3)]
+ReadSchema: struct<key:int,val:int>
+
+(2) Filter
+Input [2]: [key#x, val#x]
+Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery subquery#x, [id=#x])) AND (val#x > 3))
+
+(3) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(4) AdaptiveSparkPlan
+Output [2]: [key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT * 
+  FROM   explain_temp1 
+  WHERE  key = (SELECT max(key) 
+                FROM   explain_temp2 
+                WHERE  val > 0) 
+         OR
+         key = (SELECT avg(key)
+                FROM   explain_temp3
+                WHERE  val > 0)
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (3)
++- Filter (2)
+   +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<key:int,val:int>
+
+(2) Filter
+Input [2]: [key#x, val#x]
+Condition : ((key#x = Subquery subquery#x, [id=#x]) OR (cast(key#x as double) = Subquery subquery#x, [id=#x]))
+
+(3) AdaptiveSparkPlan
+Output [2]: [key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT (SELECT Avg(key) FROM explain_temp1) + (SELECT Avg(key) FROM explain_temp1)
+  FROM explain_temp1
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (3)
++- Project (2)
+   +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1
+Output: []
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<>
+
+(2) Project
+Output [1]: [(Subquery subquery#x, [id=#x] + Subquery subquery#x, [id=#x]) AS (scalarsubquery() + scalarsubquery())#x]
+Input: []
+
+(3) AdaptiveSparkPlan
+Output [1]: [(scalarsubquery() + scalarsubquery())#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  WITH cte1 AS (
+    SELECT *
+    FROM explain_temp1 
+    WHERE key > 10
+  )
+  SELECT * FROM cte1 a, cte1 b WHERE a.key = b.key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (9)
++- BroadcastHashJoin Inner BuildRight (8)
+   :- Project (3)
+   :  +- Filter (2)
+   :     +- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (7)
+      +- Project (6)
+         +- Filter (5)
+            +- Scan parquet default.explain_temp1 (4)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
+ReadSchema: struct<key:int,val:int>
+
+(2) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 10))
+
+(3) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(4) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
+ReadSchema: struct<key:int,val:int>
+
+(5) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 10))
+
+(6) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(7) BroadcastExchange
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+
+(8) BroadcastHashJoin
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
+Join condition: None
+
+(9) AdaptiveSparkPlan
+Output [4]: [key#x, val#x, key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  WITH cte1 AS (
+    SELECT key, max(val)
+    FROM explain_temp1 
+    WHERE key > 10
+    GROUP BY key
+  )
+  SELECT * FROM cte1 a, cte1 b WHERE a.key = b.key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (15)
++- BroadcastHashJoin Inner BuildRight (14)
+   :- HashAggregate (6)
+   :  +- Exchange (5)
+   :     +- HashAggregate (4)
+   :        +- Project (3)
+   :           +- Filter (2)
+   :              +- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (13)
+      +- HashAggregate (12)
+         +- Exchange (11)
+            +- HashAggregate (10)
+               +- Project (9)
+                  +- Filter (8)
+                     +- Scan parquet default.explain_temp1 (7)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
+ReadSchema: struct<key:int,val:int>
+
+(2) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 10))
+
+(3) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(4) HashAggregate
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+
+(5) Exchange
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
+(6) HashAggregate
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
+
+(7) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
+ReadSchema: struct<key:int,val:int>
+
+(8) Filter
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 10))
+
+(9) Project
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(10) HashAggregate
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+
+(11) Exchange
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
+(12) HashAggregate
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
+
+(13) BroadcastExchange
+Input [2]: [key#x, max(val)#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+
+(14) BroadcastHashJoin
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
+Join condition: None
+
+(15) AdaptiveSparkPlan
+Output [4]: [key#x, max(val)#x, key#x, max(val)#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  CREATE VIEW explain_view AS
+    SELECT key, val FROM explain_temp1
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+Execute CreateViewCommand (1)
+   +- CreateViewCommand (2)
+         +- Project (4)
+            +- UnresolvedRelation (3)
+
+
+(1) Execute CreateViewCommand
+Output: []
+
+(2) CreateViewCommand
+Arguments: `default`.`explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView
+
+(3) UnresolvedRelation
+Arguments: [explain_temp1]
+
+(4) Project
+Arguments: ['key, 'val]
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT
+    COUNT(val) + SUM(key) as TOTAL,
+    COUNT(key) FILTER (WHERE val > 1)
+  FROM explain_temp1
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (5)
++- HashAggregate (4)
+   +- Exchange (3)
+      +- HashAggregate (2)
+         +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<key:int,val:int>
+
+(2) HashAggregate
+Input [2]: [key#x, val#x]
+Keys: []
+Functions [3]: [partial_count(val#x), partial_sum(cast(key#x as bigint)), partial_count(key#x) FILTER (WHERE (val#x > 1))]
+Aggregate Attributes [3]: [count#xL, sum#xL, count#xL]
+Results [3]: [count#xL, sum#xL, count#xL]
+
+(3) Exchange
+Input [3]: [count#xL, sum#xL, count#xL]
+Arguments: SinglePartition, true, [id=#x]
+
+(4) HashAggregate
+Input [3]: [count#xL, sum#xL, count#xL]
+Keys: []
+Functions [3]: [count(val#x), sum(cast(key#x as bigint)), count(key#x)]
+Aggregate Attributes [3]: [count(val#x)#xL, sum(cast(key#x as bigint))#xL, count(key#x)#xL]
+Results [2]: [(count(val#x)#xL + sum(cast(key#x as bigint))#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
+
+(5) AdaptiveSparkPlan
+Output [2]: [TOTAL#xL, count(key) FILTER (WHERE (val > 1))#xL]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, sort_array(collect_set(val))[0]
+  FROM explain_temp4
+  GROUP BY key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (5)
++- ObjectHashAggregate (4)
+   +- Exchange (3)
+      +- ObjectHashAggregate (2)
+         +- Scan parquet default.explain_temp4 (1)
+
+
+(1) Scan parquet default.explain_temp4
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp4]
+ReadSchema: struct<key:int,val:string>
+
+(2) ObjectHashAggregate
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_collect_set(val#x, 0, 0)]
+Aggregate Attributes [1]: [buf#x]
+Results [2]: [key#x, buf#x]
+
+(3) Exchange
+Input [2]: [key#x, buf#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
+(4) ObjectHashAggregate
+Input [2]: [key#x, buf#x]
+Keys [1]: [key#x]
+Functions [1]: [collect_set(val#x, 0, 0)]
+Aggregate Attributes [1]: [collect_set(val#x, 0, 0)#x]
+Results [2]: [key#x, sort_array(collect_set(val#x, 0, 0)#x, true)[0] AS sort_array(collect_set(val), true)[0]#x]
+
+(5) AdaptiveSparkPlan
+Output [2]: [key#x, sort_array(collect_set(val), true)[0]#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, MIN(val)
+  FROM explain_temp4
+  GROUP BY key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (7)
++- SortAggregate (6)
+   +- Sort (5)
+      +- Exchange (4)
+         +- SortAggregate (3)
+            +- Sort (2)
+               +- Scan parquet default.explain_temp4 (1)
+
+
+(1) Scan parquet default.explain_temp4
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp4]
+ReadSchema: struct<key:int,val:string>
+
+(2) Sort
+Input [2]: [key#x, val#x]
+Arguments: [key#x ASC NULLS FIRST], false, 0
+
+(3) SortAggregate
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_min(val#x)]
+Aggregate Attributes [1]: [min#x]
+Results [2]: [key#x, min#x]
+
+(4) Exchange
+Input [2]: [key#x, min#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
+(5) Sort
+Input [2]: [key#x, min#x]
+Arguments: [key#x ASC NULLS FIRST], false, 0
+
+(6) SortAggregate
+Input [2]: [key#x, min#x]
+Keys [1]: [key#x]
+Functions [1]: [min(val#x)]
+Aggregate Attributes [1]: [min(val#x)#x]
+Results [2]: [key#x, min(val#x)#x AS min(val)#x]
+
+(7) AdaptiveSparkPlan
+Output [2]: [key#x, min(val)#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+DROP TABLE explain_temp1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP TABLE explain_temp2
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP TABLE explain_temp3
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP TABLE explain_temp4
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
index 756c14f28a657..2b07dac0e5d0a 100644
--- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 18
+-- Number of queries: 23
 
 
 -- !query
@@ -26,6 +26,14 @@ struct<>
 
 
 
+-- !query
+CREATE table  explain_temp4 (key int, val string) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
 -- !query
 SET spark.sql.codegen.wholeStage = true
 -- !query schema
@@ -56,38 +64,49 @@ struct<plan:string>
                      +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
+
 (4) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
 (5) HashAggregate [codegen id : 1]
-Input: [key#x, val#x]
-     
-(6) Exchange 
-Input: [key#x, max#x]
-     
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+
+(6) Exchange
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
 (7) HashAggregate [codegen id : 2]
-Input: [key#x, max#x]
-     
-(8) Exchange 
-Input: [key#x, max(val)#x]
-     
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
+
+(8) Exchange
+Input [2]: [key#x, max(val)#x]
+Arguments: rangepartitioning(key#x ASC NULLS FIRST, 4), true, [id=#x]
+
 (9) Sort [codegen id : 3]
-Input: [key#x, max(val)#x]
+Input [2]: [key#x, max(val)#x]
+Arguments: [key#x ASC NULLS FIRST], true, 0
 
 
 -- !query
@@ -112,40 +131,49 @@ struct<plan:string>
                      +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
+
 (4) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
 (5) HashAggregate [codegen id : 1]
-Input: [key#x, val#x]
-     
-(6) Exchange 
-Input: [key#x, max#x]
-     
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+
+(6) Exchange
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
 (7) HashAggregate [codegen id : 2]
-Input: [key#x, max#x]
-     
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [3]: [key#x, max(val#x)#x AS max(val)#x, max(val#x)#x AS max(val#x)#x]
+
 (8) Filter [codegen id : 2]
-Input     : [key#x, max(val)#x, max(val#x)#x]
+Input [3]: [key#x, max(val)#x, max(val#x)#x]
 Condition : (isnotnull(max(val#x)#x) AND (max(val#x)#x > 0))
-     
+
 (9) Project [codegen id : 2]
-Output    : [key#x, max(val)#x]
-Input     : [key#x, max(val)#x, max(val#x)#x]
+Output [2]: [key#x, max(val)#x]
+Input [3]: [key#x, max(val)#x, max(val#x)#x]
 
 
 -- !query
@@ -171,52 +199,61 @@ struct<plan:string>
                   +- Scan parquet default.explain_temp1 (5)
 
 
-(1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
+
 (4) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
-(5) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(5) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (6) ColumnarToRow [codegen id : 2]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (7) Filter [codegen id : 2]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
+
 (8) Project [codegen id : 2]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
-(9) Union 
-     
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(9) Union
+
 (10) HashAggregate [codegen id : 3]
-Input: [key#x, val#x]
-     
-(11) Exchange 
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+Keys [2]: [key#x, val#x]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [key#x, val#x]
+
+(11) Exchange
+Input [2]: [key#x, val#x]
+Arguments: hashpartitioning(key#x, val#x, 4), true, [id=#x]
+
 (12) HashAggregate [codegen id : 4]
-Input: [key#x, val#x]
+Input [2]: [key#x, val#x]
+Keys [2]: [key#x, val#x]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [key#x, val#x]
 
 
 -- !query
@@ -241,48 +278,49 @@ struct<plan:string>
             +- Scan parquet default.explain_temp2 (5)
 
 
-(1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 2]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (3) Filter [codegen id : 2]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
-     
+
 (4) Project [codegen id : 2]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
-(5) Scan parquet default.explain_temp2 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(5) Scan parquet default.explain_temp2
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (6) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (7) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
-     
+
 (8) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
-(9) BroadcastExchange 
-Input: [key#x, val#x]
-     
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(9) BroadcastExchange
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+
 (10) BroadcastHashJoin [codegen id : 2]
-Left keys: List(key#x)
-Right keys: List(key#x)
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
 Join condition: None
 
 
@@ -306,39 +344,40 @@ struct<plan:string>
             +- Scan parquet default.explain_temp2 (3)
 
 
-(1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 2]
-Input: [key#x, val#x]
-     
-(3) Scan parquet default.explain_temp2 
-Output: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(3) Scan parquet default.explain_temp2
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (4) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (5) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
-     
+
 (6) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
-(7) BroadcastExchange 
-Input: [key#x, val#x]
-     
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(7) BroadcastExchange
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+
 (8) BroadcastHashJoin [codegen id : 2]
-Left keys: List(key#x)
-Right keys: List(key#x)
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
 Join condition: None
 
 
@@ -363,24 +402,24 @@ struct<plan:string>
       +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), IsNotNull(val), GreaterThan(val,3)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery scalar-subquery#x, [id=#x])) AND (val#x > 3))
-     
+
 (4) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
 ===== Subqueries =====
 
 Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
@@ -393,33 +432,42 @@ Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
                +- Scan parquet default.explain_temp2 (5)
 
 
-(5) Scan parquet default.explain_temp2 
-Output: [key#x, val#x]
+(5) Scan parquet default.explain_temp2
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key), IsNotNull(val), EqualTo(val,2)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (6) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (7) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery scalar-subquery#x, [id=#x])) AND (val#x = 2))
-     
+
 (8) Project [codegen id : 1]
-Output    : [key#x]
-Input     : [key#x, val#x]
-     
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
+
 (9) HashAggregate [codegen id : 1]
-Input: [key#x]
-     
-(10) Exchange 
-Input: [max#x]
-     
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_max(key#x)]
+Aggregate Attributes [1]: [max#x]
+Results [1]: [max#x]
+
+(10) Exchange
+Input [1]: [max#x]
+Arguments: SinglePartition, true, [id=#x]
+
 (11) HashAggregate [codegen id : 2]
-Input: [max#x]
-     
+Input [1]: [max#x]
+Keys: []
+Functions [1]: [max(key#x)]
+Aggregate Attributes [1]: [max(key#x)#x]
+Results [1]: [max(key#x)#x AS max(key)#x]
+
 Subquery:2 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
 * HashAggregate (18)
 +- Exchange (17)
@@ -430,32 +478,41 @@ Subquery:2 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery
                +- Scan parquet default.explain_temp3 (12)
 
 
-(12) Scan parquet default.explain_temp3 
-Output: [key#x, val#x]
+(12) Scan parquet default.explain_temp3
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp3]
 PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (13) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (14) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(val#x) AND (val#x > 0))
-     
+
 (15) Project [codegen id : 1]
-Output    : [key#x]
-Input     : [key#x, val#x]
-     
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
+
 (16) HashAggregate [codegen id : 1]
-Input: [key#x]
-     
-(17) Exchange 
-Input: [max#x]
-     
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_max(key#x)]
+Aggregate Attributes [1]: [max#x]
+Results [1]: [max#x]
+
+(17) Exchange
+Input [1]: [max#x]
+Arguments: SinglePartition, true, [id=#x]
+
 (18) HashAggregate [codegen id : 2]
-Input: [max#x]
+Input [1]: [max#x]
+Keys: []
+Functions [1]: [max(key#x)]
+Aggregate Attributes [1]: [max(key#x)#x]
+Results [1]: [max(key#x)#x AS max(key)#x]
 
 
 -- !query
@@ -466,7 +523,7 @@ EXPLAIN FORMATTED
                 FROM   explain_temp2 
                 WHERE  val > 0) 
          OR
-         key = (SELECT max(key) 
+         key = (SELECT avg(key)
                 FROM   explain_temp3
                 WHERE  val > 0)
 -- !query schema
@@ -478,19 +535,19 @@ struct<plan:string>
    +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
-Condition : ((key#x = Subquery scalar-subquery#x, [id=#x]) OR (key#x = Subquery scalar-subquery#x, [id=#x]))
-     
+Input [2]: [key#x, val#x]
+Condition : ((key#x = Subquery scalar-subquery#x, [id=#x]) OR (cast(key#x as double) = Subquery scalar-subquery#x, [id=#x]))
+
 ===== Subqueries =====
 
 Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
@@ -503,33 +560,42 @@ Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
                +- Scan parquet default.explain_temp2 (4)
 
 
-(4) Scan parquet default.explain_temp2 
-Output: [key#x, val#x]
+(4) Scan parquet default.explain_temp2
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (5) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (6) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(val#x) AND (val#x > 0))
-     
+
 (7) Project [codegen id : 1]
-Output    : [key#x]
-Input     : [key#x, val#x]
-     
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
+
 (8) HashAggregate [codegen id : 1]
-Input: [key#x]
-     
-(9) Exchange 
-Input: [max#x]
-     
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_max(key#x)]
+Aggregate Attributes [1]: [max#x]
+Results [1]: [max#x]
+
+(9) Exchange
+Input [1]: [max#x]
+Arguments: SinglePartition, true, [id=#x]
+
 (10) HashAggregate [codegen id : 2]
-Input: [max#x]
-     
+Input [1]: [max#x]
+Keys: []
+Functions [1]: [max(key#x)]
+Aggregate Attributes [1]: [max(key#x)#x]
+Results [1]: [max(key#x)#x AS max(key)#x]
+
 Subquery:2 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
 * HashAggregate (17)
 +- Exchange (16)
@@ -540,32 +606,41 @@ Subquery:2 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
                +- Scan parquet default.explain_temp3 (11)
 
 
-(11) Scan parquet default.explain_temp3 
-Output: [key#x, val#x]
+(11) Scan parquet default.explain_temp3
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp3]
 PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (12) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (13) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(val#x) AND (val#x > 0))
-     
+
 (14) Project [codegen id : 1]
-Output    : [key#x]
-Input     : [key#x, val#x]
-     
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
+
 (15) HashAggregate [codegen id : 1]
-Input: [key#x]
-     
-(16) Exchange 
-Input: [max#x]
-     
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_avg(cast(key#x as bigint))]
+Aggregate Attributes [2]: [sum#x, count#xL]
+Results [2]: [sum#x, count#xL]
+
+(16) Exchange
+Input [2]: [sum#x, count#xL]
+Arguments: SinglePartition, true, [id=#x]
+
 (17) HashAggregate [codegen id : 2]
-Input: [max#x]
+Input [2]: [sum#x, count#xL]
+Keys: []
+Functions [1]: [avg(cast(key#x as bigint))]
+Aggregate Attributes [1]: [avg(cast(key#x as bigint))#x]
+Results [1]: [avg(cast(key#x as bigint))#x AS avg(key)#x]
 
 
 -- !query
@@ -581,19 +656,19 @@ struct<plan:string>
    +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output: []
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input: []
-     
+
 (3) Project [codegen id : 1]
-Output    : [(Subquery scalar-subquery#x, [id=#x] + ReusedSubquery Subquery scalar-subquery#x, [id=#x]) AS (scalarsubquery() + scalarsubquery())#x]
-Input     : []
-     
+Output [1]: [(Subquery scalar-subquery#x, [id=#x] + ReusedSubquery Subquery scalar-subquery#x, [id=#x]) AS (scalarsubquery() + scalarsubquery())#x]
+Input: []
+
 ===== Subqueries =====
 
 Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
@@ -604,24 +679,33 @@ Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
          +- Scan parquet default.explain_temp1 (4)
 
 
-(4) Scan parquet default.explain_temp1 
-Output: [key#x]
+(4) Scan parquet default.explain_temp1
+Output [1]: [key#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int>
-     
+
 (5) ColumnarToRow [codegen id : 1]
-Input: [key#x]
-     
+Input [1]: [key#x]
+
 (6) HashAggregate [codegen id : 1]
-Input: [key#x]
-     
-(7) Exchange 
-Input: [sum#x, count#xL]
-     
+Input [1]: [key#x]
+Keys: []
+Functions [1]: [partial_avg(cast(key#x as bigint))]
+Aggregate Attributes [2]: [sum#x, count#xL]
+Results [2]: [sum#x, count#xL]
+
+(7) Exchange
+Input [2]: [sum#x, count#xL]
+Arguments: SinglePartition, true, [id=#x]
+
 (8) HashAggregate [codegen id : 2]
-Input: [sum#x, count#xL]
-     
+Input [2]: [sum#x, count#xL]
+Keys: []
+Functions [1]: [avg(cast(key#x as bigint))]
+Aggregate Attributes [1]: [avg(cast(key#x as bigint))#x]
+Results [1]: [avg(cast(key#x as bigint))#x AS avg(key)#x]
+
 Subquery:2 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#x, [id=#x]
 
 
@@ -649,48 +733,49 @@ struct<plan:string>
             +- Scan parquet default.explain_temp1 (5)
 
 
-(1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 2]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (3) Filter [codegen id : 2]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
-     
+
 (4) Project [codegen id : 2]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
-(5) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(5) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (6) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (7) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
-     
+
 (8) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
-(9) BroadcastExchange 
-Input: [key#x, val#x]
-     
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
+(9) BroadcastExchange
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+
 (10) BroadcastHashJoin [codegen id : 2]
-Left keys: List(key#x)
-Right keys: List(key#x)
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
 Join condition: None
 
 
@@ -720,45 +805,59 @@ struct<plan:string>
       +- ReusedExchange (8)
 
 
-(1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
-     
+
 (4) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
-     
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+
 (5) HashAggregate [codegen id : 1]
-Input: [key#x, val#x]
-     
-(6) Exchange 
-Input: [key#x, max#x]
-     
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+
+(6) Exchange
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
 (7) HashAggregate [codegen id : 4]
-Input: [key#x, max#x]
-     
-(8) ReusedExchange  [Reuses operator id: 6]
-Output : ArrayBuffer(key#x, max#x)
-     
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
+
+(8) ReusedExchange [Reuses operator id: 6]
+Output [2]: [key#x, max#x]
+
 (9) HashAggregate [codegen id : 3]
-Input: [key#x, max#x]
-     
-(10) BroadcastExchange 
-Input: [key#x, max(val)#x]
-     
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
+
+(10) BroadcastExchange
+Input [2]: [key#x, max(val)#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+
 (11) BroadcastHashJoin [codegen id : 4]
-Left keys: List(key#x)
-Right keys: List(key#x)
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
 Join condition: None
 
 
@@ -776,14 +875,160 @@ Execute CreateViewCommand (1)
             +- UnresolvedRelation (3)
 
 
-(1) Execute CreateViewCommand 
+(1) Execute CreateViewCommand
 Output: []
-     
-(2) CreateViewCommand 
-     
-(3) UnresolvedRelation 
-     
+
+(2) CreateViewCommand
+Arguments: `default`.`explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView
+
+(3) UnresolvedRelation
+Arguments: [explain_temp1]
+
 (4) Project
+Arguments: ['key, 'val]
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT
+    COUNT(val) + SUM(key) as TOTAL,
+    COUNT(key) FILTER (WHERE val > 1)
+  FROM explain_temp1
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+* HashAggregate (5)
++- Exchange (4)
+   +- HashAggregate (3)
+      +- * ColumnarToRow (2)
+         +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<key:int,val:int>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [2]: [key#x, val#x]
+
+(3) HashAggregate
+Input [2]: [key#x, val#x]
+Keys: []
+Functions [3]: [partial_count(val#x), partial_sum(cast(key#x as bigint)), partial_count(key#x) FILTER (WHERE (val#x > 1))]
+Aggregate Attributes [3]: [count#xL, sum#xL, count#xL]
+Results [3]: [count#xL, sum#xL, count#xL]
+
+(4) Exchange
+Input [3]: [count#xL, sum#xL, count#xL]
+Arguments: SinglePartition, true, [id=#x]
+
+(5) HashAggregate [codegen id : 2]
+Input [3]: [count#xL, sum#xL, count#xL]
+Keys: []
+Functions [3]: [count(val#x), sum(cast(key#x as bigint)), count(key#x)]
+Aggregate Attributes [3]: [count(val#x)#xL, sum(cast(key#x as bigint))#xL, count(key#x)#xL]
+Results [2]: [(count(val#x)#xL + sum(cast(key#x as bigint))#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, sort_array(collect_set(val))[0]
+  FROM explain_temp4
+  GROUP BY key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+ObjectHashAggregate (5)
++- Exchange (4)
+   +- ObjectHashAggregate (3)
+      +- * ColumnarToRow (2)
+         +- Scan parquet default.explain_temp4 (1)
+
+
+(1) Scan parquet default.explain_temp4
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp4]
+ReadSchema: struct<key:int,val:string>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [2]: [key#x, val#x]
+
+(3) ObjectHashAggregate
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_collect_set(val#x, 0, 0)]
+Aggregate Attributes [1]: [buf#x]
+Results [2]: [key#x, buf#x]
+
+(4) Exchange
+Input [2]: [key#x, buf#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
+(5) ObjectHashAggregate
+Input [2]: [key#x, buf#x]
+Keys [1]: [key#x]
+Functions [1]: [collect_set(val#x, 0, 0)]
+Aggregate Attributes [1]: [collect_set(val#x, 0, 0)#x]
+Results [2]: [key#x, sort_array(collect_set(val#x, 0, 0)#x, true)[0] AS sort_array(collect_set(val), true)[0]#x]
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, MIN(val)
+  FROM explain_temp4
+  GROUP BY key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+SortAggregate (7)
++- * Sort (6)
+   +- Exchange (5)
+      +- SortAggregate (4)
+         +- * Sort (3)
+            +- * ColumnarToRow (2)
+               +- Scan parquet default.explain_temp4 (1)
+
+
+(1) Scan parquet default.explain_temp4
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp4]
+ReadSchema: struct<key:int,val:string>
+
+(2) ColumnarToRow [codegen id : 1]
+Input [2]: [key#x, val#x]
+
+(3) Sort [codegen id : 1]
+Input [2]: [key#x, val#x]
+Arguments: [key#x ASC NULLS FIRST], false, 0
+
+(4) SortAggregate
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_min(val#x)]
+Aggregate Attributes [1]: [min#x]
+Results [2]: [key#x, min#x]
+
+(5) Exchange
+Input [2]: [key#x, min#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+
+(6) Sort [codegen id : 2]
+Input [2]: [key#x, min#x]
+Arguments: [key#x ASC NULLS FIRST], false, 0
+
+(7) SortAggregate
+Input [2]: [key#x, min#x]
+Keys [1]: [key#x]
+Functions [1]: [min(val#x)]
+Aggregate Attributes [1]: [min(val#x)#x]
+Results [2]: [key#x, min(val#x)#x AS min(val)#x]
 
 
 -- !query
@@ -808,3 +1053,11 @@ DROP TABLE explain_temp3
 struct<>
 -- !query output
 
+
+
+-- !query
+DROP TABLE explain_temp4
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
index 583459f9037b8..29cbefdb38541 100644
--- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
@@ -1,9 +1,9 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 64
+-- Number of queries: 96
 
 
 -- !query
-CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c
+CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c, interval 10 year 20 month 30 day 40 hour 50 minute 6.7890 second as i
 -- !query schema
 struct<>
 -- !query output
@@ -11,510 +11,765 @@ struct<>
 
 
 -- !query
-select extract(millennium from c) from t
+select extract(year from c), extract(year from i) from t
 -- !query schema
-struct<date_part('millennium', `c`):int>
+struct<extract('year' FROM t.`c`):int,extract('year' FROM t.`i`):int>
 -- !query output
-3
+2011	11
 
 
 -- !query
-select extract(millennia from c) from t
+select extract(y from c), extract(y from i) from t
 -- !query schema
-struct<date_part('millennia', `c`):int>
+struct<extract('y' FROM t.`c`):int,extract('y' FROM t.`i`):int>
 -- !query output
-3
+2011	11
 
 
 -- !query
-select extract(mil from c) from t
+select extract(years from c), extract(years from i) from t
 -- !query schema
-struct<date_part('mil', `c`):int>
+struct<extract('years' FROM t.`c`):int,extract('years' FROM t.`i`):int>
 -- !query output
-3
+2011	11
 
 
 -- !query
-select extract(mils from c) from t
+select extract(yr from c), extract(yr from i) from t
 -- !query schema
-struct<date_part('mils', `c`):int>
+struct<extract('yr' FROM t.`c`):int,extract('yr' FROM t.`i`):int>
 -- !query output
-3
+2011	11
 
 
 -- !query
-select extract(century from c) from t
+select extract(yrs from c), extract(yrs from i) from t
 -- !query schema
-struct<date_part('century', `c`):int>
+struct<extract('yrs' FROM t.`c`):int,extract('yrs' FROM t.`i`):int>
 -- !query output
-21
+2011	11
 
 
 -- !query
-select extract(centuries from c) from t
+select extract(yearofweek from c) from t
 -- !query schema
-struct<date_part('centuries', `c`):int>
+struct<extract('yearofweek' FROM t.`c`):int>
 -- !query output
-21
+2011
 
 
 -- !query
-select extract(c from c) from t
+select extract(quarter from c) from t
 -- !query schema
-struct<date_part('c', `c`):int>
+struct<extract('quarter' FROM t.`c`):int>
 -- !query output
-21
+2
 
 
 -- !query
-select extract(cent from c) from t
+select extract(qtr from c) from t
 -- !query schema
-struct<date_part('cent', `c`):int>
+struct<extract('qtr' FROM t.`c`):int>
 -- !query output
-21
+2
 
 
 -- !query
-select extract(decade from c) from t
+select extract(month from c), extract(month from i) from t
 -- !query schema
-struct<date_part('decade', `c`):int>
+struct<extract('month' FROM t.`c`):int,extract('month' FROM t.`i`):tinyint>
 -- !query output
-201
+5	8
 
 
 -- !query
-select extract(decades from c) from t
+select extract(mon from c), extract(mon from i) from t
 -- !query schema
-struct<date_part('decades', `c`):int>
+struct<extract('mon' FROM t.`c`):int,extract('mon' FROM t.`i`):tinyint>
 -- !query output
-201
+5	8
 
 
 -- !query
-select extract(dec from c) from t
+select extract(mons from c), extract(mons from i) from t
 -- !query schema
-struct<date_part('dec', `c`):int>
+struct<extract('mons' FROM t.`c`):int,extract('mons' FROM t.`i`):tinyint>
 -- !query output
-201
+5	8
 
 
 -- !query
-select extract(decs from c) from t
+select extract(months from c), extract(months from i) from t
 -- !query schema
-struct<date_part('decs', `c`):int>
+struct<extract('months' FROM t.`c`):int,extract('months' FROM t.`i`):tinyint>
 -- !query output
-201
+5	8
 
 
 -- !query
-select extract(year from c) from t
+select extract(week from c) from t
 -- !query schema
-struct<date_part('year', `c`):int>
+struct<extract('week' FROM t.`c`):int>
 -- !query output
-2011
+18
 
 
 -- !query
-select extract(y from c) from t
+select extract(w from c) from t
 -- !query schema
-struct<date_part('y', `c`):int>
+struct<extract('w' FROM t.`c`):int>
 -- !query output
-2011
+18
 
 
 -- !query
-select extract(years from c) from t
+select extract(weeks from c) from t
 -- !query schema
-struct<date_part('years', `c`):int>
+struct<extract('weeks' FROM t.`c`):int>
 -- !query output
-2011
+18
 
 
 -- !query
-select extract(yr from c) from t
+select extract(day from c), extract(day from i) from t
 -- !query schema
-struct<date_part('yr', `c`):int>
+struct<extract('day' FROM t.`c`):int,extract('day' FROM t.`i`):int>
 -- !query output
-2011
+6	31
 
 
 -- !query
-select extract(yrs from c) from t
+select extract(d from c), extract(d from i) from t
 -- !query schema
-struct<date_part('yrs', `c`):int>
+struct<extract('d' FROM t.`c`):int,extract('d' FROM t.`i`):int>
 -- !query output
-2011
+6	31
+
+
+-- !query
+select extract(days from c), extract(days from i) from t
+-- !query schema
+struct<extract('days' FROM t.`c`):int,extract('days' FROM t.`i`):int>
+-- !query output
+6	31
+
+
+-- !query
+select extract(dayofweek from c) from t
+-- !query schema
+struct<extract('dayofweek' FROM t.`c`):int>
+-- !query output
+6
+
+
+-- !query
+select extract(dow from c) from t
+-- !query schema
+struct<extract('dow' FROM t.`c`):int>
+-- !query output
+6
+
+
+-- !query
+select extract(dayofweek_iso from c) from t
+-- !query schema
+struct<extract('dayofweek_iso' FROM t.`c`):int>
+-- !query output
+5
+
+
+-- !query
+select extract(dow_iso from c) from t
+-- !query schema
+struct<extract('dow_iso' FROM t.`c`):int>
+-- !query output
+5
+
+
+-- !query
+select extract(doy from c) from t
+-- !query schema
+struct<extract('doy' FROM t.`c`):int>
+-- !query output
+126
+
+
+-- !query
+select extract(hour from c), extract(hour from i) from t
+-- !query schema
+struct<extract('hour' FROM t.`c`):int,extract('hour' FROM t.`i`):bigint>
+-- !query output
+7	16
+
+
+-- !query
+select extract(h from c), extract(h from i) from t
+-- !query schema
+struct<extract('h' FROM t.`c`):int,extract('h' FROM t.`i`):bigint>
+-- !query output
+7	16
+
+
+-- !query
+select extract(hours from c), extract(hours from i) from t
+-- !query schema
+struct<extract('hours' FROM t.`c`):int,extract('hours' FROM t.`i`):bigint>
+-- !query output
+7	16
+
+
+-- !query
+select extract(hr from c), extract(hr from i) from t
+-- !query schema
+struct<extract('hr' FROM t.`c`):int,extract('hr' FROM t.`i`):bigint>
+-- !query output
+7	16
+
+
+-- !query
+select extract(hrs from c), extract(hrs from i) from t
+-- !query schema
+struct<extract('hrs' FROM t.`c`):int,extract('hrs' FROM t.`i`):bigint>
+-- !query output
+7	16
+
+
+-- !query
+select extract(minute from c), extract(minute from i) from t
+-- !query schema
+struct<extract('minute' FROM t.`c`):int,extract('minute' FROM t.`i`):tinyint>
+-- !query output
+8	50
+
+
+-- !query
+select extract(m from c), extract(m from i) from t
+-- !query schema
+struct<extract('m' FROM t.`c`):int,extract('m' FROM t.`i`):tinyint>
+-- !query output
+8	50
+
+
+-- !query
+select extract(min from c), extract(min from i) from t
+-- !query schema
+struct<extract('min' FROM t.`c`):int,extract('min' FROM t.`i`):tinyint>
+-- !query output
+8	50
+
+
+-- !query
+select extract(mins from c), extract(mins from i) from t
+-- !query schema
+struct<extract('mins' FROM t.`c`):int,extract('mins' FROM t.`i`):tinyint>
+-- !query output
+8	50
+
+
+-- !query
+select extract(minutes from c), extract(minutes from i) from t
+-- !query schema
+struct<extract('minutes' FROM t.`c`):int,extract('minutes' FROM t.`i`):tinyint>
+-- !query output
+8	50
+
+
+-- !query
+select extract(second from c), extract(second from i) from t
+-- !query schema
+struct<extract('second' FROM t.`c`):decimal(8,6),extract('second' FROM t.`i`):decimal(8,6)>
+-- !query output
+9.123456	6.789000
+
+
+-- !query
+select extract(s from c), extract(s from i) from t
+-- !query schema
+struct<extract('s' FROM t.`c`):decimal(8,6),extract('s' FROM t.`i`):decimal(8,6)>
+-- !query output
+9.123456	6.789000
+
+
+-- !query
+select extract(sec from c), extract(sec from i) from t
+-- !query schema
+struct<extract('sec' FROM t.`c`):decimal(8,6),extract('sec' FROM t.`i`):decimal(8,6)>
+-- !query output
+9.123456	6.789000
+
+
+-- !query
+select extract(seconds from c), extract(seconds from i) from t
+-- !query schema
+struct<extract('seconds' FROM t.`c`):decimal(8,6),extract('seconds' FROM t.`i`):decimal(8,6)>
+-- !query output
+9.123456	6.789000
+
+
+-- !query
+select extract(secs from c), extract(secs from i) from t
+-- !query schema
+struct<extract('secs' FROM t.`c`):decimal(8,6),extract('secs' FROM t.`i`):decimal(8,6)>
+-- !query output
+9.123456	6.789000
+
+
+-- !query
+select extract(not_supported from c) from t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'not_supported' are currently not supported for the string type.;; line 1 pos 7
+
+
+-- !query
+select extract(not_supported from i) from t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'not_supported' are currently not supported for the interval type.;; line 1 pos 7
+
+
+-- !query
+select date_part('year', c), date_part('year', i) from t
+-- !query schema
+struct<date_part('year', t.`c`):int,date_part('year', t.`i`):int>
+-- !query output
+2011	11
+
+
+-- !query
+select date_part('y', c), date_part('y', i) from t
+-- !query schema
+struct<date_part('y', t.`c`):int,date_part('y', t.`i`):int>
+-- !query output
+2011	11
 
 
 -- !query
-select extract(isoyear from c) from t
+select date_part('years', c), date_part('years', i) from t
 -- !query schema
-struct<date_part('isoyear', `c`):int>
+struct<date_part('years', t.`c`):int,date_part('years', t.`i`):int>
+-- !query output
+2011	11
+
+
+-- !query
+select date_part('yr', c), date_part('yr', i) from t
+-- !query schema
+struct<date_part('yr', t.`c`):int,date_part('yr', t.`i`):int>
+-- !query output
+2011	11
+
+
+-- !query
+select date_part('yrs', c), date_part('yrs', i) from t
+-- !query schema
+struct<date_part('yrs', t.`c`):int,date_part('yrs', t.`i`):int>
+-- !query output
+2011	11
+
+
+-- !query
+select date_part('yearofweek', c) from t
+-- !query schema
+struct<date_part('yearofweek', t.`c`):int>
 -- !query output
 2011
 
 
 -- !query
-select extract(quarter from c) from t
+select date_part('quarter', c) from t
 -- !query schema
-struct<date_part('quarter', `c`):int>
+struct<date_part('quarter', t.`c`):int>
 -- !query output
 2
 
 
 -- !query
-select extract(qtr from c) from t
+select date_part('qtr', c) from t
 -- !query schema
-struct<date_part('qtr', `c`):int>
+struct<date_part('qtr', t.`c`):int>
 -- !query output
 2
 
 
 -- !query
-select extract(month from c) from t
+select date_part('month', c), date_part('month', i) from t
 -- !query schema
-struct<date_part('month', `c`):int>
+struct<date_part('month', t.`c`):int,date_part('month', t.`i`):tinyint>
 -- !query output
-5
+5	8
 
 
 -- !query
-select extract(mon from c) from t
+select date_part('mon', c), date_part('mon', i) from t
 -- !query schema
-struct<date_part('mon', `c`):int>
+struct<date_part('mon', t.`c`):int,date_part('mon', t.`i`):tinyint>
 -- !query output
-5
+5	8
 
 
 -- !query
-select extract(mons from c) from t
+select date_part('mons', c), date_part('mons', i) from t
 -- !query schema
-struct<date_part('mons', `c`):int>
+struct<date_part('mons', t.`c`):int,date_part('mons', t.`i`):tinyint>
 -- !query output
-5
+5	8
 
 
 -- !query
-select extract(months from c) from t
+select date_part('months', c), date_part('months', i) from t
 -- !query schema
-struct<date_part('months', `c`):int>
+struct<date_part('months', t.`c`):int,date_part('months', t.`i`):tinyint>
 -- !query output
-5
+5	8
 
 
 -- !query
-select extract(week from c) from t
+select date_part('week', c) from t
 -- !query schema
-struct<date_part('week', `c`):int>
+struct<date_part('week', t.`c`):int>
 -- !query output
 18
 
 
 -- !query
-select extract(w from c) from t
+select date_part('w', c) from t
 -- !query schema
-struct<date_part('w', `c`):int>
+struct<date_part('w', t.`c`):int>
 -- !query output
 18
 
 
 -- !query
-select extract(weeks from c) from t
+select date_part('weeks', c) from t
 -- !query schema
-struct<date_part('weeks', `c`):int>
+struct<date_part('weeks', t.`c`):int>
 -- !query output
 18
 
 
 -- !query
-select extract(day from c) from t
+select date_part('day', c), date_part('day', i) from t
 -- !query schema
-struct<date_part('day', `c`):int>
+struct<date_part('day', t.`c`):int,date_part('day', t.`i`):int>
 -- !query output
-6
+6	31
 
 
 -- !query
-select extract(d from c) from t
+select date_part('d', c), date_part('d', i) from t
 -- !query schema
-struct<date_part('d', `c`):int>
+struct<date_part('d', t.`c`):int,date_part('d', t.`i`):int>
 -- !query output
-6
+6	31
 
 
 -- !query
-select extract(days from c) from t
+select date_part('days', c), date_part('days', i) from t
 -- !query schema
-struct<date_part('days', `c`):int>
+struct<date_part('days', t.`c`):int,date_part('days', t.`i`):int>
+-- !query output
+6	31
+
+
+-- !query
+select date_part('dayofweek', c) from t
+-- !query schema
+struct<date_part('dayofweek', t.`c`):int>
 -- !query output
 6
 
 
 -- !query
-select extract(dayofweek from c) from t
+select date_part('dow', c) from t
 -- !query schema
-struct<date_part('dayofweek', `c`):int>
+struct<date_part('dow', t.`c`):int>
 -- !query output
 6
 
 
 -- !query
-select extract(dow from c) from t
+select date_part('dayofweek_iso', c) from t
 -- !query schema
-struct<date_part('dow', `c`):int>
+struct<date_part('dayofweek_iso', t.`c`):int>
 -- !query output
 5
 
 
 -- !query
-select extract(isodow from c) from t
+select date_part('dow_iso', c) from t
 -- !query schema
-struct<date_part('isodow', `c`):int>
+struct<date_part('dow_iso', t.`c`):int>
 -- !query output
 5
 
 
 -- !query
-select extract(doy from c) from t
+select date_part('doy', c) from t
 -- !query schema
-struct<date_part('doy', `c`):int>
+struct<date_part('doy', t.`c`):int>
 -- !query output
 126
 
 
 -- !query
-select extract(hour from c) from t
+select date_part('hour', c), date_part('hour', i) from t
 -- !query schema
-struct<date_part('hour', `c`):int>
+struct<date_part('hour', t.`c`):int,date_part('hour', t.`i`):bigint>
 -- !query output
-7
+7	16
 
 
 -- !query
-select extract(h from c) from t
+select date_part('h', c), date_part('h', i) from t
 -- !query schema
-struct<date_part('h', `c`):int>
+struct<date_part('h', t.`c`):int,date_part('h', t.`i`):bigint>
 -- !query output
-7
+7	16
 
 
 -- !query
-select extract(hours from c) from t
+select date_part('hours', c), date_part('hours', i) from t
 -- !query schema
-struct<date_part('hours', `c`):int>
+struct<date_part('hours', t.`c`):int,date_part('hours', t.`i`):bigint>
 -- !query output
-7
+7	16
 
 
 -- !query
-select extract(hr from c) from t
+select date_part('hr', c), date_part('hr', i) from t
 -- !query schema
-struct<date_part('hr', `c`):int>
+struct<date_part('hr', t.`c`):int,date_part('hr', t.`i`):bigint>
 -- !query output
-7
+7	16
 
 
 -- !query
-select extract(hrs from c) from t
+select date_part('hrs', c), date_part('hrs', i) from t
 -- !query schema
-struct<date_part('hrs', `c`):int>
+struct<date_part('hrs', t.`c`):int,date_part('hrs', t.`i`):bigint>
 -- !query output
-7
+7	16
 
 
 -- !query
-select extract(minute from c) from t
+select date_part('minute', c), date_part('minute', i) from t
 -- !query schema
-struct<date_part('minute', `c`):int>
+struct<date_part('minute', t.`c`):int,date_part('minute', t.`i`):tinyint>
 -- !query output
-8
+8	50
 
 
 -- !query
-select extract(m from c) from t
+select date_part('m', c), date_part('m', i) from t
 -- !query schema
-struct<date_part('m', `c`):int>
+struct<date_part('m', t.`c`):int,date_part('m', t.`i`):tinyint>
 -- !query output
-8
+8	50
 
 
 -- !query
-select extract(min from c) from t
+select date_part('min', c), date_part('min', i) from t
 -- !query schema
-struct<date_part('min', `c`):int>
+struct<date_part('min', t.`c`):int,date_part('min', t.`i`):tinyint>
 -- !query output
-8
+8	50
 
 
 -- !query
-select extract(mins from c) from t
+select date_part('mins', c), date_part('mins', i) from t
 -- !query schema
-struct<date_part('mins', `c`):int>
+struct<date_part('mins', t.`c`):int,date_part('mins', t.`i`):tinyint>
 -- !query output
-8
+8	50
 
 
 -- !query
-select extract(minutes from c) from t
+select date_part('minutes', c), date_part('minutes', i) from t
 -- !query schema
-struct<date_part('minutes', `c`):int>
+struct<date_part('minutes', t.`c`):int,date_part('minutes', t.`i`):tinyint>
 -- !query output
-8
+8	50
 
 
 -- !query
-select extract(second from c) from t
+select date_part('second', c), date_part('second', i) from t
 -- !query schema
-struct<date_part('second', `c`):decimal(8,6)>
+struct<date_part('second', t.`c`):decimal(8,6),date_part('second', t.`i`):decimal(8,6)>
 -- !query output
-9.123456
+9.123456	6.789000
 
 
 -- !query
-select extract(s from c) from t
+select date_part('s', c), date_part('s', i) from t
 -- !query schema
-struct<date_part('s', `c`):decimal(8,6)>
+struct<date_part('s', t.`c`):decimal(8,6),date_part('s', t.`i`):decimal(8,6)>
 -- !query output
-9.123456
+9.123456	6.789000
 
 
 -- !query
-select extract(sec from c) from t
+select date_part('sec', c), date_part('sec', i) from t
 -- !query schema
-struct<date_part('sec', `c`):decimal(8,6)>
+struct<date_part('sec', t.`c`):decimal(8,6),date_part('sec', t.`i`):decimal(8,6)>
 -- !query output
-9.123456
+9.123456	6.789000
 
 
 -- !query
-select extract(seconds from c) from t
+select date_part('seconds', c), date_part('seconds', i) from t
 -- !query schema
-struct<date_part('seconds', `c`):decimal(8,6)>
+struct<date_part('seconds', t.`c`):decimal(8,6),date_part('seconds', t.`i`):decimal(8,6)>
 -- !query output
-9.123456
+9.123456	6.789000
 
 
 -- !query
-select extract(secs from c) from t
+select date_part('secs', c), date_part('secs', i) from t
 -- !query schema
-struct<date_part('secs', `c`):decimal(8,6)>
+struct<date_part('secs', t.`c`):decimal(8,6),date_part('secs', t.`i`):decimal(8,6)>
 -- !query output
-9.123456
+9.123456	6.789000
+
+
+-- !query
+select date_part('not_supported', c) from t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'not_supported' are currently not supported for the string type.;; line 1 pos 7
 
 
 -- !query
-select extract(milliseconds from c) from t
+select date_part(c, c) from t
 -- !query schema
-struct<date_part('milliseconds', `c`):decimal(8,3)>
+struct<>
 -- !query output
-9123.456
+org.apache.spark.sql.AnalysisException
+The field parameter needs to be a foldable string value.;; line 1 pos 7
 
 
 -- !query
-select extract(msec from c) from t
+select date_part(null, c) from t
 -- !query schema
-struct<date_part('msec', `c`):decimal(8,3)>
+struct<date_part(NULL, t.`c`):double>
 -- !query output
-9123.456
+NULL
 
 
 -- !query
-select extract(msecs from c) from t
+select date_part(i, i) from t
 -- !query schema
-struct<date_part('msecs', `c`):decimal(8,3)>
+struct<>
 -- !query output
-9123.456
+org.apache.spark.sql.AnalysisException
+The field parameter needs to be a foldable string value.;; line 1 pos 7
 
 
 -- !query
-select extract(millisecon from c) from t
+select date_part(null, i) from t
 -- !query schema
-struct<date_part('millisecon', `c`):decimal(8,3)>
+struct<date_part(NULL, t.`i`):double>
 -- !query output
-9123.456
+NULL
 
 
 -- !query
-select extract(mseconds from c) from t
+select extract('year', c) from t
 -- !query schema
-struct<date_part('mseconds', `c`):decimal(8,3)>
+struct<extract('year' FROM t.`c`):int>
 -- !query output
-9123.456
+2011
 
 
 -- !query
-select extract(ms from c) from t
+select extract('quarter', c) from t
 -- !query schema
-struct<date_part('ms', `c`):decimal(8,3)>
+struct<extract('quarter' FROM t.`c`):int>
 -- !query output
-9123.456
+2
 
 
 -- !query
-select extract(microseconds from c) from t
+select extract('month', c) from t
 -- !query schema
-struct<date_part('microseconds', `c`):int>
+struct<extract('month' FROM t.`c`):int>
 -- !query output
-9123456
+5
 
 
 -- !query
-select extract(usec from c) from t
+select extract('week', c) from t
 -- !query schema
-struct<date_part('usec', `c`):int>
+struct<extract('week' FROM t.`c`):int>
 -- !query output
-9123456
+18
 
 
 -- !query
-select extract(usecs from c) from t
+select extract('day', c) from t
 -- !query schema
-struct<date_part('usecs', `c`):int>
+struct<extract('day' FROM t.`c`):int>
 -- !query output
-9123456
+6
 
 
 -- !query
-select extract(useconds from c) from t
+select extract('days', c) from t
 -- !query schema
-struct<date_part('useconds', `c`):int>
+struct<extract('days' FROM t.`c`):int>
 -- !query output
-9123456
+6
 
 
 -- !query
-select extract(microsecon from c) from t
+select extract('dayofweek', c) from t
 -- !query schema
-struct<date_part('microsecon', `c`):int>
+struct<extract('dayofweek' FROM t.`c`):int>
 -- !query output
-9123456
+6
 
 
 -- !query
-select extract(us from c) from t
+select extract('dow', c) from t
 -- !query schema
-struct<date_part('us', `c`):int>
+struct<extract('dow' FROM t.`c`):int>
 -- !query output
-9123456
+6
 
 
 -- !query
-select extract(epoch from c) from t
+select extract('doy', c) from t
 -- !query schema
-struct<date_part('epoch', `c`):decimal(20,6)>
+struct<extract('doy' FROM t.`c`):int>
 -- !query output
-1304665689.123456
+126
 
 
 -- !query
-select extract(not_supported from c) from t
+select extract('hour', c) from t
 -- !query schema
-struct<>
+struct<extract('hour' FROM t.`c`):int>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
+7
 
-Literals of type 'not_supported' are currently not supported.(line 1, pos 7)
 
-== SQL ==
-select extract(not_supported from c) from t
--------^^^
+-- !query
+select extract('minute', c) from t
+-- !query schema
+struct<extract('minute' FROM t.`c`):int>
+-- !query output
+8
+
+
+-- !query
+select extract('second', c) from t
+-- !query schema
+struct<extract('second' FROM t.`c`):decimal(8,6)>
+-- !query output
+9.123456
diff --git a/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out b/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
index a032678e90fe8..9ff7f711ea296 100644
--- a/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
@@ -272,7 +272,7 @@ struct<foo:string,approx_count_distinct(a) FILTER (WHERE (b >= 0)):bigint>
 -- !query
 SELECT 'foo', MAX(STRUCT(a)) FILTER (WHERE b >= 1) FROM testData WHERE a = 0 GROUP BY 1
 -- !query schema
-struct<foo:string,max(named_struct(a, a)) FILTER (WHERE (b >= 1)):struct<a:int>>
+struct<foo:string,max(struct(a)) FILTER (WHERE (b >= 1)):struct<a:int>>
 -- !query output
 
 
@@ -369,13 +369,13 @@ org.apache.spark.sql.AnalysisException
 IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few commands: Aggregate [dept_id#x], [dept_id#x, avg(salary#x) AS avg(salary)#x, avg(salary#x) FILTER (WHERE exists#x [dept_id#x]) AS avg(salary) FILTER (WHERE exists(dept_id))#x]
 :  +- Project [state#x]
 :     +- Filter (dept_id#x = outer(dept_id#x))
-:        +- SubqueryAlias `dept`
+:        +- SubqueryAlias dept
 :           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias `DEPT`
+:              +- SubqueryAlias DEPT
 :                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
-+- SubqueryAlias `emp`
++- SubqueryAlias emp
    +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias `EMP`
+      +- SubqueryAlias EMP
          +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 ;
 
@@ -395,13 +395,13 @@ org.apache.spark.sql.AnalysisException
 IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few commands: Aggregate [dept_id#x], [dept_id#x, sum(salary#x) AS sum(salary)#x, sum(salary#x) FILTER (WHERE NOT exists#x [dept_id#x]) AS sum(salary) FILTER (WHERE (NOT exists(dept_id)))#x]
 :  +- Project [state#x]
 :     +- Filter (dept_id#x = outer(dept_id#x))
-:        +- SubqueryAlias `dept`
+:        +- SubqueryAlias dept
 :           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias `DEPT`
+:              +- SubqueryAlias DEPT
 :                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
-+- SubqueryAlias `emp`
++- SubqueryAlias emp
    +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias `EMP`
+      +- SubqueryAlias EMP
          +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 ;
 
@@ -420,13 +420,13 @@ org.apache.spark.sql.AnalysisException
 IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few commands: Aggregate [dept_id#x], [dept_id#x, avg(salary#x) AS avg(salary)#x, avg(salary#x) FILTER (WHERE dept_id#x IN (list#x [])) AS avg(salary) FILTER (WHERE (dept_id IN (listquery())))#x]
 :  +- Distinct
 :     +- Project [dept_id#x]
-:        +- SubqueryAlias `dept`
+:        +- SubqueryAlias dept
 :           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias `DEPT`
+:              +- SubqueryAlias DEPT
 :                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
-+- SubqueryAlias `emp`
++- SubqueryAlias emp
    +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias `EMP`
+      +- SubqueryAlias EMP
          +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 ;
 
@@ -445,13 +445,13 @@ org.apache.spark.sql.AnalysisException
 IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few commands: Aggregate [dept_id#x], [dept_id#x, sum(salary#x) AS sum(salary)#x, sum(salary#x) FILTER (WHERE NOT dept_id#x IN (list#x [])) AS sum(salary) FILTER (WHERE (NOT (dept_id IN (listquery()))))#x]
 :  +- Distinct
 :     +- Project [dept_id#x]
-:        +- SubqueryAlias `dept`
+:        +- SubqueryAlias dept
 :           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias `DEPT`
+:              +- SubqueryAlias DEPT
 :                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
-+- SubqueryAlias `emp`
++- SubqueryAlias emp
    +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias `EMP`
+      +- SubqueryAlias EMP
          +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 ;
 
diff --git a/sql/core/src/test/resources/sql-tests/results/group-by.sql.out b/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
index 7bfdd0ad53a95..e5b705895340e 100644
--- a/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 56
+-- Number of queries: 61
 
 
 -- !query
@@ -87,7 +87,7 @@ struct<foo:string,approx_count_distinct(a):bigint>
 -- !query
 SELECT 'foo', MAX(STRUCT(a)) FROM testData WHERE a = 0 GROUP BY 1
 -- !query schema
-struct<foo:string,max(named_struct(a, a)):struct<a:int>>
+struct<foo:string,max(struct(a)):struct<a:int>>
 -- !query output
 
 
@@ -277,6 +277,67 @@ org.apache.spark.sql.AnalysisException
 grouping expressions sequence is empty, and '`id`' is not an aggregate function. Wrap '()' in windowing function(s) or wrap '`id`' in first() (or first_value) if you don't care which value you get.;
 
 
+-- !query
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=true
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.parser.havingWithoutGroupByAsWhere	true
+
+
+-- !query
+SELECT 1 FROM range(10) HAVING true
+-- !query schema
+struct<1:int>
+-- !query output
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+
+
+-- !query
+SELECT 1 FROM range(10) HAVING MAX(id) > 0
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+
+Aggregate/Window/Generate expressions are not valid in where clause of the query.
+Expression in where clause: [(max(`id`) > CAST(0 AS BIGINT))]
+Invalid expressions: [max(`id`)];
+
+
+-- !query
+SELECT id FROM range(10) HAVING id > 0
+-- !query schema
+struct<id:bigint>
+-- !query output
+1
+2
+3
+4
+5
+6
+7
+8
+9
+
+
+-- !query
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=false
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.parser.havingWithoutGroupByAsWhere	false
+
+
 -- !query
 CREATE OR REPLACE TEMPORARY VIEW test_agg AS SELECT * FROM VALUES
   (1, true), (1, false),
diff --git a/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out b/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
index 8eeabb34b4fab..0a5fe7a727241 100644
--- a/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
@@ -138,7 +138,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-extraneous input 'ROLLUP' expecting <EOF>(line 1, pos 53)
+extraneous input 'ROLLUP' expecting {<EOF>, ';'}(line 1, pos 53)
 
 == SQL ==
 SELECT a, b, c, count(d) FROM grouping GROUP BY WITH ROLLUP
@@ -152,7 +152,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-extraneous input 'CUBE' expecting <EOF>(line 1, pos 53)
+extraneous input 'CUBE' expecting {<EOF>, ';'}(line 1, pos 53)
 
 == SQL ==
 SELECT a, b, c, count(d) FROM grouping GROUP BY WITH CUBE
diff --git a/sql/core/src/test/resources/sql-tests/results/having.sql.out b/sql/core/src/test/resources/sql-tests/results/having.sql.out
index 5bd185d7b815d..6508143e6f9fe 100644
--- a/sql/core/src/test/resources/sql-tests/results/having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/having.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 5
+-- Number of queries: 9
 
 
 -- !query
@@ -47,3 +47,69 @@ struct<(a + CAST(b AS BIGINT)):bigint>
 -- !query output
 3
 7
+
+
+-- !query
+SELECT SUM(a) AS b, CAST('2020-01-01' AS DATE) AS fake FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY b HAVING b > 10
+-- !query schema
+struct<b:bigint,fake:date>
+-- !query output
+2	2020-01-01
+
+
+-- !query
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY GROUPING SETS ((b), (a, b)) HAVING b > 10
+-- !query schema
+struct<b:bigint>
+-- !query output
+2
+2
+
+
+-- !query
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY CUBE(a, b) HAVING b > 10
+-- !query schema
+struct<b:bigint>
+-- !query output
+2
+2
+
+
+-- !query
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY ROLLUP(a, b) HAVING b > 10
+-- !query schema
+struct<b:bigint>
+-- !query output
+2
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY GROUPING SETS(t.c1) HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY CUBE(t.c1) HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY ROLLUP(t.c1) HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY t.c1 HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
diff --git a/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out
index d35d0d5d944bb..7b31b5690998c 100644
--- a/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 29
+-- Number of queries: 30
 
 
 -- !query
@@ -270,3 +270,11 @@ select transform(ys, (all, i) -> all + i) as v from values (array(32, 97)) as t(
 struct<v:array<int>>
 -- !query output
 [32,98]
+
+
+-- !query
+select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)))
+-- !query schema
+struct<aggregate(split(abcdefgh, , -1), array(array()), lambdafunction(array(array(namedlambdavariable())), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):array<array<string>>>
+-- !query output
+[[""]]
diff --git a/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out b/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
index 4762082dc3be2..b99f63393cc4d 100644
--- a/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 22
+-- Number of queries: 26
 
 
 -- !query
@@ -291,6 +291,38 @@ struct<key:string,value:string>
 spark.sql.legacy.setopsPrecedence.enabled	false
 
 
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW tab3 AS VALUES (decimal(1)), (decimal(2)) tbl3(v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT t.v FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t
+-- !query schema
+struct<v:decimal(11,0)>
+-- !query output
+2
+
+
+-- !query
+SELECT SUM(t.v) FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t
+-- !query schema
+struct<sum(v):decimal(21,0)>
+-- !query output
+2
+
+
 -- !query
 DROP VIEW IF EXISTS tab1
 -- !query schema
@@ -305,3 +337,11 @@ DROP VIEW IF EXISTS tab2
 struct<>
 -- !query output
 
+
+
+-- !query
+DROP VIEW IF EXISTS tab3
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 8f523a35f3c19..3068e8ede70f9 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 101
+-- Number of queries: 107
 
 
 -- !query
@@ -26,13 +26,44 @@ struct<divide_interval(subtracttimestamps(TIMESTAMP '2019-10-15 00:00:00', TIMES
 16 hours
 
 
+-- !query
+select interval 2147483647 month * 2
+-- !query schema
+struct<multiply_interval(INTERVAL '178956970 years 7 months', CAST(2 AS DOUBLE)):interval>
+-- !query output
+178956970 years 7 months
+
+
+-- !query
+select interval 2147483647 month / 0.5
+-- !query schema
+struct<divide_interval(INTERVAL '178956970 years 7 months', CAST(0.5 AS DOUBLE)):interval>
+-- !query output
+178956970 years 7 months
+
+
+-- !query
+select interval 2147483647 day * 2
+-- !query schema
+struct<multiply_interval(INTERVAL '2147483647 days', CAST(2 AS DOUBLE)):interval>
+-- !query output
+2147483647 days 2562047788 hours 54.775807 seconds
+
+
+-- !query
+select interval 2147483647 day / 0.5
+-- !query schema
+struct<divide_interval(INTERVAL '2147483647 days', CAST(0.5 AS DOUBLE)):interval>
+-- !query output
+2147483647 days 2562047788 hours 54.775807 seconds
+
+
 -- !query
 select interval '2 seconds' / 0
 -- !query schema
-struct<>
+struct<divide_interval(INTERVAL '2 seconds', CAST(0 AS DOUBLE)):interval>
 -- !query output
-java.lang.ArithmeticException
-divide by zero
+NULL
 
 
 -- !query
@@ -142,11 +173,27 @@ struct<make_interval(1, 2, 3, 4, 5, 6, 0.000000):interval>
 -- !query
 select make_interval(1, 2, 3, 4, 5, 6, 7.008009)
 -- !query schema
-struct<make_interval(1, 2, 3, 4, 5, 6, CAST(7.008009 AS DECIMAL(8,6))):interval>
+struct<make_interval(1, 2, 3, 4, 5, 6, CAST(7.008009 AS DECIMAL(18,6))):interval>
 -- !query output
 1 years 2 months 25 days 5 hours 6 minutes 7.008009 seconds
 
 
+-- !query
+select make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456)
+-- !query schema
+struct<make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456):interval>
+-- !query output
+1 years 2 months 25 days 34293552 hours 30 minutes 12.123456 seconds
+
+
+-- !query
+select make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789)
+-- !query schema
+struct<make_interval(0, 0, 0, 0, 0, 0, CAST(1234567890123456789 AS DECIMAL(18,6))):interval>
+-- !query output
+NULL
+
+
 -- !query
 select cast('1 second' as interval)
 -- !query schema
@@ -307,6 +354,14 @@ struct<day:interval>
 30 days
 
 
+-- !query
+select interval 30 days days
+-- !query schema
+struct<days:interval>
+-- !query output
+30 days
+
+
 -- !query
 select interval '20 15:40:32.99899999' day to hour
 -- !query schema
@@ -314,7 +369,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 15:40:32.99899999' day to hour
@@ -328,7 +383,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 15:40:32.99899999' day to minute
@@ -342,7 +397,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40:32.99899999' hour to minute
@@ -356,7 +411,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40.99899999' hour to second
@@ -370,7 +425,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40' hour to second
@@ -384,7 +439,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 40:32.99899999' minute to second
@@ -429,6 +484,14 @@ struct<INTERVAL '3 years 1 hours':interval>
 3 years 1 hours
 
 
+-- !query
+SELECT interval '1 year 3 months 2 weeks 2 days 1 hour 3 minutes 2 seconds 100 millisecond 200 microseconds'
+-- !query schema
+struct<INTERVAL '1 years 3 months 16 days 1 hours 3 minutes 2.1002 seconds':interval>
+-- !query output
+1 years 3 months 16 days 1 hours 3 minutes 2.1002 seconds
+
+
 -- !query
 select interval
 -- !query schema
@@ -608,7 +671,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-extraneous input 'day' expecting <EOF>(line 1, pos 27)
+extraneous input 'day' expecting {<EOF>, ';'}(line 1, pos 27)
 
 == SQL ==
 select interval 30 day day day
@@ -616,177 +679,35 @@ select interval 30 day day day
 
 
 -- !query
-select sum(cast(null as interval))
+select interval (-30) days
 -- !query schema
-struct<sum(CAST(NULL AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('1 seconds') t(v) where 1=0
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-3 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-1 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
--3 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
--7 days 2 seconds
-
-
--- !query
-select
-    i,
-    sum(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i
--- !query schema
-struct<i:int,sum(CAST(v AS INTERVAL)):interval>
--- !query output
-1	-2 days
-2	2 seconds
-3	NULL
-
-
--- !query
-select
-    sum(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null
--- !query schema
-struct<sv:interval>
--- !query output
--2 days 2 seconds
-
-
--- !query
-SELECT
-    i,
-    sum(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES(1, '1 seconds'), (1, '2 seconds'), (2, NULL), (2, NULL) t(i,v)
--- !query schema
-struct<i:int,sum(CAST(v AS INTERVAL)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):interval>
--- !query output
-1	2 seconds
-1	3 seconds
-2	NULL
-2	NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v) where 1=0
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-1.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-0.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
--1.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
+struct<>
 -- !query output
--3 days -11 hours -59 minutes -59 seconds
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
 
 
 -- !query
-select
-    i,
-    avg(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i
+select interval (a + 1) days
 -- !query schema
-struct<i:int,avg(CAST(v AS INTERVAL)):interval>
+struct<>
 -- !query output
-1	-1 days
-2	2 seconds
-3	NULL
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
 
 
 -- !query
-select
-    avg(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null
+select interval 30 days days days
 -- !query schema
-struct<sv:interval>
+struct<>
 -- !query output
--15 hours -59 minutes -59.333333 seconds
+org.apache.spark.sql.catalyst.parser.ParseException
 
+extraneous input 'days' expecting {<EOF>, ';'}(line 1, pos 29)
 
--- !query
-SELECT
-    i,
-    avg(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES (1,'1 seconds'), (1,'2 seconds'), (2,NULL), (2,NULL) t(i,v)
--- !query schema
-struct<i:int,avg(CAST(v AS INTERVAL)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):interval>
--- !query output
-1	1.5 seconds
-1	2 seconds
-2	NULL
-2	NULL
+== SQL ==
+select interval 30 days days days
+-----------------------------^^^
 
 
 -- !query
@@ -811,7 +732,7 @@ select
   interval '2-2' year to month + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '2 years 2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '-2 years -2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '2 years 2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '-2 years -2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + (- INTERVAL '2 years 2 months') AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '2 years 2 months' AS DATE):date>
+struct<dateval:date,dateval + (- INTERVAL '2 years 2 months'):date,dateval + (- INTERVAL '-2 years -2 months'):date,dateval + INTERVAL '2 years 2 months':date,dateval + INTERVAL '-2 years -2 months':date,dateval + (- INTERVAL '2 years 2 months'):date,dateval + INTERVAL '2 years 2 months':date>
 -- !query output
 2012-01-01	2009-11-01	2014-03-01	2014-03-01	2009-11-01	2009-11-01	2014-03-01
 
@@ -854,7 +775,7 @@ select
   interval '99 11:22:33.123456789' day to second + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds') AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date>
+struct<dateval:date,dateval + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds'):date,dateval + (- INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds'):date,dateval + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds':date,dateval + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds':date,dateval + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds'):date,dateval + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds':date>
 -- !query output
 2012-01-01	2011-09-23	2012-04-09	2012-04-09	2011-09-23	2011-09-23	2012-04-09
 
@@ -926,6 +847,65 @@ struct<INTERVAL '1 days':interval>
 1 days
 
 
+-- !query
+select interval '2-2\t' year to month
+-- !query schema
+struct<INTERVAL '2 years 2 months':interval>
+-- !query output
+2 years 2 months
+
+
+-- !query
+select interval '-\t2-2\t' year to month
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Interval string does not match year-month format of 'y-m': -	2-2	(line 1, pos 16)
+
+== SQL ==
+select interval '-\t2-2\t' year to month
+----------------^^^
+
+
+-- !query
+select interval '\n0 12:34:46.789\t' day to second
+-- !query schema
+struct<INTERVAL '12 hours 34 minutes 46.789 seconds':interval>
+-- !query output
+12 hours 34 minutes 46.789 seconds
+
+
+-- !query
+select interval '\n-\t10\t 12:34:46.789\t' day to second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 
+-	10	 12:34:46.789	, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
+
+== SQL ==
+select interval '\n-\t10\t 12:34:46.789\t' day to second
+----------------^^^
+
+
+-- !query
+select interval '中文 interval 1 day'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 中文 interval 1 day(line 1, pos 7)
+
+== SQL ==
+select interval '中文 interval 1 day'
+-------^^^
+
+
 -- !query
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
@@ -953,48 +933,172 @@ struct<(b + INTERVAL '1 months'):interval>
 -- !query
 select a * 1.1 from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
-struct<>
+struct<multiply_interval(a, CAST(1.1 AS DOUBLE)):interval>
 -- !query output
-java.lang.ArithmeticException
-integer overflow
+-178956970 years -8 months
 
 
 -- !query
 select a / 0.5 from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
+struct<divide_interval(a, CAST(0.5 AS DOUBLE)):interval>
+-- !query output
+-178956970 years -8 months
+
+
+-- !query
+SELECT
+  from_csv('1, 1 day', 'a INT, b interval'),
+  to_csv(from_csv('1, 1 day', 'a INT, b interval')),
+  to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)),
+  from_csv(to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)), 'a interval, b interval')
+-- !query schema
+struct<from_csv(1, 1 day):struct<a:int,b:interval>,to_csv(from_csv(1, 1 day)):string,to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes')):string,from_csv(to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes'))):struct<a:interval,b:interval>>
+-- !query output
+{"a":1,"b":1 days}	1,1 days	2 years 8 months,1 hours 10 minutes	{"a":2 years 8 months,"b":1 hours 10 minutes}
+
+
+-- !query
+SELECT
+  from_json('{"a":"1 days"}', 'a interval'),
+  to_json(from_json('{"a":"1 days"}', 'a interval')),
+  to_json(map('a', interval 25 month 100 day 130 minute)),
+  from_json(to_json(map('a', interval 25 month 100 day 130 minute)), 'a interval')
+-- !query schema
+struct<from_json({"a":"1 days"}):struct<a:interval>,to_json(from_json({"a":"1 days"})):string,to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string,from_json(to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes'))):struct<a:interval>>
+-- !query output
+{"a":1 days}	{"a":"1 days"}	{"a":"2 years 1 months 100 days 2 hours 10 minutes"}	{"a":2 years 1 months 100 days 2 hours 10 minutes}
+
+
+-- !query
+select interval '+'
+-- !query schema
 struct<>
 -- !query output
-java.lang.ArithmeticException
-integer overflow
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +(line 1, pos 7)
+
+== SQL ==
+select interval '+'
+-------^^^
 
 
 -- !query
-SELECT from_csv('1, 1 day', 'a INT, b interval')
+select interval '+.'
 -- !query schema
-struct<from_csv(1, 1 day):struct<a:int,b:interval>>
+struct<>
 -- !query output
-{"a":1,"b":1 days}
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +.(line 1, pos 7)
+
+== SQL ==
+select interval '+.'
+-------^^^
 
 
 -- !query
-SELECT to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute))
+select interval '1'
 -- !query schema
-struct<to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes')):string>
+struct<>
 -- !query output
-2 years 8 months,1 hours 10 minutes
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1(line 1, pos 7)
+
+== SQL ==
+select interval '1'
+-------^^^
 
 
 -- !query
-SELECT from_json('{"a":"1 days"}', 'a interval')
+select interval '1.2'
 -- !query schema
-struct<from_json({"a":"1 days"}):struct<a:interval>>
+struct<>
 -- !query output
-{"a":1 days}
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1.2(line 1, pos 7)
+
+== SQL ==
+select interval '1.2'
+-------^^^
+
+
+-- !query
+select interval '- 2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: - 2(line 1, pos 7)
+
+== SQL ==
+select interval '- 2'
+-------^^^
+
+
+-- !query
+select interval '1 day -'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day -(line 1, pos 7)
+
+== SQL ==
+select interval '1 day -'
+-------^^^
+
+
+-- !query
+select interval '1 day 1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day 1(line 1, pos 7)
+
+== SQL ==
+select interval '1 day 1'
+-------^^^
+
+
+-- !query
+select interval '1 day 2' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: 1 day 2(line 1, pos 16)
+
+== SQL ==
+select interval '1 day 2' day
+----------------^^^
+
+
+-- !query
+select interval 'interval 1' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: interval 1(line 1, pos 16)
+
+== SQL ==
+select interval 'interval 1' day
+----------------^^^
 
 
 -- !query
-SELECT to_json(map('a', interval 25 month 100 day 130 minute))
+select interval '-\t 1' day
 -- !query schema
-struct<to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string>
+struct<INTERVAL '-1 days':interval>
 -- !query output
-{"a":"2 years 1 months 100 days 2 hours 10 minutes"}
+-1 days
diff --git a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
index 21a3531caf732..665c79c4753bc 100644
--- a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 42
+-- Number of queries: 46
 
 
 -- !query
@@ -288,6 +288,49 @@ struct<from_json([{"a": 1}, 2]):array<map<string,int>>>
 NULL
 
 
+-- !query
+select from_json('{"d": "2012-12-15", "t": "2012-12-15 15:15:15"}', 'd date, t timestamp')
+-- !query schema
+struct<from_json({"d": "2012-12-15", "t": "2012-12-15 15:15:15"}):struct<d:date,t:timestamp>>
+-- !query output
+{"d":2012-12-15,"t":2012-12-15 15:15:15}
+
+
+-- !query
+select from_json(
+  '{"d": "12/15 2012", "t": "12/15 2012 15:15:15"}',
+  'd date, t timestamp',
+  map('dateFormat', 'MM/dd yyyy', 'timestampFormat', 'MM/dd yyyy HH:mm:ss'))
+-- !query schema
+struct<from_json({"d": "12/15 2012", "t": "12/15 2012 15:15:15"}):struct<d:date,t:timestamp>>
+-- !query output
+{"d":2012-12-15,"t":2012-12-15 15:15:15}
+
+
+-- !query
+select from_json(
+  '{"d": "02-29"}',
+  'd date',
+  map('dateFormat', 'MM-dd'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '02-29' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select from_json(
+  '{"t": "02-29"}',
+  't timestamp',
+  map('timestampFormat', 'MM-dd'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '02-29' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
 -- !query
 select to_json(array('1', '2', '3'))
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/limit.sql.out b/sql/core/src/test/resources/sql-tests/results/limit.sql.out
index 281326e22a97a..074e7a6d28c47 100644
--- a/sql/core/src/test/resources/sql-tests/results/limit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/limit.sql.out
@@ -88,7 +88,7 @@ SELECT * FROM testdata LIMIT key > 3
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The limit expression must evaluate to a constant value, but got (testdata.`key` > 3);
+The limit expression must evaluate to a constant value, but got (spark_catalog.default.testdata.`key` > 3);
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/operator-div.sql.out b/sql/core/src/test/resources/sql-tests/results/operator-div.sql.out
deleted file mode 100644
index 3f933f4c0e449..0000000000000
--- a/sql/core/src/test/resources/sql-tests/results/operator-div.sql.out
+++ /dev/null
@@ -1,146 +0,0 @@
--- Automatically generated by SQLQueryTestSuite
--- Number of queries: 18
-
-
--- !query
-set spark.sql.legacy.integralDivide.returnBigint=true
--- !query schema
-struct<key:string,value:string>
--- !query output
-spark.sql.legacy.integralDivide.returnBigint	true
-
-
--- !query
-select 5 div 2
--- !query schema
-struct<(5 div 2):bigint>
--- !query output
-2
-
-
--- !query
-select 5 div 0
--- !query schema
-struct<(5 div 0):bigint>
--- !query output
-NULL
-
-
--- !query
-select 5 div null
--- !query schema
-struct<(5 div CAST(NULL AS INT)):bigint>
--- !query output
-NULL
-
-
--- !query
-select null div 5
--- !query schema
-struct<(CAST(NULL AS INT) div 5):bigint>
--- !query output
-NULL
-
-
--- !query
-select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(51 AS DECIMAL(10,0)) AS DECIMAL(10,0)) div CAST(CAST(2 AS DECIMAL(2,0)) AS DECIMAL(10,0))):bigint>
--- !query output
-25
-
-
--- !query
-select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(0 AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
--- !query output
-NULL
-
-
--- !query
-select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(NULL AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
--- !query output
-NULL
-
-
--- !query
-select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(NULL AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(5 AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
--- !query output
-NULL
-
-
--- !query
-set spark.sql.legacy.integralDivide.returnBigint=false
--- !query schema
-struct<key:string,value:string>
--- !query output
-spark.sql.legacy.integralDivide.returnBigint	false
-
-
--- !query
-select 5 div 2
--- !query schema
-struct<(5 div 2):int>
--- !query output
-2
-
-
--- !query
-select 5 div 0
--- !query schema
-struct<(5 div 0):int>
--- !query output
-NULL
-
-
--- !query
-select 5 div null
--- !query schema
-struct<(5 div CAST(NULL AS INT)):int>
--- !query output
-NULL
-
-
--- !query
-select null div 5
--- !query schema
-struct<(CAST(NULL AS INT) div 5):int>
--- !query output
-NULL
-
-
--- !query
-select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(51 AS DECIMAL(10,0)) AS DECIMAL(10,0)) div CAST(CAST(2 AS DECIMAL(2,0)) AS DECIMAL(10,0))):decimal(10,0)>
--- !query output
-25
-
-
--- !query
-select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(0 AS DECIMAL(2,0)) AS DECIMAL(2,0))):decimal(1,0)>
--- !query output
-NULL
-
-
--- !query
-select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(NULL AS DECIMAL(2,0)) AS DECIMAL(2,0))):decimal(1,0)>
--- !query output
-NULL
-
-
--- !query
-select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(NULL AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(5 AS DECIMAL(2,0)) AS DECIMAL(2,0))):decimal(1,0)>
--- !query output
-NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/operators.sql.out b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
index 548281014afd7..9accc57d0bf60 100644
--- a/sql/core/src/test/resources/sql-tests/results/operators.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 49
+-- Number of queries: 57
 
 
 -- !query
@@ -154,6 +154,70 @@ struct<(CAST(NULL AS DOUBLE) / CAST(5 AS DOUBLE)):double>
 NULL
 
 
+-- !query
+select 5 div 2
+-- !query schema
+struct<(CAST(5 AS BIGINT) div CAST(2 AS BIGINT)):bigint>
+-- !query output
+2
+
+
+-- !query
+select 5 div 0
+-- !query schema
+struct<(CAST(5 AS BIGINT) div CAST(0 AS BIGINT)):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select 5 div null
+-- !query schema
+struct<(CAST(5 AS BIGINT) div CAST(NULL AS BIGINT)):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select null div 5
+-- !query schema
+struct<(CAST(NULL AS BIGINT) div CAST(5 AS BIGINT)):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0))
+-- !query schema
+struct<(CAST(CAST(51 AS DECIMAL(10,0)) AS DECIMAL(10,0)) div CAST(CAST(2 AS DECIMAL(2,0)) AS DECIMAL(10,0))):bigint>
+-- !query output
+25
+
+
+-- !query
+select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0))
+-- !query schema
+struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(0 AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0))
+-- !query schema
+struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(NULL AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0))
+-- !query schema
+struct<(CAST(CAST(NULL AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(5 AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
+-- !query output
+NULL
+
+
 -- !query
 select 1 + 2
 -- !query schema
@@ -229,7 +293,7 @@ struct<COT(CAST(-1 AS DOUBLE)):double>
 -- !query
 select ceiling(0)
 -- !query schema
-struct<CEIL(CAST(0 AS DOUBLE)):bigint>
+struct<ceiling(CAST(0 AS DOUBLE)):bigint>
 -- !query output
 0
 
@@ -237,7 +301,7 @@ struct<CEIL(CAST(0 AS DOUBLE)):bigint>
 -- !query
 select ceiling(1)
 -- !query schema
-struct<CEIL(CAST(1 AS DOUBLE)):bigint>
+struct<ceiling(CAST(1 AS DOUBLE)):bigint>
 -- !query output
 1
 
@@ -253,7 +317,7 @@ struct<CEIL(1234567890123456):bigint>
 -- !query
 select ceiling(1234567890123456)
 -- !query schema
-struct<CEIL(1234567890123456):bigint>
+struct<ceiling(1234567890123456):bigint>
 -- !query output
 1234567890123456
 
@@ -269,7 +333,7 @@ struct<CEIL(0.01):decimal(1,0)>
 -- !query
 select ceiling(-0.10)
 -- !query schema
-struct<CEIL(-0.10):decimal(1,0)>
+struct<ceiling(-0.10):decimal(1,0)>
 -- !query output
 0
 
@@ -325,7 +389,7 @@ true
 -- !query
 select mod(7, 2), mod(7, 0), mod(0, 2), mod(7, null), mod(null, 2), mod(null, null)
 -- !query schema
-struct<(7 % 2):int,(7 % 0):int,(0 % 2):int,(7 % CAST(NULL AS INT)):int,(CAST(NULL AS INT) % 2):int,(CAST(NULL AS DOUBLE) % CAST(NULL AS DOUBLE)):double>
+struct<mod(7, 2):int,mod(7, 0):int,mod(0, 2):int,mod(7, CAST(NULL AS INT)):int,mod(CAST(NULL AS INT), 2):int,mod(CAST(NULL AS DOUBLE), CAST(NULL AS DOUBLE)):double>
 -- !query output
 1	NULL	0	NULL	NULL	NULL
 
@@ -341,7 +405,7 @@ struct<bit_length(abc):int>
 -- !query
 select CHAR_LENGTH('abc')
 -- !query schema
-struct<length(abc):int>
+struct<char_length(abc):int>
 -- !query output
 3
 
@@ -349,7 +413,7 @@ struct<length(abc):int>
 -- !query
 select CHARACTER_LENGTH('abc')
 -- !query schema
-struct<length(abc):int>
+struct<character_length(abc):int>
 -- !query output
 3
 
@@ -373,7 +437,7 @@ struct<abs(-3.13):decimal(3,2),abs(CAST(-2.19 AS DOUBLE)):double>
 -- !query
 select positive('-1.11'), positive(-1.11), negative('-1.11'), negative(-1.11)
 -- !query schema
-struct<(+ CAST(-1.11 AS DOUBLE)):double,(+ -1.11):decimal(3,2),(- CAST(-1.11 AS DOUBLE)):double,(- -1.11):decimal(3,2)>
+struct<(+ CAST(-1.11 AS DOUBLE)):double,(+ -1.11):decimal(3,2),negative(CAST(-1.11 AS DOUBLE)):double,negative(-1.11):decimal(3,2)>
 -- !query output
 -1.11	-1.11	1.11	1.11
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out
index 5efb58c7fc1b0..f7bba96738eab 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out
@@ -381,8 +381,8 @@ struct<>
 org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
-Expression in where clause: [(sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT)) = CAST(b.`four` AS BIGINT))]
-Invalid expressions: [sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT))];
+Expression in where clause: [(sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT)) = CAST(b.`four` AS BIGINT))]
+Invalid expressions: [sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT))];
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/comments.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/comments.sql.out
index 4ea49013a62d1..637c5561bd940 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/comments.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/comments.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
+-- Number of queries: 7
 
 
 -- !query
@@ -36,129 +36,32 @@ before multi-line
 
 -- !query
 /* This is an example of SQL which should not execute:
- * select 'multi-line'
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-mismatched input '/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-/* This is an example of SQL which should not execute:
-^^^
- * select 'multi-line'
-
-
--- !query
-*/
+ * select 'multi-line';
+ */
 SELECT 'after multi-line' AS fifth
 -- !query schema
-struct<>
+struct<fifth:string>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-extraneous input '*/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-*/
-^^^
-SELECT 'after multi-line' AS fifth
+after multi-line
 
 
 -- !query
 /*
-SELECT 'trailing' as x1
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-mismatched input '/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-/*
-^^^
-SELECT 'trailing' as x1
-
-
--- !query
+SELECT 'trailing' as x1; -- inside block comment
 */
 
 /* This block comment surrounds a query which itself has a block comment...
-SELECT /* embedded single line */ 'embedded' AS x2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-mismatched input '*/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-*/
-^^^
-
-/* This block comment surrounds a query which itself has a block comment...
-SELECT /* embedded single line */ 'embedded' AS x2
-
-
--- !query
+SELECT /* embedded single line */ 'embedded' AS x2;
 */
 
 SELECT -- continued after the following block comments...
 /* Deeply nested comment.
    This includes a single apostrophe to make sure we aren't decoding this part as a string.
-SELECT 'deep nest' AS n1
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-extraneous input '*/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-*/
-^^^
-
-SELECT -- continued after the following block comments...
-/* Deeply nested comment.
-   This includes a single apostrophe to make sure we aren't decoding this part as a string.
-SELECT 'deep nest' AS n1
-
-
--- !query
+SELECT 'deep nest' AS n1;
 /* Second level of nesting...
-SELECT 'deeper nest' as n2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-mismatched input '/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-/* Second level of nesting...
-^^^
-SELECT 'deeper nest' as n2
-
-
--- !query
+SELECT 'deeper nest' as n2;
 /* Third level of nesting...
-SELECT 'deepest nest' as n3
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-mismatched input '/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-/* Third level of nesting...
-^^^
-SELECT 'deepest nest' as n3
-
-
--- !query
+SELECT 'deepest nest' as n3;
 */
 Hoo boy. Still two deep...
 */
@@ -170,11 +73,27 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-mismatched input '*/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
+mismatched input ''embedded'' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 6, pos 34)
 
 == SQL ==
+/*
+SELECT 'trailing' as x1; -- inside block comment
+*/
+
+/* This block comment surrounds a query which itself has a block comment...
+SELECT /* embedded single line */ 'embedded' AS x2;
+----------------------------------^^^
+*/
+
+SELECT -- continued after the following block comments...
+/* Deeply nested comment.
+   This includes a single apostrophe to make sure we aren't decoding this part as a string.
+SELECT 'deep nest' AS n1;
+/* Second level of nesting...
+SELECT 'deeper nest' as n2;
+/* Third level of nesting...
+SELECT 'deepest nest' as n3;
 */
-^^^
 Hoo boy. Still two deep...
 */
 Now just one deep...
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
index 436b33ce43980..ae1cb2f171704 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
@@ -42,7 +42,7 @@ struct<>
 
 
 -- !query
-CREATE TABLE view_base_table (key int /* PRIMARY KEY */, data varchar(20))
+CREATE TABLE view_base_table (key int /* PRIMARY KEY */, data varchar(20)) USING PARQUET
 -- !query schema
 struct<>
 -- !query output
@@ -56,7 +56,7 @@ CREATE VIEW key_dependent_view AS
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'default.view_base_table.`data`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'spark_catalog.default.view_base_table.`data`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
 
 
 -- !query
@@ -266,7 +266,7 @@ CREATE VIEW v1_temp AS SELECT * FROM temp_table
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v1_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v1_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -371,7 +371,7 @@ CREATE VIEW v4_temp AS
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v4_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v4_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -383,7 +383,7 @@ CREATE VIEW v5_temp AS
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v5_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v5_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -542,7 +542,7 @@ CREATE VIEW v6_temp AS SELECT * FROM base_table WHERE id IN (SELECT id FROM temp
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v6_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v6_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -551,7 +551,7 @@ CREATE VIEW v7_temp AS SELECT t1.id, t2.a FROM base_table t1, (SELECT * FROM tem
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v7_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v7_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -560,7 +560,7 @@ CREATE VIEW v8_temp AS SELECT * FROM base_table WHERE EXISTS (SELECT 1 FROM temp
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v8_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v8_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -569,7 +569,7 @@ CREATE VIEW v9_temp AS SELECT * FROM base_table WHERE NOT EXISTS (SELECT 1 FROM
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v9_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v9_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -678,7 +678,7 @@ CREATE VIEW temporal1 AS SELECT * FROM t1 CROSS JOIN tt
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temporal1` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal1` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -719,7 +719,7 @@ CREATE VIEW temporal2 AS SELECT * FROM t1 INNER JOIN tt ON t1.num = tt.num2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temporal2` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal2` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -760,7 +760,7 @@ CREATE VIEW temporal3 AS SELECT * FROM t1 LEFT JOIN tt ON t1.num = tt.num2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temporal3` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal3` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -801,7 +801,7 @@ CREATE VIEW temporal4 AS SELECT * FROM t1 LEFT JOIN tt ON t1.num = tt.num2 AND t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temporal4` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal4` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -810,7 +810,7 @@ CREATE VIEW temporal5 AS SELECT * FROM t1 WHERE num IN (SELECT num FROM t1 WHERE
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temporal5` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal5` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
index fd5dc42632176..151fa1e28d725 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 98
+-- Number of queries: 60
 
 
 -- !query
@@ -571,310 +571,6 @@ struct<Two days:interval>
 2 days
 
 
--- !query
-SELECT EXTRACT(EPOCH FROM DATE        '1970-01-01')
--- !query schema
-struct<date_part('EPOCH', DATE '1970-01-01'):decimal(20,6)>
--- !query output
-0.000000
-
-
--- !query
-SELECT EXTRACT(EPOCH FROM TIMESTAMP   '1970-01-01')
--- !query schema
-struct<date_part('EPOCH', TIMESTAMP '1970-01-01 00:00:00'):decimal(20,6)>
--- !query output
-0.000000
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_part('CENTURY', TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--2
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_part('CENTURY', TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--1
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_part('CENTURY', TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--1
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '0001-01-01')
--- !query schema
-struct<date_part('CENTURY', DATE '0001-01-01'):int>
--- !query output
-1
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD')
--- !query schema
-struct<date_part('CENTURY', DATE '0001-01-01'):int>
--- !query output
-1
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '1900-12-31')
--- !query schema
-struct<date_part('CENTURY', DATE '1900-12-31'):int>
--- !query output
-19
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '1901-01-01')
--- !query schema
-struct<date_part('CENTURY', DATE '1901-01-01'):int>
--- !query output
-20
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '2000-12-31')
--- !query schema
-struct<date_part('CENTURY', DATE '2000-12-31'):int>
--- !query output
-20
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '2001-01-01')
--- !query schema
-struct<date_part('CENTURY', DATE '2001-01-01'):int>
--- !query output
-21
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM CURRENT_DATE)>=21 AS True
--- !query schema
-struct<True:boolean>
--- !query output
-true
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_part('MILLENNIUM', TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--1
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD')
--- !query schema
-struct<date_part('MILLENNIUM', DATE '0001-01-01'):int>
--- !query output
-1
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31')
--- !query schema
-struct<date_part('MILLENNIUM', DATE '1000-12-31'):int>
--- !query output
-1
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01')
--- !query schema
-struct<date_part('MILLENNIUM', DATE '1001-01-01'):int>
--- !query output
-2
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31')
--- !query schema
-struct<date_part('MILLENNIUM', DATE '2000-12-31'):int>
--- !query output
-2
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01')
--- !query schema
-struct<date_part('MILLENNIUM', DATE '2001-01-01'):int>
--- !query output
-3
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE)
--- !query schema
-struct<date_part('MILLENNIUM', current_date()):int>
--- !query output
-3
-
-
--- !query
-SELECT EXTRACT(DECADE FROM DATE '1994-12-25')
--- !query schema
-struct<date_part('DECADE', DATE '1994-12-25'):int>
--- !query output
-199
-
-
--- !query
-SELECT EXTRACT(DECADE FROM DATE '0010-01-01')
--- !query schema
-struct<date_part('DECADE', DATE '0010-01-01'):int>
--- !query output
-1
-
-
--- !query
-SELECT EXTRACT(DECADE FROM DATE '0009-12-31')
--- !query schema
-struct<date_part('DECADE', DATE '0009-12-31'):int>
--- !query output
-0
-
-
--- !query
-SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_part('DECADE', TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')):int>
--- !query output
-0
-
-
--- !query
-SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_part('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--1
-
-
--- !query
-SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_part('DECADE', TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')):int>
--- !query output
--1
-
-
--- !query
-SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_part('DECADE', TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--2
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM NOW())>=21 AS True
--- !query schema
-struct<True:boolean>
--- !query output
-true
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000')
--- !query schema
-struct<date_part('CENTURY', TIMESTAMP '1970-03-20 04:30:00'):int>
--- !query output
-20
-
-
--- !query
-SELECT DATE_TRUNC('MILLENNIUM', TIMESTAMP '1970-03-20 04:30:00.00000')
--- !query schema
-struct<date_trunc(MILLENNIUM, TIMESTAMP '1970-03-20 04:30:00'):timestamp>
--- !query output
-1001-01-01 00:07:02
-
-
--- !query
-SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20')
--- !query schema
-struct<date_trunc(MILLENNIUM, CAST(DATE '1970-03-20' AS TIMESTAMP)):timestamp>
--- !query output
-1001-01-01 00:07:02
-
-
--- !query
-SELECT DATE_TRUNC('CENTURY', TIMESTAMP '1970-03-20 04:30:00.00000')
--- !query schema
-struct<date_trunc(CENTURY, TIMESTAMP '1970-03-20 04:30:00'):timestamp>
--- !query output
-1901-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('CENTURY', DATE '1970-03-20')
--- !query schema
-struct<date_trunc(CENTURY, CAST(DATE '1970-03-20' AS TIMESTAMP)):timestamp>
--- !query output
-1901-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('CENTURY', DATE '2004-08-10')
--- !query schema
-struct<date_trunc(CENTURY, CAST(DATE '2004-08-10' AS TIMESTAMP)):timestamp>
--- !query output
-2001-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('CENTURY', DATE '0002-02-04')
--- !query schema
-struct<date_trunc(CENTURY, CAST(DATE '0002-02-04' AS TIMESTAMP)):timestamp>
--- !query output
-0001-01-01 00:07:02
-
-
--- !query
-SELECT DATE_TRUNC('CENTURY', TO_DATE('0055-08-10 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_trunc(CENTURY, CAST(to_date('0055-08-10 BC', 'yyyy-MM-dd G') AS TIMESTAMP)):timestamp>
--- !query output
--0099-01-01 00:07:02
-
-
--- !query
-SELECT DATE_TRUNC('DECADE', DATE '1993-12-25')
--- !query schema
-struct<date_trunc(DECADE, CAST(DATE '1993-12-25' AS TIMESTAMP)):timestamp>
--- !query output
-1990-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('DECADE', DATE '0004-12-25')
--- !query schema
-struct<date_trunc(DECADE, CAST(DATE '0004-12-25' AS TIMESTAMP)):timestamp>
--- !query output
-0000-01-01 00:07:02
-
-
--- !query
-SELECT DATE_TRUNC('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_trunc(DECADE, CAST(to_date('0002-12-31 BC', 'yyyy-MM-dd G') AS TIMESTAMP)):timestamp>
--- !query output
--0010-01-01 00:07:02
-
-
 -- !query
 select make_date(2013, 7, 15)
 -- !query schema
@@ -888,7 +584,7 @@ select make_date(-44, 3, 15)
 -- !query schema
 struct<make_date(-44, 3, 15):date>
 -- !query output
--0044-03-15
+0045-03-15
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
index ba913789d5623..fe8375c5eab8f 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
@@ -322,9 +322,10 @@ struct<CAST(CAST(2147483520 AS FLOAT) AS INT):int>
 -- !query
 SELECT int(float('2147483647'))
 -- !query schema
-struct<CAST(CAST(2147483647 AS FLOAT) AS INT):int>
+struct<>
 -- !query output
-2147483647
+java.lang.ArithmeticException
+Casting 2.14748365E9 to int causes overflow
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/insert.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/insert.sql.out
index 1046d0ec86bbd..63ad74aac32ec 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/insert.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/insert.sql.out
@@ -64,7 +64,7 @@ struct<>
 -- !query
 select col1, col2, char_length(col3) from inserttest
 -- !query schema
-struct<col1:int,col2:int,length(col3):int>
+struct<col1:int,col2:int,char_length(col3):int>
 -- !query output
 30	50	10000
 NULL	3	7
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out
index 4bd846d3ff923..62d47410aab65 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out
@@ -105,7 +105,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03' day to hour
@@ -119,7 +119,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03:04(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03:04' day to hour
@@ -141,7 +141,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03:04' day to minute
@@ -155,7 +155,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03' day to second
@@ -177,7 +177,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03' hour to minute
@@ -191,7 +191,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03:04' hour to minute
@@ -205,7 +205,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03' hour to second
@@ -219,7 +219,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03:04' hour to second
@@ -233,7 +233,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03' minute to second
@@ -247,7 +247,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03:04' minute to second
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
index 5332dfff9f101..20f4f6b1f213f 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
@@ -536,7 +536,7 @@ SELECT '' AS `xxx`, i, k, t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i' is ambiguous, could be: default.j1_tbl.i, default.j2_tbl.i.; line 1 pos 20
+Reference 'i' is ambiguous, could be: spark_catalog.default.j1_tbl.i, spark_catalog.default.j2_tbl.i.; line 1 pos 20
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
index bdb605e406b8a..7b7aeb4ec7934 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
@@ -4404,7 +4404,7 @@ struct<>
 -- !query
 SELECT a, ceil(a), ceiling(a), floor(a), round(a) FROM ceil_floor_round
 -- !query schema
-struct<a:decimal(38,18),CEIL(a):decimal(21,0),CEIL(a):decimal(21,0),FLOOR(a):decimal(21,0),round(a, 0):decimal(38,0)>
+struct<a:decimal(38,18),CEIL(a):decimal(21,0),ceiling(a):decimal(21,0),FLOOR(a):decimal(21,0),round(a, 0):decimal(38,0)>
 -- !query output
 -0.000001000000000000	0	0	-1	0
 -5.499999000000000000	-5	-5	-6	-5
@@ -4494,7 +4494,7 @@ struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) / CA
 -- !query
 select div(cast(999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))):decimal(38,0)>
+struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))):bigint>
 -- !query output
 0
 
@@ -4510,7 +4510,7 @@ struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) % CA
 -- !query
 select div(cast(-9999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))):decimal(38,0)>
+struct<(CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))):bigint>
 -- !query output
 -9
 
@@ -4526,7 +4526,7 @@ struct<(CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) %
 -- !query
 select div(cast(-9999999999999999999999 as decimal(38, 0)),1000000000000000000000)*1000000000000000000000 + mod(cast(-9999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST((CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(38,0)) * CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(38,0)) + CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) % CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(38,0))):decimal(38,0)>
+struct<(CAST((CAST(CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(20,0)) AS DECIMAL(22,0)) * CAST(1000000000000000000000 AS DECIMAL(22,0))) AS DECIMAL(38,0)) + CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) % CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(38,0))):decimal(38,0)>
 -- !query output
 -9999999999999999999999
 
@@ -4542,7 +4542,7 @@ struct<(CAST(70.0 AS DECIMAL(3,1)) % CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,
 -- !query
 select div (70.0,70)
 -- !query schema
-struct<(CAST(70.0 AS DECIMAL(3,1)) div CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,1))):decimal(2,0)>
+struct<(CAST(70.0 AS DECIMAL(3,1)) div CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,1))):bigint>
 -- !query output
 1
 
@@ -4654,7 +4654,7 @@ struct<id:bigint>
 -- !query
 select ln(1.2345678e-28)
 -- !query schema
-struct<LOG(1.2345678E-28):double>
+struct<ln(1.2345678E-28):double>
 -- !query output
 -64.26166165451762
 
@@ -4662,7 +4662,7 @@ struct<LOG(1.2345678E-28):double>
 -- !query
 select ln(0.0456789)
 -- !query schema
-struct<LOG(CAST(0.0456789 AS DOUBLE)):double>
+struct<ln(CAST(0.0456789 AS DOUBLE)):double>
 -- !query output
 -3.0861187944847437
 
@@ -4670,7 +4670,7 @@ struct<LOG(CAST(0.0456789 AS DOUBLE)):double>
 -- !query
 select ln(0.99949452)
 -- !query schema
-struct<LOG(CAST(0.99949452 AS DOUBLE)):double>
+struct<ln(CAST(0.99949452 AS DOUBLE)):double>
 -- !query output
 -5.056077980832118E-4
 
@@ -4678,7 +4678,7 @@ struct<LOG(CAST(0.99949452 AS DOUBLE)):double>
 -- !query
 select ln(1.00049687395)
 -- !query schema
-struct<LOG(CAST(1.00049687395 AS DOUBLE)):double>
+struct<ln(CAST(1.00049687395 AS DOUBLE)):double>
 -- !query output
 4.967505490136803E-4
 
@@ -4686,7 +4686,7 @@ struct<LOG(CAST(1.00049687395 AS DOUBLE)):double>
 -- !query
 select ln(1234.567890123456789)
 -- !query schema
-struct<LOG(CAST(1234.567890123456789 AS DOUBLE)):double>
+struct<ln(CAST(1234.567890123456789 AS DOUBLE)):double>
 -- !query output
 7.11847630129779
 
@@ -4694,7 +4694,7 @@ struct<LOG(CAST(1234.567890123456789 AS DOUBLE)):double>
 -- !query
 select ln(5.80397490724e5)
 -- !query schema
-struct<LOG(580397.490724):double>
+struct<ln(580397.490724):double>
 -- !query output
 13.271468476626518
 
@@ -4702,7 +4702,7 @@ struct<LOG(580397.490724):double>
 -- !query
 select ln(9.342536355e34)
 -- !query schema
-struct<LOG(9.342536355E34):double>
+struct<ln(9.342536355E34):double>
 -- !query output
 80.52247093552418
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
index cbf4cfa58cdb9..d8d33d92a7cc4 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
@@ -143,7 +143,7 @@ SELECT a FROM test_having HAVING min(a) < max(a)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'default.test_having.`a`' is not an aggregate function. Wrap '(min(default.test_having.`a`) AS `min(a#x)`, max(default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'spark_catalog.default.test_having.`a`' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.`a`) AS `min(a#x)`, max(spark_catalog.default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
index c30eea8ab689d..e8a3a9b9731a6 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 122
+-- Number of queries: 121
 
 
 -- !query
@@ -627,7 +627,7 @@ struct<>
 -- !query
 SELECT substr(f1, 99995) from toasttest
 -- !query schema
-struct<substring(f1, 99995, 2147483647):string>
+struct<substr(f1, 99995, 2147483647):string>
 -- !query output
 567890
 567890
@@ -638,7 +638,7 @@ struct<substring(f1, 99995, 2147483647):string>
 -- !query
 SELECT substr(f1, 99995, 10) from toasttest
 -- !query schema
-struct<substring(f1, 99995, 10):string>
+struct<substr(f1, 99995, 10):string>
 -- !query output
 567890
 567890
@@ -910,14 +910,6 @@ struct<rpad(hi, 5, ):string>
 hi
 
 
--- !query
-SELECT ltrim('zzzytrim', 'xyz')
--- !query schema
-struct<ltrim(zzzytrim, xyz):string>
--- !query output
-trim
-
-
 -- !query
 SELECT translate('', '14', 'ax')
 -- !query schema
@@ -985,7 +977,7 @@ struct<repeat(Pg, -4):string>
 -- !query
 SELECT trim(binary('\\000') from binary('\\000Tom\\000'))
 -- !query schema
-struct<trim(CAST(CAST(\000Tom\000 AS BINARY) AS STRING), CAST(CAST(\000 AS BINARY) AS STRING)):string>
+struct<TRIM(BOTH CAST(CAST(\000 AS BINARY) AS STRING) FROM CAST(CAST(\000Tom\000 AS BINARY) AS STRING)):string>
 -- !query output
 Tom
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
index 75ea3f3c42932..5b0b636ff0c29 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 32
+-- Number of queries: 30
 
 
 -- !query
@@ -265,36 +265,6 @@ struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute:
 	2001-09-22 18:19:20	2001	9	22	18	19	20.000000
 
 
--- !query
-SELECT '' AS `54`, d1 as `timestamp`,
-    date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec,
-    date_part( 'usec', d1) AS usec
-    FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'
--- !query schema
-struct<54:string,timestamp:timestamp,quarter:int,msec:decimal(8,3),usec:int>
--- !query output
-	1969-12-31 16:00:00	4	0.000	0
-	1997-01-02 00:00:00	1	0.000	0
-	1997-01-02 03:04:05	1	5000.000	5000000
-	1997-02-10 17:32:01	1	1000.000	1000000
-	2001-09-22 18:19:20	3	20000.000	20000000
-
-
--- !query
-SELECT '' AS `54`, d1 as `timestamp`,
-    date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week,
-    date_part( 'dow', d1) AS dow
-    FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'
--- !query schema
-struct<54:string,timestamp:timestamp,isoyear:int,week:int,dow:int>
--- !query output
-	1969-12-31 16:00:00	1970	1	3
-	1997-01-02 00:00:00	1997	1	4
-	1997-01-02 03:04:05	1997	1	4
-	1997-02-10 17:32:01	1997	7	1
-	2001-09-22 18:19:20	2001	38	6
-
-
 -- !query
 SELECT make_timestamp(2014,12,28,6,30,45.887)
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out
index 2b1de87a6be5e..76567b689445a 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out
@@ -270,7 +270,7 @@ struct<lead((ten * 2), 1, -1) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIR
 -- !query
 SELECT first(ten) OVER (PARTITION BY four ORDER BY ten), ten, four FROM tenk1 WHERE unique2 < 10
 -- !query schema
-struct<first(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
+struct<first(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
 -- !query output
 0	0	0
 0	0	0
@@ -287,7 +287,7 @@ struct<first(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RA
 -- !query
 SELECT last(four) OVER (ORDER BY ten), ten, four FROM tenk1 WHERE unique2 < 10
 -- !query schema
-struct<last(four, false) OVER (ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
+struct<last(four) OVER (ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
 -- !query output
 0	4	0
 1	1	1
@@ -306,7 +306,7 @@ SELECT last(ten) OVER (PARTITION BY four), ten, four FROM
 (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten)s
 ORDER BY four, ten
 -- !query schema
-struct<last(ten, false) OVER (PARTITION BY four ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int,ten:int,four:int>
+struct<last(ten) OVER (PARTITION BY four ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int,ten:int,four:int>
 -- !query output
 4	0	0
 4	0	0
@@ -476,7 +476,7 @@ sum(ten) over (partition by four order by ten),
 last(ten) over (partition by four order by ten)
 FROM (select distinct ten, four from tenk1) ss
 -- !query schema
-struct<four:int,ten:int,sum(CAST(ten AS BIGINT)) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
+struct<four:int,ten:int,sum(CAST(ten AS BIGINT)) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
 -- !query output
 0	0	0	0
 0	2	2	2
@@ -506,7 +506,7 @@ sum(ten) over (partition by four order by ten range between unbounded preceding
 last(ten) over (partition by four order by ten range between unbounded preceding and current row)
 FROM (select distinct ten, four from tenk1) ss
 -- !query schema
-struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
+struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
 -- !query output
 0	0	0	0
 0	2	2	2
@@ -536,7 +536,7 @@ sum(ten) over (partition by four order by ten range between unbounded preceding
 last(ten) over (partition by four order by ten range between unbounded preceding and unbounded following)
 FROM (select distinct ten, four from tenk1) ss
 -- !query schema
-struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,last(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int>
+struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,last(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int>
 -- !query output
 0	0	20	8
 0	2	20	8
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
index 0015740a0638e..ccddf9db172a6 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
@@ -15,6 +15,24 @@ struct<>
 
 
 
+-- !query
+INSERT INTO empsalary VALUES
+  ('develop', 10, 5200, date '2007-08-01'),
+  ('sales', 1, 5000, date '2006-10-01'),
+  ('personnel', 5, 3500, date '2007-12-10'),
+  ('sales', 4, 4800, date '2007-08-08'),
+  ('personnel', 2, 3900, date '2006-12-23'),
+  ('develop', 7, 4200, date '2008-01-01'),
+  ('develop', 9, 4500, date '2008-01-01'),
+  ('sales', 3, 4800, date '2007-08-01'),
+  ('develop', 8, 6000, date '2006-10-01'),
+  ('develop', 11, 5200, date '2007-08-15')
+-- !query schema
+struct<>
+-- !query output
+
+
+
 -- !query
 SELECT sum(unique1) over (order by four range between 2 preceding and 1 preceding),
 unique1, four
@@ -72,17 +90,6 @@ struct<sum(unique1) OVER (PARTITION BY four ORDER BY unique1 ASC NULLS FIRST RAN
 8	6	2
 
 
--- !query
-select last(salary) over(order by salary range between 1000 preceding and 1000 following),
-lag(salary) over(order by salary range between 1000 preceding and 1000 following),
-salary from empsalary
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Window Frame specifiedwindowframe(RangeFrame, -1000, 1000) must match the required frame specifiedwindowframe(RowFrame, -1, -1);
-
-
 -- !query
 select ss.id, ss.y,
        first(ss.y) over w,
@@ -94,7 +101,7 @@ from
 window w as
   (order by ss.id asc nulls first range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 1	1	1	3
 2	2	1	4
@@ -116,7 +123,7 @@ from
 window w as
   (order by ss.id asc nulls last range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 1	1	1	3
 2	2	1	4
@@ -138,7 +145,7 @@ from
 window w as
   (order by ss.id desc nulls first range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 1	1	3	1
 2	2	4	1
@@ -160,7 +167,7 @@ from
 window w as
   (order by ss.id desc nulls last range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 1	1	3	1
 2	2	4	1
@@ -175,7 +182,7 @@ NULL	43	42	43
 select x.id, last(x.id) over (order by x.id range between current row and 2147450884 following)
 from range(32764, 32767) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(2147450884 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(2147450884 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 32764	32766
 32765	32766
@@ -186,7 +193,7 @@ struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN
 select x.id, last(x.id) over (order by x.id desc range between current row and 2147450885 following)
 from range(-32766, -32765) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(2147450885 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(2147450885 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 -32766	-32766
 
@@ -195,7 +202,7 @@ struct<id:bigint,last(id, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN
 select x.id, last(x.id) over (order by x.id range between current row and 4 following)
 from range(2147483644, 2147483647) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(4 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(4 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 2147483644	2147483646
 2147483645	2147483646
@@ -206,7 +213,7 @@ struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN
 select x.id, last(x.id) over (order by x.id desc range between current row and 5 following)
 from range(-2147483646, -2147483645) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(5 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(5 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 -2147483646	-2147483646
 
@@ -265,7 +272,7 @@ from numerics
 window w as (order by f_float4 range between
              1 preceding and 1 following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -282,7 +289,7 @@ from numerics
 window w as (order by f_float4 range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -299,7 +306,7 @@ from numerics
 window w as (order by f_float4 range between
              'inf' preceding and 'inf' following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	1	7
@@ -316,7 +323,7 @@ from numerics
 window w as (order by f_float4 range between
              1.1 preceding and 'NaN' following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	2	7
@@ -333,7 +340,7 @@ from numerics
 window w as (order by f_float8 range between
              1 preceding and 1 following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -350,7 +357,7 @@ from numerics
 window w as (order by f_float8 range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -367,7 +374,7 @@ from numerics
 window w as (order by f_float8 range between
              'inf' preceding and 'inf' following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	1	7
@@ -384,7 +391,7 @@ from numerics
 window w as (order by f_float8 range between
              1.1 preceding and 'NaN' following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	2	7
@@ -401,7 +408,7 @@ from numerics
 window w as (order by f_numeric range between
              1 preceding and 1 following)
 -- !query schema
-struct<id:int,f_numeric:int,first(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int,last(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int>
+struct<id:int,f_numeric:int,first(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int,last(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int>
 -- !query output
 1	-3	1	1
 2	-1	2	3
@@ -418,7 +425,7 @@ from numerics
 window w as (order by f_numeric range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_numeric:int,first(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int>
+struct<id:int,f_numeric:int,first(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int,last(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int>
 -- !query output
 1	-3	1	1
 2	-1	2	3
@@ -435,7 +442,7 @@ from numerics
 window w as (order by f_numeric range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_numeric:int,first(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int>
+struct<id:int,f_numeric:int,first(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int,last(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int>
 -- !query output
 1	-3	1	1
 2	-1	2	3
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
index 5a52358fe1c53..08eba6797b01d 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
@@ -244,7 +244,7 @@ from t1 where f1 = f2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(PARTITION BY default.t1.`f1` RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 24
+cannot resolve '(PARTITION BY spark_catalog.default.t1.`f1` RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 24
 
 
 -- !query
@@ -294,7 +294,7 @@ SELECT * FROM empsalary WHERE row_number() OVER (ORDER BY salary) < 10
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use window functions inside WHERE and HAVING clauses;
+It is not allowed to use window functions inside WHERE clause;
 
 
 -- !query
@@ -306,7 +306,7 @@ org.apache.spark.sql.AnalysisException
 
 The query operator `Join` contains one or more unsupported
 expression types Aggregate, Window or Generate.
-Invalid expressions: [row_number() OVER (ORDER BY default.empsalary.`salary` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)];
+Invalid expressions: [row_number() OVER (ORDER BY spark_catalog.default.empsalary.`salary` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)];
 
 
 -- !query
@@ -341,7 +341,7 @@ SELECT * FROM empsalary WHERE (rank() OVER (ORDER BY random())) > 10
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use window functions inside WHERE and HAVING clauses;
+It is not allowed to use window functions inside WHERE clause;
 
 
 -- !query
@@ -350,7 +350,7 @@ SELECT * FROM empsalary WHERE rank() OVER (ORDER BY random())
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use window functions inside WHERE and HAVING clauses;
+It is not allowed to use window functions inside WHERE clause;
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out
new file mode 100644
index 0000000000000..64aa6053d8d70
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out
@@ -0,0 +1,87 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 10
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 0, but the specified group index is 1
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+', 0)
+-- !query schema
+struct<regexp_extract(1a 2b 14m, \d+, 0):string>
+-- !query output
+1
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+', 1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 0, but the specified group index is 1
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+', 2)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 0, but the specified group index is 2
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+', -1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+The specified group index cannot be less than zero
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)')
+-- !query schema
+struct<regexp_extract(1a 2b 14m, (\d+)([a-z]+), 1):string>
+-- !query output
+1
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 0)
+-- !query schema
+struct<regexp_extract(1a 2b 14m, (\d+)([a-z]+), 0):string>
+-- !query output
+1a
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 1)
+-- !query schema
+struct<regexp_extract(1a 2b 14m, (\d+)([a-z]+), 1):string>
+-- !query output
+1
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 2)
+-- !query schema
+struct<regexp_extract(1a 2b 14m, (\d+)([a-z]+), 2):string>
+-- !query output
+a
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', -1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+The specified group index cannot be less than zero
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
index e8ee07171651d..88fef8908638e 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
@@ -15,7 +15,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -44,7 +44,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -75,7 +75,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -105,7 +105,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -135,7 +135,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `b` STRING,
   `c` INT,
   `a` INT)
@@ -165,7 +165,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -197,7 +197,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -227,7 +227,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -257,7 +257,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` FLOAT,
   `b` DECIMAL(10,0),
   `c` DECIMAL(10,0),
@@ -295,7 +295,18 @@ SHOW CREATE TABLE view_SPARK_30302 AS SERDE
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE VIEW `view_SPARK_30302`(
+CREATE VIEW `default`.`view_SPARK_30302`(
+  `aaa`,
+  `bbb`)
+AS SELECT a, b FROM tbl
+
+
+-- !query
+SHOW CREATE TABLE view_SPARK_30302
+-- !query schema
+struct<createtab_stmt:string>
+-- !query output
+CREATE VIEW `default`.`view_SPARK_30302` (
   `aaa`,
   `bbb`)
 AS SELECT a, b FROM tbl
@@ -324,7 +335,19 @@ SHOW CREATE TABLE view_SPARK_30302 AS SERDE
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE VIEW `view_SPARK_30302`(
+CREATE VIEW `default`.`view_SPARK_30302`(
+  `aaa` COMMENT 'comment with \'quoted text\' for aaa',
+  `bbb`)
+COMMENT 'This is a comment with \'quoted text\' for view'
+AS SELECT a, b FROM tbl
+
+
+-- !query
+SHOW CREATE TABLE view_SPARK_30302
+-- !query schema
+struct<createtab_stmt:string>
+-- !query output
+CREATE VIEW `default`.`view_SPARK_30302` (
   `aaa` COMMENT 'comment with \'quoted text\' for aaa',
   `bbb`)
 COMMENT 'This is a comment with \'quoted text\' for view'
@@ -354,7 +377,7 @@ SHOW CREATE TABLE view_SPARK_30302 AS SERDE
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE VIEW `view_SPARK_30302`(
+CREATE VIEW `default`.`view_SPARK_30302`(
   `aaa`,
   `bbb`)
 TBLPROPERTIES (
@@ -363,30 +386,18 @@ TBLPROPERTIES (
 AS SELECT a, b FROM tbl
 
 
--- !query
-DROP VIEW view_SPARK_30302
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-CREATE VIEW view_SPARK_30302 (aaa, bbb)
-AS SELECT a, b FROM tbl
--- !query schema
-struct<>
--- !query output
-
-
-
 -- !query
 SHOW CREATE TABLE view_SPARK_30302
 -- !query schema
-struct<>
+struct<createtab_stmt:string>
 -- !query output
-org.apache.spark.sql.AnalysisException
-Hive view isn't supported by SHOW CREATE TABLE;
+CREATE VIEW `default`.`view_SPARK_30302` (
+  `aaa`,
+  `bbb`)
+TBLPROPERTIES (
+  'a' = '1',
+  'b' = '2')
+AS SELECT a, b FROM tbl
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
index 501e185b07f7a..60c5e6d5642b7 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 26
+-- Number of queries: 27
 
 
 -- !query
@@ -63,15 +63,9 @@ SHOW TABLES
 -- !query schema
 struct<database:string,tableName:string,isTemporary:boolean>
 -- !query output
-aggtest
-arraydata
-mapdata
-onek
 show_t1
 show_t2
 show_t3
-tenk1
-testdata
 
 
 -- !query
@@ -79,15 +73,9 @@ SHOW TABLES IN showdb
 -- !query schema
 struct<database:string,tableName:string,isTemporary:boolean>
 -- !query output
-aggtest
-arraydata
-mapdata
-onek
 show_t1
 show_t2
 show_t3
-tenk1
-testdata
 
 
 -- !query
@@ -119,6 +107,16 @@ show_t2
 show_t3
 
 
+-- !query
+SHOW TABLES IN showdb LIKE 'show_t*'
+-- !query schema
+struct<database:string,tableName:string,isTemporary:boolean>
+-- !query output
+show_t1
+show_t2
+show_t3
+
+
 -- !query
 SHOW TABLE EXTENDED LIKE 'show_t*'
 -- !query schema
@@ -226,7 +224,7 @@ SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(a='Us', d=1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Partition spec is invalid. The spec (a, d) must match the partition spec (c, d) defined in table '`showdb`.`show_t1`';
+a is not a valid partition column in table `showdb`.`show_t1`.;
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out
new file mode 100644
index 0000000000000..6984b34c365ec
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out
@@ -0,0 +1,114 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 13
+
+
+-- !query
+CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
+TBLPROPERTIES('p1'='v1', 'p2'='v2')
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SHOW TBLPROPERTIES tbl
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+p1	v1
+p2	v2
+
+
+-- !query
+SHOW TBLPROPERTIES tbl("p1")
+-- !query schema
+struct<value:string>
+-- !query output
+v1
+
+
+-- !query
+SHOW TBLPROPERTIES tbl("p3")
+-- !query schema
+struct<value:string>
+-- !query output
+Table default.tbl does not have property: p3
+
+
+-- !query
+DROP TABLE tbl
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE VIEW view TBLPROPERTIES('p1'='v1', 'p2'='v2') AS SELECT 1 AS c1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SHOW TBLPROPERTIES view
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+p1	v1
+p2	v2
+view.catalogAndNamespace.numParts	2
+view.catalogAndNamespace.part.0	spark_catalog
+view.catalogAndNamespace.part.1	default
+view.query.out.col.0	c1
+view.query.out.numCols	1
+
+
+-- !query
+SHOW TBLPROPERTIES view("p1")
+-- !query schema
+struct<value:string>
+-- !query output
+v1
+
+
+-- !query
+SHOW TBLPROPERTIES view("p3")
+-- !query schema
+struct<value:string>
+-- !query output
+Table default.view does not have property: p3
+
+
+-- !query
+DROP VIEW view
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE TEMPORARY VIEW tv TBLPROPERTIES('p1'='v1') AS SELECT 1 AS c1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SHOW TBLPROPERTIES tv
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+
+
+
+-- !query
+DROP VIEW tv
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/show-views.sql.out b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
new file mode 100644
index 0000000000000..d88790d8b5ec8
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
@@ -0,0 +1,177 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 20
+
+
+-- !query
+CREATE DATABASE showdb
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+USE showdb
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE TABLE tbl(a STRING, b INT, c STRING, d STRING) USING parquet
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE VIEW view_1 AS SELECT * FROM tbl
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE VIEW view_2 AS SELECT * FROM tbl WHERE c='a'
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE GLOBAL TEMP VIEW view_3 AS SELECT 1 as col1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE TEMPORARY VIEW view_4(e INT) USING parquet
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SHOW VIEWS
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS FROM showdb
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS IN showdb
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS IN global_temp
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_3
+view_4
+
+
+-- !query
+SHOW VIEWS 'view_*'
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS LIKE 'view_1*|view_2*'
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+
+
+-- !query
+SHOW VIEWS IN showdb 'view_*'
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS IN showdb LIKE 'view_*'
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS IN wrongdb LIKE 'view_*'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException
+Database 'wrongdb' not found;
+
+
+-- !query
+DROP VIEW global_temp.view_3
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP VIEW view_4
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+USE default
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP DATABASE showdb CASCADE
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
index 33d1b25aee483..c9aad35bec98a 100644
--- a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 33
+-- Number of queries: 36
 
 
 -- !query
@@ -55,7 +55,7 @@ struct<length(uuid()):int,(NOT (uuid() = uuid())):boolean>
 -- !query
 select position('bar' in 'foobarbar'), position(null, 'foobarbar'), position('aaads', null)
 -- !query schema
-struct<locate(bar, foobarbar, 1):int,locate(CAST(NULL AS STRING), foobarbar, 1):int,locate(aaads, CAST(NULL AS STRING), 1):int>
+struct<locate(bar, foobarbar, 1):int,position(CAST(NULL AS STRING), foobarbar, 1):int,position(aaads, CAST(NULL AS STRING), 1):int>
 -- !query output
 4	NULL	NULL
 
@@ -111,7 +111,7 @@ struct<split(aa1cc2ee3, [1-9]+, 2):array<string>>
 -- !query
 SELECT substr('Spark SQL', 5)
 -- !query schema
-struct<substring(Spark SQL, 5, 2147483647):string>
+struct<substr(Spark SQL, 5, 2147483647):string>
 -- !query output
 k SQL
 
@@ -119,7 +119,7 @@ k SQL
 -- !query
 SELECT substr('Spark SQL', -3)
 -- !query schema
-struct<substring(Spark SQL, -3, 2147483647):string>
+struct<substr(Spark SQL, -3, 2147483647):string>
 -- !query output
 SQL
 
@@ -127,7 +127,7 @@ SQL
 -- !query
 SELECT substr('Spark SQL', 5, 1)
 -- !query schema
-struct<substring(Spark SQL, 5, 1):string>
+struct<substr(Spark SQL, 5, 1):string>
 -- !query output
 k
 
@@ -205,64 +205,88 @@ k
 
 
 -- !query
-SELECT trim('yxTomxx', 'xyz'), trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx')
+SELECT trim(" xyz "), ltrim(" xyz "), rtrim(" xyz ")
+-- !query schema
+struct<trim( xyz ):string,ltrim( xyz ):string,rtrim( xyz ):string>
+-- !query output
+xyz	xyz 	 xyz
+
+
+-- !query
+SELECT trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx')
+-- !query schema
+struct<TRIM(BOTH xyz FROM yxTomxx):string,TRIM(BOTH xyz FROM yxTomxx):string>
+-- !query output
+Tom	Tom
+
+
+-- !query
+SELECT trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx')
+-- !query schema
+struct<TRIM(BOTH x FROM xxxbarxxx):string,TRIM(BOTH x FROM xxxbarxxx):string>
+-- !query output
+bar	bar
+
+
+-- !query
+SELECT trim(LEADING 'xyz' FROM 'zzzytest')
 -- !query schema
-struct<trim(yxTomxx, xyz):string,trim(yxTomxx, xyz):string,trim(yxTomxx, xyz):string>
+struct<TRIM(LEADING xyz FROM zzzytest):string>
 -- !query output
-Tom	Tom	Tom
+test
 
 
 -- !query
-SELECT trim('xxxbarxxx', 'x'), trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx')
+SELECT trim(LEADING 'xyz' FROM 'zzzytestxyz')
 -- !query schema
-struct<trim(xxxbarxxx, x):string,trim(xxxbarxxx, x):string,trim(xxxbarxxx, x):string>
+struct<TRIM(LEADING xyz FROM zzzytestxyz):string>
 -- !query output
-bar	bar	bar
+testxyz
 
 
 -- !query
-SELECT ltrim('zzzytest', 'xyz'), trim(LEADING 'xyz' FROM 'zzzytest')
+SELECT trim(LEADING 'xy' FROM 'xyxXxyLAST WORD')
 -- !query schema
-struct<ltrim(zzzytest, xyz):string,ltrim(zzzytest, xyz):string>
+struct<TRIM(LEADING xy FROM xyxXxyLAST WORD):string>
 -- !query output
-test	test
+XxyLAST WORD
 
 
 -- !query
-SELECT ltrim('zzzytestxyz', 'xyz'), trim(LEADING 'xyz' FROM 'zzzytestxyz')
+SELECT trim(TRAILING 'xyz' FROM 'testxxzx')
 -- !query schema
-struct<ltrim(zzzytestxyz, xyz):string,ltrim(zzzytestxyz, xyz):string>
+struct<TRIM(TRAILING xyz FROM testxxzx):string>
 -- !query output
-testxyz	testxyz
+test
 
 
 -- !query
-SELECT ltrim('xyxXxyLAST WORD', 'xy'), trim(LEADING 'xy' FROM 'xyxXxyLAST WORD')
+SELECT trim(TRAILING 'xyz' FROM 'xyztestxxzx')
 -- !query schema
-struct<ltrim(xyxXxyLAST WORD, xy):string,ltrim(xyxXxyLAST WORD, xy):string>
+struct<TRIM(TRAILING xyz FROM xyztestxxzx):string>
 -- !query output
-XxyLAST WORD	XxyLAST WORD
+xyztest
 
 
 -- !query
-SELECT rtrim('testxxzx', 'xyz'), trim(TRAILING 'xyz' FROM 'testxxzx')
+SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy')
 -- !query schema
-struct<rtrim(testxxzx, xyz):string,rtrim(testxxzx, xyz):string>
+struct<TRIM(TRAILING xy FROM TURNERyxXxy):string>
 -- !query output
-test	test
+TURNERyxX
 
 
 -- !query
-SELECT rtrim('xyztestxxzx', 'xyz'), trim(TRAILING 'xyz' FROM 'xyztestxxzx')
+SELECT lpad('hi', 'invalid_length')
 -- !query schema
-struct<rtrim(xyztestxxzx, xyz):string,rtrim(xyztestxxzx, xyz):string>
+struct<lpad(hi, CAST(invalid_length AS INT),  ):string>
 -- !query output
-xyztest	xyztest
+NULL
 
 
 -- !query
-SELECT rtrim('TURNERyxXxy', 'xy'), trim(TRAILING 'xy' FROM 'TURNERyxXxy')
+SELECT rpad('hi', 'invalid_length')
 -- !query schema
-struct<rtrim(TURNERyxXxy, xy):string,rtrim(TURNERyxXxy, xy):string>
+struct<rpad(hi, CAST(invalid_length AS INT),  ):string>
 -- !query output
-TURNERyxX	TURNERyxX
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/struct.sql.out b/sql/core/src/test/resources/sql-tests/results/struct.sql.out
index f294c5213d319..3b610edc47169 100644
--- a/sql/core/src/test/resources/sql-tests/results/struct.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/struct.sql.out
@@ -83,7 +83,7 @@ struct<ID:int,NST:string>
 -- !query
 SELECT ID, STRUCT(ST.C as STC, ST.D as STD).STD FROM tbl_x
 -- !query schema
-struct<ID:int,named_struct(STC, ST.C AS `C` AS `STC`, STD, ST.D AS `D` AS `STD`).STD:string>
+struct<ID:int,struct(ST.C AS `C` AS `STC`, ST.D AS `D` AS `STD`).STD:string>
 -- !query output
 1	delta
 2	eta
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out
index 1c335445114c7..e24538b9138ba 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out
@@ -103,7 +103,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d IN (SELECT t2d
                FROM   t2
-               ORDER  BY t2c
+               ORDER  BY t2c, t2d
                LIMIT 2)
 GROUP  BY t1b
 ORDER  BY t1b DESC NULLS FIRST
@@ -136,7 +136,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d NOT IN (SELECT t2d
                    FROM   t2
-                   ORDER  BY t2b DESC nulls first
+                   ORDER  BY t2b DESC nulls first, t2d
                    LIMIT 1)
 GROUP  BY t1b
 ORDER BY t1b NULLS last
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
index 1599634ff9efb..d703d4e9112e9 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
@@ -109,9 +109,9 @@ struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
 Expressions referencing the outer query are not supported outside of WHERE/HAVING clauses:
-Aggregate [min(outer(t2a#x)) AS min(outer())#x]
-+- SubqueryAlias `t3`
+Aggregate [min(outer(t2a#x)) AS min(outer(t2.`t2a`))#x]
++- SubqueryAlias t3
    +- Project [t3a#x, t3b#x, t3c#x]
-      +- SubqueryAlias `t3`
+      +- SubqueryAlias t3
          +- LocalRelation [t3a#x, t3b#x, t3c#x]
 ;
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out
index d5c27ade8e152..7b1fcddfdac7d 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out
@@ -118,7 +118,7 @@ struct<CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) + INTERVAL '2 days' AS TIME
 -- !query
 select cast('2017-12-11 09:30:00' as date) + interval 2 day
 -- !query schema
-struct<CAST(CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP) + INTERVAL '2 days' AS DATE):date>
+struct<CAST(2017-12-11 09:30:00 AS DATE) + INTERVAL '2 days':date>
 -- !query output
 2017-12-13
 
@@ -231,7 +231,7 @@ struct<CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) + INTERVAL '2 days' AS TIME
 -- !query
 select interval 2 day + cast('2017-12-11 09:30:00' as date)
 -- !query schema
-struct<CAST(CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP) + INTERVAL '2 days' AS DATE):date>
+struct<CAST(2017-12-11 09:30:00 AS DATE) + INTERVAL '2 days':date>
 -- !query output
 2017-12-13
 
@@ -344,6 +344,6 @@ struct<CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) - INTERVAL '2 days' AS TIME
 -- !query
 select cast('2017-12-11 09:30:00' as date) - interval 2 day
 -- !query schema
-struct<CAST(CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP) - INTERVAL '2 days' AS DATE):date>
+struct<CAST(2017-12-11 09:30:00 AS DATE) + (- INTERVAL '2 days'):date>
 -- !query output
 2017-12-09
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out
index f841adf89612e..e47decbd33920 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out
@@ -285,7 +285,7 @@ struct<month(CAST(1996-01-10 AS DATE)):int>
 -- !query
 SELECT day( '1996-01-10') FROM t
 -- !query schema
-struct<dayofmonth(CAST(1996-01-10 AS DATE)):int>
+struct<day(CAST(1996-01-10 AS DATE)):int>
 -- !query output
 10
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
index ed7ab5a342c12..d046ff249379f 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
@@ -85,7 +85,7 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_zip_with(various_maps.`decimal_map1`, various_maps.`decimal_map2`, lambdafunction(named_struct(NamePlaceholder(), k, NamePlaceholder(), v1, NamePlaceholder(), v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,0), decimal(36,35)].; line 1 pos 7
+cannot resolve 'map_zip_with(various_maps.`decimal_map1`, various_maps.`decimal_map2`, lambdafunction(struct(k, v1, v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,0), decimal(36,35)].; line 1 pos 7
 
 
 -- !query
@@ -113,7 +113,7 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_zip_with(various_maps.`decimal_map2`, various_maps.`int_map`, lambdafunction(named_struct(NamePlaceholder(), k, NamePlaceholder(), v1, NamePlaceholder(), v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,35), int].; line 1 pos 7
+cannot resolve 'map_zip_with(various_maps.`decimal_map2`, various_maps.`int_map`, lambdafunction(struct(k, v1, v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,35), int].; line 1 pos 7
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out
index 31353bdedc69f..b8c190beeae19 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out
@@ -107,7 +107,7 @@ SELECT '1' + cast('2017-12-11 09:30:00' as date)        FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), '1')' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, ''1'' is of string type.; line 1 pos 7
+cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
 
 
 -- !query
@@ -698,7 +698,7 @@ SELECT cast('2017-12-11 09:30:00' as date)        + '1' FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), '1')' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, ''1'' is of string type.; line 1 pos 7
+cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
 
 
 -- !query
@@ -790,7 +790,7 @@ SELECT cast('2017-12-11 09:30:00' as date)        - '1' FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_sub(CAST('2017-12-11 09:30:00' AS DATE), '1')' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, ''1'' is of string type.; line 1 pos 7
+cannot resolve 'date_sub(CAST('2017-12-11 09:30:00' AS DATE), CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
index 7b419c6702586..b4c052585ef7c 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
@@ -136,15 +136,16 @@ NULL
 -- !query
 select to_timestamp('2018-01-01', a) from t
 -- !query schema
-struct<to_timestamp('2018-01-01', t.`a`):timestamp>
+struct<>
 -- !query output
-NULL
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
 select to_unix_timestamp(a) from t
 -- !query schema
-struct<to_unix_timestamp(a, uuuu-MM-dd HH:mm:ss):bigint>
+struct<to_unix_timestamp(a, yyyy-MM-dd HH:mm:ss):bigint>
 -- !query output
 NULL
 
@@ -152,15 +153,16 @@ NULL
 -- !query
 select to_unix_timestamp('2018-01-01', a) from t
 -- !query schema
-struct<to_unix_timestamp(2018-01-01, a):bigint>
+struct<>
 -- !query output
-NULL
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
 select unix_timestamp(a) from t
 -- !query schema
-struct<unix_timestamp(a, uuuu-MM-dd HH:mm:ss):bigint>
+struct<unix_timestamp(a, yyyy-MM-dd HH:mm:ss):bigint>
 -- !query output
 NULL
 
@@ -168,15 +170,16 @@ NULL
 -- !query
 select unix_timestamp('2018-01-01', a) from t
 -- !query schema
-struct<unix_timestamp(2018-01-01, a):bigint>
+struct<>
 -- !query output
-NULL
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
 select from_unixtime(a) from t
 -- !query schema
-struct<from_unixtime(CAST(a AS BIGINT), uuuu-MM-dd HH:mm:ss):string>
+struct<from_unixtime(CAST(a AS BIGINT), yyyy-MM-dd HH:mm:ss):string>
 -- !query output
 NULL
 
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
index d65c56774eafd..76637bf578e6f 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
@@ -77,7 +77,7 @@ struct<max_324_78:float>
 -- !query
 SELECT stddev_pop(udf(b)) FROM aggtest
 -- !query schema
-struct<stddev_pop(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE)):double>
+struct<stddev_pop(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE)):double>
 -- !query output
 131.10703231895047
 
@@ -85,7 +85,7 @@ struct<stddev_pop(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE)):double>
 -- !query
 SELECT udf(stddev_samp(b)) FROM aggtest
 -- !query schema
-struct<CAST(udf(cast(stddev_samp(cast(b as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast(stddev_samp(ansi_cast(b as double)) as string)) AS DOUBLE):double>
 -- !query output
 151.38936080399804
 
@@ -93,7 +93,7 @@ struct<CAST(udf(cast(stddev_samp(cast(b as double)) as string)) AS DOUBLE):doubl
 -- !query
 SELECT var_pop(udf(b)) FROM aggtest
 -- !query schema
-struct<var_pop(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE)):double>
+struct<var_pop(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE)):double>
 -- !query output
 17189.053923482323
 
@@ -101,7 +101,7 @@ struct<var_pop(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE)):double>
 -- !query
 SELECT udf(var_samp(b)) FROM aggtest
 -- !query schema
-struct<CAST(udf(cast(var_samp(cast(b as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast(var_samp(ansi_cast(b as double)) as string)) AS DOUBLE):double>
 -- !query output
 22918.738564643096
 
@@ -109,7 +109,7 @@ struct<CAST(udf(cast(var_samp(cast(b as double)) as string)) AS DOUBLE):double>
 -- !query
 SELECT udf(stddev_pop(CAST(b AS Decimal(38,0)))) FROM aggtest
 -- !query schema
-struct<CAST(udf(cast(stddev_pop(cast(cast(b as decimal(38,0)) as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast(stddev_pop(ansi_cast(ansi_cast(b as decimal(38,0)) as double)) as string)) AS DOUBLE):double>
 -- !query output
 131.18117242958306
 
@@ -117,7 +117,7 @@ struct<CAST(udf(cast(stddev_pop(cast(cast(b as decimal(38,0)) as double)) as str
 -- !query
 SELECT stddev_samp(CAST(udf(b) AS Decimal(38,0))) FROM aggtest
 -- !query schema
-struct<stddev_samp(CAST(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<stddev_samp(CAST(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DECIMAL(38,0)) AS DOUBLE)):double>
 -- !query output
 151.47497042966097
 
@@ -125,7 +125,7 @@ struct<stddev_samp(CAST(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DECIMAL(38
 -- !query
 SELECT udf(var_pop(CAST(b AS Decimal(38,0)))) FROM aggtest
 -- !query schema
-struct<CAST(udf(cast(var_pop(cast(cast(b as decimal(38,0)) as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast(var_pop(ansi_cast(ansi_cast(b as decimal(38,0)) as double)) as string)) AS DOUBLE):double>
 -- !query output
 17208.5
 
@@ -133,7 +133,7 @@ struct<CAST(udf(cast(var_pop(cast(cast(b as decimal(38,0)) as double)) as string
 -- !query
 SELECT var_samp(udf(CAST(b AS Decimal(38,0)))) FROM aggtest
 -- !query schema
-struct<var_samp(CAST(CAST(udf(cast(cast(b as decimal(38,0)) as string)) AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<var_samp(CAST(CAST(udf(ansi_cast(ansi_cast(b as decimal(38,0)) as string)) AS DECIMAL(38,0)) AS DOUBLE)):double>
 -- !query output
 22944.666666666668
 
@@ -141,7 +141,7 @@ struct<var_samp(CAST(CAST(udf(cast(cast(b as decimal(38,0)) as string)) AS DECIM
 -- !query
 SELECT udf(var_pop(1.0)), var_samp(udf(2.0))
 -- !query schema
-struct<CAST(udf(cast(var_pop(cast(1.0 as double)) as string)) AS DOUBLE):double,var_samp(CAST(CAST(udf(cast(2.0 as string)) AS DECIMAL(2,1)) AS DOUBLE)):double>
+struct<CAST(udf(ansi_cast(var_pop(ansi_cast(1.0 as double)) as string)) AS DOUBLE):double,var_samp(CAST(CAST(udf(ansi_cast(2.0 as string)) AS DECIMAL(2,1)) AS DOUBLE)):double>
 -- !query output
 0.0	NaN
 
@@ -149,7 +149,7 @@ struct<CAST(udf(cast(var_pop(cast(1.0 as double)) as string)) AS DOUBLE):double,
 -- !query
 SELECT stddev_pop(udf(CAST(3.0 AS Decimal(38,0)))), stddev_samp(CAST(udf(4.0) AS Decimal(38,0)))
 -- !query schema
-struct<stddev_pop(CAST(CAST(udf(cast(cast(3.0 as decimal(38,0)) as string)) AS DECIMAL(38,0)) AS DOUBLE)):double,stddev_samp(CAST(CAST(CAST(udf(cast(4.0 as string)) AS DECIMAL(2,1)) AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<stddev_pop(CAST(CAST(udf(ansi_cast(ansi_cast(3.0 as decimal(38,0)) as string)) AS DECIMAL(38,0)) AS DOUBLE)):double,stddev_samp(CAST(CAST(CAST(udf(ansi_cast(4.0 as string)) AS DECIMAL(2,1)) AS DECIMAL(38,0)) AS DOUBLE)):double>
 -- !query output
 0.0	NaN
 
@@ -157,7 +157,7 @@ struct<stddev_pop(CAST(CAST(udf(cast(cast(3.0 as decimal(38,0)) as string)) AS D
 -- !query
 select sum(udf(CAST(null AS int))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(cast(cast(null as int) as string)) AS INT)):bigint>
+struct<sum(CAST(udf(ansi_cast(ansi_cast(null as int) as string)) AS INT)):bigint>
 -- !query output
 NULL
 
@@ -165,7 +165,7 @@ NULL
 -- !query
 select sum(udf(CAST(null AS long))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(cast(cast(null as bigint) as string)) AS BIGINT)):bigint>
+struct<sum(CAST(udf(ansi_cast(ansi_cast(null as bigint) as string)) AS BIGINT)):bigint>
 -- !query output
 NULL
 
@@ -173,7 +173,7 @@ NULL
 -- !query
 select sum(udf(CAST(null AS Decimal(38,0)))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(cast(cast(null as decimal(38,0)) as string)) AS DECIMAL(38,0))):decimal(38,0)>
+struct<sum(CAST(udf(ansi_cast(ansi_cast(null as decimal(38,0)) as string)) AS DECIMAL(38,0))):decimal(38,0)>
 -- !query output
 NULL
 
@@ -181,7 +181,7 @@ NULL
 -- !query
 select sum(udf(CAST(null AS DOUBLE))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(cast(cast(null as double) as string)) AS DOUBLE)):double>
+struct<sum(CAST(udf(ansi_cast(ansi_cast(null as double) as string)) AS DOUBLE)):double>
 -- !query output
 NULL
 
@@ -189,7 +189,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS int))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(null as int) as string)) AS INT)):double>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(null as int) as string)) AS INT)):double>
 -- !query output
 NULL
 
@@ -197,7 +197,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS long))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(null as bigint) as string)) AS BIGINT)):double>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(null as bigint) as string)) AS BIGINT)):double>
 -- !query output
 NULL
 
@@ -205,7 +205,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS Decimal(38,0)))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(null as decimal(38,0)) as string)) AS DECIMAL(38,0))):decimal(38,4)>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(null as decimal(38,0)) as string)) AS DECIMAL(38,0))):decimal(38,4)>
 -- !query output
 NULL
 
@@ -213,7 +213,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS DOUBLE))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(null as double) as string)) AS DOUBLE)):double>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(null as double) as string)) AS DOUBLE)):double>
 -- !query output
 NULL
 
@@ -221,7 +221,7 @@ NULL
 -- !query
 select sum(CAST(udf('NaN') AS DOUBLE)) from range(1,4)
 -- !query schema
-struct<sum(CAST(CAST(udf(cast(NaN as string)) AS STRING) AS DOUBLE)):double>
+struct<sum(CAST(CAST(udf(ansi_cast(NaN as string)) AS STRING) AS DOUBLE)):double>
 -- !query output
 NaN
 
@@ -229,7 +229,7 @@ NaN
 -- !query
 select avg(CAST(udf('NaN') AS DOUBLE)) from range(1,4)
 -- !query schema
-struct<avg(CAST(CAST(udf(cast(NaN as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(CAST(udf(ansi_cast(NaN as string)) AS STRING) AS DOUBLE)):double>
 -- !query output
 NaN
 
@@ -238,7 +238,7 @@ NaN
 SELECT avg(CAST(udf(x) AS DOUBLE)), var_pop(CAST(udf(x) AS DOUBLE))
 FROM (VALUES ('Infinity'), ('1')) v(x)
 -- !query schema
-struct<avg(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double>
 -- !query output
 Infinity	NaN
 
@@ -247,7 +247,7 @@ Infinity	NaN
 SELECT avg(CAST(udf(x) AS DOUBLE)), var_pop(CAST(udf(x) AS DOUBLE))
 FROM (VALUES ('Infinity'), ('Infinity')) v(x)
 -- !query schema
-struct<avg(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double>
 -- !query output
 Infinity	NaN
 
@@ -256,7 +256,7 @@ Infinity	NaN
 SELECT avg(CAST(udf(x) AS DOUBLE)), var_pop(CAST(udf(x) AS DOUBLE))
 FROM (VALUES ('-Infinity'), ('Infinity')) v(x)
 -- !query schema
-struct<avg(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double>
 -- !query output
 NaN	NaN
 
@@ -265,7 +265,7 @@ NaN	NaN
 SELECT avg(udf(CAST(x AS DOUBLE))), udf(var_pop(CAST(x AS DOUBLE)))
 FROM (VALUES (100000003), (100000004), (100000006), (100000007)) v(x)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(x as double) as string)) AS DOUBLE)):double,CAST(udf(cast(var_pop(cast(x as double)) as string)) AS DOUBLE):double>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(x as double) as string)) AS DOUBLE)):double,CAST(udf(ansi_cast(var_pop(ansi_cast(x as double)) as string)) AS DOUBLE):double>
 -- !query output
 1.00000005E8	2.5
 
@@ -274,7 +274,7 @@ struct<avg(CAST(udf(cast(cast(x as double) as string)) AS DOUBLE)):double,CAST(u
 SELECT avg(udf(CAST(x AS DOUBLE))), udf(var_pop(CAST(x AS DOUBLE)))
 FROM (VALUES (7000000000005), (7000000000007)) v(x)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(x as double) as string)) AS DOUBLE)):double,CAST(udf(cast(var_pop(cast(x as double)) as string)) AS DOUBLE):double>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(x as double) as string)) AS DOUBLE)):double,CAST(udf(ansi_cast(var_pop(ansi_cast(x as double)) as string)) AS DOUBLE):double>
 -- !query output
 7.000000000006E12	1.0
 
@@ -282,7 +282,7 @@ struct<avg(CAST(udf(cast(cast(x as double) as string)) AS DOUBLE)):double,CAST(u
 -- !query
 SELECT udf(covar_pop(b, udf(a))), covar_samp(udf(b), a) FROM aggtest
 -- !query schema
-struct<CAST(udf(cast(covar_pop(cast(b as double), cast(cast(udf(cast(a as string)) as int) as double)) as string)) AS DOUBLE):double,covar_samp(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE), CAST(a AS DOUBLE)):double>
+struct<CAST(udf(ansi_cast(covar_pop(ansi_cast(b as double), ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as double)) as string)) AS DOUBLE):double,covar_samp(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE), CAST(a AS DOUBLE)):double>
 -- !query output
 653.6289553875104	871.5052738500139
 
@@ -290,7 +290,7 @@ struct<CAST(udf(cast(covar_pop(cast(b as double), cast(cast(udf(cast(a as string
 -- !query
 SELECT corr(b, udf(a)) FROM aggtest
 -- !query schema
-struct<corr(CAST(b AS DOUBLE), CAST(CAST(udf(cast(a as string)) AS INT) AS DOUBLE)):double>
+struct<corr(CAST(b AS DOUBLE), CAST(CAST(udf(ansi_cast(a as string)) AS INT) AS DOUBLE)):double>
 -- !query output
 0.1396345165178734
 
@@ -315,7 +315,7 @@ struct<cnt_4:bigint>
 select ten, udf(count(*)), sum(udf(four)) from onek
 group by ten order by ten
 -- !query schema
-struct<ten:int,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint,sum(CAST(udf(cast(four as string)) AS INT)):bigint>
+struct<ten:int,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint,sum(CAST(udf(ansi_cast(four as string)) AS INT)):bigint>
 -- !query output
 0	100	100
 1	100	200
@@ -333,7 +333,7 @@ struct<ten:int,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint,sum(CAST(udf
 select ten, count(udf(four)), udf(sum(DISTINCT four)) from onek
 group by ten order by ten
 -- !query schema
-struct<ten:int,count(CAST(udf(cast(four as string)) AS INT)):bigint,CAST(udf(cast(sum(distinct cast(four as bigint)) as string)) AS BIGINT):bigint>
+struct<ten:int,count(CAST(udf(ansi_cast(four as string)) AS INT)):bigint,CAST(udf(ansi_cast(sum(distinct ansi_cast(four as bigint)) as string)) AS BIGINT):bigint>
 -- !query output
 0	100	2
 1	100	4
@@ -352,7 +352,7 @@ select ten, udf(sum(distinct four)) from onek a
 group by ten
 having exists (select 1 from onek b where udf(sum(distinct a.four)) = b.four)
 -- !query schema
-struct<ten:int,CAST(udf(cast(sum(distinct cast(four as bigint)) as string)) AS BIGINT):bigint>
+struct<ten:int,CAST(udf(ansi_cast(sum(distinct ansi_cast(four as bigint)) as string)) AS BIGINT):bigint>
 -- !query output
 0	2
 2	2
@@ -372,8 +372,8 @@ struct<>
 org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
-Expression in where clause: [(sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT)) = CAST(CAST(udf(cast(four as string)) AS INT) AS BIGINT))]
-Invalid expressions: [sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT))];
+Expression in where clause: [(sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT)) = CAST(CAST(udf(ansi_cast(four as string)) AS INT) AS BIGINT))]
+Invalid expressions: [sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT))];
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out
index c10fe9b51dd72..d4941d0a0b768 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out
@@ -161,7 +161,7 @@ true	true	false	NULL	false	true
 -- !query
 select min(udf(unique1)) from tenk1
 -- !query schema
-struct<min(CAST(udf(cast(unique1 as string)) AS INT)):int>
+struct<min(CAST(udf(ansi_cast(unique1 as string)) AS INT)):int>
 -- !query output
 0
 
@@ -169,7 +169,7 @@ struct<min(CAST(udf(cast(unique1 as string)) AS INT)):int>
 -- !query
 select udf(max(unique1)) from tenk1
 -- !query schema
-struct<CAST(udf(cast(max(unique1) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(max(unique1) as string)) AS INT):int>
 -- !query output
 9999
 
@@ -217,7 +217,7 @@ struct<min(tenthous):int>
 -- !query
 select distinct max(udf(unique2)) from tenk1
 -- !query schema
-struct<max(CAST(udf(cast(unique2 as string)) AS INT)):int>
+struct<max(CAST(udf(ansi_cast(unique2 as string)) AS INT)):int>
 -- !query output
 9999
 
@@ -241,7 +241,7 @@ struct<max(unique2):int>
 -- !query
 select udf(max(udf(unique2))) from tenk1 order by udf(max(unique2))+1
 -- !query schema
-struct<CAST(udf(cast(max(cast(udf(cast(unique2 as string)) as int)) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(max(ansi_cast(udf(ansi_cast(unique2 as string)) as int)) as string)) AS INT):int>
 -- !query output
 9999
 
@@ -249,7 +249,7 @@ struct<CAST(udf(cast(max(cast(udf(cast(unique2 as string)) as int)) as string))
 -- !query
 select t1.max_unique2, udf(g) from (select max(udf(unique2)) as max_unique2 FROM tenk1) t1 LATERAL VIEW explode(array(1,2,3)) t2 AS g order by g desc
 -- !query schema
-struct<max_unique2:int,CAST(udf(cast(g as string)) AS INT):int>
+struct<max_unique2:int,CAST(udf(ansi_cast(g as string)) AS INT):int>
 -- !query output
 9999	3
 9999	2
@@ -259,6 +259,6 @@ struct<max_unique2:int,CAST(udf(cast(g as string)) AS INT):int>
 -- !query
 select udf(max(100)) from tenk1
 -- !query schema
-struct<CAST(udf(cast(max(100) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(max(100) as string)) AS INT):int>
 -- !query output
 100
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out
index 04c4f54b02a3e..6c733e916d734 100755
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out
@@ -176,7 +176,7 @@ struct<None:string,NULL on no matches:int>
 -- !query
 SELECT CASE WHEN udf(1=0) THEN 1/0 WHEN 1=1 THEN 1 ELSE 2/0 END
 -- !query schema
-struct<CASE WHEN CAST(udf(cast((1 = 0) as string)) AS BOOLEAN) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
+struct<CASE WHEN CAST(udf(ansi_cast((1 = 0) as string)) AS BOOLEAN) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
 -- !query output
 1.0
 
@@ -184,7 +184,7 @@ struct<CASE WHEN CAST(udf(cast((1 = 0) as string)) AS BOOLEAN) THEN (CAST(1 AS D
 -- !query
 SELECT CASE 1 WHEN 0 THEN 1/udf(0) WHEN 1 THEN 1 ELSE 2/0 END
 -- !query schema
-struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(cast(0 as string)) AS INT) AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
+struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(ansi_cast(0 as string)) AS INT) AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
 -- !query output
 1.0
 
@@ -192,7 +192,7 @@ struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(cast(0 as strin
 -- !query
 SELECT CASE WHEN i > 100 THEN udf(1/0) ELSE udf(0) END FROM case_tbl
 -- !query schema
-struct<CASE WHEN (i > 100) THEN CAST(udf(cast((cast(1 as double) / cast(0 as double)) as string)) AS DOUBLE) ELSE CAST(CAST(udf(cast(0 as string)) AS INT) AS DOUBLE) END:double>
+struct<CASE WHEN (i > 100) THEN CAST(udf(ansi_cast((ansi_cast(1 as double) / ansi_cast(0 as double)) as string)) AS DOUBLE) ELSE CAST(CAST(udf(ansi_cast(0 as string)) AS INT) AS DOUBLE) END:double>
 -- !query output
 0.0
 0.0
@@ -203,7 +203,7 @@ struct<CASE WHEN (i > 100) THEN CAST(udf(cast((cast(1 as double) / cast(0 as dou
 -- !query
 SELECT CASE 'a' WHEN 'a' THEN udf(1) ELSE udf(2) END
 -- !query schema
-struct<CASE WHEN (a = a) THEN CAST(udf(cast(1 as string)) AS INT) ELSE CAST(udf(cast(2 as string)) AS INT) END:int>
+struct<CASE WHEN (a = a) THEN CAST(udf(ansi_cast(1 as string)) AS INT) ELSE CAST(udf(ansi_cast(2 as string)) AS INT) END:int>
 -- !query output
 1
 
@@ -294,7 +294,7 @@ struct<i:int,f:double>
 SELECT udf(COALESCE(a.f, b.i, b.j))
   FROM CASE_TBL a, CASE2_TBL b
 -- !query schema
-struct<CAST(udf(cast(coalesce(f, cast(i as double), cast(j as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast(coalesce(f, ansi_cast(i as double), ansi_cast(j as double)) as string)) AS DOUBLE):double>
 -- !query output
 -30.3
 -30.3
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
index f113aee6d3b51..188b57ffd58d5 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
@@ -243,7 +243,7 @@ struct<>
 SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t)
   FROM J1_TBL AS tx
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -262,7 +262,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as str
 SELECT udf(udf('')) AS `xxx`, udf(udf(i)), udf(j), udf(t)
   FROM J1_TBL tx
 -- !query schema
-struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string>
+struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -281,7 +281,7 @@ struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string))
 SELECT udf('') AS `xxx`, a, udf(udf(b)), c
   FROM J1_TBL AS t1 (a, b, c)
 -- !query schema
-struct<xxx:string,a:int,CAST(udf(cast(cast(udf(cast(b as string)) as int) as string)) AS INT):int,c:string>
+struct<xxx:string,a:int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(b as string)) as int) as string)) AS INT):int,c:string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -300,7 +300,7 @@ struct<xxx:string,a:int,CAST(udf(cast(cast(udf(cast(b as string)) as int) as str
 SELECT udf('') AS `xxx`, udf(a), udf(b), udf(udf(c))
   FROM J1_TBL t1 (a, b, c)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(c as string)) as string) as string)) AS STRING):string>
+struct<xxx:string,CAST(udf(ansi_cast(a as string)) AS INT):int,CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(c as string)) as string) as string)) AS STRING):string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -319,7 +319,7 @@ struct<xxx:string,CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(b as str
 SELECT udf('') AS `xxx`, udf(a), b, udf(c), udf(d), e
   FROM J1_TBL t1 (a, b, c), J2_TBL t2 (d, e)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(a as string)) AS INT):int,b:int,CAST(udf(cast(c as string)) AS STRING):string,CAST(udf(cast(d as string)) AS INT):int,e:int>
+struct<xxx:string,CAST(udf(ansi_cast(a as string)) AS INT):int,b:int,CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(d as string)) AS INT):int,e:int>
 -- !query output
 	0	NULL	zero	0	NULL
 	0	NULL	zero	1	-1
@@ -536,14 +536,14 @@ SELECT udf('') AS `xxx`, udf(i) AS i, udf(k), udf(t) AS t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i' is ambiguous, could be: default.j1_tbl.i, default.j2_tbl.i.; line 1 pos 29
+Reference 'i' is ambiguous, could be: spark_catalog.default.j1_tbl.i, spark_catalog.default.j2_tbl.i.; line 1 pos 29
 
 
 -- !query
 SELECT udf('') AS `xxx`, udf(t1.i) AS i, udf(k), udf(t)
   FROM J1_TBL t1 CROSS JOIN J2_TBL t2
 -- !query schema
-struct<xxx:string,i:int,CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string>
+struct<xxx:string,i:int,CAST(udf(ansi_cast(k as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string>
 -- !query output
 	0	-1	zero
 	0	-3	zero
@@ -651,7 +651,7 @@ SELECT udf(udf('')) AS `xxx`, udf(udf(ii)) AS ii, udf(udf(tt)) AS tt, udf(udf(kk
   FROM (J1_TBL CROSS JOIN J2_TBL)
     AS tx (ii, jj, tt, ii2, kk)
 -- !query schema
-struct<xxx:string,ii:int,tt:string,CAST(udf(cast(cast(udf(cast(kk as string)) as int) as string)) AS INT):int>
+struct<xxx:string,ii:int,tt:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(kk as string)) as int) as string)) AS INT):int>
 -- !query output
 	0	zero	-1
 	0	zero	-3
@@ -758,7 +758,7 @@ struct<xxx:string,ii:int,tt:string,CAST(udf(cast(cast(udf(cast(kk as string)) as
 SELECT udf('') AS `xxx`, udf(udf(j1_tbl.i)), udf(j), udf(t), udf(a.i), udf(a.k), udf(b.i),  udf(b.k)
   FROM J1_TBL CROSS JOIN J2_TBL a CROSS JOIN J2_TBL b
 -- !query schema
-struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	0	NULL	0	NULL
 	0	NULL	zero	0	NULL	1	-1
@@ -1657,7 +1657,7 @@ struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string))
 SELECT udf('') AS `xxx`, udf(i) AS i, udf(j), udf(t) AS t, udf(k)
   FROM J1_TBL INNER JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,i:int,CAST(udf(cast(j as string)) AS INT):int,t:string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,i:int,CAST(udf(ansi_cast(j as string)) AS INT):int,t:string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1672,7 +1672,7 @@ struct<xxx:string,i:int,CAST(udf(cast(j as string)) AS INT):int,t:string,CAST(ud
 SELECT udf(udf('')) AS `xxx`, udf(i), udf(j) AS j, udf(t), udf(k) AS k
   FROM J1_TBL JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,j:int,CAST(udf(cast(t as string)) AS STRING):string,k:int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,j:int,CAST(udf(ansi_cast(t as string)) AS STRING):string,k:int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1703,7 +1703,7 @@ struct<xxx:string,a:int,b:int,c:string,d:int>
 SELECT udf(udf('')) AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL NATURAL JOIN J2_TBL
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1718,7 +1718,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as str
 SELECT udf('') AS `xxx`, udf(udf(udf(a))) AS a, udf(b), udf(c), udf(d)
   FROM J1_TBL t1 (a, b, c) NATURAL JOIN J2_TBL t2 (a, d)
 -- !query schema
-struct<xxx:string,a:int,CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(c as string)) AS STRING):string,CAST(udf(cast(d as string)) AS INT):int>
+struct<xxx:string,a:int,CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(d as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1733,7 +1733,7 @@ struct<xxx:string,a:int,CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(c
 SELECT udf('') AS `xxx`, udf(udf(a)), udf(udf(b)), udf(udf(c)) AS c, udf(udf(udf(d))) AS d
   FROM J1_TBL t1 (a, b, c) NATURAL JOIN J2_TBL t2 (d, a)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(b as string)) as int) as string)) AS INT):int,c:string,d:int>
+struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(b as string)) as int) as string)) AS INT):int,c:string,d:int>
 -- !query output
 	0	NULL	zero	NULL
 	2	3	two	2
@@ -1744,7 +1744,7 @@ struct<xxx:string,CAST(udf(cast(cast(udf(cast(a as string)) as int) as string))
 SELECT udf('') AS `xxx`, udf(J1_TBL.i), udf(udf(J1_TBL.j)), udf(J1_TBL.t), udf(J2_TBL.i), udf(J2_TBL.k)
   FROM J1_TBL JOIN J2_TBL ON (udf(J1_TBL.i) = J2_TBL.i)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(j as string)) as int) as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(j as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	0	NULL
 	1	4	one	1	-1
@@ -1759,7 +1759,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(cast(udf
 SELECT udf('') AS `xxx`, udf(udf(J1_TBL.i)), udf(udf(J1_TBL.j)), udf(udf(J1_TBL.t)), J2_TBL.i, J2_TBL.k
   FROM J1_TBL JOIN J2_TBL ON (J1_TBL.i = udf(J2_TBL.k))
 -- !query schema
-struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(j as string)) as int) as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(t as string)) as string) as string)) AS STRING):string,i:int,k:int>
+struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(j as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(t as string)) as string) as string)) AS STRING):string,i:int,k:int>
 -- !query output
 	0	NULL	zero	NULL	0
 	2	3	two	2	2
@@ -1770,7 +1770,7 @@ struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string))
 SELECT udf('') AS `xxx`, udf(J1_TBL.i), udf(J1_TBL.j), udf(J1_TBL.t), udf(J2_TBL.i), udf(J2_TBL.k)
   FROM J1_TBL JOIN J2_TBL ON (udf(J1_TBL.i) <= udf(udf(J2_TBL.k)))
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	2	2
 	0	NULL	zero	2	4
@@ -1788,7 +1788,7 @@ SELECT udf(udf('')) AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL LEFT OUTER JOIN J2_TBL USING (i)
   ORDER BY udf(udf(i)), udf(k), udf(t)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	NULL	NULL	null	NULL
 	NULL	0	zero	NULL
@@ -1810,7 +1810,7 @@ SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL LEFT JOIN J2_TBL USING (i)
   ORDER BY udf(i), udf(udf(k)), udf(t)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	NULL	NULL	null	NULL
 	NULL	0	zero	NULL
@@ -1831,7 +1831,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as str
 SELECT udf('') AS `xxx`, udf(udf(i)), udf(j), udf(t), udf(k)
   FROM J1_TBL RIGHT OUTER JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1848,7 +1848,7 @@ struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string))
 SELECT udf('') AS `xxx`, udf(i), udf(udf(j)), udf(t), udf(k)
   FROM J1_TBL RIGHT JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(j as string)) as int) as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(j as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1866,7 +1866,7 @@ SELECT udf('') AS `xxx`, udf(i), udf(j), udf(udf(t)), udf(k)
   FROM J1_TBL FULL OUTER JOIN J2_TBL USING (i)
   ORDER BY udf(udf(i)), udf(k), udf(t)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(t as string)) as string) as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(t as string)) as string) as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	NULL	NULL	NULL	NULL
 	NULL	NULL	null	NULL
@@ -1890,7 +1890,7 @@ SELECT udf('') AS `xxx`, udf(i), udf(j), t, udf(udf(k))
   FROM J1_TBL FULL JOIN J2_TBL USING (i)
   ORDER BY udf(udf(i)), udf(k), udf(udf(t))
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,t:string,CAST(udf(cast(cast(udf(cast(k as string)) as int) as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,t:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(k as string)) as int) as string)) AS INT):int>
 -- !query output
 	NULL	NULL	NULL	NULL
 	NULL	NULL	null	NULL
@@ -1913,7 +1913,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as str
 SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t), udf(udf(k))
   FROM J1_TBL LEFT JOIN J2_TBL USING (i) WHERE (udf(k) = 1)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(cast(udf(cast(k as string)) as int) as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(k as string)) as int) as string)) AS INT):int>
 -- !query output
 
 
@@ -1922,7 +1922,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as str
 SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL LEFT JOIN J2_TBL USING (i) WHERE (udf(udf(i)) = udf(1))
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	1	4	one	-1
 
@@ -2052,7 +2052,7 @@ FULL JOIN
 (SELECT * FROM t3) s3
 USING (name)
 -- !query schema
-struct<CAST(udf(cast(name as string)) AS STRING):string,CAST(udf(cast(cast(udf(cast(n as string)) as int) as string)) AS INT):int,CAST(udf(cast(n as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(name as string)) AS STRING):string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(n as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(n as string)) AS INT):int>
 -- !query output
 bb	12	13
 cc	22	23
@@ -2138,7 +2138,7 @@ NATURAL FULL JOIN
     (SELECT name, udf(udf(n)) as s3_n FROM t3) as s3
   ) ss2
 -- !query schema
-struct<name:string,CAST(udf(cast(cast(udf(cast(s1_n as string)) as int) as string)) AS INT):int,CAST(udf(cast(s2_n as string)) AS INT):int,CAST(udf(cast(s3_n as string)) AS INT):int>
+struct<name:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(s1_n as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(s2_n as string)) AS INT):int,CAST(udf(ansi_cast(s3_n as string)) AS INT):int>
 -- !query output
 bb	11	12	13
 cc	NULL	22	23
@@ -2171,7 +2171,7 @@ FULL JOIN
   (SELECT name, 2 as s2_n FROM t2) as s2
 ON (udf(udf(s1_n)) = udf(s2_n))
 -- !query schema
-struct<name:string,CAST(udf(cast(s1_n as string)) AS INT):int,name:string,CAST(udf(cast(cast(udf(cast(s2_n as string)) as int) as string)) AS INT):int>
+struct<name:string,CAST(udf(ansi_cast(s1_n as string)) AS INT):int,name:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(s2_n as string)) as int) as string)) AS INT):int>
 -- !query output
 NULL	NULL	bb	2
 NULL	NULL	cc	2
@@ -2202,7 +2202,7 @@ struct<>
 -- !query
 select udf(udf(x1)), udf(x2) from x
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(x1 as string)) as int) as string)) AS INT):int,CAST(udf(cast(x2 as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(x1 as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(x2 as string)) AS INT):int>
 -- !query output
 1	11
 2	22
@@ -2214,7 +2214,7 @@ struct<CAST(udf(cast(cast(udf(cast(x1 as string)) as int) as string)) AS INT):in
 -- !query
 select udf(y1), udf(udf(y2)) from y
 -- !query schema
-struct<CAST(udf(cast(y1 as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(y2 as string)) as int) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(y1 as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(y2 as string)) as int) as string)) AS INT):int>
 -- !query output
 1	111
 2	222
@@ -2336,7 +2336,7 @@ select udf(udf(count(*))) from tenk1 a where udf(udf(unique1)) in
   (select udf(unique1) from tenk1 b join tenk1 c using (unique1)
    where udf(udf(b.unique2)) = udf(42))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(count(1) as string)) as bigint) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(count(1) as string)) as bigint) as string)) AS BIGINT):bigint>
 -- !query output
 1
 
@@ -2347,7 +2347,7 @@ select udf(count(*)) from tenk1 x where
   udf(x.unique1) = 0 and
   udf(x.unique1) in (select aa.f1 from int4_tbl aa,float8_tbl bb where aa.f1=udf(udf(bb.f1)))
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1
 
@@ -2358,7 +2358,7 @@ select udf(udf(count(*))) from tenk1 x where
   udf(x.unique1) = 0 and
   udf(udf(x.unique1)) in (select udf(aa.f1) from int4_tbl aa,float8_tbl bb where udf(aa.f1)=udf(udf(bb.f1)))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(count(1) as string)) as bigint) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(count(1) as string)) as bigint) as string)) AS BIGINT):bigint>
 -- !query output
 1
 
@@ -2388,7 +2388,7 @@ from
   tenk1 t5
 where udf(t4.thousand) = udf(t5.unique1) and udf(udf(ss.x1)) = t4.tenthous and udf(ss.x2) = udf(udf(t5.stringu1))
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1000
 
@@ -2400,7 +2400,7 @@ select udf(a.f1), udf(b.f1), udf(t.thousand), udf(t.tenthous) from
   (select udf(sum(udf(f1))) as f1 from int4_tbl i4b) b
 where b.f1 = udf(t.thousand) and udf(a.f1) = udf(b.f1) and udf((udf(a.f1)+udf(b.f1)+999)) = udf(udf(t.tenthous))
 -- !query schema
-struct<CAST(udf(cast(f1 as string)) AS BIGINT):bigint,CAST(udf(cast(f1 as string)) AS BIGINT):bigint,CAST(udf(cast(thousand as string)) AS INT):int,CAST(udf(cast(tenthous as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(f1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(f1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(thousand as string)) AS INT):int,CAST(udf(ansi_cast(tenthous as string)) AS INT):int>
 -- !query output
 
 
@@ -2441,7 +2441,7 @@ select udf(count(*)) from
   (select * from tenk1 y order by udf(y.unique2)) y
   on udf(x.thousand) = y.unique2 and x.twothousand = udf(y.hundred) and x.fivethous = y.unique2
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 10000
 
@@ -2530,7 +2530,7 @@ struct<tt1_id:int,joincol:int,tt2_id:int,joincol:int>
 select udf(count(*)) from tenk1 a, tenk1 b
   where udf(a.hundred) = b.thousand and udf(udf((b.fivethous % 10)) < 10)
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 100000
 
@@ -2727,7 +2727,7 @@ from tenk1 a left join tenk1 b on a.unique2 = udf(b.tenthous)
 where udf(a.unique1) = 42 and
       ((udf(b.unique2) is null and udf(a.ten) = 2) or udf(udf(b.hundred)) = udf(udf(3)))
 -- !query schema
-struct<CAST(udf(cast(unique2 as string)) AS INT):int,CAST(udf(cast(ten as string)) AS INT):int,CAST(udf(cast(tenthous as string)) AS INT):int,CAST(udf(cast(unique2 as string)) AS INT):int,CAST(udf(cast(hundred as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CAST(udf(ansi_cast(ten as string)) AS INT):int,CAST(udf(ansi_cast(tenthous as string)) AS INT):int,CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CAST(udf(ansi_cast(hundred as string)) AS INT):int>
 -- !query output
 
 
@@ -2761,7 +2761,7 @@ select udf(t1.q2), udf(count(t2.*))
 from int8_tbl t1 left join int8_tbl t2 on (udf(udf(t1.q2)) = t2.q1)
 group by udf(t1.q2) order by 1
 -- !query schema
-struct<CAST(udf(cast(q2 as string)) AS BIGINT):bigint,CAST(udf(cast(count(q1, q2) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(q2 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(count(q1, q2) as string)) AS BIGINT):bigint>
 -- !query output
 -4567890123456789	0
 123	2
@@ -2774,7 +2774,7 @@ select udf(udf(t1.q2)), udf(count(t2.*))
 from int8_tbl t1 left join (select * from int8_tbl) t2 on (udf(udf(t1.q2)) = udf(t2.q1))
 group by udf(udf(t1.q2)) order by 1
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(q2 as string)) as bigint) as string)) AS BIGINT):bigint,CAST(udf(cast(count(q1, q2) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(q2 as string)) as bigint) as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(count(q1, q2) as string)) AS BIGINT):bigint>
 -- !query output
 -4567890123456789	0
 123	2
@@ -2789,7 +2789,7 @@ from int8_tbl t1 left join
   on (udf(t1.q2) = udf(t2.q1))
 group by t1.q2 order by 1
 -- !query schema
-struct<q2:bigint,CAST(udf(cast(cast(udf(cast(count(q1, q2) as string)) as bigint) as string)) AS BIGINT):bigint>
+struct<q2:bigint,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(count(q1, q2) as string)) as bigint) as string)) AS BIGINT):bigint>
 -- !query output
 -4567890123456789	0
 123	2
@@ -2838,7 +2838,7 @@ from c left join
   on (udf(udf(c.a)) = udf(ss.code))
 order by c.name
 -- !query schema
-struct<CAST(udf(cast(name as string)) AS STRING):string,CAST(udf(cast(code as string)) AS STRING):string,CAST(udf(cast(b_cnt as string)) AS BIGINT):bigint,CAST(udf(cast(const as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(name as string)) AS STRING):string,CAST(udf(ansi_cast(code as string)) AS STRING):string,CAST(udf(ansi_cast(b_cnt as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(const as string)) AS INT):int>
 -- !query output
 A	p	2	-1
 B	q	0	-1
@@ -2884,7 +2884,7 @@ LEFT JOIN
 ) sub2
 ON sub1.key1 = udf(udf(sub2.key3))
 -- !query schema
-struct<key1:int,key3:int,CAST(udf(cast(value2 as string)) AS INT):int,value3:int>
+struct<key1:int,key3:int,CAST(udf(ansi_cast(value2 as string)) AS INT):int,value3:int>
 -- !query output
 1	1	1	1
 
@@ -2898,7 +2898,7 @@ SELECT udf(qq), udf(udf(unique1))
   USING (qq)
   INNER JOIN tenk1 c ON udf(qq) = udf(unique2)
 -- !query schema
-struct<CAST(udf(cast(qq as string)) AS BIGINT):bigint,CAST(udf(cast(cast(udf(cast(unique1 as string)) as int) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(qq as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(unique1 as string)) as int) as string)) AS INT):int>
 -- !query output
 123	4596
 123	4596
@@ -2948,7 +2948,7 @@ from nt3 as nt3
     on udf(ss2.id) = nt3.nt2_id
 where udf(nt3.id) = 1 and udf(ss2.b3)
 -- !query schema
-struct<CAST(udf(cast(id as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(id as string)) AS INT):int>
 -- !query output
 1
 
@@ -3008,7 +3008,7 @@ select udf(count(*)) from
   left join tenk1 c on udf(a.unique2) = udf(b.unique1) and udf(c.thousand) = udf(udf(a.thousand))
   join int4_tbl on udf(b.thousand) = f1
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 10
 
@@ -3021,7 +3021,7 @@ select udf(b.unique1) from
   right join int4_tbl i2 on udf(udf(i2.f1)) = udf(b.tenthous)
   order by udf(1)
 -- !query schema
-struct<CAST(udf(cast(unique1 as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(unique1 as string)) AS INT):int>
 -- !query output
 NULL
 NULL
@@ -3039,7 +3039,7 @@ select * from
 where udf(fault) = udf(122)
 order by udf(fault)
 -- !query schema
-struct<CAST(udf(cast(unique1 as string)) AS INT):int,CAST(udf(cast(q1 as string)) AS BIGINT):bigint,fault:bigint>
+struct<CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint,fault:bigint>
 -- !query output
 NULL	123	122
 
@@ -3049,7 +3049,7 @@ select udf(q1), udf(unique2), udf(thousand), udf(hundred)
   from int8_tbl a left join tenk1 b on udf(q1) = udf(unique2)
   where udf(coalesce(thousand,123)) = udf(q1) and udf(q1) = udf(udf(coalesce(hundred,123)))
 -- !query schema
-struct<CAST(udf(cast(q1 as string)) AS BIGINT):bigint,CAST(udf(cast(unique2 as string)) AS INT):int,CAST(udf(cast(thousand as string)) AS INT):int,CAST(udf(cast(hundred as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CAST(udf(ansi_cast(thousand as string)) AS INT):int,CAST(udf(ansi_cast(hundred as string)) AS INT):int>
 -- !query output
 
 
@@ -3059,7 +3059,7 @@ select udf(f1), udf(unique2), case when udf(udf(unique2)) is null then udf(f1) e
   from int4_tbl a left join tenk1 b on udf(f1) = udf(udf(unique2))
   where (case when udf(unique2) is null then udf(f1) else 0 end) = 0
 -- !query schema
-struct<CAST(udf(cast(f1 as string)) AS INT):int,CAST(udf(cast(unique2 as string)) AS INT):int,CASE WHEN (CAST(udf(cast(cast(udf(cast(unique2 as string)) as int) as string)) AS INT) IS NULL) THEN CAST(udf(cast(f1 as string)) AS INT) ELSE 0 END:int>
+struct<CAST(udf(ansi_cast(f1 as string)) AS INT):int,CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CASE WHEN (CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(unique2 as string)) as int) as string)) AS INT) IS NULL) THEN CAST(udf(ansi_cast(f1 as string)) AS INT) ELSE 0 END:int>
 -- !query output
 0	0	0
 
@@ -3069,7 +3069,7 @@ select udf(a.unique1), udf(b.unique1), udf(c.unique1), udf(coalesce(b.twothousan
   from tenk1 a left join tenk1 b on udf(b.thousand) = a.unique1                       left join tenk1 c on udf(c.unique2) = udf(coalesce(b.twothousand, a.twothousand))
   where a.unique2 < udf(10) and udf(udf(coalesce(b.twothousand, a.twothousand))) = udf(44)
 -- !query schema
-struct<CAST(udf(cast(unique1 as string)) AS INT):int,CAST(udf(cast(unique1 as string)) AS INT):int,CAST(udf(cast(unique1 as string)) AS INT):int,CAST(udf(cast(coalesce(twothousand, twothousand) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(coalesce(twothousand, twothousand) as string)) AS INT):int>
 -- !query output
 
 
@@ -3107,7 +3107,7 @@ select udf(a.q2), udf(b.q1)
   from int8_tbl a left join int8_tbl b on udf(a.q2) = coalesce(b.q1, 1)
   where udf(udf(coalesce(b.q1, 1)) > 0)
 -- !query schema
-struct<CAST(udf(cast(q2 as string)) AS BIGINT):bigint,CAST(udf(cast(q1 as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(q2 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint>
 -- !query output
 -4567890123456789	NULL
 123	123
@@ -3237,7 +3237,7 @@ SELECT * FROM
      FROM int8_tbl LEFT JOIN innertab ON udf(udf(q2)) = id) ss2
   ON true
 -- !query schema
-struct<x:int,CAST(udf(cast(q1 as string)) AS BIGINT):bigint,CAST(udf(cast(q2 as string)) AS BIGINT):bigint,y:bigint>
+struct<x:int,CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(q2 as string)) AS BIGINT):bigint,y:bigint>
 -- !query output
 1	123	456	123
 1	123	4567890123456789	123
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
index 68113afdfae30..50b6e60086747 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
@@ -94,7 +94,7 @@ struct<>
 SELECT udf(b), udf(c) FROM test_having
 	GROUP BY b, c HAVING udf(count(*)) = 1 ORDER BY udf(b), udf(c)
 -- !query schema
-struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(c as string)) AS STRING):string>
+struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(c as string)) AS STRING):string>
 -- !query output
 1	XXXX
 3	bbbb
@@ -104,7 +104,7 @@ struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(c as string)) AS ST
 SELECT udf(b), udf(c) FROM test_having
 	GROUP BY b, c HAVING udf(b) = 3 ORDER BY udf(b), udf(c)
 -- !query schema
-struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(c as string)) AS STRING):string>
+struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(c as string)) AS STRING):string>
 -- !query output
 3	BBBB
 3	bbbb
@@ -115,7 +115,7 @@ SELECT udf(c), max(udf(a)) FROM test_having
 	GROUP BY c HAVING udf(count(*)) > 2 OR udf(min(a)) = udf(max(a))
 	ORDER BY c
 -- !query schema
-struct<CAST(udf(cast(c as string)) AS STRING):string,max(CAST(udf(cast(a as string)) AS INT)):int>
+struct<CAST(udf(ansi_cast(c as string)) AS STRING):string,max(CAST(udf(ansi_cast(a as string)) AS INT)):int>
 -- !query output
 XXXX	0
 bbbb	5
@@ -124,7 +124,7 @@ bbbb	5
 -- !query
 SELECT udf(udf(min(udf(a)))), udf(udf(max(udf(a)))) FROM test_having HAVING udf(udf(min(udf(a)))) = udf(udf(max(udf(a))))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(min(cast(udf(cast(a as string)) as int)) as string)) as int) as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(max(cast(udf(cast(a as string)) as int)) as string)) as int) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(min(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(max(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) as int) as string)) AS INT):int>
 -- !query output
 
 
@@ -132,7 +132,7 @@ struct<CAST(udf(cast(cast(udf(cast(min(cast(udf(cast(a as string)) as int)) as s
 -- !query
 SELECT udf(min(udf(a))), udf(udf(max(a))) FROM test_having HAVING udf(min(a)) < udf(max(udf(a)))
 -- !query schema
-struct<CAST(udf(cast(min(cast(udf(cast(a as string)) as int)) as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(max(a) as string)) as int) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(min(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(max(a) as string)) as int) as string)) AS INT):int>
 -- !query output
 0	9
 
@@ -143,7 +143,7 @@ SELECT udf(a) FROM test_having HAVING udf(min(a)) < udf(max(a))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'default.test_having.`a`' is not an aggregate function. Wrap '(min(default.test_having.`a`) AS `min(a#x)`, max(default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'spark_catalog.default.test_having.`a`' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.`a`) AS `min(a#x)`, max(spark_catalog.default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out
index 11cb682ee1494..66e6c20a2f6f2 100755
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out
@@ -95,7 +95,7 @@ SELECT udf(c), udf(count(*)) FROM test_missing_target GROUP BY
 udf(test_missing_target.c)
 ORDER BY udf(c)
 -- !query schema
-struct<CAST(udf(cast(c as string)) AS STRING):string,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 ABAB	2
 BBBB	2
@@ -109,7 +109,7 @@ cccc	2
 SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(test_missing_target.c)
 ORDER BY udf(c)
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 2
 2
@@ -125,13 +125,13 @@ SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(a) ORDER BY udf(b)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`b`' given input columns: [CAST(udf(cast(count(1) as string)) AS BIGINT)]; line 1 pos 75
+cannot resolve '`b`' given input columns: [CAST(udf(ansi_cast(count(1) as string)) AS BIGINT)]; line 1 pos 75
 
 
 -- !query
 SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(b) ORDER BY udf(b)
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1
 2
@@ -143,7 +143,7 @@ struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
 SELECT udf(test_missing_target.b), udf(count(*))
   FROM test_missing_target GROUP BY udf(b) ORDER BY udf(b)
 -- !query schema
-struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1	1
 2	2
@@ -154,7 +154,7 @@ struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(count(1) as string)
 -- !query
 SELECT udf(c) FROM test_missing_target ORDER BY udf(a)
 -- !query schema
-struct<CAST(udf(cast(c as string)) AS STRING):string>
+struct<CAST(udf(ansi_cast(c as string)) AS STRING):string>
 -- !query output
 XXXX
 ABAB
@@ -171,7 +171,7 @@ CCCC
 -- !query
 SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(b) ORDER BY udf(b) desc
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 4
 3
@@ -182,7 +182,7 @@ struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
 -- !query
 SELECT udf(count(*)) FROM test_missing_target ORDER BY udf(1) desc
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 10
 
@@ -190,7 +190,7 @@ struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
 -- !query
 SELECT udf(c), udf(count(*)) FROM test_missing_target GROUP BY 1 ORDER BY 1
 -- !query schema
-struct<CAST(udf(cast(c as string)) AS STRING):string,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 ABAB	2
 BBBB	2
@@ -224,7 +224,7 @@ Reference 'b' is ambiguous, could be: x.b, y.b.; line 3 pos 14
 SELECT udf(a), udf(a) FROM test_missing_target
 	ORDER BY udf(a)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(a as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(a as string)) AS INT):int,CAST(udf(ansi_cast(a as string)) AS INT):int>
 -- !query output
 0	0
 1	1
@@ -242,7 +242,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(a as string)) AS IN
 SELECT udf(udf(a)/2), udf(udf(a)/2) FROM test_missing_target
 	ORDER BY udf(udf(a)/2)
 -- !query schema
-struct<CAST(udf(cast((cast(cast(udf(cast(a as string)) as int) as double) / cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(cast((cast(cast(udf(cast(a as string)) as int) as double) / cast(2 as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast((ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(ansi_cast((ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double>
 -- !query output
 0.0	0.0
 0.5	0.5
@@ -260,7 +260,7 @@ struct<CAST(udf(cast((cast(cast(udf(cast(a as string)) as int) as double) / cast
 SELECT udf(a/2), udf(a/2) FROM test_missing_target
 	GROUP BY udf(a/2) ORDER BY udf(a/2)
 -- !query schema
-struct<CAST(udf(cast((cast(a as double) / cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(cast((cast(a as double) / cast(2 as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast((ansi_cast(a as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(ansi_cast((ansi_cast(a as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double>
 -- !query output
 0.0	0.0
 0.5	0.5
@@ -279,7 +279,7 @@ SELECT udf(x.b), udf(count(*)) FROM test_missing_target x, test_missing_target y
 	WHERE udf(x.a) = udf(y.a)
 	GROUP BY udf(x.b) ORDER BY udf(x.b)
 -- !query schema
-struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1	1
 2	2
@@ -292,7 +292,7 @@ SELECT udf(count(*)) FROM test_missing_target x, test_missing_target y
 	WHERE udf(x.a) = udf(y.a)
 	GROUP BY udf(x.b) ORDER BY udf(x.b)
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1
 2
@@ -305,7 +305,7 @@ SELECT udf(a%2), udf(count(udf(b))) FROM test_missing_target
 GROUP BY udf(test_missing_target.a%2)
 ORDER BY udf(test_missing_target.a%2)
 -- !query schema
-struct<CAST(udf(cast((a % 2) as string)) AS INT):int,CAST(udf(cast(count(cast(udf(cast(b as string)) as int)) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast((a % 2) as string)) AS INT):int,CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(b as string)) as int)) as string)) AS BIGINT):bigint>
 -- !query output
 0	5
 1	5
@@ -316,7 +316,7 @@ SELECT udf(count(c)) FROM test_missing_target
 GROUP BY udf(lower(test_missing_target.c))
 ORDER BY udf(lower(test_missing_target.c))
 -- !query schema
-struct<CAST(udf(cast(count(c) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(c) as string)) AS BIGINT):bigint>
 -- !query output
 2
 3
@@ -330,13 +330,13 @@ SELECT udf(count(udf(a))) FROM test_missing_target GROUP BY udf(a) ORDER BY udf(
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`b`' given input columns: [CAST(udf(cast(count(cast(udf(cast(a as string)) as int)) as string)) AS BIGINT)]; line 1 pos 80
+cannot resolve '`b`' given input columns: [CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) AS BIGINT)]; line 1 pos 80
 
 
 -- !query
 SELECT udf(count(b)) FROM test_missing_target GROUP BY udf(b/2) ORDER BY udf(b/2)
 -- !query schema
-struct<CAST(udf(cast(count(b) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(b) as string)) AS BIGINT):bigint>
 -- !query output
 1
 2
@@ -348,7 +348,7 @@ struct<CAST(udf(cast(count(b) as string)) AS BIGINT):bigint>
 SELECT udf(lower(test_missing_target.c)), udf(count(udf(c)))
   FROM test_missing_target GROUP BY udf(lower(c)) ORDER BY udf(lower(c))
 -- !query schema
-struct<CAST(udf(cast(lower(c) as string)) AS STRING):string,CAST(udf(cast(count(cast(udf(cast(c as string)) as string)) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(lower(c) as string)) AS STRING):string,CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(c as string)) as string)) as string)) AS BIGINT):bigint>
 -- !query output
 abab	2
 bbbb	3
@@ -359,7 +359,7 @@ xxxx	1
 -- !query
 SELECT udf(a) FROM test_missing_target ORDER BY udf(upper(udf(d)))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(a as string)) AS INT):int>
 -- !query output
 0
 1
@@ -377,7 +377,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int>
 SELECT udf(count(b)) FROM test_missing_target
 	GROUP BY udf((b + 1) / 2) ORDER BY udf((b + 1) / 2) desc
 -- !query schema
-struct<CAST(udf(cast(count(b) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(b) as string)) AS BIGINT):bigint>
 -- !query output
 4
 3
@@ -402,7 +402,7 @@ test_missing_target y
 	WHERE udf(x.a) = udf(y.a)
 	GROUP BY udf(x.b/2) ORDER BY udf(x.b/2)
 -- !query schema
-struct<CAST(udf(cast((cast(b as double) / cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(cast(count(cast(udf(cast(b as string)) as int)) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast((ansi_cast(b as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(b as string)) as int)) as string)) AS BIGINT):bigint>
 -- !query output
 0.5	1
 1.0	2
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out
index 6403406413db9..da5256f5c0453 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out
@@ -87,7 +87,7 @@ struct<foo:string,CAST(udf(cast(approx_count_distinct(cast(udf(cast(a as string)
 -- !query
 SELECT 'foo', MAX(STRUCT(udf(a))) FROM testData WHERE a = 0 GROUP BY udf(1)
 -- !query schema
-struct<foo:string,max(named_struct(col1, CAST(udf(cast(a as string)) AS INT))):struct<col1:int>>
+struct<foo:string,max(struct(CAST(udf(cast(a as string)) AS INT))):struct<col1:int>>
 -- !query output
 
 
diff --git a/sql/core/src/test/resources/sql-tests/results/union.sql.out b/sql/core/src/test/resources/sql-tests/results/union.sql.out
index 44002406836a4..ce3c761bc5d2d 100644
--- a/sql/core/src/test/resources/sql-tests/results/union.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/union.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 16
+-- Number of queries: 20
 
 
 -- !query
@@ -126,6 +126,39 @@ struct<array(1, 2):array<int>,str:string>
 [1,2]	str
 
 
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT t.v FROM (
+  SELECT v FROM t3
+  UNION ALL
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<v:decimal(11,0)>
+-- !query output
+1
+2
+
+
+-- !query
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  UNION
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<sum(v):decimal(21,0)>
+-- !query output
+3
+
+
 -- !query
 DROP VIEW IF EXISTS t1
 -- !query schema
@@ -142,6 +175,14 @@ struct<>
 
 
 
+-- !query
+DROP VIEW IF EXISTS t3
+-- !query schema
+struct<>
+-- !query output
+
+
+
 -- !query
 DROP VIEW IF EXISTS p1
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/window.sql.out b/sql/core/src/test/resources/sql-tests/results/window.sql.out
index f795374735f59..625088f90ced9 100644
--- a/sql/core/src/test/resources/sql-tests/results/window.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/window.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 23
+-- Number of queries: 24
 
 
 -- !query
@@ -380,3 +380,14 @@ a	4
 b	1
 b	3
 b	6
+
+
+-- !query
+SELECT val, cate,
+count(val) FILTER (WHERE val > 1) OVER(PARTITION BY cate)
+FROM testData ORDER BY cate, val
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+window aggregate function with filter predicate is not supported yet.;
diff --git a/sql/core/src/test/resources/test-data/TestStringDictionary.testRowIndex.orc b/sql/core/src/test/resources/test-data/TestStringDictionary.testRowIndex.orc
new file mode 100644
index 0000000000000..cba483d1f070f
Binary files /dev/null and b/sql/core/src/test/resources/test-data/TestStringDictionary.testRowIndex.orc differ
diff --git a/sql/core/src/test/resources/test-data/bad_after_good.csv b/sql/core/src/test/resources/test-data/bad_after_good.csv
index 4621a7d23714d..1a7c2651a11a7 100644
--- a/sql/core/src/test/resources/test-data/bad_after_good.csv
+++ b/sql/core/src/test/resources/test-data/bad_after_good.csv
@@ -1,2 +1,2 @@
 "good record",1999-08-01
-"bad record",1999-088-01
+"bad record",1999-088_01
diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc b/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc
new file mode 100644
index 0000000000000..ebe01743b2e20
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_date_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000..edd61c9b9fec8
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_date_v2_4_5.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_date_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000..01f4887f5e994
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_date_v2_4_6.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000..c7e8d3926f63a
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000..939e2b8088eb0
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000..88a94ac482052
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000..68bfa33aac13f
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000..62e6048354dc1
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_5.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000..d7fdaa3e67212
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_6.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000..a7cef9e60f134
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_5.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000..4c213f4540a73
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_6.snappy.parquet differ
diff --git a/sql/core/src/test/resources/test-data/before_1582_ts_v2_4.snappy.orc b/sql/core/src/test/resources/test-data/before_1582_ts_v2_4.snappy.orc
new file mode 100644
index 0000000000000..af9ef040270ac
Binary files /dev/null and b/sql/core/src/test/resources/test-data/before_1582_ts_v2_4.snappy.orc differ
diff --git a/sql/core/src/test/resources/test-data/percentile_approx-input.csv.bz2 b/sql/core/src/test/resources/test-data/percentile_approx-input.csv.bz2
new file mode 100644
index 0000000000000..f85e2896b3a89
Binary files /dev/null and b/sql/core/src/test/resources/test-data/percentile_approx-input.csv.bz2 differ
diff --git a/sql/core/src/test/resources/test-data/value-malformed.csv b/sql/core/src/test/resources/test-data/value-malformed.csv
index 8945ed73d2e83..6e6f08fca6df8 100644
--- a/sql/core/src/test/resources/test-data/value-malformed.csv
+++ b/sql/core/src/test/resources/test-data/value-malformed.csv
@@ -1,2 +1,2 @@
-0,2013-111-11 12:13:14
+0,2013-111_11 12:13:14
 1,1983-08-04
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala
index 2b4abed645910..4991e397eb11c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala
@@ -150,7 +150,7 @@ class ApproximatePercentileQuerySuite extends QueryTest with SharedSparkSession
       (1 to 1000).toDF("col").createOrReplaceTempView(table)
       checkAnswer(
         spark.sql(s"SELECT percentile_approx(col, array(0.25 + 0.25D), 200 + 800) FROM $table"),
-        Row(Seq(499))
+        Row(Seq(500))
       )
     }
   }
@@ -296,4 +296,23 @@ class ApproximatePercentileQuerySuite extends QueryTest with SharedSparkSession
     buffer.quantileSummaries
     assert(buffer.isCompressed)
   }
+
+  test("SPARK-32908: maximum target error in percentile_approx") {
+    withTempView(table) {
+      spark.read
+        .schema("col int")
+        .csv(testFile("test-data/percentile_approx-input.csv.bz2"))
+        .repartition(1)
+        .createOrReplaceTempView(table)
+      checkAnswer(
+        spark.sql(
+          s"""SELECT
+             |  percentile_approx(col, 0.77, 1000),
+             |  percentile_approx(col, 0.77, 10000),
+             |  percentile_approx(col, 0.77, 100000),
+             |  percentile_approx(col, 0.77, 1000000)
+             |FROM $table""".stripMargin),
+        Row(18, 17, 17, 17))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
index 07afd4195c3d4..174e7345d56b6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
@@ -50,11 +50,17 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSparkSession {
 
   protected def checkGeneratedCode(plan: SparkPlan, checkMethodCodeSize: Boolean = true): Unit = {
     val codegenSubtrees = new collection.mutable.HashSet[WholeStageCodegenExec]()
-    plan foreach {
-      case s: WholeStageCodegenExec =>
-        codegenSubtrees += s
-      case _ =>
+
+    def findSubtrees(plan: SparkPlan): Unit = {
+      plan foreach {
+        case s: WholeStageCodegenExec =>
+          codegenSubtrees += s
+        case s =>
+          s.subqueries.foreach(findSubtrees)
+      }
     }
+
+    findSubtrees(plan)
     codegenSubtrees.toSeq.foreach { subtree =>
       val code = subtree.doCodeGen()._2
       val (_, ByteCodeStats(maxMethodCodeSize, _, _)) = try {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index cd2c681dd7e0e..dc16418d50199 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -17,9 +17,13 @@
 
 package org.apache.spark.sql
 
+import java.io.File
+
 import scala.collection.mutable.HashSet
 import scala.concurrent.duration._
 
+import org.apache.commons.io.FileUtils
+
 import org.apache.spark.CleanerListener
 import org.apache.spark.executor.DataReadMethod._
 import org.apache.spark.executor.DataReadMethod.DataReadMethod
@@ -141,27 +145,31 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("uncaching temp table") {
-    testData.select("key").createOrReplaceTempView("tempTable1")
-    testData.select("key").createOrReplaceTempView("tempTable2")
-    spark.catalog.cacheTable("tempTable1")
+    withTempView("tempTable1", "tempTable2") {
+      testData.select("key").createOrReplaceTempView("tempTable1")
+      testData.select("key").createOrReplaceTempView("tempTable2")
+      spark.catalog.cacheTable("tempTable1")
 
-    assertCached(sql("SELECT COUNT(*) FROM tempTable1"))
-    assertCached(sql("SELECT COUNT(*) FROM tempTable2"))
+      assertCached(sql("SELECT COUNT(*) FROM tempTable1"))
+      assertCached(sql("SELECT COUNT(*) FROM tempTable2"))
 
-    // Is this valid?
-    uncacheTable("tempTable2")
+      // Is this valid?
+      uncacheTable("tempTable2")
 
-    // Should this be cached?
-    assertCached(sql("SELECT COUNT(*) FROM tempTable1"), 0)
+      // Should this be cached?
+      assertCached(sql("SELECT COUNT(*) FROM tempTable1"), 0)
+    }
   }
 
   test("too big for memory") {
-    val data = "*" * 1000
-    sparkContext.parallelize(1 to 200000, 1).map(_ => BigData(data)).toDF()
-      .createOrReplaceTempView("bigData")
-    spark.table("bigData").persist(StorageLevel.MEMORY_AND_DISK)
-    assert(spark.table("bigData").count() === 200000L)
-    spark.table("bigData").unpersist(blocking = true)
+    withTempView("bigData") {
+      val data = "*" * 1000
+      sparkContext.parallelize(1 to 200000, 1).map(_ => BigData(data)).toDF()
+        .createOrReplaceTempView("bigData")
+      spark.table("bigData").persist(StorageLevel.MEMORY_AND_DISK)
+      assert(spark.table("bigData").count() === 200000L)
+      spark.table("bigData").unpersist(blocking = true)
+    }
   }
 
   test("calling .cache() should use in-memory columnar caching") {
@@ -195,7 +203,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("SPARK-1669: cacheTable should be idempotent") {
-    assume(!spark.table("testData").logicalPlan.isInstanceOf[InMemoryRelation])
+    assert(!spark.table("testData").logicalPlan.isInstanceOf[InMemoryRelation])
 
     spark.catalog.cacheTable("testData")
     assertCached(spark.table("testData"))
@@ -225,12 +233,14 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("SELECT star from cached table") {
-    sql("SELECT * FROM testData").createOrReplaceTempView("selectStar")
-    spark.catalog.cacheTable("selectStar")
-    checkAnswer(
-      sql("SELECT * FROM selectStar WHERE key = 1"),
-      Seq(Row(1, "1")))
-    uncacheTable("selectStar")
+    withTempView("selectStar") {
+      sql("SELECT * FROM testData").createOrReplaceTempView("selectStar")
+      spark.catalog.cacheTable("selectStar")
+      checkAnswer(
+        sql("SELECT * FROM selectStar WHERE key = 1"),
+        Seq(Row(1, "1")))
+      uncacheTable("selectStar")
+    }
   }
 
   test("Self-join cached") {
@@ -375,102 +385,112 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("Drops temporary table") {
-    testData.select("key").createOrReplaceTempView("t1")
-    spark.table("t1")
-    spark.catalog.dropTempView("t1")
-    intercept[AnalysisException](spark.table("t1"))
+    withTempView("t1") {
+      testData.select("key").createOrReplaceTempView("t1")
+      spark.table("t1")
+      spark.catalog.dropTempView("t1")
+      intercept[AnalysisException](spark.table("t1"))
+    }
   }
 
   test("Drops cached temporary table") {
-    testData.select("key").createOrReplaceTempView("t1")
-    testData.select("key").createOrReplaceTempView("t2")
-    spark.catalog.cacheTable("t1")
+    withTempView("t1", "t2") {
+      testData.select("key").createOrReplaceTempView("t1")
+      testData.select("key").createOrReplaceTempView("t2")
+      spark.catalog.cacheTable("t1")
 
-    assert(spark.catalog.isCached("t1"))
-    assert(spark.catalog.isCached("t2"))
+      assert(spark.catalog.isCached("t1"))
+      assert(spark.catalog.isCached("t2"))
 
-    spark.catalog.dropTempView("t1")
-    intercept[AnalysisException](spark.table("t1"))
-    assert(!spark.catalog.isCached("t2"))
+      spark.catalog.dropTempView("t1")
+      intercept[AnalysisException](spark.table("t1"))
+      assert(!spark.catalog.isCached("t2"))
+    }
   }
 
   test("Clear all cache") {
-    sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
-    sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
-    spark.catalog.cacheTable("t1")
-    spark.catalog.cacheTable("t2")
-    spark.catalog.clearCache()
-    assert(spark.sharedState.cacheManager.isEmpty)
+    withTempView("t1", "t2") {
+      sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
+      sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
+      spark.catalog.cacheTable("t1")
+      spark.catalog.cacheTable("t2")
+      spark.catalog.clearCache()
+      assert(spark.sharedState.cacheManager.isEmpty)
 
-    sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
-    sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
-    spark.catalog.cacheTable("t1")
-    spark.catalog.cacheTable("t2")
-    sql("Clear CACHE")
-    assert(spark.sharedState.cacheManager.isEmpty)
+      sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
+      sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
+      spark.catalog.cacheTable("t1")
+      spark.catalog.cacheTable("t2")
+      sql("Clear CACHE")
+      assert(spark.sharedState.cacheManager.isEmpty)
+    }
   }
 
   test("Ensure accumulators to be cleared after GC when uncacheTable") {
-    sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
-    sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
+    withTempView("t1", "t2") {
+      sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
+      sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
 
-    spark.catalog.cacheTable("t1")
-    spark.catalog.cacheTable("t2")
+      spark.catalog.cacheTable("t1")
+      spark.catalog.cacheTable("t2")
 
-    sql("SELECT * FROM t1").count()
-    sql("SELECT * FROM t2").count()
-    sql("SELECT * FROM t1").count()
-    sql("SELECT * FROM t2").count()
+      sql("SELECT * FROM t1").count()
+      sql("SELECT * FROM t2").count()
+      sql("SELECT * FROM t1").count()
+      sql("SELECT * FROM t2").count()
+
+      val toBeCleanedAccIds = new HashSet[Long]
+
+      val accId1 = spark.table("t1").queryExecution.withCachedData.collect {
+        case i: InMemoryRelation => i.cacheBuilder.sizeInBytesStats.id
+      }.head
+      toBeCleanedAccIds += accId1
+
+      val accId2 = spark.table("t1").queryExecution.withCachedData.collect {
+        case i: InMemoryRelation => i.cacheBuilder.sizeInBytesStats.id
+      }.head
+      toBeCleanedAccIds += accId2
+
+      val cleanerListener = new CleanerListener {
+        def rddCleaned(rddId: Int): Unit = {}
+        def shuffleCleaned(shuffleId: Int): Unit = {}
+        def broadcastCleaned(broadcastId: Long): Unit = {}
+        def accumCleaned(accId: Long): Unit = {
+          toBeCleanedAccIds.synchronized { toBeCleanedAccIds -= accId }
+        }
+        def checkpointCleaned(rddId: Long): Unit = {}
+      }
+      spark.sparkContext.cleaner.get.attachListener(cleanerListener)
 
-    val toBeCleanedAccIds = new HashSet[Long]
+      uncacheTable("t1")
+      uncacheTable("t2")
 
-    val accId1 = spark.table("t1").queryExecution.withCachedData.collect {
-      case i: InMemoryRelation => i.cacheBuilder.sizeInBytesStats.id
-    }.head
-    toBeCleanedAccIds += accId1
+      System.gc()
 
-    val accId2 = spark.table("t1").queryExecution.withCachedData.collect {
-      case i: InMemoryRelation => i.cacheBuilder.sizeInBytesStats.id
-    }.head
-    toBeCleanedAccIds += accId2
-
-    val cleanerListener = new CleanerListener {
-      def rddCleaned(rddId: Int): Unit = {}
-      def shuffleCleaned(shuffleId: Int): Unit = {}
-      def broadcastCleaned(broadcastId: Long): Unit = {}
-      def accumCleaned(accId: Long): Unit = {
-        toBeCleanedAccIds.synchronized { toBeCleanedAccIds -= accId }
+      eventually(timeout(10.seconds)) {
+        assert(toBeCleanedAccIds.synchronized { toBeCleanedAccIds.isEmpty },
+          "batchStats accumulators should be cleared after GC when uncacheTable")
       }
-      def checkpointCleaned(rddId: Long): Unit = {}
-    }
-    spark.sparkContext.cleaner.get.attachListener(cleanerListener)
-
-    uncacheTable("t1")
-    uncacheTable("t2")
-
-    System.gc()
 
-    eventually(timeout(10.seconds)) {
-      assert(toBeCleanedAccIds.synchronized { toBeCleanedAccIds.isEmpty },
-        "batchStats accumulators should be cleared after GC when uncacheTable")
+      assert(AccumulatorContext.get(accId1).isEmpty)
+      assert(AccumulatorContext.get(accId2).isEmpty)
     }
-
-    assert(AccumulatorContext.get(accId1).isEmpty)
-    assert(AccumulatorContext.get(accId2).isEmpty)
   }
 
   test("SPARK-10327 Cache Table is not working while subquery has alias in its project list") {
-    sparkContext.parallelize((1, 1) :: (2, 2) :: Nil)
-      .toDF("key", "value").selectExpr("key", "value", "key+1").createOrReplaceTempView("abc")
-    spark.catalog.cacheTable("abc")
-
-    val sparkPlan = sql(
-      """select a.key, b.key, c.key from
-        |abc a join abc b on a.key=b.key
-        |join abc c on a.key=c.key""".stripMargin).queryExecution.sparkPlan
-
-    assert(sparkPlan.collect { case e: InMemoryTableScanExec => e }.size === 3)
-    assert(sparkPlan.collect { case e: RDDScanExec => e }.size === 0)
+    withTempView("abc") {
+      sparkContext.parallelize((1, 1) :: (2, 2) :: Nil)
+        .toDF("key", "value").selectExpr("key", "value", "key+1").createOrReplaceTempView("abc")
+      spark.catalog.cacheTable("abc")
+
+      val sparkPlan = sql(
+        """select a.key, b.key, c.key from
+          |abc a join abc b on a.key=b.key
+          |join abc c on a.key=c.key""".stripMargin).queryExecution.sparkPlan
+
+      assert(sparkPlan.collect { case e: InMemoryTableScanExec => e }.size === 3)
+      assert(sparkPlan.collect { case e: RDDScanExec => e }.size === 0)
+    }
   }
 
   /**
@@ -628,26 +648,30 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("SPARK-15870 DataFrame can't execute after uncacheTable") {
-    val selectStar = sql("SELECT * FROM testData WHERE key = 1")
-    selectStar.createOrReplaceTempView("selectStar")
+    withTempView("selectStar") {
+      val selectStar = sql("SELECT * FROM testData WHERE key = 1")
+      selectStar.createOrReplaceTempView("selectStar")
 
-    spark.catalog.cacheTable("selectStar")
-    checkAnswer(
-      selectStar,
-      Seq(Row(1, "1")))
+      spark.catalog.cacheTable("selectStar")
+      checkAnswer(
+        selectStar,
+        Seq(Row(1, "1")))
 
-    uncacheTable("selectStar")
-    checkAnswer(
-      selectStar,
-      Seq(Row(1, "1")))
+      uncacheTable("selectStar")
+      checkAnswer(
+        selectStar,
+        Seq(Row(1, "1")))
+    }
   }
 
   test("SPARK-15915 Logical plans should use canonicalized plan when override sameResult") {
-    val localRelation = Seq(1, 2, 3).toDF()
-    localRelation.createOrReplaceTempView("localRelation")
+    withTempView("localRelation") {
+      val localRelation = Seq(1, 2, 3).toDF()
+      localRelation.createOrReplaceTempView("localRelation")
 
-    spark.catalog.cacheTable("localRelation")
-    assert(getNumInMemoryRelations(localRelation) == 1)
+      spark.catalog.cacheTable("localRelation")
+      assert(getNumInMemoryRelations(localRelation) == 1)
+    }
   }
 
   test("SPARK-19093 Caching in side subquery") {
@@ -1122,4 +1146,201 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
       assert(!spark.catalog.isCached("t1"))
     }
   }
+
+  test("SPARK-30494 Fix the leak of cached data when replace an existing view") {
+    withTempView("tempView") {
+      spark.catalog.clearCache()
+      sql("create or replace temporary view tempView as select 1")
+      sql("cache table tempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isDefined)
+      sql("create or replace temporary view tempView as select 1, 2")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isEmpty)
+      sql("cache table tempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1, 2")).isDefined)
+    }
+
+    withGlobalTempView("tempGlobalTempView") {
+      spark.catalog.clearCache()
+      sql("create or replace global temporary view tempGlobalTempView as select 1")
+      sql("cache table global_temp.tempGlobalTempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isDefined)
+      sql("create or replace global temporary view tempGlobalTempView as select 1, 2")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isEmpty)
+      sql("cache table global_temp.tempGlobalTempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1, 2")).isDefined)
+    }
+
+    withView("view1") {
+      spark.catalog.clearCache()
+      sql("create or replace view view1 as select 1")
+      sql("cache table view1")
+      sql("create or replace view view1 as select 1, 2")
+      sql("cache table view1")
+      // the cached plan of persisted view likes below,
+      // we cannot use the same assertion of temp view.
+      // SubqueryAlias
+      //    |
+      //    + View
+      //        |
+      //        + Project[1 AS 1]
+      spark.sharedState.cacheManager.uncacheQuery(spark.table("view1"), cascade = false)
+      // make sure there is no cached data leak
+      assert(spark.sharedState.cacheManager.isEmpty)
+    }
+  }
+
+  test("SPARK-33228: Don't uncache data when replacing an existing view having the same plan") {
+    withTempView("tempView") {
+      spark.catalog.clearCache()
+      val df = spark.range(1).selectExpr("id a", "id b")
+      df.cache()
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceTempView("tempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceTempView("tempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+    }
+
+    withTempView("tempGlobalTempView") {
+      spark.catalog.clearCache()
+      val df = spark.range(1).selectExpr("id a", "id b")
+      df.cache()
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceGlobalTempView("tempGlobalTempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceGlobalTempView("tempGlobalTempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+    }
+  }
+
+  test("SPARK-33290: REFRESH TABLE should invalidate all caches referencing the table") {
+    withTable("t") {
+      withTempPath { path =>
+        withTempView("tempView1", "tempView2") {
+          Seq((1 -> "a")).toDF("i", "j").write.parquet(path.getCanonicalPath)
+          sql(s"CREATE TABLE t USING parquet LOCATION '${path.toURI}'")
+          sql("CREATE TEMPORARY VIEW tempView1 AS SELECT * FROM t")
+          sql("CACHE TABLE tempView2 AS SELECT i FROM tempView1")
+          checkAnswer(sql("SELECT * FROM tempView1"), Seq(Row(1, "a")))
+          checkAnswer(sql("SELECT * FROM tempView2"), Seq(Row(1)))
+
+          Utils.deleteRecursively(path)
+          sql("REFRESH TABLE tempView1")
+          checkAnswer(sql("SELECT * FROM tempView1"), Seq.empty)
+          checkAnswer(sql("SELECT * FROM tempView2"), Seq.empty)
+        }
+      }
+    }
+  }
+
+  test("SPARK-33290: querying temporary view after REFRESH TABLE fails with FNFE") {
+    withTable("t") {
+      withTempPath { path =>
+        withTempView("tempView1") {
+          Seq((1 -> "a")).toDF("i", "j").write.parquet(path.getCanonicalPath)
+          sql(s"CREATE TABLE t USING parquet LOCATION '${path.toURI}'")
+          sql("CREATE TEMPORARY VIEW tempView1 AS SELECT * FROM t")
+          checkAnswer(sql("SELECT * FROM tempView1"), Seq(Row(1, "a")))
+
+          Utils.deleteRecursively(path)
+          sql("REFRESH TABLE t")
+          checkAnswer(sql("SELECT * FROM t"), Seq.empty)
+          val exception = intercept[Exception] {
+            checkAnswer(sql("SELECT * FROM tempView1"), Seq.empty)
+          }
+          assert(exception.getMessage.contains("FileNotFoundException"))
+          assert(exception.getMessage.contains("REFRESH TABLE"))
+        }
+      }
+    }
+  }
+
+  test("SPARK-33786: Cache's storage level should be respected when a table name is altered.") {
+    withTable("old", "new") {
+      withTempPath { path =>
+        def getStorageLevel(tableName: String): StorageLevel = {
+          val table = spark.table(tableName)
+          val cachedData = spark.sharedState.cacheManager.lookupCachedData(table).get
+          cachedData.cachedRepresentation.cacheBuilder.storageLevel
+        }
+        Seq(1 -> "a").toDF("i", "j").write.parquet(path.getCanonicalPath)
+        sql(s"CREATE TABLE old USING parquet LOCATION '${path.toURI}'")
+        sql("CACHE TABLE old OPTIONS('storageLevel' 'MEMORY_ONLY')")
+        val oldStorageLevel = getStorageLevel("old")
+
+        sql("ALTER TABLE old RENAME TO new")
+        val newStorageLevel = getStorageLevel("new")
+        assert(oldStorageLevel === newStorageLevel)
+      }
+    }
+  }
+
+  test("SPARK-33950: refresh cache after partition dropping") {
+    withTable("t") {
+      sql(s"CREATE TABLE t (id int, part int) USING parquet PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      sql("INSERT INTO t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(1, 1)))
+      sql("ALTER TABLE t DROP PARTITION (part=0)")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 1)))
+    }
+  }
+
+  test("SPARK-34011: refresh cache after partition renaming") {
+    withTable("t") {
+      sql("CREATE TABLE t (id int, part int) USING parquet PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      sql("INSERT INTO t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      QueryTest.checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(1, 1)))
+      sql("ALTER TABLE t PARTITION (part=0) RENAME TO PARTITION (part=2)")
+      assert(spark.catalog.isCached("t"))
+      QueryTest.checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 2), Row(1, 1)))
+    }
+  }
+
+  private def testCacheRefreshing(cmd: String => DataFrame): Unit = {
+    withTable("t") {
+      sql("CREATE TABLE t (id int, part int) USING parquet PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0)))
+
+      // Create new partition (part = 1) in the filesystem
+      val information = sql("SHOW TABLE EXTENDED LIKE 't' PARTITION (part = 0)")
+        .select("information")
+        .first().getString(0)
+      val part0Loc = information
+        .split("\\r?\\n")
+        .filter(_.startsWith("Location:"))
+        .head
+        .replace("Location: file:", "")
+      val part1Loc = part0Loc.replace("part=0", "part=1")
+      FileUtils.copyDirectory(new File(part0Loc), new File(part1Loc))
+
+      cmd(part1Loc)
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(0, 1)))
+    }
+  }
+
+  test("SPARK-34055: refresh cache in partition adding") {
+    testCacheRefreshing { location =>
+      sql(s"ALTER TABLE t ADD PARTITION (part=1) LOCATION '$location'")
+    }
+  }
+
+  test("SPARK-34027: refresh cache in partitions recovering") {
+    testCacheRefreshing { _ =>
+      sql("ALTER TABLE t RECOVER PARTITIONS")
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index a9ee25b10dc02..fa06484a73d95 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql
 
-import java.sql.Date
+import java.sql.{Date, Timestamp}
 import java.util.Locale
 
 import scala.collection.JavaConverters._
@@ -26,12 +26,13 @@ import org.apache.hadoop.io.{LongWritable, Text}
 import org.apache.hadoop.mapreduce.lib.input.{TextInputFormat => NewTextInputFormat}
 import org.scalatest.Matchers._
 
-import org.apache.spark.sql.catalyst.expressions.{In, InSet, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.{InSet, Literal, NamedExpression}
 import org.apache.spark.sql.execution.ProjectExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
 
 class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
@@ -259,26 +260,28 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
   }
 
   test("nanvl") {
-    val testData = spark.createDataFrame(sparkContext.parallelize(
-      Row(null, 3.0, Double.NaN, Double.PositiveInfinity, 1.0f, 4) :: Nil),
-      StructType(Seq(StructField("a", DoubleType), StructField("b", DoubleType),
-        StructField("c", DoubleType), StructField("d", DoubleType),
-        StructField("e", FloatType), StructField("f", IntegerType))))
+    withTempView("t") {
+      val testData = spark.createDataFrame(sparkContext.parallelize(
+        Row(null, 3.0, Double.NaN, Double.PositiveInfinity, 1.0f, 4) :: Nil),
+        StructType(Seq(StructField("a", DoubleType), StructField("b", DoubleType),
+          StructField("c", DoubleType), StructField("d", DoubleType),
+          StructField("e", FloatType), StructField("f", IntegerType))))
 
-    checkAnswer(
-      testData.select(
-        nanvl($"a", lit(5)), nanvl($"b", lit(10)), nanvl(lit(10), $"b"),
-        nanvl($"c", lit(null).cast(DoubleType)), nanvl($"d", lit(10)),
-        nanvl($"b", $"e"), nanvl($"e", $"f")),
-      Row(null, 3.0, 10.0, null, Double.PositiveInfinity, 3.0, 1.0)
-    )
-    testData.createOrReplaceTempView("t")
-    checkAnswer(
-      sql(
-        "select nanvl(a, 5), nanvl(b, 10), nanvl(10, b), nanvl(c, null), nanvl(d, 10), " +
-          " nanvl(b, e), nanvl(e, f) from t"),
-      Row(null, 3.0, 10.0, null, Double.PositiveInfinity, 3.0, 1.0)
-    )
+      checkAnswer(
+        testData.select(
+          nanvl($"a", lit(5)), nanvl($"b", lit(10)), nanvl(lit(10), $"b"),
+          nanvl($"c", lit(null).cast(DoubleType)), nanvl($"d", lit(10)),
+          nanvl($"b", $"e"), nanvl($"e", $"f")),
+        Row(null, 3.0, 10.0, null, Double.PositiveInfinity, 3.0, 1.0)
+      )
+      testData.createOrReplaceTempView("t")
+      checkAnswer(
+        sql(
+          "select nanvl(a, 5), nanvl(b, 10), nanvl(10, b), nanvl(c, null), nanvl(d, 10), " +
+            " nanvl(b, e), nanvl(e, f) from t"),
+        Row(null, 3.0, 10.0, null, Double.PositiveInfinity, 3.0, 1.0)
+      )
+    }
   }
 
   test("===") {
@@ -453,35 +456,81 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
   }
 
   test("isInCollection: Scala Collection") {
-    val df = Seq((1, "x"), (2, "y"), (3, "z")).toDF("a", "b")
+    Seq(0, 1, 10).foreach { optThreshold =>
+      Seq(0, 1, 10).foreach { switchThreshold =>
+        withSQLConf(
+          SQLConf.OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> optThreshold.toString,
+          SQLConf.OPTIMIZER_INSET_SWITCH_THRESHOLD.key -> switchThreshold.toString) {
+          val df = Seq((1, "x"), (2, "y"), (3, "z")).toDF("a", "b")
+          // Test with different types of collections
+          checkAnswer(df.filter($"a".isInCollection(Seq(3, 1))),
+            df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
+          checkAnswer(df.filter($"a".isInCollection(Seq(1, 2).toSet)),
+            df.collect().toSeq.filter(r => r.getInt(0) == 1 || r.getInt(0) == 2))
+          checkAnswer(df.filter($"a".isInCollection(Seq(3, 2).toArray)),
+            df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 2))
+          checkAnswer(df.filter($"a".isInCollection(Seq(3, 1).toList)),
+            df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
+
+          val df2 = Seq((1, Seq(1)), (2, Seq(2)), (3, Seq(3))).toDF("a", "b")
 
-    Seq(1, 2).foreach { conf =>
-      withSQLConf(SQLConf.OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> conf.toString) {
-        if (conf <= 1) {
-          assert($"a".isInCollection(Seq(3, 1)).expr.isInstanceOf[InSet], "Expect expr to be InSet")
-        } else {
-          assert($"a".isInCollection(Seq(3, 1)).expr.isInstanceOf[In], "Expect expr to be In")
+          val e = intercept[AnalysisException] {
+            df2.filter($"a".isInCollection(Seq($"b")))
+          }
+          Seq("cannot resolve", "due to data type mismatch: Arguments must be same type but were")
+            .foreach { s =>
+              assert(e.getMessage.toLowerCase(Locale.ROOT).contains(s.toLowerCase(Locale.ROOT)))
+            }
         }
+      }
+    }
+  }
 
-        // Test with different types of collections
-        checkAnswer(df.filter($"a".isInCollection(Seq(3, 1))),
-          df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
-        checkAnswer(df.filter($"a".isInCollection(Seq(1, 2).toSet)),
-          df.collect().toSeq.filter(r => r.getInt(0) == 1 || r.getInt(0) == 2))
-        checkAnswer(df.filter($"a".isInCollection(Seq(3, 2).toArray)),
-          df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 2))
-        checkAnswer(df.filter($"a".isInCollection(Seq(3, 1).toList)),
-          df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
-
-        val df2 = Seq((1, Seq(1)), (2, Seq(2)), (3, Seq(3))).toDF("a", "b")
-
-        val e = intercept[AnalysisException] {
-          df2.filter($"a".isInCollection(Seq($"b")))
+  test("SPARK-31553: isInCollection - collection element types") {
+    val expected = Seq(Row(true), Row(false))
+    Seq(0, 1, 10).foreach { optThreshold =>
+      Seq(0, 1, 10).foreach { switchThreshold =>
+        withSQLConf(
+          SQLConf.OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> optThreshold.toString,
+          SQLConf.OPTIMIZER_INSET_SWITCH_THRESHOLD.key -> switchThreshold.toString) {
+          checkAnswer(Seq(0).toDS.select($"value".isInCollection(Seq(null))), Seq(Row(null)))
+          checkAnswer(
+            Seq(true).toDS.select($"value".isInCollection(Seq(true, false))),
+            Seq(Row(true)))
+          checkAnswer(
+            Seq(0.toByte, 1.toByte).toDS.select($"value".isInCollection(Seq(0.toByte, 2.toByte))),
+            expected)
+          checkAnswer(
+            Seq(0.toShort, 1.toShort).toDS
+              .select($"value".isInCollection(Seq(0.toShort, 2.toShort))),
+            expected)
+          checkAnswer(Seq(0, 1).toDS.select($"value".isInCollection(Seq(0, 2))), expected)
+          checkAnswer(Seq(0L, 1L).toDS.select($"value".isInCollection(Seq(0L, 2L))), expected)
+          checkAnswer(Seq(0.0f, 1.0f).toDS
+            .select($"value".isInCollection(Seq(0.0f, 2.0f))), expected)
+          checkAnswer(Seq(0.0D, 1.0D).toDS
+            .select($"value".isInCollection(Seq(0.0D, 2.0D))), expected)
+          checkAnswer(
+            Seq(BigDecimal(0), BigDecimal(2)).toDS
+              .select($"value".isInCollection(Seq(BigDecimal(0), BigDecimal(1)))),
+            expected)
+          checkAnswer(
+            Seq("abc", "def").toDS.select($"value".isInCollection(Seq("abc", "xyz"))),
+            expected)
+          checkAnswer(
+            Seq(Date.valueOf("2020-04-29"), Date.valueOf("2020-05-01")).toDS
+              .select($"value".isInCollection(
+                Seq(Date.valueOf("2020-04-29"), Date.valueOf("2020-04-30")))),
+            expected)
+          checkAnswer(
+            Seq(new Timestamp(0), new Timestamp(2)).toDS
+              .select($"value".isInCollection(Seq(new Timestamp(0), new Timestamp(1)))),
+            expected)
+          checkAnswer(
+            Seq(Array("a", "b"), Array("c", "d")).toDS
+              .select($"value".isInCollection(Seq(Array("a", "b"), Array("x", "z")))),
+            expected)
         }
-        Seq("cannot resolve",
-          "due to data type mismatch: Arguments must be same type but were").foreach { s =>
-            assert(e.getMessage.toLowerCase(Locale.ROOT).contains(s.toLowerCase(Locale.ROOT)))
-          }
       }
     }
   }
@@ -869,4 +918,9 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
       df.select(typedLit(("a", 2, 1.0))),
       Row(Row("a", 2, 1.0)) :: Nil)
   }
+
+  test("SPARK-31563: sql of InSet for UTF8String collection") {
+    val inSet = InSet(Literal("a"), Set("a", "b").map(UTF8String.fromString))
+    assert(inSet.sql === "('a' IN ('a', 'b'))")
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
index 6b503334f9f23..bdcf7230e3211 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
@@ -17,11 +17,15 @@
 
 package org.apache.spark.sql
 
+import scala.collection.JavaConverters._
+
 import org.apache.spark.sql.catalyst.expressions.CreateNamedStruct
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{ArrayType, StructType}
 
 class ComplexTypesSuite extends QueryTest with SharedSparkSession {
+  import testImplicits._
 
   override def beforeAll(): Unit = {
     super.beforeAll()
@@ -106,4 +110,11 @@ class ComplexTypesSuite extends QueryTest with SharedSparkSession {
     checkAnswer(df1, Row(10, 12) :: Row(11, 13) :: Nil)
     checkNamedStruct(df.queryExecution.optimizedPlan, expectedCount = 0)
   }
+
+  test("SPARK-32167: get field from an array of struct") {
+    val innerStruct = new StructType().add("i", "int", nullable = true)
+    val schema = new StructType().add("arr", ArrayType(innerStruct, containsNull = false))
+    val df = spark.createDataFrame(List(Row(Seq(Row(1), Row(null)))).asJava, schema)
+    checkAnswer(df.select($"arr".getField("i")), Row(Seq(1, null)))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala
index c3dbbb325d842..36989efbe870d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql
 
 import org.apache.commons.math3.stat.inference.ChiSquareTest
 
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -27,7 +28,8 @@ class ConfigBehaviorSuite extends QueryTest with SharedSparkSession {
 
   import testImplicits._
 
-  test("SPARK-22160 spark.sql.execution.rangeExchange.sampleSizePerPartition") {
+  test("SPARK-22160 spark.sql.execution.rangeExchange.sampleSizePerPartition",
+    DisableAdaptiveExecution("Post shuffle partition number can be different")) {
     // In this test, we run a sort and compute the histogram for partition size post shuffle.
     // With a high sample count, the partition size should be more evenly distributed, and has a
     // low chi-sq test value.
@@ -53,11 +55,8 @@ class ConfigBehaviorSuite extends QueryTest with SharedSparkSession {
         dist)
     }
 
-    // When enable AQE, the post partition number is changed.
     // And the ChiSquareTest result is also need updated. So disable AQE.
-    withSQLConf(
-        SQLConf.SHUFFLE_PARTITIONS.key -> numPartitions.toString,
-        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> numPartitions.toString) {
       // The default chi-sq value should be low
       assert(computeChiSquareTest() < 100)
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
index 61f0e138cc358..b9e0d509eeece 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
@@ -59,10 +59,22 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession {
     val df2 = df
       .select(from_csv($"value", schemaWithCorrField1, Map(
         "mode" -> "Permissive", "columnNameOfCorruptRecord" -> columnNameOfCorruptRecord)))
-
-    checkAnswer(df2, Seq(
-      Row(Row(0, null, "0,2013-111-11 12:13:14")),
-      Row(Row(1, java.sql.Date.valueOf("1983-08-04"), null))))
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "corrected") {
+      checkAnswer(df2, Seq(
+        Row(Row(0, null, "0,2013-111-11 12:13:14")),
+        Row(Row(1, java.sql.Date.valueOf("1983-08-04"), null))))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") {
+      checkAnswer(df2, Seq(
+        Row(Row(0, java.sql.Date.valueOf("2022-03-11"), null)),
+        Row(Row(1, java.sql.Date.valueOf("1983-08-04"), null))))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "exception") {
+      val msg = intercept[SparkException] {
+        df2.collect()
+      }.getCause.getMessage
+      assert(msg.contains("Fail to parse"))
+    }
   }
 
   test("schema_of_csv - infers schemas") {
@@ -200,4 +212,16 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession {
       assert(readback(0).getAs[Row](0).getAs[Date](0).getTime >= 0)
     }
   }
+
+  test("optional datetime parser does not affect csv time formatting") {
+    val s = "2015-08-26 12:34:46"
+    def toDF(p: String): DataFrame = sql(
+      s"""
+         |SELECT
+         | to_csv(
+         |   named_struct('time', timestamp'$s'), map('timestampFormat', "$p")
+         | )
+         | """.stripMargin)
+    checkAnswer(toDF("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), toDF("yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index dc1767a6852f6..2cb7790028b33 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -530,6 +530,22 @@ class DataFrameAggregateSuite extends QueryTest
     )
   }
 
+  test("SPARK-31500: collect_set() of BinaryType returns duplicate elements") {
+    val bytesTest1 = "test1".getBytes
+    val bytesTest2 = "test2".getBytes
+    val df = Seq(bytesTest1, bytesTest1, bytesTest2).toDF("a")
+    checkAnswer(df.select(size(collect_set($"a"))), Row(2) :: Nil)
+
+    val a = "aa".getBytes
+    val b = "bb".getBytes
+    val c = "cc".getBytes
+    val d = "dd".getBytes
+    val df1 = Seq((a, b), (a, b), (c, d))
+      .toDF("x", "y")
+      .select(struct($"x", $"y").as("a"))
+    checkAnswer(df1.select(size(collect_set($"a"))), Row(2) :: Nil)
+  }
+
   test("collect_set functions cannot have maps") {
     val df = Seq((1, 3, 0), (2, 3, 0), (3, 4, 1))
       .toDF("a", "x", "y")
@@ -615,34 +631,33 @@ class DataFrameAggregateSuite extends QueryTest
          Seq((true, true), (true, false), (false, true), (false, false))) {
       withSQLConf(
         (SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, wholeStage.toString),
-        (SQLConf.USE_OBJECT_HASH_AGG.key, useObjectHashAgg.toString),
-        (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false")) {
-        // When enable AQE, the WholeStageCodegenExec is added during QueryStageExec.
+        (SQLConf.USE_OBJECT_HASH_AGG.key, useObjectHashAgg.toString)) {
 
         val df = Seq(("1", 1), ("1", 2), ("2", 3), ("2", 4)).toDF("x", "y")
 
         // test case for HashAggregate
         val hashAggDF = df.groupBy("x").agg(c, sum("y"))
+        hashAggDF.collect()
         val hashAggPlan = hashAggDF.queryExecution.executedPlan
         if (wholeStage) {
-          assert(hashAggPlan.find {
+          assert(find(hashAggPlan) {
             case WholeStageCodegenExec(_: HashAggregateExec) => true
             case _ => false
           }.isDefined)
         } else {
-          assert(hashAggPlan.isInstanceOf[HashAggregateExec])
+          assert(stripAQEPlan(hashAggPlan).isInstanceOf[HashAggregateExec])
         }
-        hashAggDF.collect()
 
         // test case for ObjectHashAggregate and SortAggregate
         val objHashAggOrSortAggDF = df.groupBy("x").agg(c, collect_list("y"))
-        val objHashAggOrSortAggPlan = objHashAggOrSortAggDF.queryExecution.executedPlan
+        objHashAggOrSortAggDF.collect()
+        val objHashAggOrSortAggPlan =
+          stripAQEPlan(objHashAggOrSortAggDF.queryExecution.executedPlan)
         if (useObjectHashAgg) {
           assert(objHashAggOrSortAggPlan.isInstanceOf[ObjectHashAggregateExec])
         } else {
           assert(objHashAggOrSortAggPlan.isInstanceOf[SortAggregateExec])
         }
-        objHashAggOrSortAggDF.collect()
       }
     }
   }
@@ -959,16 +974,85 @@ class DataFrameAggregateSuite extends QueryTest
     }
   }
 
-  test("calendar interval agg support hash aggregate") {
-    val df1 = Seq((1, "1 day"), (2, "2 day"), (3, "3 day"), (3, null)).toDF("a", "b")
-    val df2 = df1.select(avg($"b" cast CalendarIntervalType))
-    checkAnswer(df2, Row(new CalendarInterval(0, 2, 0)) :: Nil)
-    assert(find(df2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
-    val df3 = df1.groupBy($"a").agg(avg($"b" cast CalendarIntervalType))
-    checkAnswer(df3,
-      Row(1, new CalendarInterval(0, 1, 0)) ::
-        Row(2, new CalendarInterval(0, 2, 0)) ::
-        Row(3, new CalendarInterval(0, 3, 0)) :: Nil)
-    assert(find(df3.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
+  Seq(true, false).foreach { value =>
+    test(s"SPARK-31620: agg with subquery (whole-stage-codegen = $value)") {
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> value.toString) {
+        withTempView("t1", "t2") {
+          sql("create temporary view t1 as select * from values (1, 2) as t1(a, b)")
+          sql("create temporary view t2 as select * from values (3, 4) as t2(c, d)")
+
+          // test without grouping keys
+          checkAnswer(sql("select sum(if(c > (select a from t1), d, 0)) as csum from t2"),
+            Row(4) :: Nil)
+
+          // test with grouping keys
+          checkAnswer(sql("select c, sum(if(c > (select a from t1), d, 0)) as csum from " +
+            "t2 group by c"), Row(3, 4) :: Nil)
+
+          // test with distinct
+          checkAnswer(sql("select avg(distinct(d)), sum(distinct(if(c > (select a from t1)," +
+            " d, 0))) as csum from t2 group by c"), Row(4, 4) :: Nil)
+
+          // test subquery with agg
+          checkAnswer(sql("select sum(distinct(if(c > (select sum(distinct(a)) from t1)," +
+            " d, 0))) as csum from t2 group by c"), Row(4) :: Nil)
+
+          // test SortAggregateExec
+          var df = sql("select max(if(c > (select a from t1), 'str1', 'str2')) as csum from t2")
+          assert(df.queryExecution.executedPlan
+            .find { case _: SortAggregateExec => true }.isDefined)
+          checkAnswer(df, Row("str1") :: Nil)
+
+          // test ObjectHashAggregateExec
+          df = sql("select collect_list(d), sum(if(c > (select a from t1), d, 0)) as csum from t2")
+          assert(df.queryExecution.executedPlan
+            .find { case _: ObjectHashAggregateExec => true }.isDefined)
+          checkAnswer(df, Row(Array(4), 4) :: Nil)
+        }
+      }
+    }
+  }
+
+  test("SPARK-32038: NormalizeFloatingNumbers should work on distinct aggregate") {
+    withTempView("view") {
+      val nan1 = java.lang.Float.intBitsToFloat(0x7f800001)
+      val nan2 = java.lang.Float.intBitsToFloat(0x7fffffff)
+
+      Seq(("mithunr", Float.NaN),
+        ("mithunr", nan1),
+        ("mithunr", nan2),
+        ("abellina", 1.0f),
+        ("abellina", 2.0f)).toDF("uid", "score").createOrReplaceTempView("view")
+
+      val df = spark.sql("select uid, count(distinct score) from view group by 1 order by 1 asc")
+      checkAnswer(df, Row("abellina", 2) :: Row("mithunr", 1) :: Nil)
+    }
+  }
+
+  test("SPARK-32136: NormalizeFloatingNumbers should work on null struct") {
+    val df = Seq(
+      A(None),
+      A(Some(B(None))),
+      A(Some(B(Some(1.0))))).toDF
+    val groupBy = df.groupBy("b").agg(count("*"))
+    checkAnswer(groupBy, Row(null, 1) :: Row(Row(null), 1) :: Row(Row(1.0), 1) :: Nil)
+  }
+
+  test("SPARK-32344: Unevaluable's set to FIRST/LAST ignoreNullsExpr in distinct aggregates") {
+    val queryTemplate = (agg: String) =>
+      s"SELECT $agg(DISTINCT v) FROM (SELECT v FROM VALUES 1, 2, 3 t(v) ORDER BY v)"
+    checkAnswer(sql(queryTemplate("FIRST")), Row(1))
+    checkAnswer(sql(queryTemplate("LAST")), Row(3))
+  }
+
+  test("SPARK-32906: struct field names should not change after normalizing floats") {
+    val df = Seq(Tuple1(Tuple2(-0.0d, Double.NaN)), Tuple1(Tuple2(0.0d, Double.NaN))).toDF("k")
+    val aggs = df.distinct().queryExecution.sparkPlan.collect { case a: HashAggregateExec => a }
+    assert(aggs.length == 2)
+    assert(aggs.head.output.map(_.dataType.simpleString).head ===
+      aggs.last.output.map(_.dataType.simpleString).head)
   }
 }
+
+case class B(c: Option[Double])
+case class A(b: Option[B])
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index 7fce03658fc16..ac98d3f0c7095 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -23,11 +23,13 @@ import java.util.TimeZone
 
 import scala.util.Random
 
+import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.catalyst.plans.logical.OneRowRelation
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -489,6 +491,12 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     withSQLConf(SQLConf.LEGACY_SIZE_OF_NULL.key -> "false") {
       testSizeOfArray(sizeOfNull = null)
     }
+    // size(null) should return null under ansi mode.
+    withSQLConf(
+      SQLConf.LEGACY_SIZE_OF_NULL.key -> "true",
+      SQLConf.ANSI_ENABLED.key -> "true") {
+      testSizeOfArray(sizeOfNull = null)
+    }
   }
 
   test("dataframe arrays_zip function") {
@@ -568,6 +576,12 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     withSQLConf(SQLConf.LEGACY_SIZE_OF_NULL.key -> "false") {
       testSizeOfMap(sizeOfNull = null)
     }
+    // size(null) should return null under ansi mode.
+    withSQLConf(
+      SQLConf.LEGACY_SIZE_OF_NULL.key -> "true",
+      SQLConf.ANSI_ENABLED.key -> "true") {
+      testSizeOfMap(sizeOfNull = null)
+    }
   }
 
   test("map_keys/map_values function") {
@@ -651,8 +665,12 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
       Row(null)
     )
 
-    checkAnswer(df1.selectExpr("map_concat(map1, map2)"), expected1a)
-    checkAnswer(df1.select(map_concat($"map1", $"map2")), expected1a)
+    intercept[SparkException](df1.selectExpr("map_concat(map1, map2)").collect())
+    intercept[SparkException](df1.select(map_concat($"map1", $"map2")).collect())
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+      checkAnswer(df1.selectExpr("map_concat(map1, map2)"), expected1a)
+      checkAnswer(df1.select(map_concat($"map1", $"map2")), expected1a)
+    }
 
     val expected1b = Seq(
       Row(Map(1 -> 100, 2 -> 200)),
@@ -1011,7 +1029,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
         Timestamp.valueOf("2018-01-01 12:00:00"),
         Timestamp.valueOf("2018-01-02 00:00:00")))))
 
-    DateTimeTestUtils.withDefaultTimeZone(TimeZone.getTimeZone("UTC")) {
+    DateTimeTestUtils.withDefaultTimeZone(UTC) {
       checkAnswer(
         spark.sql("select sequence(" +
           "   cast('2018-01-01' as date)" +
@@ -1515,6 +1533,13 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     assert(e.getMessage.contains("string, binary or array"))
   }
 
+  test("SPARK-31227: Non-nullable null type should not coerce to nullable type in concat") {
+    val actual = spark.range(1).selectExpr("concat(array(), array(1)) as arr")
+    val expected = spark.range(1).selectExpr("array(1) as arr")
+    checkAnswer(actual, expected)
+    assert(actual.schema === expected.schema)
+  }
+
   test("flatten function") {
     // Test cases with a primitive type
     val intDF = Seq(
@@ -3068,11 +3093,19 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
       checkAnswer(dfExample2.select(transform_keys(col("j"), (k, v) => k + v)),
         Seq(Row(Map(2.0 -> 1.0, 3.4 -> 1.4, 4.7 -> 1.7))))
 
-      checkAnswer(dfExample3.selectExpr("transform_keys(x, (k, v) ->  k % 2 = 0 OR v)"),
-        Seq(Row(Map(true -> true, true -> false))))
+      intercept[SparkException] {
+        dfExample3.selectExpr("transform_keys(x, (k, v) ->  k % 2 = 0 OR v)").collect()
+      }
+      intercept[SparkException] {
+        dfExample3.select(transform_keys(col("x"), (k, v) => k % 2 === 0 || v)).collect()
+      }
+      withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+        checkAnswer(dfExample3.selectExpr("transform_keys(x, (k, v) ->  k % 2 = 0 OR v)"),
+          Seq(Row(Map(true -> true, true -> false))))
 
-      checkAnswer(dfExample3.select(transform_keys(col("x"), (k, v) => k % 2 === 0 || v)),
-        Seq(Row(Map(true -> true, true -> false))))
+        checkAnswer(dfExample3.select(transform_keys(col("x"), (k, v) => k % 2 === 0 || v)),
+          Seq(Row(Map(true -> true, true -> false))))
+      }
 
       checkAnswer(dfExample3.selectExpr("transform_keys(x, (k, v) -> if(v, 2 * k, 3 * k))"),
         Seq(Row(Map(50 -> true, 78 -> false))))
@@ -3499,16 +3532,6 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     ).foreach(assertValuesDoNotChangeAfterCoalesceOrUnion(_))
   }
 
-  test("SPARK-21281 use string types by default if array and map have no argument") {
-    val ds = spark.range(1)
-    var expectedSchema = new StructType()
-      .add("x", ArrayType(StringType, containsNull = false), nullable = false)
-    assert(ds.select(array().as("x")).schema == expectedSchema)
-    expectedSchema = new StructType()
-      .add("x", MapType(StringType, StringType, valueContainsNull = false), nullable = false)
-    assert(ds.select(map().as("x")).schema == expectedSchema)
-  }
-
   test("SPARK-21281 fails if functions have no argument") {
     val df = Seq(1).toDF("a")
 
@@ -3563,19 +3586,40 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
       Seq(Row(1)))
   }
 
-  test("the like function with the escape parameter") {
-    val df = Seq(("abc", "a_c", "!")).toDF("str", "pattern", "escape")
-    checkAnswer(df.selectExpr("like(str, pattern, '@')"), Row(true))
+  test("SPARK-29462: Empty array of NullType for array function with no arguments") {
+    Seq((true, StringType), (false, NullType)).foreach {
+      case (arrayDefaultToString, expectedType) =>
+        withSQLConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE.key ->
+          arrayDefaultToString.toString) {
+          val schema = spark.range(1).select(array()).schema
+          assert(schema.nonEmpty && schema.head.dataType.isInstanceOf[ArrayType])
+          val actualType = schema.head.dataType.asInstanceOf[ArrayType].elementType
+          assert(actualType === expectedType)
+        }
+    }
+  }
 
-    val longEscapeError = intercept[AnalysisException] {
-      df.selectExpr("like(str, pattern, '@%')").collect()
-    }.getMessage
-    assert(longEscapeError.contains("The 'escape' parameter must be a string literal of one char"))
+  test("SPARK-30790: Empty map with NullType as key/value type for map function with no argument") {
+    Seq((true, StringType), (false, NullType)).foreach {
+      case (mapDefaultToString, expectedType) =>
+        withSQLConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE.key ->
+          mapDefaultToString.toString) {
+          val schema = spark.range(1).select(map()).schema
+          assert(schema.nonEmpty && schema.head.dataType.isInstanceOf[MapType])
+          val actualKeyType = schema.head.dataType.asInstanceOf[MapType].keyType
+          val actualValueType = schema.head.dataType.asInstanceOf[MapType].valueType
+          assert(actualKeyType === expectedType)
+          assert(actualValueType === expectedType)
+        }
+    }
+  }
 
-    val nonFoldableError = intercept[AnalysisException] {
-      df.selectExpr("like(str, pattern, escape)").collect()
-    }.getMessage
-    assert(nonFoldableError.contains("The 'escape' parameter must be a string literal"))
+  test("SPARK-26071: convert map to array and use as map key") {
+    val df = Seq(Map(1 -> "a")).toDF("m")
+    intercept[AnalysisException](df.select(map($"m", lit(1))))
+    checkAnswer(
+      df.select(map(map_entries($"m"), lit(1))),
+      Row(Map(Seq(Row(1, "a")) -> 1)))
   }
 }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index c7545bcad8962..a49f95f1ed134 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -17,14 +17,21 @@
 
 package org.apache.spark.sql
 
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.plans.{Inner, InnerLike, LeftOuter, RightOuter}
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, Join, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, Filter, HintInfo, Join, JoinHint, LogicalPlan, Project}
+import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types._
 
 class DataFrameJoinSuite extends QueryTest
   with SharedSparkSession
@@ -322,4 +329,132 @@ class DataFrameJoinSuite extends QueryTest
       }
     }
   }
+
+  test("Supports multi-part names for broadcast hint resolution") {
+    val (table1Name, table2Name) = ("t1", "t2")
+
+    withTempDatabase { dbName =>
+      withTable(table1Name, table2Name) {
+        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+          spark.range(50).write.saveAsTable(s"$dbName.$table1Name")
+          spark.range(100).write.saveAsTable(s"$dbName.$table2Name")
+
+          def checkIfHintApplied(df: DataFrame): Unit = {
+            val sparkPlan = df.queryExecution.executedPlan
+            val broadcastHashJoins = sparkPlan.collect { case p: BroadcastHashJoinExec => p }
+            assert(broadcastHashJoins.size == 1)
+            val broadcastExchanges = broadcastHashJoins.head.collect {
+              case p: BroadcastExchangeExec => p
+            }
+            assert(broadcastExchanges.size == 1)
+            val tables = broadcastExchanges.head.collect {
+              case FileSourceScanExec(_, _, _, _, _, _, Some(tableIdent)) => tableIdent
+            }
+            assert(tables.size == 1)
+            assert(tables.head === TableIdentifier(table1Name, Some(dbName)))
+          }
+
+          def checkIfHintNotApplied(df: DataFrame): Unit = {
+            val sparkPlan = df.queryExecution.executedPlan
+            val broadcastHashJoins = sparkPlan.collect { case p: BroadcastHashJoinExec => p }
+            assert(broadcastHashJoins.isEmpty)
+          }
+
+          def sqlTemplate(tableName: String, hintTableName: String): DataFrame = {
+            sql(s"SELECT /*+ BROADCASTJOIN($hintTableName) */ * " +
+              s"FROM $tableName, $dbName.$table2Name " +
+              s"WHERE $tableName.id = $table2Name.id")
+          }
+
+          def dfTemplate(tableName: String, hintTableName: String): DataFrame = {
+            spark.table(tableName).join(spark.table(s"$dbName.$table2Name"), "id")
+              .hint("broadcast", hintTableName)
+          }
+
+          sql(s"USE $dbName")
+
+          checkIfHintApplied(sqlTemplate(table1Name, table1Name))
+          checkIfHintApplied(sqlTemplate(s"$dbName.$table1Name", s"$dbName.$table1Name"))
+          checkIfHintApplied(sqlTemplate(s"$dbName.$table1Name", table1Name))
+          checkIfHintNotApplied(sqlTemplate(table1Name, s"$dbName.$table1Name"))
+
+          checkIfHintApplied(dfTemplate(table1Name, table1Name))
+          checkIfHintApplied(dfTemplate(s"$dbName.$table1Name", s"$dbName.$table1Name"))
+          checkIfHintApplied(dfTemplate(s"$dbName.$table1Name", table1Name))
+          checkIfHintApplied(dfTemplate(table1Name, s"$dbName.$table1Name"))
+          checkIfHintApplied(dfTemplate(table1Name,
+            s"${CatalogManager.SESSION_CATALOG_NAME}.$dbName.$table1Name"))
+
+          withView("tv") {
+            sql(s"CREATE VIEW tv AS SELECT * FROM $dbName.$table1Name")
+            checkIfHintApplied(sqlTemplate("tv", "tv"))
+            checkIfHintNotApplied(sqlTemplate("tv", s"$dbName.tv"))
+
+            checkIfHintApplied(dfTemplate("tv", "tv"))
+            checkIfHintApplied(dfTemplate("tv", s"$dbName.tv"))
+          }
+        }
+      }
+    }
+  }
+
+  test("The same table name exists in two databases for broadcast hint resolution") {
+    val (db1Name, db2Name) = ("db1", "db2")
+
+    withDatabase(db1Name, db2Name) {
+      withTable("t") {
+        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+          sql(s"CREATE DATABASE $db1Name")
+          sql(s"CREATE DATABASE $db2Name")
+          spark.range(1).write.saveAsTable(s"$db1Name.t")
+          spark.range(1).write.saveAsTable(s"$db2Name.t")
+
+          // Checks if a broadcast hint applied in both sides
+          val statement = s"SELECT /*+ BROADCASTJOIN(t) */ * FROM $db1Name.t, $db2Name.t " +
+            s"WHERE $db1Name.t.id = $db2Name.t.id"
+          sql(statement).queryExecution.optimizedPlan match {
+            case Join(_, _, _, _, JoinHint(Some(HintInfo(Some(BROADCAST))),
+              Some(HintInfo(Some(BROADCAST))))) =>
+            case _ => fail("broadcast hint not found in both tables")
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-32693: Compare two dataframes with same schema except nullable property") {
+    val schema1 = StructType(
+      StructField("a", IntegerType, false) ::
+        StructField("b", IntegerType, false) ::
+        StructField("c", IntegerType, false) :: Nil)
+    val rowSeq1: List[Row] = List(Row(10, 1, 1), Row(10, 50, 2))
+    val df1 = spark.createDataFrame(rowSeq1.asJava, schema1)
+
+    val schema2 = StructType(
+      StructField("a", IntegerType) ::
+        StructField("b", IntegerType) ::
+        StructField("c", IntegerType) :: Nil)
+    val rowSeq2: List[Row] = List(Row(10, 1, 1))
+    val df2 = spark.createDataFrame(rowSeq2.asJava, schema2)
+
+    checkAnswer(df1.except(df2), Row(10, 50, 2))
+
+    val schema3 = StructType(
+      StructField("a", IntegerType, false) ::
+        StructField("b", IntegerType, false) ::
+        StructField("c", IntegerType, false) ::
+        StructField("d", schema1, false) :: Nil)
+    val rowSeq3: List[Row] = List(Row(10, 1, 1, Row(10, 1, 1)), Row(10, 50, 2, Row(10, 50, 2)))
+    val df3 = spark.createDataFrame(rowSeq3.asJava, schema3)
+
+    val schema4 = StructType(
+      StructField("a", IntegerType) ::
+        StructField("b", IntegerType) ::
+        StructField("b", IntegerType) ::
+        StructField("d", schema2) :: Nil)
+    val rowSeq4: List[Row] = List(Row(10, 1, 1, Row(10, 1, 1)))
+    val df4 = spark.createDataFrame(rowSeq4.asJava, schema4)
+
+    checkAnswer(df3.except(df4), Row(10, 50, 2, Row(10, 50, 2)))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
index fb1ca69b6f73f..091877f7cac37 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
@@ -45,6 +45,16 @@ class DataFrameNaFunctionsSuite extends QueryTest with SharedSparkSession {
     ).toDF("int", "long", "short", "byte", "float", "double")
   }
 
+  def createDFWithNestedColumns: DataFrame = {
+    val schema = new StructType()
+      .add("c1", new StructType()
+        .add("c1-1", StringType)
+        .add("c1-2", StringType))
+    val data = Seq(Row(Row(null, "a2")), Row(Row("b1", "b2")), Row(null))
+    spark.createDataFrame(
+      spark.sparkContext.parallelize(data), schema)
+  }
+
   test("drop") {
     val input = createDF()
     val rows = input.collect()
@@ -275,33 +285,35 @@ class DataFrameNaFunctionsSuite extends QueryTest with SharedSparkSession {
     assert(message.contains("Reference 'f2' is ambiguous"))
   }
 
-  test("fill/drop with col(*)") {
+  test("fill with col(*)") {
     val df = createDF()
     // If columns are specified with "*", they are ignored.
     checkAnswer(df.na.fill("new name", Seq("*")), df.collect())
-    checkAnswer(df.na.drop("any", Seq("*")), df.collect())
   }
 
-  test("fill/drop with nested columns") {
-    val schema = new StructType()
-      .add("c1", new StructType()
-        .add("c1-1", StringType)
-        .add("c1-2", StringType))
+  test("drop with col(*)") {
+    val df = createDF()
+    val exception = intercept[AnalysisException] {
+      df.na.drop("any", Seq("*"))
+    }
+    assert(exception.getMessage.contains("Cannot resolve column name \"*\""))
+  }
 
-    val data = Seq(
-      Row(Row(null, "a2")),
-      Row(Row("b1", "b2")),
-      Row(null))
+  test("fill with nested columns") {
+    val df = createDFWithNestedColumns
 
-    val df = spark.createDataFrame(
-      spark.sparkContext.parallelize(data), schema)
+    // Nested columns are ignored for fill().
+    checkAnswer(df.na.fill("a1", Seq("c1.c1-1")), df)
+  }
 
-    checkAnswer(df.select("c1.c1-1"),
-      Row(null) :: Row("b1") :: Row(null) :: Nil)
+  test("drop with nested columns") {
+    val df = createDFWithNestedColumns
 
-    // Nested columns are ignored for fill() and drop().
-    checkAnswer(df.na.fill("a1", Seq("c1.c1-1")), data)
-    checkAnswer(df.na.drop("any", Seq("c1.c1-1")), data)
+    // Rows with the specified nested columns whose null values are dropped.
+    assert(df.count == 3)
+    checkAnswer(
+      df.na.drop("any", Seq("c1.c1-1")),
+      Seq(Row(Row("b1", "b2"))))
   }
 
   test("replace") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
index 51c6a835d58d6..51d861ec69b23 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
@@ -258,7 +258,7 @@ class DataFramePivotSuite extends QueryTest with SharedSparkSession {
     val ts = "2012-12-31 16:00:10.011"
     val tsWithZone = "2013-01-01 00:00:10.011"
 
-    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
       val df = Seq(java.sql.Timestamp.valueOf(ts)).toDF("a").groupBy("a").pivot("a").count()
       val expected = StructType(
         StructField("a", TimestampType) ::
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
index 250ec7dc0ba5a..3b3b54f75da57 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql
 
+import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.functions.{count, sum}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -202,4 +203,20 @@ class DataFrameSelfJoinSuite extends QueryTest with SharedSparkSession {
       assertAmbiguousSelfJoin(df1.join(df4).join(df2).select(df2("id")))
     }
   }
+
+  test("SPARK-28344: don't fail if there is no ambiguous self join") {
+    withSQLConf(
+      SQLConf.FAIL_AMBIGUOUS_SELF_JOIN_ENABLED.key -> "true") {
+      val df = Seq(1, 1, 2, 2).toDF("a")
+      val w = Window.partitionBy(df("a"))
+      checkAnswer(
+        df.select(df("a").alias("x"), sum(df("a")).over(w)),
+        Seq((1, 2), (1, 2), (2, 4), (2, 4)).map(Row.fromTuple))
+
+      val joined = df.join(spark.range(1)).select($"a")
+      checkAnswer(
+        joined.select(joined("a").alias("x"), sum(joined("a")).over(w)),
+        Seq((1, 2), (1, 2), (2, 4), (2, 4)).map(Row.fromTuple))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala
index 394bad751b5ce..19601726fa393 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala
@@ -126,6 +126,32 @@ class DataFrameStatSuite extends QueryTest with SharedSparkSession {
     assert(math.abs(corr3 - 0.95723391394758572) < 1e-12)
   }
 
+  test("SPARK-30532 stat functions to understand fully-qualified column name") {
+    val df1 = spark.sparkContext.parallelize(0 to 10).toDF("num").as("table1")
+    val df2 = spark.sparkContext.parallelize(0 to 10).toDF("num").as("table2")
+    val dfx = df2.crossJoin(df1)
+
+    assert(dfx.stat.corr("table1.num", "table2.num") != 0.0)
+    assert(dfx.stat.cov("table1.num", "table2.num") != 0.0)
+    assert(dfx.stat.approxQuantile("table1.num", Array(0.1), 0.0).length == 1)
+    assert(dfx.stat.approxQuantile("table2.num", Array(0.1), 0.0).length == 1)
+    assert(dfx.stat.freqItems(Array("table1.num", "table2.num")).collect()(0).length == 2)
+
+    // this should throw "Reference 'num' is ambiguous"
+    intercept[AnalysisException] {
+      dfx.stat.freqItems(Array("num"))
+    }
+    intercept[AnalysisException] {
+      dfx.stat.approxQuantile("num", Array(0.1), 0.0)
+    }
+    intercept[AnalysisException] {
+      dfx.stat.cov("num", "num")
+    }
+    intercept[AnalysisException] {
+      dfx.stat.corr("num", "num")
+    }
+  }
+
   test("covariance") {
     val df = Seq.tabulate(10)(i => (i, 2.0 * i, toLetter(i))).toDF("singles", "doubles", "letters")
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index d2d58a83ded5d..7c410e862e10c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -23,6 +23,7 @@ import java.sql.{Date, Timestamp}
 import java.util.UUID
 import java.util.concurrent.atomic.AtomicLong
 
+import scala.reflect.runtime.universe.TypeTag
 import scala.util.Random
 
 import org.scalatest.Matchers._
@@ -30,10 +31,12 @@ import org.scalatest.Matchers._
 import org.apache.spark.SparkException
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobEnd}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.encoders.RowEncoder
+import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
 import org.apache.spark.sql.catalyst.expressions.Uuid
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
-import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, OneRowRelation, Union}
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, OneRowRelation}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.connector.FakeV2Provider
 import org.apache.spark.sql.execution.{FilterExec, QueryExecution, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
@@ -41,8 +44,9 @@ import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExc
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession}
-import org.apache.spark.sql.test.SQLTestData.{DecimalData, NullStrings, TestData2}
+import org.apache.spark.sql.test.SQLTestData.{DecimalData, TestData2}
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
 import org.apache.spark.util.Utils
 import org.apache.spark.util.random.XORShiftRandom
 
@@ -109,6 +113,31 @@ class DataFrameSuite extends QueryTest
     dfAlias.col("t2.c")
   }
 
+  test("simple explode") {
+    val df = Seq(Tuple1("a b c"), Tuple1("d e")).toDF("words")
+
+    checkAnswer(
+      df.explode("words", "word") { word: String => word.split(" ").toSeq }.select('word),
+      Row("a") :: Row("b") :: Row("c") :: Row("d") ::Row("e") :: Nil
+    )
+  }
+
+  test("explode") {
+    val df = Seq((1, "a b c"), (2, "a b"), (3, "a")).toDF("number", "letters")
+    val df2 =
+      df.explode('letters) {
+        case Row(letters: String) => letters.split(" ").map(Tuple1(_)).toSeq
+      }
+
+    checkAnswer(
+      df2
+        .select('_1 as 'letter, 'number)
+        .groupBy('letter)
+        .agg(countDistinct('number)),
+      Row("a", 3) :: Row("b", 2) :: Row("c", 1) :: Nil
+    )
+  }
+
   test("Star Expansion - CreateStruct and CreateArray") {
     val structDf = testData2.select("a", "b").as("record")
     // CreateStruct and CreateArray in aggregateExpressions
@@ -185,6 +214,27 @@ class DataFrameSuite extends QueryTest
     }
   }
 
+  test("Star Expansion - ds.explode should fail with a meaningful message if it takes a star") {
+    val df = Seq(("1", "1,2"), ("2", "4"), ("3", "7,8,9")).toDF("prefix", "csv")
+    val e = intercept[AnalysisException] {
+      df.explode($"*") { case Row(prefix: String, csv: String) =>
+        csv.split(",").map(v => Tuple1(prefix + ":" + v)).toSeq
+      }.queryExecution.assertAnalyzed()
+    }
+    assert(e.getMessage.contains("Invalid usage of '*' in explode/json_tuple/UDTF"))
+
+    checkAnswer(
+      df.explode('prefix, 'csv) { case Row(prefix: String, csv: String) =>
+        csv.split(",").map(v => Tuple1(prefix + ":" + v)).toSeq
+      },
+      Row("1", "1,2", "1:1") ::
+        Row("1", "1,2", "1:2") ::
+        Row("2", "4", "2:4") ::
+        Row("3", "7,8,9", "3:7") ::
+        Row("3", "7,8,9", "3:8") ::
+        Row("3", "7,8,9", "3:9") :: Nil)
+  }
+
   test("Star Expansion - explode should fail with a meaningful message if it takes a star") {
     val df = Seq(("1,2"), ("4"), ("7,8,9")).toDF("csv")
     val e = intercept[AnalysisException] {
@@ -196,7 +246,7 @@ class DataFrameSuite extends QueryTest
   test("explode on output of array-valued function") {
     val df = Seq(("1,2"), ("4"), ("7,8,9")).toDF("csv")
     checkAnswer(
-      df.select(explode(split($"csv", ","))),
+      df.select(explode(split($"csv", pattern = ","))),
       Row("1") :: Row("2") :: Row("4") :: Row("7") :: Row("8") :: Row("9") :: Nil)
   }
 
@@ -330,7 +380,7 @@ class DataFrameSuite extends QueryTest
       testData.select("key").coalesce(1).select("key"),
       testData.select("key").collect().toSeq)
 
-    assert(spark.emptyDataFrame.coalesce(1).rdd.partitions.size === 1)
+    assert(spark.emptyDataFrame.coalesce(1).rdd.partitions.size === 0)
   }
 
   test("convert $\"attribute name\" into unresolved attribute") {
@@ -1167,7 +1217,7 @@ class DataFrameSuite extends QueryTest
                            |""".stripMargin
     assert(df.showString(1, truncate = 0) === expectedAnswer)
 
-    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
 
       val expectedAnswer = """+----------+-------------------+
                              ||d         |ts                 |
@@ -1188,7 +1238,7 @@ class DataFrameSuite extends QueryTest
                          " ts  | 2016-12-01 00:00:00 \n"
     assert(df.showString(1, truncate = 0, vertical = true) === expectedAnswer)
 
-    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
 
       val expectedAnswer = "-RECORD 0------------------\n" +
                            " d   | 2016-12-01          \n" +
@@ -1335,47 +1385,48 @@ class DataFrameSuite extends QueryTest
 
   test("SPARK-6941: Better error message for inserting into RDD-based Table") {
     withTempDir { dir =>
+      withTempView("parquet_base", "json_base", "rdd_base", "indirect_ds", "one_row") {
+        val tempParquetFile = new File(dir, "tmp_parquet")
+        val tempJsonFile = new File(dir, "tmp_json")
+
+        val df = Seq(Tuple1(1)).toDF()
+        val insertion = Seq(Tuple1(2)).toDF("col")
+
+        // pass case: parquet table (HadoopFsRelation)
+        df.write.mode(SaveMode.Overwrite).parquet(tempParquetFile.getCanonicalPath)
+        val pdf = spark.read.parquet(tempParquetFile.getCanonicalPath)
+        pdf.createOrReplaceTempView("parquet_base")
+
+        insertion.write.insertInto("parquet_base")
+
+        // pass case: json table (InsertableRelation)
+        df.write.mode(SaveMode.Overwrite).json(tempJsonFile.getCanonicalPath)
+        val jdf = spark.read.json(tempJsonFile.getCanonicalPath)
+        jdf.createOrReplaceTempView("json_base")
+        insertion.write.mode(SaveMode.Overwrite).insertInto("json_base")
+
+        // error cases: insert into an RDD
+        df.createOrReplaceTempView("rdd_base")
+        val e1 = intercept[AnalysisException] {
+          insertion.write.insertInto("rdd_base")
+        }
+        assert(e1.getMessage.contains("Inserting into an RDD-based table is not allowed."))
 
-      val tempParquetFile = new File(dir, "tmp_parquet")
-      val tempJsonFile = new File(dir, "tmp_json")
-
-      val df = Seq(Tuple1(1)).toDF()
-      val insertion = Seq(Tuple1(2)).toDF("col")
-
-      // pass case: parquet table (HadoopFsRelation)
-      df.write.mode(SaveMode.Overwrite).parquet(tempParquetFile.getCanonicalPath)
-      val pdf = spark.read.parquet(tempParquetFile.getCanonicalPath)
-      pdf.createOrReplaceTempView("parquet_base")
-
-      insertion.write.insertInto("parquet_base")
-
-      // pass case: json table (InsertableRelation)
-      df.write.mode(SaveMode.Overwrite).json(tempJsonFile.getCanonicalPath)
-      val jdf = spark.read.json(tempJsonFile.getCanonicalPath)
-      jdf.createOrReplaceTempView("json_base")
-      insertion.write.mode(SaveMode.Overwrite).insertInto("json_base")
-
-      // error cases: insert into an RDD
-      df.createOrReplaceTempView("rdd_base")
-      val e1 = intercept[AnalysisException] {
-        insertion.write.insertInto("rdd_base")
-      }
-      assert(e1.getMessage.contains("Inserting into an RDD-based table is not allowed."))
-
-      // error case: insert into a logical plan that is not a LeafNode
-      val indirectDS = pdf.select("_1").filter($"_1" > 5)
-      indirectDS.createOrReplaceTempView("indirect_ds")
-      val e2 = intercept[AnalysisException] {
-        insertion.write.insertInto("indirect_ds")
-      }
-      assert(e2.getMessage.contains("Inserting into an RDD-based table is not allowed."))
+        // error case: insert into a logical plan that is not a LeafNode
+        val indirectDS = pdf.select("_1").filter($"_1" > 5)
+        indirectDS.createOrReplaceTempView("indirect_ds")
+        val e2 = intercept[AnalysisException] {
+          insertion.write.insertInto("indirect_ds")
+        }
+        assert(e2.getMessage.contains("Inserting into an RDD-based table is not allowed."))
 
-      // error case: insert into an OneRowRelation
-      Dataset.ofRows(spark, OneRowRelation()).createOrReplaceTempView("one_row")
-      val e3 = intercept[AnalysisException] {
-        insertion.write.insertInto("one_row")
+        // error case: insert into an OneRowRelation
+        Dataset.ofRows(spark, OneRowRelation()).createOrReplaceTempView("one_row")
+        val e3 = intercept[AnalysisException] {
+          insertion.write.insertInto("one_row")
+        }
+        assert(e3.getMessage.contains("Inserting into an RDD-based table is not allowed."))
       }
-      assert(e3.getMessage.contains("Inserting into an RDD-based table is not allowed."))
     }
   }
 
@@ -1747,13 +1798,17 @@ class DataFrameSuite extends QueryTest
     val df = Seq("foo", "bar").map(Tuple1.apply).toDF("col")
     // invalid table names
     Seq("11111", "t~", "#$@sum", "table!#").foreach { name =>
-      val m = intercept[AnalysisException](df.createOrReplaceTempView(name)).getMessage
-      assert(m.contains(s"Invalid view name: $name"))
+      withTempView(name) {
+        val m = intercept[AnalysisException](df.createOrReplaceTempView(name)).getMessage
+        assert(m.contains(s"Invalid view name: $name"))
+      }
     }
 
     // valid table names
     Seq("table1", "`11111`", "`t~`", "`#$@sum`", "`table!#`").foreach { name =>
-      df.createOrReplaceTempView(name)
+      withTempView(name) {
+        df.createOrReplaceTempView(name)
+      }
     }
   }
 
@@ -2298,6 +2353,133 @@ class DataFrameSuite extends QueryTest
         fail("emptyDataFrame should be foldable")
     }
   }
+
+  test("SPARK-30811: CTE should not cause stack overflow when " +
+    "it refers to non-existent table with same name") {
+    val e = intercept[AnalysisException] {
+      sql("WITH t AS (SELECT 1 FROM nonexist.t) SELECT * FROM t")
+    }
+    assert(e.getMessage.contains("Table or view not found:"))
+  }
+
+  test("SPARK-32680: Don't analyze CTAS with unresolved query") {
+    val v2Source = classOf[FakeV2Provider].getName
+    val e = intercept[AnalysisException] {
+      sql(s"CREATE TABLE t USING $v2Source AS SELECT * from nonexist")
+    }
+    assert(e.getMessage.contains("Table or view not found:"))
+  }
+
+  test("CalendarInterval reflection support") {
+    val df = Seq((1, new CalendarInterval(1, 2, 3))).toDF("a", "b")
+    checkAnswer(df.selectExpr("b"), Row(new CalendarInterval(1, 2, 3)))
+  }
+
+  test("SPARK-31552: array encoder with different types") {
+    // primitives
+    val booleans = Array(true, false)
+    checkAnswer(Seq(booleans).toDF(), Row(booleans))
+
+    val bytes = Array(1.toByte, 2.toByte)
+    checkAnswer(Seq(bytes).toDF(), Row(bytes))
+    val shorts = Array(1.toShort, 2.toShort)
+    checkAnswer(Seq(shorts).toDF(), Row(shorts))
+    val ints = Array(1, 2)
+    checkAnswer(Seq(ints).toDF(), Row(ints))
+    val longs = Array(1L, 2L)
+    checkAnswer(Seq(longs).toDF(), Row(longs))
+
+    val floats = Array(1.0F, 2.0F)
+    checkAnswer(Seq(floats).toDF(), Row(floats))
+    val doubles = Array(1.0D, 2.0D)
+    checkAnswer(Seq(doubles).toDF(), Row(doubles))
+
+    val strings = Array("2020-04-24", "2020-04-25")
+    checkAnswer(Seq(strings).toDF(), Row(strings))
+
+    // tuples
+    val decOne = Decimal(1, 38, 18)
+    val decTwo = Decimal(2, 38, 18)
+    val tuple1 = (1, 2.2, "3.33", decOne, Date.valueOf("2012-11-22"))
+    val tuple2 = (2, 3.3, "4.44", decTwo, Date.valueOf("2022-11-22"))
+    checkAnswer(Seq(Array(tuple1, tuple2)).toDF(), Seq(Seq(tuple1, tuple2)).toDF())
+
+    // case classes
+    val gbks = Array(GroupByKey(1, 2), GroupByKey(4, 5))
+    checkAnswer(Seq(gbks).toDF(), Row(Array(Row(1, 2), Row(4, 5))))
+
+    // We can move this implicit def to [[SQLImplicits]] when we eventually make fully
+    // support for array encoder like Seq and Set
+    // For now cases below, decimal/datetime/interval/binary/nested types, etc,
+    // are not supported by array
+    implicit def newArrayEncoder[T <: Array[_] : TypeTag]: Encoder[T] = ExpressionEncoder()
+
+    // decimals
+    val decSpark = Array(decOne, decTwo)
+    val decScala = decSpark.map(_.toBigDecimal)
+    val decJava = decSpark.map(_.toJavaBigDecimal)
+    checkAnswer(Seq(decSpark).toDF(), Row(decJava))
+    checkAnswer(Seq(decScala).toDF(), Row(decJava))
+    checkAnswer(Seq(decJava).toDF(), Row(decJava))
+
+    // datetimes and intervals
+    val dates = strings.map(Date.valueOf)
+    checkAnswer(Seq(dates).toDF(), Row(dates))
+    val localDates = dates.map(d => DateTimeUtils.daysToLocalDate(DateTimeUtils.fromJavaDate(d)))
+    checkAnswer(Seq(localDates).toDF(), Row(dates))
+
+    val timestamps =
+      Array(Timestamp.valueOf("2020-04-24 12:34:56"), Timestamp.valueOf("2020-04-24 11:22:33"))
+    checkAnswer(Seq(timestamps).toDF(), Row(timestamps))
+    val instants =
+      timestamps.map(t => DateTimeUtils.microsToInstant(DateTimeUtils.fromJavaTimestamp(t)))
+    checkAnswer(Seq(instants).toDF(), Row(timestamps))
+
+    val intervals = Array(new CalendarInterval(1, 2, 3), new CalendarInterval(4, 5, 6))
+    checkAnswer(Seq(intervals).toDF(), Row(intervals))
+
+    // binary
+    val bins = Array(Array(1.toByte), Array(2.toByte), Array(3.toByte), Array(4.toByte))
+    checkAnswer(Seq(bins).toDF(), Row(bins))
+
+    // nested
+    val nestedIntArray = Array(Array(1), Array(2))
+    checkAnswer(Seq(nestedIntArray).toDF(), Row(nestedIntArray.map(wrapIntArray)))
+    val nestedDecArray = Array(decSpark)
+    checkAnswer(Seq(nestedDecArray).toDF(), Row(Array(wrapRefArray(decJava))))
+  }
+
+  test("SPARK-31750: eliminate UpCast if child's dataType is DecimalType") {
+    withTempPath { f =>
+      sql("select cast(1 as decimal(38, 0)) as d")
+        .write.mode("overwrite")
+        .parquet(f.getAbsolutePath)
+
+      val df = spark.read.parquet(f.getAbsolutePath).as[BigDecimal]
+      assert(df.schema === new StructType().add(StructField("d", DecimalType(38, 0))))
+    }
+  }
+
+  test("SPARK-32764: -0.0 and 0.0 should be equal") {
+    val df = Seq(0.0 -> -0.0).toDF("pos", "neg")
+    checkAnswer(df.select($"pos" > $"neg"), Row(false))
+  }
+
+  test("SPARK-32635: Replace references with foldables coming only from the node's children") {
+    val a = Seq("1").toDF("col1").withColumn("col2", lit("1"))
+    val b = Seq("2").toDF("col1").withColumn("col2", lit("2"))
+    val aub = a.union(b)
+    val c = aub.filter($"col1" === "2").cache()
+    val d = Seq("2").toDF("col4")
+    val r = d.join(aub, $"col2" === $"col4").select("col4")
+    val l = c.select("col2")
+    val df = l.join(r, $"col2" === $"col4", "LeftOuter")
+    checkAnswer(df, Row("2", "2"))
+  }
+
+  test("SPARK-32761: aggregating multiple distinct CONSTANT columns") {
+     checkAnswer(sql("select count(distinct 2), count(distinct 2,3)"), Row(1, 1))
+  }
 }
 
 case class GroupByKey(a: Int, b: Int)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
index d398657ec0b6e..09ce43e4e2b27 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
@@ -61,26 +61,28 @@ class DataFrameWindowFunctionsSuite extends QueryTest
   }
 
   test("rank functions in unspecific window") {
-    val df = Seq((1, "1"), (2, "2"), (1, "2"), (2, "2")).toDF("key", "value")
-    df.createOrReplaceTempView("window_table")
-    checkAnswer(
-      df.select(
-        $"key",
-        max("key").over(Window.partitionBy("value").orderBy("key")),
-        min("key").over(Window.partitionBy("value").orderBy("key")),
-        mean("key").over(Window.partitionBy("value").orderBy("key")),
-        count("key").over(Window.partitionBy("value").orderBy("key")),
-        sum("key").over(Window.partitionBy("value").orderBy("key")),
-        ntile(2).over(Window.partitionBy("value").orderBy("key")),
-        row_number().over(Window.partitionBy("value").orderBy("key")),
-        dense_rank().over(Window.partitionBy("value").orderBy("key")),
-        rank().over(Window.partitionBy("value").orderBy("key")),
-        cume_dist().over(Window.partitionBy("value").orderBy("key")),
-        percent_rank().over(Window.partitionBy("value").orderBy("key"))),
-      Row(1, 1, 1, 1.0d, 1, 1, 1, 1, 1, 1, 1.0d, 0.0d) ::
-        Row(1, 1, 1, 1.0d, 1, 1, 1, 1, 1, 1, 1.0d / 3.0d, 0.0d) ::
-        Row(2, 2, 1, 5.0d / 3.0d, 3, 5, 1, 2, 2, 2, 1.0d, 0.5d) ::
-        Row(2, 2, 1, 5.0d / 3.0d, 3, 5, 2, 3, 2, 2, 1.0d, 0.5d) :: Nil)
+    withTempView("window_table") {
+      val df = Seq((1, "1"), (2, "2"), (1, "2"), (2, "2")).toDF("key", "value")
+      df.createOrReplaceTempView("window_table")
+      checkAnswer(
+        df.select(
+          $"key",
+          max("key").over(Window.partitionBy("value").orderBy("key")),
+          min("key").over(Window.partitionBy("value").orderBy("key")),
+          mean("key").over(Window.partitionBy("value").orderBy("key")),
+          count("key").over(Window.partitionBy("value").orderBy("key")),
+          sum("key").over(Window.partitionBy("value").orderBy("key")),
+          ntile(2).over(Window.partitionBy("value").orderBy("key")),
+          row_number().over(Window.partitionBy("value").orderBy("key")),
+          dense_rank().over(Window.partitionBy("value").orderBy("key")),
+          rank().over(Window.partitionBy("value").orderBy("key")),
+          cume_dist().over(Window.partitionBy("value").orderBy("key")),
+          percent_rank().over(Window.partitionBy("value").orderBy("key"))),
+        Row(1, 1, 1, 1.0d, 1, 1, 1, 1, 1, 1, 1.0d, 0.0d) ::
+          Row(1, 1, 1, 1.0d, 1, 1, 1, 1, 1, 1, 1.0d / 3.0d, 0.0d) ::
+          Row(2, 2, 1, 5.0d / 3.0d, 3, 5, 1, 2, 2, 2, 1.0d, 0.5d) ::
+          Row(2, 2, 1, 5.0d / 3.0d, 3, 5, 2, 3, 2, 2, 1.0d, 0.5d) :: Nil)
+    }
   }
 
   test("window function should fail if order by clause is not specified") {
@@ -348,15 +350,17 @@ class DataFrameWindowFunctionsSuite extends QueryTest
   }
 
   test("SPARK-16195 empty over spec") {
-    val df = Seq(("a", 1), ("a", 1), ("a", 2), ("b", 2)).
-      toDF("key", "value")
-    df.createOrReplaceTempView("window_table")
-    checkAnswer(
-      df.select($"key", $"value", sum($"value").over(), avg($"value").over()),
-      Seq(Row("a", 1, 6, 1.5), Row("a", 1, 6, 1.5), Row("a", 2, 6, 1.5), Row("b", 2, 6, 1.5)))
-    checkAnswer(
-      sql("select key, value, sum(value) over(), avg(value) over() from window_table"),
-      Seq(Row("a", 1, 6, 1.5), Row("a", 1, 6, 1.5), Row("a", 2, 6, 1.5), Row("b", 2, 6, 1.5)))
+    withTempView("window_table") {
+      val df = Seq(("a", 1), ("a", 1), ("a", 2), ("b", 2)).
+        toDF("key", "value")
+      df.createOrReplaceTempView("window_table")
+      checkAnswer(
+        df.select($"key", $"value", sum($"value").over(), avg($"value").over()),
+        Seq(Row("a", 1, 6, 1.5), Row("a", 1, 6, 1.5), Row("a", 2, 6, 1.5), Row("b", 2, 6, 1.5)))
+      checkAnswer(
+        sql("select key, value, sum(value) over(), avg(value) over() from window_table"),
+        Seq(Row("a", 1, 6, 1.5), Row("a", 1, 6, 1.5), Row("a", 2, 6, 1.5), Row("b", 2, 6, 1.5)))
+    }
   }
 
   test("window function with udaf") {
@@ -548,37 +552,41 @@ class DataFrameWindowFunctionsSuite extends QueryTest
   }
 
   test("aggregation and rows between with unbounded + predicate pushdown") {
-    val df = Seq((1, "1"), (2, "2"), (2, "3"), (1, "3"), (3, "2"), (4, "3")).toDF("key", "value")
-    df.createOrReplaceTempView("window_table")
-    val selectList = Seq($"key", $"value",
-      last("key").over(
-        Window.partitionBy($"value").orderBy($"key").rowsBetween(0, Long.MaxValue)),
-      last("key").over(
-        Window.partitionBy($"value").orderBy($"key").rowsBetween(Long.MinValue, 0)),
-      last("key").over(Window.partitionBy($"value").orderBy($"key").rowsBetween(-1, 1)))
-
-    checkAnswer(
-      df.select(selectList: _*).where($"value" < "3"),
-      Seq(Row(1, "1", 1, 1, 1), Row(2, "2", 3, 2, 3), Row(3, "2", 3, 3, 3)))
+    withTempView("window_table") {
+      val df = Seq((1, "1"), (2, "2"), (2, "3"), (1, "3"), (3, "2"), (4, "3")).toDF("key", "value")
+      df.createOrReplaceTempView("window_table")
+      val selectList = Seq($"key", $"value",
+        last("key").over(
+          Window.partitionBy($"value").orderBy($"key").rowsBetween(0, Long.MaxValue)),
+        last("key").over(
+          Window.partitionBy($"value").orderBy($"key").rowsBetween(Long.MinValue, 0)),
+        last("key").over(Window.partitionBy($"value").orderBy($"key").rowsBetween(-1, 1)))
+
+      checkAnswer(
+        df.select(selectList: _*).where($"value" < "3"),
+        Seq(Row(1, "1", 1, 1, 1), Row(2, "2", 3, 2, 3), Row(3, "2", 3, 3, 3)))
+    }
   }
 
   test("aggregation and range between with unbounded + predicate pushdown") {
-    val df = Seq((5, "1"), (5, "2"), (4, "2"), (6, "2"), (3, "1"), (2, "2")).toDF("key", "value")
-    df.createOrReplaceTempView("window_table")
-    val selectList = Seq($"key", $"value",
-      last("value").over(
-        Window.partitionBy($"value").orderBy($"key").rangeBetween(-2, -1)).equalTo("2")
-        .as("last_v"),
-      avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(Long.MinValue, 1))
-        .as("avg_key1"),
-      avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(0, Long.MaxValue))
-        .as("avg_key2"),
-      avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(-1, 1))
-        .as("avg_key3"))
-
-    checkAnswer(
-      df.select(selectList: _*).where($"value" < 2),
-      Seq(Row(3, "1", null, 3.0, 4.0, 3.0), Row(5, "1", false, 4.0, 5.0, 5.0)))
+    withTempView("window_table") {
+      val df = Seq((5, "1"), (5, "2"), (4, "2"), (6, "2"), (3, "1"), (2, "2")).toDF("key", "value")
+      df.createOrReplaceTempView("window_table")
+      val selectList = Seq($"key", $"value",
+        last("value").over(
+          Window.partitionBy($"value").orderBy($"key").rangeBetween(-2, -1)).equalTo("2")
+          .as("last_v"),
+        avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(Long.MinValue, 1))
+          .as("avg_key1"),
+        avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(0, Long.MaxValue))
+          .as("avg_key2"),
+        avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(-1, 1))
+          .as("avg_key3"))
+
+      checkAnswer(
+        df.select(selectList: _*).where($"value" < 2),
+        Seq(Row(3, "1", null, 3.0, 4.0, 3.0), Row(5, "1", false, 4.0, 5.0, 5.0)))
+    }
   }
 
   test("Window spill with less than the inMemoryThreshold") {
@@ -665,40 +673,46 @@ class DataFrameWindowFunctionsSuite extends QueryTest
   }
 
   test("SPARK-24575: Window functions inside WHERE and HAVING clauses") {
-    def checkAnalysisError(df: => DataFrame): Unit = {
+    def checkAnalysisError(df: => DataFrame, clause: String): Unit = {
       val thrownException = the[AnalysisException] thrownBy {
         df.queryExecution.analyzed
       }
-      assert(thrownException.message.contains("window functions inside WHERE and HAVING clauses"))
+      assert(thrownException.message.contains(s"window functions inside $clause clause"))
     }
 
-    checkAnalysisError(testData2.select("a").where(rank().over(Window.orderBy($"b")) === 1))
-    checkAnalysisError(testData2.where($"b" === 2 && rank().over(Window.orderBy($"b")) === 1))
+    checkAnalysisError(
+      testData2.select("a").where(rank().over(Window.orderBy($"b")) === 1), "WHERE")
+    checkAnalysisError(
+      testData2.where($"b" === 2 && rank().over(Window.orderBy($"b")) === 1), "WHERE")
     checkAnalysisError(
       testData2.groupBy($"a")
         .agg(avg($"b").as("avgb"))
-        .where($"a" > $"avgb" && rank().over(Window.orderBy($"a")) === 1))
+        .where($"a" > $"avgb" && rank().over(Window.orderBy($"a")) === 1), "WHERE")
     checkAnalysisError(
       testData2.groupBy($"a")
         .agg(max($"b").as("maxb"), sum($"b").as("sumb"))
-        .where(rank().over(Window.orderBy($"a")) === 1))
+        .where(rank().over(Window.orderBy($"a")) === 1), "WHERE")
     checkAnalysisError(
       testData2.groupBy($"a")
         .agg(max($"b").as("maxb"), sum($"b").as("sumb"))
-        .where($"sumb" === 5 && rank().over(Window.orderBy($"a")) === 1))
+        .where($"sumb" === 5 && rank().over(Window.orderBy($"a")) === 1), "WHERE")
 
-    checkAnalysisError(sql("SELECT a FROM testData2 WHERE RANK() OVER(ORDER BY b) = 1"))
-    checkAnalysisError(sql("SELECT * FROM testData2 WHERE b = 2 AND RANK() OVER(ORDER BY b) = 1"))
+    checkAnalysisError(sql("SELECT a FROM testData2 WHERE RANK() OVER(ORDER BY b) = 1"), "WHERE")
+    checkAnalysisError(
+      sql("SELECT * FROM testData2 WHERE b = 2 AND RANK() OVER(ORDER BY b) = 1"), "WHERE")
     checkAnalysisError(
-      sql("SELECT * FROM testData2 GROUP BY a HAVING a > AVG(b) AND RANK() OVER(ORDER BY a) = 1"))
+      sql("SELECT * FROM testData2 GROUP BY a HAVING a > AVG(b) AND RANK() OVER(ORDER BY a) = 1"),
+      "HAVING")
     checkAnalysisError(
-      sql("SELECT a, MAX(b), SUM(b) FROM testData2 GROUP BY a HAVING RANK() OVER(ORDER BY a) = 1"))
+      sql("SELECT a, MAX(b), SUM(b) FROM testData2 GROUP BY a HAVING RANK() OVER(ORDER BY a) = 1"),
+      "HAVING")
     checkAnalysisError(
       sql(
         s"""SELECT a, MAX(b)
            |FROM testData2
            |GROUP BY a
-           |HAVING SUM(b) = 5 AND RANK() OVER(ORDER BY a) = 1""".stripMargin))
+           |HAVING SUM(b) = 5 AND RANK() OVER(ORDER BY a) = 1""".stripMargin),
+      "HAVING")
   }
 
   test("window functions in multiple selects") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
index d49dc58e93ddb..ff5c6242987de 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
@@ -17,20 +17,23 @@
 
 package org.apache.spark.sql
 
+import java.sql.Timestamp
+
 import scala.collection.JavaConverters._
 
 import org.scalatest.BeforeAndAfter
 
-import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NoSuchTableException, TableAlreadyExistsException}
-import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic}
-import org.apache.spark.sql.connector.InMemoryTableCatalog
+import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NamedRelation, NoSuchTableException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, V2WriteCommand}
+import org.apache.spark.sql.connector.{InMemoryTable, InMemoryTableCatalog}
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
 import org.apache.spark.sql.connector.expressions.{BucketTransform, DaysTransform, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, YearsTransform}
 import org.apache.spark.sql.execution.QueryExecution
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType}
+import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType, LongType, MapType, StringType, StructField, StructType, TimestampType}
 import org.apache.spark.sql.util.QueryExecutionListener
+import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
 
 class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with BeforeAndAfter {
@@ -67,7 +70,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
         plan = qe.analyzed
 
       }
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
     }
     spark.listenerManager.register(listener)
 
@@ -97,6 +100,86 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     assert(v2.catalog.exists(_ == catalogPlugin))
   }
 
+  case class FakeV2WriteCommand(table: NamedRelation, query: LogicalPlan) extends V2WriteCommand
+
+  test("SPARK-33136 output resolved on complex types for V2 write commands") {
+    val tableCatalog = catalog("testcat")
+
+    def assertTypeCompatibility(name: String, fromType: DataType, toType: DataType): Unit = {
+      val fromTableName = s"from_table_$name"
+      tableCatalog.createTable(
+        Identifier.of(Array(), fromTableName),
+        StructType(Array(StructField("col", fromType))),
+        Array.empty,
+        new java.util.HashMap[String, String]())
+
+      val toTable = tableCatalog.createTable(
+        Identifier.of(Array(), s"to_table_$name"),
+        StructType(Array(StructField("col", toType))),
+        Array.empty,
+        new java.util.HashMap[String, String]())
+
+      val df = spark.table(s"testcat.$fromTableName")
+
+      val relation = DataSourceV2Relation.create(toTable, Some(tableCatalog), None)
+      val writeCommand = FakeV2WriteCommand(relation, df.queryExecution.analyzed)
+
+      assert(writeCommand.outputResolved, s"Unable to write from type $fromType to type $toType.")
+    }
+
+    // The major difference between `from` and `to` is that `from` is a complex type
+    // with non-nullable, whereas `to` is same data type with flipping nullable.
+
+    // nested struct type
+    val fromStructType = StructType(Array(
+      StructField("s", StringType),
+      StructField("i_nonnull", IntegerType, nullable = false),
+      StructField("st", StructType(Array(
+        StructField("l", LongType),
+        StructField("s_nonnull", StringType, nullable = false))))))
+
+    val toStructType = StructType(Array(
+      StructField("s", StringType),
+      StructField("i_nonnull", IntegerType),
+      StructField("st", StructType(Array(
+        StructField("l", LongType),
+        StructField("s_nonnull", StringType))))))
+
+    assertTypeCompatibility("struct", fromStructType, toStructType)
+
+    // array type
+    assertTypeCompatibility("array", ArrayType(LongType, containsNull = false),
+      ArrayType(LongType, containsNull = true))
+
+    // array type with struct type
+    val fromArrayWithStructType = ArrayType(
+      StructType(Array(StructField("s", StringType, nullable = false))),
+      containsNull = false)
+
+    val toArrayWithStructType = ArrayType(
+      StructType(Array(StructField("s", StringType))),
+      containsNull = true)
+
+    assertTypeCompatibility("array_struct", fromArrayWithStructType, toArrayWithStructType)
+
+    // map type
+    assertTypeCompatibility("map", MapType(IntegerType, StringType, valueContainsNull = false),
+      MapType(IntegerType, StringType, valueContainsNull = true))
+
+    // map type with struct type
+    val fromMapWithStructType = MapType(
+      IntegerType,
+      StructType(Array(StructField("s", StringType, nullable = false))),
+      valueContainsNull = false)
+
+    val toMapWithStructType = MapType(
+      IntegerType,
+      StructType(Array(StructField("s", StringType))),
+      valueContainsNull = true)
+
+    assertTypeCompatibility("map_struct", fromMapWithStructType, toMapWithStructType)
+  }
+
   test("Append: basic append") {
     spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo")
 
@@ -332,7 +415,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(years($"ts"))
         .create()
 
@@ -346,7 +428,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(months($"ts"))
         .create()
 
@@ -360,7 +441,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(days($"ts"))
         .create()
 
@@ -374,7 +454,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(hours($"ts"))
         .create()
 
@@ -387,7 +466,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
   test("Create: partitioned by bucket(4, id)") {
     spark.table("source")
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(bucket(4, $"id"))
         .create()
 
@@ -550,4 +628,82 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     assert(replaced.partitioning.isEmpty)
     assert(replaced.properties === defaultOwnership.asJava)
   }
+
+  test("SPARK-30289 Create: partitioned by nested column") {
+    val schema = new StructType().add("ts", new StructType()
+      .add("created", TimestampType)
+      .add("modified", TimestampType)
+      .add("timezone", StringType))
+
+    val data = Seq(
+      Row(Row(Timestamp.valueOf("2019-06-01 10:00:00"), Timestamp.valueOf("2019-09-02 07:00:00"),
+        "America/Los_Angeles")),
+      Row(Row(Timestamp.valueOf("2019-08-26 18:00:00"), Timestamp.valueOf("2019-09-26 18:00:00"),
+        "America/Los_Angeles")),
+      Row(Row(Timestamp.valueOf("2018-11-23 18:00:00"), Timestamp.valueOf("2018-12-22 18:00:00"),
+        "America/New_York")))
+    val df = spark.createDataFrame(spark.sparkContext.parallelize(data, 1), schema)
+
+    df.writeTo("testcat.table_name")
+      .partitionedBy($"ts.timezone")
+      .create()
+
+    val table = catalog("testcat").loadTable(Identifier.of(Array(), "table_name"))
+      .asInstanceOf[InMemoryTable]
+
+    assert(table.name === "testcat.table_name")
+    assert(table.partitioning === Seq(IdentityTransform(FieldReference(Array("ts", "timezone")))))
+    checkAnswer(spark.table(table.name), data)
+    assert(table.dataMap.toArray.length == 2)
+    assert(table.dataMap(Seq(UTF8String.fromString("America/Los_Angeles"))).rows.size == 2)
+    assert(table.dataMap(Seq(UTF8String.fromString("America/New_York"))).rows.size == 1)
+
+    // TODO: `DataSourceV2Strategy` can not translate nested fields into source filter yet
+    // so the following sql will fail.
+    // sql("DELETE FROM testcat.table_name WHERE ts.timezone = \"America/Los_Angeles\"")
+  }
+
+  test("SPARK-30289 Create: partitioned by multiple transforms on nested columns") {
+    spark.table("source")
+      .withColumn("ts", struct(
+        lit("2019-06-01 10:00:00.000000").cast("timestamp") as "created",
+        lit("2019-09-02 07:00:00.000000").cast("timestamp") as "modified",
+        lit("America/Los_Angeles") as "timezone"))
+      .writeTo("testcat.table_name")
+      .partitionedBy(
+        years($"ts.created"), months($"ts.created"), days($"ts.created"), hours($"ts.created"),
+        years($"ts.modified"), months($"ts.modified"), days($"ts.modified"), hours($"ts.modified")
+      )
+      .create()
+
+    val table = catalog("testcat").loadTable(Identifier.of(Array(), "table_name"))
+
+    assert(table.name === "testcat.table_name")
+    assert(table.partitioning === Seq(
+      YearsTransform(FieldReference(Array("ts", "created"))),
+      MonthsTransform(FieldReference(Array("ts", "created"))),
+      DaysTransform(FieldReference(Array("ts", "created"))),
+      HoursTransform(FieldReference(Array("ts", "created"))),
+      YearsTransform(FieldReference(Array("ts", "modified"))),
+      MonthsTransform(FieldReference(Array("ts", "modified"))),
+      DaysTransform(FieldReference(Array("ts", "modified"))),
+      HoursTransform(FieldReference(Array("ts", "modified")))))
+  }
+
+  test("SPARK-30289 Create: partitioned by bucket(4, ts.timezone)") {
+    spark.table("source")
+      .withColumn("ts", struct(
+        lit("2019-06-01 10:00:00.000000").cast("timestamp") as "created",
+        lit("2019-09-02 07:00:00.000000").cast("timestamp") as "modified",
+        lit("America/Los_Angeles") as "timezone"))
+      .writeTo("testcat.table_name")
+      .partitionedBy(bucket(4, $"ts.timezone"))
+      .create()
+
+    val table = catalog("testcat").loadTable(Identifier.of(Array(), "table_name"))
+
+    assert(table.name === "testcat.table_name")
+    assert(table.partitioning === Seq(BucketTransform(LiteralValue(4, IntegerType),
+      Seq(FieldReference(Seq("ts", "timezone"))))))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala
index 6ffe133ee652b..a22abd505ca00 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala
@@ -219,6 +219,15 @@ case class OptionBooleanIntAggregator(colName: String)
   def OptionalBoolIntEncoder: Encoder[Option[(Boolean, Int)]] = ExpressionEncoder()
 }
 
+case class FooAgg(s: Int) extends Aggregator[Row, Int, Int] {
+  def zero: Int = s
+  def reduce(b: Int, r: Row): Int = b + r.getAs[Int](0)
+  def merge(b1: Int, b2: Int): Int = b1 + b2
+  def finish(b: Int): Int = b
+  def bufferEncoder: Encoder[Int] = Encoders.scalaInt
+  def outputEncoder: Encoder[Int] = Encoders.scalaInt
+}
+
 class DatasetAggregatorSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
 
@@ -394,4 +403,19 @@ class DatasetAggregatorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(group, Row("bob", Row(true, 3)) :: Nil)
     checkDataset(group.as[OptionBooleanIntData], OptionBooleanIntData("bob", Some((true, 3))))
   }
+
+  test("SPARK-30590: untyped select should not accept typed column that needs input type") {
+    val df = Seq((1, 2, 3, 4, 5, 6)).toDF("a", "b", "c", "d", "e", "f")
+    val fooAgg = (i: Int) => FooAgg(i).toColumn.name(s"foo_agg_$i")
+
+    val agg1 = df.select(fooAgg(1), fooAgg(2), fooAgg(3), fooAgg(4), fooAgg(5))
+    checkDataset(agg1, (3, 5, 7, 9, 11))
+
+    // Passes typed columns to untyped `Dataset.select` API.
+    val err = intercept[AnalysisException] {
+      df.select(fooAgg(1), fooAgg(2), fooAgg(3), fooAgg(4), fooAgg(5), fooAgg(6))
+    }.getMessage
+    assert(err.contains("cannot be passed in untyped `select` API. " +
+      "Use the typed `Dataset.select` API instead."))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
index 0ac99905f35f4..5b8c80b471bb4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
@@ -18,12 +18,13 @@
 package org.apache.spark.sql
 
 import org.apache.spark.metrics.source.CodegenMetrics
+import org.apache.spark.sql.catalyst.encoders.RowEncoder
 import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression}
-import org.apache.spark.sql.catalyst.expressions.objects.ExternalMapToCatalyst
 import org.apache.spark.sql.catalyst.plans.logical.SerializeFromObject
-import org.apache.spark.sql.functions.expr
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.StructType
 
 class DatasetOptimizationSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
@@ -195,4 +196,12 @@ class DatasetOptimizationSuite extends QueryTest with SharedSparkSession {
       checkCodegenCache(() => Seq(Seq(Map("abc" -> 1))).toDS())
     }
   }
+
+  test("SPARK-32652: Pruned nested serializers: RowEncoder") {
+    val df = Seq(("a", 1), ("b", 2), ("c", 3)).toDF("i", "j")
+    val encoder = RowEncoder(new StructType().add("s", df.schema))
+    val query = df.map(row => Row(row))(encoder).select("s.i")
+    testSerializer(query, Seq(Seq("i")))
+    checkAnswer(query, Seq(Row("a"), Row("b"), Row("c")))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala
index 43de2663b1236..b20d050f2fc4a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala
@@ -34,7 +34,7 @@ class DatasetSerializerRegistratorSuite extends QueryTest with SharedSparkSessio
 
   override protected def sparkConf: SparkConf = {
     // Make sure we use the KryoRegistrator
-    super.sparkConf.set(KRYO_USER_REGISTRATORS, TestRegistrator().getClass.getCanonicalName)
+    super.sparkConf.set(KRYO_USER_REGISTRATORS, Seq(TestRegistrator().getClass.getCanonicalName))
   }
 
   test("Kryo registrator") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 233d67898f909..a1e813227c9c0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -1222,14 +1222,6 @@ class DatasetSuite extends QueryTest
     assert(result == Set(ClassData("a", 1) -> null, ClassData("b", 2) -> ClassData("x", 2)))
   }
 
-  test("better error message when use java reserved keyword as field name") {
-    val e = intercept[UnsupportedOperationException] {
-      Seq(InvalidInJava(1)).toDS()
-    }
-    assert(e.getMessage.contains(
-      "`abstract` is a reserved keyword and cannot be used as field name"))
-  }
-
   test("Dataset should support flat input object to be null") {
     checkDataset(Seq("a", null).toDS(), "a", null)
   }
@@ -1899,6 +1891,26 @@ class DatasetSuite extends QueryTest
     val e = intercept[AnalysisException](spark.range(1).tail(-1))
     e.getMessage.contains("tail expression must be equal to or greater than 0")
   }
+
+  test("SparkSession.active should be the same instance after dataset operations") {
+    val active = SparkSession.getActiveSession.get
+    val clone = active.cloneSession()
+    val ds = new Dataset(clone, spark.range(10).queryExecution.logical, Encoders.INT)
+
+    ds.queryExecution.analyzed
+
+    assert(active eq SparkSession.getActiveSession.get)
+  }
+
+  test("SPARK-31854: Invoke in MapElementsExec should not propagate null") {
+    Seq("true", "false").foreach { wholeStage =>
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> wholeStage) {
+        val ds = Seq(1.asInstanceOf[Integer], null.asInstanceOf[Integer]).toDS()
+        val expectedAnswer = Seq[(Integer, Integer)]((1, 1), (null, null))
+        checkDataset(ds.map(v => (v, v)), expectedAnswer: _*)
+      }
+    }
+  }
 }
 
 object AssertExecutionId {
@@ -1939,8 +1951,6 @@ case class ClassNullableData(a: String, b: Integer)
 case class NestedStruct(f: ClassData)
 case class DeepNestedStruct(f: NestedStruct)
 
-case class InvalidInJava(`abstract`: Int)
-
 /**
  * A class used to test serialization using encoders. This class throws exceptions when using
  * Java serialization -- so the only way it can be "serialized" is through our encoders.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index d7d8c2c52d12b..247f8153dfde5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -19,11 +19,13 @@ package org.apache.spark.sql
 
 import java.sql.{Date, Timestamp}
 import java.text.SimpleDateFormat
-import java.time.Instant
-import java.util.Locale
+import java.time.{Instant, LocalDateTime}
+import java.util.{Locale, TimeZone}
 import java.util.concurrent.TimeUnit
 
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils}
+import org.apache.spark.{SparkException, SparkUpgradeException}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{CEST, LA}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -96,15 +98,19 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("date format") {
-    val df = Seq((d, sdf.format(d), ts)).toDF("a", "b", "c")
-
-    checkAnswer(
-      df.select(date_format($"a", "y"), date_format($"b", "y"), date_format($"c", "y")),
-      Row("2015", "2015", "2013"))
-
-    checkAnswer(
-      df.selectExpr("date_format(a, 'y')", "date_format(b, 'y')", "date_format(c, 'y')"),
-      Row("2015", "2015", "2013"))
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+        val df = Seq((d, sdf.format(d), ts)).toDF("a", "b", "c")
+
+        checkAnswer(
+          df.select(date_format($"a", "y"), date_format($"b", "y"), date_format($"c", "y")),
+          Row("2015", "2015", "2013"))
+
+        checkAnswer(
+          df.selectExpr("date_format(a, 'y')", "date_format(b, 'y')", "date_format(c, 'y')"),
+          Row("2015", "2015", "2013"))
+      }
+    }
   }
 
   test("year") {
@@ -373,6 +379,13 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       Seq(Row(Date.valueOf("2015-07-30")), Row(Date.valueOf("2015-07-30"))))
   }
 
+  def checkExceptionMessage(df: DataFrame): Unit = {
+    val message = intercept[SparkException] {
+      df.collect()
+    }.getCause.getMessage
+    assert(message.contains("Fail to parse"))
+  }
+
   test("function to_date") {
     val d1 = Date.valueOf("2015-07-22")
     val d2 = Date.valueOf("2015-07-01")
@@ -418,9 +431,15 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       df.select(to_date(col("d"), "yyyy-MM-dd")),
       Seq(Row(Date.valueOf("2015-07-22")), Row(Date.valueOf("2015-07-01")),
         Row(Date.valueOf("2014-12-31"))))
-    checkAnswer(
-      df.select(to_date(col("s"), "yyyy-MM-dd")),
-      Seq(Row(null), Row(Date.valueOf("2014-12-31")), Row(null)))
+    val confKey = SQLConf.LEGACY_TIME_PARSER_POLICY.key
+    withSQLConf(confKey -> "corrected") {
+      checkAnswer(
+        df.select(to_date(col("s"), "yyyy-MM-dd")),
+        Seq(Row(null), Row(Date.valueOf("2014-12-31")), Row(null)))
+    }
+    withSQLConf(confKey -> "exception") {
+      checkExceptionMessage(df.select(to_date(col("s"), "yyyy-MM-dd")))
+    }
 
     // now switch format
     checkAnswer(
@@ -431,9 +450,9 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df.select(to_date(col("s"), "yyyy-hh-MM")),
       Seq(Row(null), Row(null), Row(null)))
-    checkAnswer(
-      df.select(to_date(col("s"), "yyyy-dd-aa")),
-      Seq(Row(null), Row(null), Row(null)))
+    val e = intercept[SparkUpgradeException](df.select(to_date(col("s"), "yyyy-dd-aa")).collect())
+    assert(e.getCause.isInstanceOf[IllegalArgumentException])
+    assert(e.getMessage.contains("You may get a different result due to the upgrading of Spark"))
 
     // february
     val x1 = "2016-02-29"
@@ -455,10 +474,6 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df.selectExpr("trunc(t, 'Month')"),
       Seq(Row(Date.valueOf("2015-07-01")), Row(Date.valueOf("2014-12-01"))))
-
-    checkAnswer(
-      df.selectExpr("trunc(t, 'decade')"),
-      Seq(Row(Date.valueOf("2010-01-01")), Row(Date.valueOf("2010-01-01"))))
   }
 
   test("function date_trunc") {
@@ -510,185 +525,215 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       df.selectExpr("date_trunc('MILLISECOND', t)"),
       Seq(Row(Timestamp.valueOf("2015-07-22 10:01:40.123")),
         Row(Timestamp.valueOf("2014-12-31 05:29:06.123"))))
+  }
 
+  test("unsupported fmt fields for trunc/date_trunc results null") {
+    Seq("INVALID", "decade", "century", "millennium", "whatever", null).foreach { f =>
     checkAnswer(
-      df.selectExpr("date_trunc('DECADE', t)"),
-      Seq(Row(Timestamp.valueOf("2010-01-01 00:00:00")),
-        Row(Timestamp.valueOf("2010-01-01 00:00:00"))))
-
-    Seq("century", "millennium").foreach { level =>
-      checkAnswer(
-        df.selectExpr(s"date_trunc('$level', t)"),
-        Seq(Row(Timestamp.valueOf("2001-01-01 00:00:00")),
-          Row(Timestamp.valueOf("2001-01-01 00:00:00"))))
+      Seq(Date.valueOf("2014-12-31"))
+        .toDF("dt")
+        .selectExpr(s"date_trunc('$f', dt)", "trunc(dt, '$f')"),
+      Row(null, null))
     }
   }
 
   test("from_unixtime") {
-    val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
-    val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
-    val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
-    val fmt3 = "yy-MM-dd HH-mm-ss"
-    val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
-    val df = Seq((1000, "yyyy-MM-dd HH:mm:ss.SSS"), (-1000, "yy-MM-dd HH-mm-ss")).toDF("a", "b")
-    checkAnswer(
-      df.select(from_unixtime(col("a"))),
-      Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new Timestamp(-1000000)))))
-    checkAnswer(
-      df.select(from_unixtime(col("a"), fmt2)),
-      Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new Timestamp(-1000000)))))
-    checkAnswer(
-      df.select(from_unixtime(col("a"), fmt3)),
-      Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new Timestamp(-1000000)))))
-    checkAnswer(
-      df.selectExpr("from_unixtime(a)"),
-      Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new Timestamp(-1000000)))))
-    checkAnswer(
-      df.selectExpr(s"from_unixtime(a, '$fmt2')"),
-      Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new Timestamp(-1000000)))))
-    checkAnswer(
-      df.selectExpr(s"from_unixtime(a, '$fmt3')"),
-      Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new Timestamp(-1000000)))))
+    Seq("corrected", "legacy").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+        val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
+        val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+        val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
+        val fmt3 = "yy-MM-dd HH-mm-ss"
+        val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
+        val df = Seq((1000, "yyyy-MM-dd HH:mm:ss.SSS"), (-1000, "yy-MM-dd HH-mm-ss")).toDF("a", "b")
+        checkAnswer(
+          df.select(from_unixtime(col("a"))),
+          Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new Timestamp(-1000000)))))
+        checkAnswer(
+          df.select(from_unixtime(col("a"), fmt2)),
+          Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new Timestamp(-1000000)))))
+        checkAnswer(
+          df.select(from_unixtime(col("a"), fmt3)),
+          Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new Timestamp(-1000000)))))
+        checkAnswer(
+          df.selectExpr("from_unixtime(a)"),
+          Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new Timestamp(-1000000)))))
+        checkAnswer(
+          df.selectExpr(s"from_unixtime(a, '$fmt2')"),
+          Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new Timestamp(-1000000)))))
+        checkAnswer(
+          df.selectExpr(s"from_unixtime(a, '$fmt3')"),
+          Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new Timestamp(-1000000)))))
+      }
+    }
   }
 
   private def secs(millis: Long): Long = TimeUnit.MILLISECONDS.toSeconds(millis)
 
   test("unix_timestamp") {
-    val date1 = Date.valueOf("2015-07-24")
-    val date2 = Date.valueOf("2015-07-25")
-    val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
-    val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2")
-    val s1 = "2015/07/24 10:00:00.5"
-    val s2 = "2015/07/25 02:02:02.6"
-    val ss1 = "2015-07-24 10:00:00"
-    val ss2 = "2015-07-25 02:02:02"
-    val fmt = "yyyy/MM/dd HH:mm:ss.S"
-    val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
-    checkAnswer(df.select(unix_timestamp(col("ts"))), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.select(unix_timestamp(col("ss"))), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.select(unix_timestamp(col("d"), fmt)), Seq(
-      Row(secs(date1.getTime)), Row(secs(date2.getTime))))
-    checkAnswer(df.select(unix_timestamp(col("s"), fmt)), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.selectExpr("unix_timestamp(ts)"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.selectExpr("unix_timestamp(ss)"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.selectExpr(s"unix_timestamp(d, '$fmt')"), Seq(
-      Row(secs(date1.getTime)), Row(secs(date2.getTime))))
-    checkAnswer(df.selectExpr(s"unix_timestamp(s, '$fmt')"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-
-    val x1 = "2015-07-24 10:00:00"
-    val x2 = "2015-25-07 02:02:02"
-    val x3 = "2015-07-24 25:02:02"
-    val x4 = "2015-24-07 26:02:02"
-    val ts3 = Timestamp.valueOf("2015-07-24 02:25:02")
-    val ts4 = Timestamp.valueOf("2015-07-24 00:10:00")
-
-    val df1 = Seq(x1, x2, x3, x4).toDF("x")
-    checkAnswer(df1.select(unix_timestamp(col("x"))), Seq(
-      Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
-    checkAnswer(df1.selectExpr("unix_timestamp(x)"), Seq(
-      Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
-    checkAnswer(df1.select(unix_timestamp(col("x"), "yyyy-dd-MM HH:mm:ss")), Seq(
-      Row(null), Row(secs(ts2.getTime)), Row(null), Row(null)))
-    checkAnswer(df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(
-      Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
-
-    // invalid format
-    checkAnswer(df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')"), Seq(
-      Row(null), Row(null), Row(null), Row(null)))
-
-    // february
-    val y1 = "2016-02-29"
-    val y2 = "2017-02-29"
-    val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
-    val df2 = Seq(y1, y2).toDF("y")
-    checkAnswer(df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(
-      Row(secs(ts5.getTime)), Row(null)))
-
-    val now = sql("select unix_timestamp()").collect().head.getLong(0)
-    checkAnswer(
-      sql(s"select cast ($now as timestamp)"),
-      Row(new java.util.Date(TimeUnit.SECONDS.toMillis(now))))
+    Seq("corrected", "legacy").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+        val date1 = Date.valueOf("2015-07-24")
+        val date2 = Date.valueOf("2015-07-25")
+        val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
+        val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2")
+        val s1 = "2015/07/24 10:00:00.5"
+        val s2 = "2015/07/25 02:02:02.6"
+        val ss1 = "2015-07-24 10:00:00"
+        val ss2 = "2015-07-25 02:02:02"
+        val fmt = "yyyy/MM/dd HH:mm:ss.S"
+        val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
+        checkAnswer(df.select(unix_timestamp(col("ts"))), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.select(unix_timestamp(col("ss"))), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.select(unix_timestamp(col("d"), fmt)), Seq(
+          Row(secs(date1.getTime)), Row(secs(date2.getTime))))
+        checkAnswer(df.select(unix_timestamp(col("s"), fmt)), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.selectExpr("unix_timestamp(ts)"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.selectExpr("unix_timestamp(ss)"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.selectExpr(s"unix_timestamp(d, '$fmt')"), Seq(
+          Row(secs(date1.getTime)), Row(secs(date2.getTime))))
+        checkAnswer(df.selectExpr(s"unix_timestamp(s, '$fmt')"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+
+        val x1 = "2015-07-24 10:00:00"
+        val x2 = "2015-25-07 02:02:02"
+        val x3 = "2015-07-24 25:02:02"
+        val x4 = "2015-24-07 26:02:02"
+        val ts3 = Timestamp.valueOf("2015-07-24 02:25:02")
+        val ts4 = Timestamp.valueOf("2015-07-24 00:10:00")
+
+        val df1 = Seq(x1, x2, x3, x4).toDF("x")
+        checkAnswer(df1.select(unix_timestamp(col("x"))), Seq(
+          Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
+        checkAnswer(df1.selectExpr("unix_timestamp(x)"), Seq(
+          Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
+        checkAnswer(df1.select(unix_timestamp(col("x"), "yyyy-dd-MM HH:mm:ss")), Seq(
+          Row(null), Row(secs(ts2.getTime)), Row(null), Row(null)))
+        checkAnswer(df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(
+          Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
+
+        // invalid format
+        val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')")
+        if (legacyParserPolicy == "legacy") {
+          checkAnswer(invalid,
+            Seq(Row(null), Row(null), Row(null), Row(null)))
+        } else {
+          val e = intercept[SparkUpgradeException](invalid.collect())
+          assert(e.getCause.isInstanceOf[IllegalArgumentException])
+          assert(
+            e.getMessage.contains("You may get a different result due to the upgrading of Spark"))
+        }
+
+        // february
+        val y1 = "2016-02-29"
+        val y2 = "2017-02-29"
+        val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
+        val df2 = Seq(y1, y2).toDF("y")
+        checkAnswer(df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(
+          Row(secs(ts5.getTime)), Row(null)))
+
+        val now = sql("select unix_timestamp()").collect().head.getLong(0)
+        checkAnswer(
+          sql(s"select cast ($now as timestamp)"),
+          Row(new java.util.Date(TimeUnit.SECONDS.toMillis(now))))
+      }
+    }
   }
 
   test("to_unix_timestamp") {
-    val date1 = Date.valueOf("2015-07-24")
-    val date2 = Date.valueOf("2015-07-25")
-    val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
-    val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2")
-    val s1 = "2015/07/24 10:00:00.5"
-    val s2 = "2015/07/25 02:02:02.6"
-    val ss1 = "2015-07-24 10:00:00"
-    val ss2 = "2015-07-25 02:02:02"
-    val fmt = "yyyy/MM/dd HH:mm:ss.S"
-    val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
-    checkAnswer(df.selectExpr("to_unix_timestamp(ts)"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.selectExpr("to_unix_timestamp(ss)"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.selectExpr(s"to_unix_timestamp(d, '$fmt')"), Seq(
-      Row(secs(date1.getTime)), Row(secs(date2.getTime))))
-    checkAnswer(df.selectExpr(s"to_unix_timestamp(s, '$fmt')"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-
-    val x1 = "2015-07-24 10:00:00"
-    val x2 = "2015-25-07 02:02:02"
-    val x3 = "2015-07-24 25:02:02"
-    val x4 = "2015-24-07 26:02:02"
-    val ts3 = Timestamp.valueOf("2015-07-24 02:25:02")
-    val ts4 = Timestamp.valueOf("2015-07-24 00:10:00")
-
-    val df1 = Seq(x1, x2, x3, x4).toDF("x")
-    checkAnswer(df1.selectExpr("to_unix_timestamp(x)"), Seq(
-      Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
-    checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(
-      Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
-
-    // february
-    val y1 = "2016-02-29"
-    val y2 = "2017-02-29"
-    val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
-    val df2 = Seq(y1, y2).toDF("y")
-    checkAnswer(df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(
-      Row(secs(ts5.getTime)), Row(null)))
-
-    // invalid format
-    checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')"), Seq(
-      Row(null), Row(null), Row(null), Row(null)))
+    Seq("corrected", "legacy").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+        val date1 = Date.valueOf("2015-07-24")
+        val date2 = Date.valueOf("2015-07-25")
+        val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
+        val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2")
+        val s1 = "2015/07/24 10:00:00.5"
+        val s2 = "2015/07/25 02:02:02.6"
+        val ss1 = "2015-07-24 10:00:00"
+        val ss2 = "2015-07-25 02:02:02"
+        val fmt = "yyyy/MM/dd HH:mm:ss.S"
+        val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
+        checkAnswer(df.selectExpr("to_unix_timestamp(ts)"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.selectExpr("to_unix_timestamp(ss)"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.selectExpr(s"to_unix_timestamp(d, '$fmt')"), Seq(
+          Row(secs(date1.getTime)), Row(secs(date2.getTime))))
+        checkAnswer(df.selectExpr(s"to_unix_timestamp(s, '$fmt')"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+
+        val x1 = "2015-07-24 10:00:00"
+        val x2 = "2015-25-07 02:02:02"
+        val x3 = "2015-07-24 25:02:02"
+        val x4 = "2015-24-07 26:02:02"
+        val ts3 = Timestamp.valueOf("2015-07-24 02:25:02")
+        val ts4 = Timestamp.valueOf("2015-07-24 00:10:00")
+
+        val df1 = Seq(x1, x2, x3, x4).toDF("x")
+        checkAnswer(df1.selectExpr("to_unix_timestamp(x)"), Seq(
+          Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
+        checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(
+          Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
+
+        // february
+        val y1 = "2016-02-29"
+        val y2 = "2017-02-29"
+        val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
+        val df2 = Seq(y1, y2).toDF("y")
+        checkAnswer(df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(
+          Row(secs(ts5.getTime)), Row(null)))
+
+        // invalid format
+        checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')"), Seq(
+          Row(null), Row(null), Row(null), Row(null)))
+      }
+    }
   }
 
 
   test("to_timestamp") {
-    val date1 = Date.valueOf("2015-07-24")
-    val date2 = Date.valueOf("2015-07-25")
-    val ts_date1 = Timestamp.valueOf("2015-07-24 00:00:00")
-    val ts_date2 = Timestamp.valueOf("2015-07-25 00:00:00")
-    val ts1 = Timestamp.valueOf("2015-07-24 10:00:00")
-    val ts2 = Timestamp.valueOf("2015-07-25 02:02:02")
-    val s1 = "2015/07/24 10:00:00.5"
-    val s2 = "2015/07/25 02:02:02.6"
-    val ts1m = Timestamp.valueOf("2015-07-24 10:00:00.5")
-    val ts2m = Timestamp.valueOf("2015-07-25 02:02:02.6")
-    val ss1 = "2015-07-24 10:00:00"
-    val ss2 = "2015-07-25 02:02:02"
-    val fmt = "yyyy/MM/dd HH:mm:ss.S"
-    val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
-
-    checkAnswer(df.select(to_timestamp(col("ss"))),
-      df.select(unix_timestamp(col("ss")).cast("timestamp")))
-    checkAnswer(df.select(to_timestamp(col("ss"))), Seq(
-      Row(ts1), Row(ts2)))
-    checkAnswer(df.select(to_timestamp(col("s"), fmt)), Seq(
-      Row(ts1m), Row(ts2m)))
-    checkAnswer(df.select(to_timestamp(col("ts"), fmt)), Seq(
-      Row(ts1), Row(ts2)))
-    checkAnswer(df.select(to_timestamp(col("d"), "yyyy-MM-dd")), Seq(
-      Row(ts_date1), Row(ts_date2)))
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+        val date1 = Date.valueOf("2015-07-24")
+        val date2 = Date.valueOf("2015-07-25")
+        val ts_date1 = Timestamp.valueOf("2015-07-24 00:00:00")
+        val ts_date2 = Timestamp.valueOf("2015-07-25 00:00:00")
+        val ts1 = Timestamp.valueOf("2015-07-24 10:00:00")
+        val ts2 = Timestamp.valueOf("2015-07-25 02:02:02")
+        val s1 = "2015/07/24 10:00:00.5"
+        val s2 = "2015/07/25 02:02:02.6"
+        val ts1m = Timestamp.valueOf("2015-07-24 10:00:00.5")
+        val ts2m = Timestamp.valueOf("2015-07-25 02:02:02.6")
+        val ss1 = "2015-07-24 10:00:00"
+        val ss2 = "2015-07-25 02:02:02"
+        val fmt = "yyyy/MM/dd HH:mm:ss.S"
+        val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
+
+        checkAnswer(df.select(to_timestamp(col("ss"))),
+          df.select(unix_timestamp(col("ss")).cast("timestamp")))
+        checkAnswer(df.select(to_timestamp(col("ss"))), Seq(
+          Row(ts1), Row(ts2)))
+        if (legacyParserPolicy == "legacy") {
+          // In Spark 2.4 and earlier, to_timestamp() parses in seconds precision and cuts off
+          // the fractional part of seconds. The behavior was changed by SPARK-27438.
+          val legacyFmt = "yyyy/MM/dd HH:mm:ss"
+          checkAnswer(df.select(to_timestamp(col("s"), legacyFmt)), Seq(
+            Row(ts1), Row(ts2)))
+        } else {
+          checkAnswer(df.select(to_timestamp(col("s"), fmt)), Seq(
+            Row(ts1m), Row(ts2m)))
+        }
+        checkAnswer(df.select(to_timestamp(col("ts"), fmt)), Seq(
+          Row(ts1), Row(ts2)))
+        checkAnswer(df.select(to_timestamp(col("d"), "yyyy-MM-dd")), Seq(
+          Row(ts_date1), Row(ts_date2)))
+      }
+    }
   }
 
   test("datediff") {
@@ -719,12 +764,12 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
     ).toDF("a", "b")
     checkAnswer(
-      df.select(from_utc_timestamp(col("a"), "PST")),
+      df.select(from_utc_timestamp(col("a"), LA.getId)),
       Seq(
         Row(Timestamp.valueOf("2015-07-23 17:00:00")),
         Row(Timestamp.valueOf("2015-07-24 17:00:00"))))
     checkAnswer(
-      df.select(from_utc_timestamp(col("b"), "PST")),
+      df.select(from_utc_timestamp(col("b"), LA.getId)),
       Seq(
         Row(Timestamp.valueOf("2015-07-23 17:00:00")),
         Row(Timestamp.valueOf("2015-07-24 17:00:00"))))
@@ -732,8 +777,8 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
 
   test("from_utc_timestamp with column zone") {
     val df = Seq(
-      (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "CET"),
-      (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "PST")
+      (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", CEST.getId),
+      (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", LA.getId)
     ).toDF("a", "b", "c")
     checkAnswer(
       df.select(from_utc_timestamp(col("a"), col("c"))),
@@ -762,12 +807,12 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
     ).toDF("a", "b")
     checkAnswer(
-      df.select(to_utc_timestamp(col("a"), "PST")),
+      df.select(to_utc_timestamp(col("a"), LA.getId)),
       Seq(
         Row(Timestamp.valueOf("2015-07-24 07:00:00")),
         Row(Timestamp.valueOf("2015-07-25 07:00:00"))))
     checkAnswer(
-      df.select(to_utc_timestamp(col("b"), "PST")),
+      df.select(to_utc_timestamp(col("b"), LA.getId)),
       Seq(
         Row(Timestamp.valueOf("2015-07-24 07:00:00")),
         Row(Timestamp.valueOf("2015-07-25 07:00:00"))))
@@ -775,8 +820,8 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
 
   test("to_utc_timestamp with column zone") {
     val df = Seq(
-      (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "PST"),
-      (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "CET")
+      (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", LA.getId),
+      (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", CEST.getId)
     ).toDF("a", "b", "c")
     checkAnswer(
       df.select(to_utc_timestamp(col("a"), col("c"))),
@@ -789,4 +834,71 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
         Row(Timestamp.valueOf("2015-07-24 07:00:00")),
         Row(Timestamp.valueOf("2015-07-24 22:00:00"))))
   }
+
+  test("SPARK-30668: use legacy timestamp parser in to_timestamp") {
+    val confKey = SQLConf.LEGACY_TIME_PARSER_POLICY.key
+    val df = Seq("2020-01-27T20:06:11.847-0800").toDF("ts")
+    withSQLConf(confKey -> "legacy") {
+      val expected = Timestamp.valueOf("2020-01-27 20:06:11.847")
+      checkAnswer(df.select(to_timestamp(col("ts"), "yyyy-MM-dd'T'HH:mm:ss.SSSz")),
+        Row(expected))
+    }
+    withSQLConf(confKey -> "corrected") {
+      checkAnswer(df.select(to_timestamp(col("ts"), "yyyy-MM-dd'T'HH:mm:ss.SSSz")), Row(null))
+    }
+    withSQLConf(confKey -> "exception") {
+      checkExceptionMessage(df.select(to_timestamp(col("ts"), "yyyy-MM-dd'T'HH:mm:ss.SSSz")))
+    }
+  }
+
+  test("SPARK-30752: convert time zones on a daylight saving day") {
+    val systemTz = LA.getId
+    val sessionTz = "UTC"
+    val fromTz = "Asia/Hong_Kong"
+    val fromTs = "2019-11-03T12:00:00" // daylight saving date in America/Los_Angeles
+    val utsTs = "2019-11-03T04:00:00"
+    val defaultTz = TimeZone.getDefault
+    try {
+      TimeZone.setDefault(DateTimeUtils.getTimeZone(systemTz))
+      withSQLConf(
+        SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true",
+        SQLConf.SESSION_LOCAL_TIMEZONE.key -> sessionTz) {
+        val expected = LocalDateTime.parse(utsTs)
+          .atZone(DateTimeUtils.getZoneId(sessionTz))
+          .toInstant
+        val df = Seq(fromTs).toDF("localTs")
+        checkAnswer(
+          df.select(to_utc_timestamp(col("localTs"), fromTz)),
+          Row(expected))
+      }
+    } finally {
+      TimeZone.setDefault(defaultTz)
+    }
+  }
+
+  test("SPARK-30766: date_trunc of old timestamps to hours and days") {
+    def checkTrunc(level: String, expected: String): Unit = {
+      val df = Seq("0010-01-01 01:02:03.123456")
+        .toDF()
+        .select($"value".cast("timestamp").as("ts"))
+        .select(date_trunc(level, $"ts").cast("string"))
+      checkAnswer(df, Row(expected))
+    }
+
+    checkTrunc("HOUR", "0010-01-01 01:00:00")
+    checkTrunc("DAY", "0010-01-01 00:00:00")
+  }
+
+  test("SPARK-30793: truncate timestamps before the epoch to seconds and minutes") {
+    def checkTrunc(level: String, expected: String): Unit = {
+      val df = Seq("1961-04-12 00:01:02.345")
+        .toDF()
+        .select($"value".cast("timestamp").as("ts"))
+        .select(date_trunc(level, $"ts").cast("string"))
+      checkAnswer(df, Row(expected))
+    }
+
+    checkTrunc("SECOND", "1961-04-12 00:01:02")
+    checkTrunc("MINUTE", "1961-04-12 00:01:00")
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala
new file mode 100644
index 0000000000000..25b8849d61248
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
+
+class DeprecatedAPISuite extends QueryTest with SharedSparkSession {
+  import MathFunctionsTestData.DoubleData
+  import testImplicits._
+
+  private lazy val doubleData = (1 to 10).map(i => DoubleData(i * 0.2 - 1, i * -0.2 + 1)).toDF()
+
+  private def testOneToOneMathFunction[
+    @specialized(Int, Long, Float, Double) T,
+    @specialized(Int, Long, Float, Double) U](
+        c: Column => Column,
+        f: T => U): Unit = {
+    checkAnswer(
+      doubleData.select(c('a)),
+      (1 to 10).map(n => Row(f((n * 0.2 - 1).asInstanceOf[T])))
+    )
+
+    checkAnswer(
+      doubleData.select(c('b)),
+      (1 to 10).map(n => Row(f((-n * 0.2 + 1).asInstanceOf[T])))
+    )
+
+    checkAnswer(
+      doubleData.select(c(lit(null))),
+      (1 to 10).map(_ => Row(null))
+    )
+  }
+
+  test("functions.toDegrees") {
+    testOneToOneMathFunction(toDegrees, math.toDegrees)
+    withView("t") {
+      val df = Seq(0, 1, 1.5).toDF("a")
+      df.createOrReplaceTempView("t")
+
+      checkAnswer(
+        sql("SELECT degrees(0), degrees(1), degrees(1.5)"),
+        Seq(0).toDF().select(toDegrees(lit(0)), toDegrees(lit(1)), toDegrees(lit(1.5)))
+      )
+      checkAnswer(
+        sql("SELECT degrees(a) FROM t"),
+        df.select(toDegrees("a"))
+      )
+    }
+  }
+
+  test("functions.toRadians") {
+    testOneToOneMathFunction(toRadians, math.toRadians)
+    withView("t") {
+      val df = Seq(0, 1, 1.5).toDF("a")
+      df.createOrReplaceTempView("t")
+
+      checkAnswer(
+        sql("SELECT radians(0), radians(1), radians(1.5)"),
+        Seq(0).toDF().select(toRadians(lit(0)), toRadians(lit(1)), toRadians(lit(1.5)))
+      )
+      checkAnswer(
+        sql("SELECT radians(a) FROM t"),
+        df.select(toRadians("a"))
+      )
+    }
+  }
+
+  test("functions.approxCountDistinct") {
+    withView("t") {
+      val df = Seq(0, 1, 2).toDF("a")
+      df.createOrReplaceTempView("t")
+      checkAnswer(
+        sql("SELECT approx_count_distinct(a) FROM t"),
+        df.select(approxCountDistinct("a")))
+    }
+  }
+
+  test("functions.monotonicallyIncreasingId") {
+    // Make sure we have 2 partitions, each with 2 records.
+    val df = sparkContext.parallelize(Seq[Int](), 2).mapPartitions { _ =>
+      Iterator(Tuple1(1), Tuple1(2))
+    }.toDF("a")
+    checkAnswer(
+      df.select(monotonicallyIncreasingId(), expr("monotonically_increasing_id()")),
+      Row(0L, 0L) ::
+        Row(1L, 1L) ::
+        Row((1L << 33) + 0L, (1L << 33) + 0L) ::
+        Row((1L << 33) + 1L, (1L << 33) + 1L) :: Nil
+    )
+  }
+
+  test("Column.!==") {
+    val nullData = Seq(
+      (Some(1), Some(1)), (Some(1), Some(2)), (Some(1), None), (None, None)).toDF("a", "b")
+    checkAnswer(
+      nullData.filter($"b" !== 1),
+      Row(1, 2) :: Nil)
+
+    checkAnswer(nullData.filter($"b" !== null), Nil)
+
+    checkAnswer(
+      nullData.filter($"a" !== $"b"),
+      Row(1, 2) :: Nil)
+  }
+
+  test("Dataset.registerTempTable") {
+    withTempView("t") {
+      Seq(1).toDF().registerTempTable("t")
+      assert(spark.catalog.tableExists("t"))
+    }
+  }
+
+  test("SQLContext.setActive/clearActive") {
+    val sc = spark.sparkContext
+    val sqlContext = new SQLContext(sc)
+    SQLContext.setActive(sqlContext)
+    assert(SparkSession.getActiveSession === Some(spark))
+    SQLContext.clearActive()
+    assert(SparkSession.getActiveSession === None)
+  }
+
+  test("SQLContext.applySchema") {
+    val rowRdd = sparkContext.parallelize(Seq(Row("Jack", 20), Row("Marry", 18)))
+    val schema = StructType(StructField("name", StringType, false) ::
+      StructField("age", IntegerType, true) :: Nil)
+    val sqlContext = spark.sqlContext
+    checkAnswer(sqlContext.applySchema(rowRdd, schema), Row("Jack", 20) :: Row("Marry", 18) :: Nil)
+    checkAnswer(sqlContext.applySchema(rowRdd.toJavaRDD(), schema),
+      Row("Jack", 20) :: Row("Marry", 18) :: Nil)
+  }
+
+  test("SQLContext.parquetFile") {
+    val sqlContext = spark.sqlContext
+    withTempDir { dir =>
+      val parquetFile = s"${dir.toString}/${System.currentTimeMillis()}"
+      val expectDF = spark.range(10).toDF()
+      expectDF.write.parquet(parquetFile)
+      val parquetDF = sqlContext.parquetFile(parquetFile)
+      checkAnswer(parquetDF, expectDF)
+    }
+  }
+
+  test("SQLContext.jsonFile") {
+    val sqlContext = spark.sqlContext
+    withTempDir { dir =>
+      val jsonFile = s"${dir.toString}/${System.currentTimeMillis()}"
+      val expectDF = spark.range(10).toDF()
+      expectDF.write.json(jsonFile)
+      var jsonDF = sqlContext.jsonFile(jsonFile)
+      checkAnswer(jsonDF, expectDF)
+      assert(jsonDF.schema === expectDF.schema.asNullable)
+
+      var schema = expectDF.schema
+      jsonDF = sqlContext.jsonFile(jsonFile, schema)
+      checkAnswer(jsonDF, expectDF)
+      assert(jsonDF.schema === schema.asNullable)
+
+      jsonDF = sqlContext.jsonFile(jsonFile, 0.9)
+      checkAnswer(jsonDF, expectDF)
+
+      val jsonRDD = sparkContext.parallelize(Seq("{\"name\":\"Jack\",\"age\":20}",
+        "{\"name\":\"Marry\",\"age\":18}"))
+      jsonDF = sqlContext.jsonRDD(jsonRDD)
+      checkAnswer(jsonDF, Row(18, "Marry") :: Row(20, "Jack") :: Nil)
+      jsonDF = sqlContext.jsonRDD(jsonRDD.toJavaRDD())
+      checkAnswer(jsonDF, Row(18, "Marry") :: Row(20, "Jack") :: Nil)
+
+      schema = StructType(StructField("name", StringType, false) ::
+        StructField("age", IntegerType, false) :: Nil)
+      jsonDF = sqlContext.jsonRDD(jsonRDD, schema)
+      checkAnswer(jsonDF, Row("Jack", 20) :: Row("Marry", 18) :: Nil)
+      jsonDF = sqlContext.jsonRDD(jsonRDD.toJavaRDD(), schema)
+      checkAnswer(jsonDF, Row("Jack", 20) :: Row("Marry", 18) :: Nil)
+
+
+      jsonDF = sqlContext.jsonRDD(jsonRDD, 0.9)
+      checkAnswer(jsonDF, Row(18, "Marry") :: Row(20, "Jack") :: Nil)
+      jsonDF = sqlContext.jsonRDD(jsonRDD.toJavaRDD(), 0.9)
+      checkAnswer(jsonDF, Row(18, "Marry") :: Row(20, "Jack") :: Nil)
+    }
+  }
+
+  test("SQLContext.load") {
+    withTempDir { dir =>
+      val path = s"${dir.toString}/${System.currentTimeMillis()}"
+      val expectDF = spark.range(10).toDF()
+      expectDF.write.parquet(path)
+      val sqlContext = spark.sqlContext
+
+      var loadDF = sqlContext.load(path)
+      checkAnswer(loadDF, expectDF)
+
+      loadDF = sqlContext.load(path, "parquet")
+      checkAnswer(loadDF, expectDF)
+
+      loadDF = sqlContext.load("parquet", Map("path" -> path))
+      checkAnswer(loadDF, expectDF)
+
+      loadDF = sqlContext.load("parquet", expectDF.schema, Map("path" -> path))
+      checkAnswer(loadDF, expectDF)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index e1f9bcc4e008d..94163315b34f9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -20,9 +20,10 @@ package org.apache.spark.sql
 import org.scalatest.GivenWhenThen
 
 import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
+import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._
 import org.apache.spark.sql.catalyst.plans.ExistenceJoin
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper}
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec}
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
 import org.apache.spark.sql.execution.streaming.{MemoryStream, StreamingQueryWrapper}
@@ -33,7 +34,7 @@ import org.apache.spark.sql.test.SharedSparkSession
 /**
  * Test suite for the filtering ratio policy used to trigger dynamic partition pruning (DPP).
  */
-class DynamicPartitionPruningSuite
+abstract class DynamicPartitionPruningSuiteBase
     extends QueryTest
     with SharedSparkSession
     with GivenWhenThen
@@ -43,9 +44,14 @@ class DynamicPartitionPruningSuite
 
   import testImplicits._
 
+  val adaptiveExecutionOn: Boolean
+
   override def beforeAll(): Unit = {
     super.beforeAll()
 
+    spark.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, adaptiveExecutionOn)
+    spark.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY, true)
+
     val factData = Seq[(Int, Int, Int, Int)](
       (1000, 1, 1, 10),
       (1010, 2, 1, 10),
@@ -153,6 +159,8 @@ class DynamicPartitionPruningSuite
       sql("DROP TABLE IF EXISTS fact_stats")
       sql("DROP TABLE IF EXISTS dim_stats")
     } finally {
+      spark.sessionState.conf.unsetConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED)
+      spark.sessionState.conf.unsetConf(SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY)
       super.afterAll()
     }
   }
@@ -195,6 +203,11 @@ class DynamicPartitionPruningSuite
           fail(s"Invalid child node found in\n$s")
       }
     }
+
+    val isMainQueryAdaptive = plan.isInstanceOf[AdaptiveSparkPlanExec]
+    subqueriesAll(plan).filterNot(subqueryBroadcast.contains).foreach { s =>
+      assert(s.find(_.isInstanceOf[AdaptiveSparkPlanExec]).isDefined == isMainQueryAdaptive)
+    }
   }
 
   /**
@@ -239,30 +252,17 @@ class DynamicPartitionPruningSuite
    */
   test("simple inner join triggers DPP with mock-up tables") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
-      withTable("df1", "df2") {
-        spark.range(1000)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df1")
-
-        spark.range(100)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df2")
-
-        val df = sql("SELECT df1.id, df2.k FROM df1 JOIN df2 ON df1.k = df2.k AND df2.id < 2")
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
+      val df = sql(
+        """
+          |SELECT f.date_id, f.store_id FROM fact_sk f
+          |JOIN dim_store s ON f.store_id = s.store_id AND s.country = 'NL'
+        """.stripMargin)
 
-        checkPartitionPruningPredicate(df, true, false)
+      checkPartitionPruningPredicate(df, true, false)
 
-        checkAnswer(df, Row(0, 0) :: Row(1, 1) :: Nil)
-      }
+      checkAnswer(df, Row(1000, 1) :: Row(1010, 2) :: Row(1020, 2) :: Nil)
     }
   }
 
@@ -271,7 +271,8 @@ class DynamicPartitionPruningSuite
    */
   test("self-join on a partitioned table should not trigger DPP") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       withTable("fact") {
         sql(
           s"""
@@ -302,8 +303,11 @@ class DynamicPartitionPruningSuite
    */
   test("static scan metrics") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       withTable("fact", "dim") {
+        val numPartitions = 10
+
         spark.range(10)
           .map { x => Tuple3(x, x + 1, 0) }
           .toDF("did", "d1", "d2")
@@ -313,7 +317,7 @@ class DynamicPartitionPruningSuite
           .saveAsTable("dim")
 
         spark.range(100)
-          .map { x => Tuple2(x, x % 10) }
+          .map { x => Tuple2(x, x % numPartitions) }
           .toDF("f1", "fid")
           .write.partitionBy("fid")
           .format(tableFormat)
@@ -340,6 +344,8 @@ class DynamicPartitionPruningSuite
         assert(!scan1.metrics.contains("staticFilesSize"))
         val allFilesNum = scan1.metrics("numFiles").value
         val allFilesSize = scan1.metrics("filesSize").value
+        assert(scan1.metrics("numPartitions").value === numPartitions)
+        assert(scan1.metrics("pruningTime").value === -1)
 
         // No dynamic partition pruning, so no static metrics
         // Only files from fid = 5 partition are scanned
@@ -352,6 +358,8 @@ class DynamicPartitionPruningSuite
         val partFilesSize = scan2.metrics("filesSize").value
         assert(0 < partFilesNum && partFilesNum < allFilesNum)
         assert(0 < partFilesSize && partFilesSize < allFilesSize)
+        assert(scan2.metrics("numPartitions").value === 1)
+        assert(scan2.metrics("pruningTime").value === -1)
 
         // Dynamic partition pruning is used
         // Static metrics are as-if reading the whole fact table
@@ -363,6 +371,8 @@ class DynamicPartitionPruningSuite
         assert(scan3.metrics("staticFilesSize").value == allFilesSize)
         assert(scan3.metrics("numFiles").value == partFilesNum)
         assert(scan3.metrics("filesSize").value == partFilesSize)
+        assert(scan3.metrics("numPartitions").value === 1)
+        assert(scan3.metrics("pruningTime").value !== -1)
       }
     }
   }
@@ -370,7 +380,8 @@ class DynamicPartitionPruningSuite
   test("DPP should not be rewritten as an existential join") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
       SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "1.5",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       val df = sql(
         s"""
            |SELECT * FROM product p WHERE p.store_id NOT IN
@@ -395,7 +406,7 @@ class DynamicPartitionPruningSuite
    */
   test("DPP triggers only for certain types of query") {
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false") {
       Given("dynamic partition pruning disabled")
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "false") {
         val df = sql(
@@ -433,7 +444,8 @@ class DynamicPartitionPruningSuite
       }
 
       Given("left-semi join with partition column on the left side")
-      withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
+      withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
+        SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
         val df = sql(
           """
             |SELECT * FROM fact_sk f
@@ -457,7 +469,8 @@ class DynamicPartitionPruningSuite
       }
 
       Given("right outer join with partition column on the left side")
-      withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
+      withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
+        SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
         val df = sql(
           """
             |SELECT * FROM fact_sk f RIGHT OUTER JOIN dim_store s
@@ -474,7 +487,8 @@ class DynamicPartitionPruningSuite
    */
   test("filtering ratio policy fallback") {
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       Given("no stats and selective predicate")
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
         SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "true") {
@@ -543,7 +557,8 @@ class DynamicPartitionPruningSuite
    */
   test("filtering ratio policy with stats when the broadcast pruning is disabled") {
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       Given("disabling the use of stats in the DPP heuristic")
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
         SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false") {
@@ -613,10 +628,7 @@ class DynamicPartitionPruningSuite
 
   test("partition pruning in broadcast hash joins with non-deterministic probe part") {
     Given("alias with simple join condition, and non-deterministic query")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -630,10 +642,7 @@ class DynamicPartitionPruningSuite
     }
 
     Given("alias over multiple sub-queries with simple join condition")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -651,10 +660,7 @@ class DynamicPartitionPruningSuite
 
   test("partition pruning in broadcast hash joins with aliases") {
     Given("alias with simple join condition, using attribute names only")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -674,10 +680,7 @@ class DynamicPartitionPruningSuite
     }
 
     Given("alias with expr as join condition")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -697,10 +700,7 @@ class DynamicPartitionPruningSuite
     }
 
     Given("alias over multiple sub-queries with simple join condition")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -722,10 +722,7 @@ class DynamicPartitionPruningSuite
     }
 
     Given("alias over multiple sub-queries with simple join condition")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid_d as pid, f.sid_d as sid FROM
@@ -754,10 +751,8 @@ class DynamicPartitionPruningSuite
   test("partition pruning in broadcast hash joins") {
     Given("disable broadcast pruning and disable subquery duplication")
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false",
-      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       val df = sql(
         """
           |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
@@ -777,9 +772,10 @@ class DynamicPartitionPruningSuite
 
     Given("disable reuse broadcast results and enable subquery duplication")
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false",
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0.5") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0.5",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       val df = sql(
         """
           |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
@@ -798,52 +794,47 @@ class DynamicPartitionPruningSuite
     }
 
     Given("enable reuse broadcast results and disable query duplication")
-      withSQLConf(
-        SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-        SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-        SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
-        val df = sql(
-          """
-            |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
-            |JOIN dim_stats s
-            |ON f.store_id = s.store_id WHERE s.country = 'DE'
-          """.stripMargin)
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
+      val df = sql(
+        """
+          |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
+          |JOIN dim_stats s
+          |ON f.store_id = s.store_id WHERE s.country = 'DE'
+        """.stripMargin)
 
-        checkPartitionPruningPredicate(df, false, true)
+      checkPartitionPruningPredicate(df, false, true)
 
-        checkAnswer(df,
-          Row(1030, 2, 10, 3) ::
-          Row(1040, 2, 50, 3) ::
-          Row(1050, 2, 50, 3) ::
-          Row(1060, 2, 50, 3) :: Nil
-        )
+      checkAnswer(df,
+        Row(1030, 2, 10, 3) ::
+        Row(1040, 2, 50, 3) ::
+        Row(1050, 2, 50, 3) ::
+        Row(1060, 2, 50, 3) :: Nil
+      )
     }
 
     Given("disable broadcast hash join and disable query duplication")
-      withSQLConf(
-        SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-        SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-        SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-        SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
-        val df = sql(
-          """
-            |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
-            |JOIN dim_stats s
-            |ON f.store_id = s.store_id WHERE s.country = 'DE'
-          """.stripMargin)
+    withSQLConf(
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      val df = sql(
+        """
+          |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
+          |JOIN dim_stats s
+          |ON f.store_id = s.store_id WHERE s.country = 'DE'
+        """.stripMargin)
 
-        checkPartitionPruningPredicate(df, false, false)
+      checkPartitionPruningPredicate(df, false, false)
 
-        checkAnswer(df,
-          Row(1030, 2, 10, 3) ::
-          Row(1040, 2, 50, 3) ::
-          Row(1050, 2, 50, 3) ::
-          Row(1060, 2, 50, 3) :: Nil
-        )
+      checkAnswer(df,
+        Row(1030, 2, 10, 3) ::
+        Row(1040, 2, 50, 3) ::
+        Row(1050, 2, 50, 3) ::
+        Row(1060, 2, 50, 3) :: Nil
+      )
     }
 
     Given("disable broadcast hash join and enable query duplication")
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
       SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "true") {
       val df = sql(
@@ -865,9 +856,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("broadcast a single key in a HashedRelation") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
         spark.range(100).select(
           $"id",
@@ -925,9 +914,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("broadcast multiple keys in a LongHashedRelation") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
         spark.range(100).select(
           $"id",
@@ -962,9 +949,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("broadcast multiple keys in an UnsafeHashedRelation") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
         spark.range(100).select(
           $"id",
@@ -999,9 +984,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("different broadcast subqueries with identical children") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
         spark.range(100).select(
           $"id",
@@ -1073,7 +1056,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("avoid reordering broadcast join keys to match input hash partitioning") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
       withTable("large", "dimTwo", "dimThree") {
         spark.range(100).select(
@@ -1123,9 +1106,7 @@ class DynamicPartitionPruningSuite
    * duplicated partitioning keys, also used to uniquely identify the dynamic pruning filters.
    */
   test("dynamic partition pruning ambiguity issue across nested joins") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("store", "date", "item") {
         spark.range(500)
           .select((($"id" + 30) % 50).as("ss_item_sk"),
@@ -1163,11 +1144,9 @@ class DynamicPartitionPruningSuite
   }
 
   test("cleanup any DPP filter that isn't pushed down due to expression id clashes") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
-        spark.range(1000).select($"id".as("A"), $"id".as("AA"))
+        spark.range(20).select($"id".as("A"), $"id".as("AA"))
           .write.partitionBy("A").format(tableFormat).mode("overwrite").saveAsTable("fact")
         spark.range(10).select($"id".as("B"), $"id".as("BB"))
           .write.format(tableFormat).mode("overwrite").saveAsTable("dim")
@@ -1186,10 +1165,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("cleanup any DPP filter that isn't pushed down due to non-determinism") {
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -1204,10 +1180,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("join key with multiple references on the filtering plan") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0",
-      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       // when enable AQE, the reusedExchange is inserted when executed.
       withTable("fact", "dim") {
         spark.range(100).select(
@@ -1240,9 +1213,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("Make sure dynamic pruning works on uncorrelated queries") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT d.store_id,
@@ -1258,18 +1229,39 @@ class DynamicPartitionPruningSuite
 
       val plan = df.queryExecution.executedPlan
       val countSubqueryBroadcasts =
-        plan.collectInPlanAndSubqueries({ case _: SubqueryBroadcastExec => 1 }).sum
+        plan.collectWithSubqueries({ case _: SubqueryBroadcastExec => 1 }).sum
 
       assert(countSubqueryBroadcasts == 2)
     }
   }
 
+  test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
+    "canonicalization and exchange reuse") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
+      withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+        val df = sql(
+          """ WITH view1 as (
+            |   SELECT f.store_id FROM fact_stats f WHERE f.units_sold = 70
+            | )
+            |
+            | SELECT * FROM view1 v1 join view1 v2 WHERE v1.store_id = v2.store_id
+          """.stripMargin)
+
+        checkPartitionPruningPredicate(df, false, false)
+        val reuseExchangeNodes = df.queryExecution.executedPlan.collect {
+          case se: ReusedExchangeExec => se
+        }
+        assert(reuseExchangeNodes.size == 1, "Expected plan to contain 1 ReusedExchangeExec " +
+          s"nodes. Found ${reuseExchangeNodes.size}")
+
+        checkAnswer(df, Row(15, 15) :: Nil)
+      }
+    }
+  }
+
   test("Plan broadcast pruning only when the broadcast can be reused") {
     Given("dynamic pruning filter on the build side")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.store_id, f.product_id, f.units_sold FROM fact_np f
@@ -1288,10 +1280,7 @@ class DynamicPartitionPruningSuite
     }
 
     Given("dynamic pruning filter on the probe side")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT /*+ BROADCAST(f)*/
@@ -1310,4 +1299,44 @@ class DynamicPartitionPruningSuite
       )
     }
   }
+
+  test("SPARK-32659: Fix the data issue when pruning DPP on non-atomic type") {
+    Seq(NO_CODEGEN, CODEGEN_ONLY).foreach { mode =>
+      Seq(true, false).foreach { pruning =>
+        withSQLConf(
+          SQLConf.CODEGEN_FACTORY_MODE.key -> mode.toString,
+          SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> s"$pruning") {
+          Seq("struct", "array").foreach { dataType =>
+            val df = sql(
+              s"""
+                 |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
+                 |JOIN dim_stats s
+                 |ON $dataType(f.store_id) = $dataType(s.store_id) WHERE s.country = 'DE'
+              """.stripMargin)
+
+            if (pruning) {
+              checkPartitionPruningPredicate(df, false, true)
+            } else {
+              checkPartitionPruningPredicate(df, false, false)
+            }
+
+            checkAnswer(df,
+              Row(1030, 2, 10, 3) ::
+              Row(1040, 2, 50, 3) ::
+              Row(1050, 2, 50, 3) ::
+              Row(1060, 2, 50, 3) :: Nil
+            )
+          }
+        }
+      }
+    }
+  }
+}
+
+class DynamicPartitionPruningSuiteAEOff extends DynamicPartitionPruningSuiteBase {
+  override val adaptiveExecutionOn: Boolean = false
+}
+
+class DynamicPartitionPruningSuiteAEOn extends DynamicPartitionPruningSuiteBase {
+  override val adaptiveExecutionOn: Boolean = true
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
index d9f4d6d5132ae..158d9392c0f5c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
@@ -18,27 +18,15 @@
 package org.apache.spark.sql
 
 import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.adaptive.{DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.StructType
 
-class ExplainSuite extends QueryTest with SharedSparkSession {
-  import testImplicits._
-
-  var originalValue: String = _
-  protected override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  protected override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
+trait ExplainSuiteHelper extends QueryTest with SharedSparkSession {
 
-  private def getNormalizedExplain(df: DataFrame, mode: ExplainMode): String = {
+  protected def getNormalizedExplain(df: DataFrame, mode: ExplainMode): String = {
     val output = new java.io.ByteArrayOutputStream()
     Console.withOut(output) {
       df.explain(mode.name)
@@ -49,7 +37,7 @@ class ExplainSuite extends QueryTest with SharedSparkSession {
   /**
    * Get the explain from a DataFrame and run the specified action on it.
    */
-  private def withNormalizedExplain(df: DataFrame, mode: ExplainMode)(f: String => Unit) = {
+  protected def withNormalizedExplain(df: DataFrame, mode: ExplainMode)(f: String => Unit) = {
     f(getNormalizedExplain(df, mode))
   }
 
@@ -57,7 +45,7 @@ class ExplainSuite extends QueryTest with SharedSparkSession {
    * Get the explain by running the sql. The explain mode should be part of the
    * sql text itself.
    */
-  private def withNormalizedExplain(queryText: String)(f: String => Unit) = {
+  protected def withNormalizedExplain(queryText: String)(f: String => Unit) = {
     val output = new java.io.ByteArrayOutputStream()
     Console.withOut(output) {
       sql(queryText).show(false)
@@ -69,7 +57,7 @@ class ExplainSuite extends QueryTest with SharedSparkSession {
   /**
    * Runs the plan and makes sure the plans contains all of the keywords.
    */
-  private def checkKeywordsExistsInExplain(
+  protected def checkKeywordsExistsInExplain(
       df: DataFrame, mode: ExplainMode, keywords: String*): Unit = {
     withNormalizedExplain(df, mode) { normalizedOutput =>
       for (key <- keywords) {
@@ -78,9 +66,13 @@ class ExplainSuite extends QueryTest with SharedSparkSession {
     }
   }
 
-  private def checkKeywordsExistsInExplain(df: DataFrame, keywords: String*): Unit = {
+  protected def checkKeywordsExistsInExplain(df: DataFrame, keywords: String*): Unit = {
     checkKeywordsExistsInExplain(df, ExtendedMode, keywords: _*)
   }
+}
+
+class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite {
+  import testImplicits._
 
   test("SPARK-23034 show rdd names in RDD scan nodes (Dataset)") {
     val rddWithName = spark.sparkContext.parallelize(Row(1, "abc") :: Nil).setName("testRdd")
@@ -116,8 +108,8 @@ class ExplainSuite extends QueryTest with SharedSparkSession {
       // plan should show the rewritten aggregate expression.
       val df = sql("SELECT k, every(v), some(v), any(v) FROM test_agg GROUP BY k")
       checkKeywordsExistsInExplain(df,
-        "Aggregate [k#x], [k#x, min(v#x) AS every(v)#x, max(v#x) AS some(v)#x, " +
-          "max(v#x) AS any(v)#x]")
+        "Aggregate [k#x], [k#x, every(v#x) AS every(v)#x, some(v#x) AS some(v)#x, " +
+          "any(v#x) AS any(v)#x]")
     }
   }
 
@@ -236,10 +228,27 @@ class ExplainSuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-33853: explain codegen - check presence of subquery") {
+    withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
+      withTempView("df") {
+        val df1 = spark.range(1, 100)
+        df1.createTempView("df")
+
+        val sqlText = "EXPLAIN CODEGEN SELECT (SELECT min(id) FROM df)"
+        val expectedText = "Found 3 WholeStageCodegen subtrees."
+
+        withNormalizedExplain(sqlText) { normalizedOutput =>
+          assert(normalizedOutput.contains(expectedText))
+        }
+      }
+    }
+  }
+
   test("explain formatted - check presence of subquery in case of DPP") {
     withTable("df1", "df2") {
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
-        SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+        SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+        SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
         withTable("df1", "df2") {
           spark.range(1000).select(col("id"), col("id").as("k"))
             .write
@@ -340,6 +349,67 @@ class ExplainSuite extends QueryTest with SharedSparkSession {
     }.getMessage
     assert(errMsg.contains("Unknown explain mode: unknown"))
   }
+
+  test("SPARK-31504: Output fields in formatted Explain should have determined order") {
+    withTempPath { path =>
+      spark.range(10).selectExpr("id as a", "id as b", "id as c", "id as d", "id as e")
+        .write.mode("overwrite").parquet(path.getAbsolutePath)
+      val df1 = spark.read.parquet(path.getAbsolutePath)
+      val df2 = spark.read.parquet(path.getAbsolutePath)
+      assert(getNormalizedExplain(df1, FormattedMode) === getNormalizedExplain(df2, FormattedMode))
+    }
+  }
+}
+
+class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuite {
+  import testImplicits._
+
+  test("Explain formatted") {
+    val df1 = Seq((1, 2), (2, 3)).toDF("k", "v1")
+    val df2 = Seq((2, 3), (1, 1)).toDF("k", "v2")
+    val testDf = df1.join(df2, "k").groupBy("k").agg(count("v1"), sum("v1"), avg("v2"))
+    // trigger the final plan for AQE
+    testDf.collect()
+    //   == Physical Plan ==
+    //   AdaptiveSparkPlan (14)
+    //   +- * HashAggregate (13)
+    //      +- CustomShuffleReader (12)
+    //         +- ShuffleQueryStage (11)
+    //            +- Exchange (10)
+    //               +- * HashAggregate (9)
+    //                  +- * Project (8)
+    //                     +- * BroadcastHashJoin Inner BuildRight (7)
+    //                        :- * Project (2)
+    //                        :  +- * LocalTableScan (1)
+    //                        +- BroadcastQueryStage (6)
+    //                           +- BroadcastExchange (5)
+    //                              +- * Project (4)
+    //                                 +- * LocalTableScan (3)
+    checkKeywordsExistsInExplain(
+      testDf,
+      FormattedMode,
+      s"""
+         |(6) BroadcastQueryStage
+         |Output [2]: [k#x, v2#x]
+         |Arguments: 0
+         |""".stripMargin,
+      s"""
+         |(11) ShuffleQueryStage
+         |Output [5]: [k#x, count#xL, sum#xL, sum#x, count#xL]
+         |Arguments: 1
+         |""".stripMargin,
+      s"""
+         |(12) CustomShuffleReader
+         |Input [5]: [k#x, count#xL, sum#xL, sum#x, count#xL]
+         |Arguments: coalesced
+         |""".stripMargin,
+      s"""
+         |(14) AdaptiveSparkPlan
+         |Output [4]: [k#x, count(v1)#xL, sum(v1)#xL, avg(v2)#x]
+         |Arguments: isFinalPlan=true
+         |""".stripMargin
+    )
+  }
 }
 
 case class ExplainSingleData(id: Int)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
new file mode 100644
index 0000000000000..81c09d169efb2
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import java.io.File
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile}
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.tags.ExtendedSQLTest
+
+// scalastyle:off line.size.limit
+/**
+ * End-to-end test cases for SQL schemas of expression examples.
+ * The golden result file is "spark/sql/core/src/test/resources/sql-functions/sql-expression-schema.md".
+ *
+ * To run the entire test suite:
+ * {{{
+ *   build/sbt "sql/test-only *ExpressionsSchemaSuite"
+ * }}}
+ *
+ * To re-generate golden files for entire suite, run:
+ * {{{
+ *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/test-only *ExpressionsSchemaSuite"
+ * }}}
+ *
+ * For example:
+ * {{{
+ *   ...
+ *   @ExpressionDescription(
+ *     usage = "_FUNC_(str, n) - Returns the string which repeats the given string value n times.",
+ *     examples = """
+ *       Examples:
+ *         > SELECT _FUNC_('123', 2);
+ *          123123
+ *     """,
+ *     since = "1.5.0")
+ *   case class StringRepeat(str: Expression, times: Expression)
+ *   ...
+ * }}}
+ *
+ * The format for golden result files look roughly like:
+ * {{{
+ *   ...
+ *   | org.apache.spark.sql.catalyst.expressions.StringRepeat | repeat | SELECT repeat('123', 2) | struct<repeat(123, 2):string> |
+ *   ...
+ * }}}
+ */
+// scalastyle:on line.size.limit
+@ExtendedSQLTest
+class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
+
+  private val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1"
+
+  private val baseResourcePath = {
+    // We use a path based on Spark home for 2 reasons:
+    //   1. Maven can't get correct resource directory when resources in other jars.
+    //   2. We test subclasses in the hive-thriftserver module.
+    java.nio.file.Paths.get(sparkHome,
+      "sql", "core", "src", "test", "resources", "sql-functions").toFile
+  }
+
+  private val resultFile = new File(baseResourcePath, "sql-expression-schema.md")
+
+  private val ignoreSet = Set(
+    // Output name with a random seed
+    "org.apache.spark.sql.catalyst.expressions.Rand",
+    "org.apache.spark.sql.catalyst.expressions.Randn")
+
+  /** A single SQL query's SQL and schema. */
+  protected case class QueryOutput(
+      className: String,
+      funcName: String,
+      sql: String = "N/A",
+      schema: String = "N/A") {
+    override def toString: String = {
+      s"| $className | $funcName | $sql | $schema |"
+    }
+  }
+
+  test("Check schemas for expression examples") {
+    val exampleRe = """^(.+);\n(?s)(.+)$""".r
+    val funInfos = spark.sessionState.functionRegistry.listFunction().map { funcId =>
+      spark.sessionState.catalog.lookupFunctionInfo(funcId)
+    }
+
+    val classFunsMap = funInfos.groupBy(_.getClassName).toSeq.sortBy(_._1)
+    val outputBuffer = new ArrayBuffer[String]
+    val outputs = new ArrayBuffer[QueryOutput]
+    val missingExamples = new ArrayBuffer[String]
+
+    classFunsMap.filterNot(v => ignoreSet.contains(v._1)).foreach { kv =>
+      val className = kv._1
+      kv._2.foreach { funInfo =>
+        val example = funInfo.getExamples
+        val funcName = funInfo.getName.replaceAll("\\|", "&#124;")
+        if (example == "") {
+          val queryOutput = QueryOutput(className, funcName)
+          outputBuffer += queryOutput.toString
+          outputs += queryOutput
+          missingExamples += funcName
+        }
+
+        // If expression exists 'Examples' segment, the first element is 'Examples'. Because
+        // this test case is only used to print aliases of expressions for double checking.
+        // Therefore, we only need to output the first SQL and its corresponding schema.
+        // Note: We need to filter out the commands that set the parameters, such as:
+        // SET spark.sql.parser.escapedStringLiterals=true
+        example.split("  > ").tail.filterNot(_.trim.startsWith("SET")).take(1).foreach {
+          case exampleRe(sql, _) =>
+            val df = spark.sql(sql)
+            val escapedSql = sql.replaceAll("\\|", "&#124;")
+            val schema = df.schema.catalogString.replaceAll("\\|", "&#124;")
+            val queryOutput = QueryOutput(className, funcName, escapedSql, schema)
+            outputBuffer += queryOutput.toString
+            outputs += queryOutput
+          case _ =>
+        }
+      }
+    }
+
+    val header = Seq(
+      s"<!-- Automatically generated by ${getClass.getSimpleName} -->",
+      "## Summary",
+      s"  - Number of queries: ${outputs.size}",
+      s"  - Number of expressions that missing example: ${missingExamples.size}",
+      s"  - Expressions missing examples: ${missingExamples.mkString(",")}",
+      "## Schema of Built-in Functions",
+      "| Class name | Function name or alias | Query example | Output schema |",
+      "| ---------- | ---------------------- | ------------- | ------------- |"
+    )
+
+    if (regenerateGoldenFiles) {
+      val goldenOutput = (header ++ outputBuffer).mkString("\n")
+      val parent = resultFile.getParentFile
+      if (!parent.exists()) {
+        assert(parent.mkdirs(), "Could not create directory: " + parent)
+      }
+      stringToFile(resultFile, goldenOutput)
+    }
+
+    val outputSize = outputs.size
+    val headerSize = header.size
+    val expectedOutputs: Seq[QueryOutput] = {
+      val expectedGoldenOutput = fileToString(resultFile)
+      val lines = expectedGoldenOutput.split("\n")
+      val expectedSize = lines.size
+
+      assert(expectedSize == outputSize + headerSize,
+        s"Expected $expectedSize blocks in result file but got " +
+          s"${outputSize + headerSize}. Try regenerate the result files.")
+
+      Seq.tabulate(outputSize) { i =>
+        val segments = lines(i + headerSize).split('|')
+        QueryOutput(
+          className = segments(1).trim,
+          funcName = segments(2).trim,
+          sql = segments(3).trim,
+          schema = segments(4).trim)
+      }
+    }
+
+    // Compare results.
+    assert(expectedOutputs.size == outputSize,
+      "The number of queries not equals the number of expected queries.")
+
+    outputs.zip(expectedOutputs).foreach { case (output, expected) =>
+      assert(expected.sql == output.sql, "SQL query did not match")
+      assert(expected.schema == output.schema, s"Schema did not match for query ${expected.sql}")
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
index c870958128483..e9bff64d72fc3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
@@ -18,12 +18,14 @@
 package org.apache.spark.sql
 
 import java.io.{File, FileNotFoundException}
+import java.net.URI
 import java.nio.file.{Files, StandardOpenOption}
 import java.util.Locale
 
 import scala.collection.mutable
 
-import org.apache.hadoop.fs.Path
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{LocalFileSystem, Path}
 
 import org.apache.spark.SparkException
 import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
@@ -631,13 +633,15 @@ class FileBasedDataSourceSuite extends QueryTest
 
       assert(fileList.toSet === expectedFileList.toSet)
 
-      val fileList2 = spark.read.format("binaryFile")
-        .option("recursiveFileLookup", true)
-        .option("pathGlobFilter", "*.bin")
-        .load(dataPath)
-        .select("path").collect().map(_.getString(0))
+      withClue("SPARK-32368: 'recursiveFileLookup' and 'pathGlobFilter' can be case insensitive") {
+        val fileList2 = spark.read.format("binaryFile")
+          .option("RecuRsivefileLookup", true)
+          .option("PaThglobFilter", "*.bin")
+          .load(dataPath)
+          .select("path").collect().map(_.getString(0))
 
-      assert(fileList2.toSet === expectedFileList.filter(_.endsWith(".bin")).toSet)
+        assert(fileList2.toSet === expectedFileList.filter(_.endsWith(".bin")).toSet)
+      }
     }
   }
 
@@ -842,6 +846,62 @@ class FileBasedDataSourceSuite extends QueryTest
       }
     }
   }
+
+  test("SPARK-31935: Hadoop file system config should be effective in data source options") {
+    Seq("parquet", "").foreach { format =>
+      withSQLConf(
+        SQLConf.USE_V1_SOURCE_LIST.key -> format,
+        "fs.file.impl" -> classOf[FakeFileSystemRequiringDSOption].getName,
+        "fs.file.impl.disable.cache" -> "true") {
+        withTempDir { dir =>
+          val path = "file:" + dir.getCanonicalPath.stripPrefix("file:")
+          spark.range(10).write.option("ds_option", "value").mode("overwrite").parquet(path)
+          checkAnswer(
+            spark.read.option("ds_option", "value").parquet(path), spark.range(10).toDF())
+        }
+      }
+    }
+  }
+
+  test("SPARK-31116: Select nested schema with case insensitive mode") {
+    // This test case failed at only Parquet. ORC is added for test coverage parity.
+    Seq("orc", "parquet").foreach { format =>
+      Seq("true", "false").foreach { nestedSchemaPruningEnabled =>
+        withSQLConf(
+          SQLConf.CASE_SENSITIVE.key -> "false",
+          SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key -> nestedSchemaPruningEnabled) {
+          withTempPath { dir =>
+            val path = dir.getCanonicalPath
+
+            // Prepare values for testing nested parquet data
+            spark
+              .range(1L)
+              .selectExpr("NAMED_STRUCT('lowercase', id, 'camelCase', id + 1) AS StructColumn")
+              .write
+              .format(format)
+              .save(path)
+
+            val exactSchema = "StructColumn struct<lowercase: LONG, camelCase: LONG>"
+
+            checkAnswer(spark.read.schema(exactSchema).format(format).load(path), Row(Row(0, 1)))
+
+            // In case insensitive manner, parquet's column cases are ignored
+            val innerColumnCaseInsensitiveSchema =
+              "StructColumn struct<Lowercase: LONG, camelcase: LONG>"
+            checkAnswer(
+              spark.read.schema(innerColumnCaseInsensitiveSchema).format(format).load(path),
+              Row(Row(0, 1)))
+
+            val rootColumnCaseInsensitiveSchema =
+              "structColumn struct<lowercase: LONG, camelCase: LONG>"
+            checkAnswer(
+              spark.read.schema(rootColumnCaseInsensitiveSchema).format(format).load(path),
+              Row(Row(0, 1)))
+          }
+        }
+      }
+    }
+  }
 }
 
 object TestingUDT {
@@ -872,3 +932,10 @@ object TestingUDT {
     override def userClass: Class[NullData] = classOf[NullData]
   }
 }
+
+class FakeFileSystemRequiringDSOption extends LocalFileSystem {
+  override def initialize(name: URI, conf: Configuration): Unit = {
+    super.initialize(name, conf)
+    require(conf.get("ds_option", "") == "value")
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
index 96a0eb3e32e9b..8f449037a5e01 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
@@ -343,6 +343,19 @@ class GeneratorFunctionSuite extends QueryTest with SharedSparkSession {
       assert(msg2.contains("Only one generator allowed per aggregate clause"))
     }
   }
+
+  test("SPARK-30998: Unsupported nested inner generators") {
+    val errMsg = intercept[AnalysisException] {
+      sql("SELECT array(array(1, 2), array(3)) v").select(explode(explode($"v"))).collect
+    }.getMessage
+    assert(errMsg.contains("Generators are not supported when it's nested in expressions, " +
+      "but got: explode(explode(v))"))
+  }
+
+  test("SPARK-30997: generators in aggregate expressions for dataframe") {
+    val df = Seq(1, 2, 3).toDF("v")
+    checkAnswer(df.select(explode(array(min($"v"), max($"v")))), Row(1) :: Row(3) :: Nil)
+  }
 }
 
 case class EmptyGenerator() extends Generator {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
index 51150a1b38b49..80346b350c142 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
@@ -75,14 +75,7 @@ object IntegratedUDFTestUtils extends SQLHelper {
   import scala.sys.process._
 
   private lazy val pythonPath = sys.env.getOrElse("PYTHONPATH", "")
-  private lazy val sparkHome = if (sys.props.contains(Tests.IS_TESTING.key)) {
-    assert(sys.props.contains("spark.test.home") ||
-      sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-    sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-  } else {
-    assert(sys.env.contains("SPARK_HOME"), "SPARK_HOME is not set.")
-    sys.env("SPARK_HOME")
-  }
+
   // Note that we will directly refer pyspark's source, not the zip from a regular build.
   // It is possible the test is being ran without the build.
   private lazy val sourcePath = Paths.get(sparkHome, "python").toAbsolutePath
@@ -204,7 +197,7 @@ object IntegratedUDFTestUtils extends SQLHelper {
 
   lazy val pythonExec: String = {
     val pythonExec = sys.env.getOrElse(
-      "PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python3.6"))
+      "PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python3"))
     if (TestUtils.testCommandAvailable(pythonExec)) {
       pythonExec
     } else {
@@ -337,7 +330,7 @@ object IntegratedUDFTestUtils extends SQLHelper {
         input.toString
       },
       StringType,
-      inputSchemas = Seq.fill(1)(None),
+      inputEncoders = Seq.fill(1)(None),
       name = Some(name)) {
 
       override def apply(exprs: Column*): Column = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
index f68c416941266..71f7a708ad681 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
@@ -570,4 +570,31 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
       assert(joinHints == expectedHints)
     }
   }
+
+  test("SPARK-32220: Non Cartesian Product Join Result Correct with SHUFFLE_REPLICATE_NL hint") {
+    withTempView("t1", "t2") {
+      Seq((1, "4"), (2, "2")).toDF("key", "value").createTempView("t1")
+      Seq((1, "1"), (2, "12.3"), (2, "123")).toDF("key", "value").createTempView("t2")
+      val df1 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t1.key = t2.key")
+      val df2 = sql("SELECT * from t1 join t2 ON t1.key = t2.key")
+      assert(df1.collect().size == df2.collect().size)
+
+      val df3 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2")
+      val df4 = sql("SELECT * from t1 join t2")
+      assert(df3.collect().size == df4.collect().size)
+
+      val df5 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t1.key < t2.key")
+      val df6 = sql("SELECT * from t1 join t2 ON t1.key < t2.key")
+      assert(df5.collect().size == df6.collect().size)
+
+      val df7 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t1.key < 2")
+      val df8 = sql("SELECT * from t1 join t2 ON t1.key < 2")
+      assert(df7.collect().size == df8.collect().size)
+
+
+      val df9 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t2.key < 2")
+      val df10 = sql("SELECT * from t1 join t2 ON t2.key < 2")
+      assert(df9.collect().size == df10.collect().size)
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
index f45bd950040ce..fe6775cc7f9b9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
@@ -401,94 +401,96 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
   }
 
   test("full outer join") {
-    upperCaseData.where('N <= 4).createOrReplaceTempView("`left`")
-    upperCaseData.where('N >= 3).createOrReplaceTempView("`right`")
+    withTempView("`left`", "`right`") {
+      upperCaseData.where('N <= 4).createOrReplaceTempView("`left`")
+      upperCaseData.where('N >= 3).createOrReplaceTempView("`right`")
 
-    val left = UnresolvedRelation(TableIdentifier("left"))
-    val right = UnresolvedRelation(TableIdentifier("right"))
+      val left = UnresolvedRelation(TableIdentifier("left"))
+      val right = UnresolvedRelation(TableIdentifier("right"))
 
-    checkAnswer(
-      left.join(right, $"left.N" === $"right.N", "full"),
-      Row(1, "A", null, null) ::
-        Row(2, "B", null, null) ::
-        Row(3, "C", 3, "C") ::
-        Row(4, "D", 4, "D") ::
-        Row(null, null, 5, "E") ::
-        Row(null, null, 6, "F") :: Nil)
+      checkAnswer(
+        left.join(right, $"left.N" === $"right.N", "full"),
+        Row(1, "A", null, null) ::
+          Row(2, "B", null, null) ::
+          Row(3, "C", 3, "C") ::
+          Row(4, "D", 4, "D") ::
+          Row(null, null, 5, "E") ::
+          Row(null, null, 6, "F") :: Nil)
 
-    checkAnswer(
-      left.join(right, ($"left.N" === $"right.N") && ($"left.N" =!= 3), "full"),
-      Row(1, "A", null, null) ::
-        Row(2, "B", null, null) ::
-        Row(3, "C", null, null) ::
-        Row(null, null, 3, "C") ::
-        Row(4, "D", 4, "D") ::
-        Row(null, null, 5, "E") ::
-        Row(null, null, 6, "F") :: Nil)
+      checkAnswer(
+        left.join(right, ($"left.N" === $"right.N") && ($"left.N" =!= 3), "full"),
+        Row(1, "A", null, null) ::
+          Row(2, "B", null, null) ::
+          Row(3, "C", null, null) ::
+          Row(null, null, 3, "C") ::
+          Row(4, "D", 4, "D") ::
+          Row(null, null, 5, "E") ::
+          Row(null, null, 6, "F") :: Nil)
 
-    checkAnswer(
-      left.join(right, ($"left.N" === $"right.N") && ($"right.N" =!= 3), "full"),
-      Row(1, "A", null, null) ::
-        Row(2, "B", null, null) ::
-        Row(3, "C", null, null) ::
-        Row(null, null, 3, "C") ::
-        Row(4, "D", 4, "D") ::
-        Row(null, null, 5, "E") ::
-        Row(null, null, 6, "F") :: Nil)
-
-    // Make sure we are UnknownPartitioning as the outputPartitioning for the outer join
-    // operator.
-    checkAnswer(
-      sql(
-        """
-        |SELECT l.a, count(*)
-        |FROM allNulls l FULL OUTER JOIN upperCaseData r ON (l.a = r.N)
-        |GROUP BY l.a
-      """.
-          stripMargin),
-      Row(null, 10))
+      checkAnswer(
+        left.join(right, ($"left.N" === $"right.N") && ($"right.N" =!= 3), "full"),
+        Row(1, "A", null, null) ::
+          Row(2, "B", null, null) ::
+          Row(3, "C", null, null) ::
+          Row(null, null, 3, "C") ::
+          Row(4, "D", 4, "D") ::
+          Row(null, null, 5, "E") ::
+          Row(null, null, 6, "F") :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT r.N, count(*)
+      // Make sure we are UnknownPartitioning as the outputPartitioning for the outer join
+      // operator.
+      checkAnswer(
+        sql(
+          """
+          |SELECT l.a, count(*)
           |FROM allNulls l FULL OUTER JOIN upperCaseData r ON (l.a = r.N)
-          |GROUP BY r.N
-        """.stripMargin),
-      Row
+          |GROUP BY l.a
+        """.
+            stripMargin),
+        Row(null, 10))
+
+      checkAnswer(
+        sql(
+          """
+            |SELECT r.N, count(*)
+            |FROM allNulls l FULL OUTER JOIN upperCaseData r ON (l.a = r.N)
+            |GROUP BY r.N
+          """.stripMargin),
+        Row
         (1, 1) ::
-        Row(2, 1) ::
-        Row(3, 1) ::
-        Row(4, 1) ::
-        Row(5, 1) ::
-        Row(6, 1) ::
-        Row(null, 4) :: Nil)
+          Row(2, 1) ::
+          Row(3, 1) ::
+          Row(4, 1) ::
+          Row(5, 1) ::
+          Row(6, 1) ::
+          Row(null, 4) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT l.N, count(*)
-          |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a)
-          |GROUP BY l.N
-        """.stripMargin),
-      Row(1
-        , 1) ::
-        Row(2, 1) ::
-        Row(3, 1) ::
-        Row(4, 1) ::
-        Row(5, 1) ::
-        Row(6, 1) ::
-        Row(null, 4) :: Nil)
+      checkAnswer(
+        sql(
+          """
+            |SELECT l.N, count(*)
+            |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a)
+            |GROUP BY l.N
+          """.stripMargin),
+        Row(1
+          , 1) ::
+          Row(2, 1) ::
+          Row(3, 1) ::
+          Row(4, 1) ::
+          Row(5, 1) ::
+          Row(6, 1) ::
+          Row(null, 4) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-        |SELECT r.a, count(*)
-        |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a)
-        |GROUP BY r.a
-      """.
-          stripMargin),
-      Row(null, 10))
+      checkAnswer(
+        sql(
+          """
+          |SELECT r.a, count(*)
+          |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a)
+          |GROUP BY r.a
+        """.
+            stripMargin),
+        Row(null, 10))
+    }
   }
 
   test("broadcasted existence join operator selection") {
@@ -614,63 +616,65 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
   }
 
   test("cross join detection") {
-    testData.createOrReplaceTempView("A")
-    testData.createOrReplaceTempView("B")
-    testData2.createOrReplaceTempView("C")
-    testData3.createOrReplaceTempView("D")
-    upperCaseData.where('N >= 3).createOrReplaceTempView("`right`")
-    val cartesianQueries = Seq(
-      /** The following should error out since there is no explicit cross join */
-      "SELECT * FROM testData inner join testData2",
-      "SELECT * FROM testData left outer join testData2",
-      "SELECT * FROM testData right outer join testData2",
-      "SELECT * FROM testData full outer join testData2",
-      "SELECT * FROM testData, testData2",
-      "SELECT * FROM testData, testData2 where testData.key = 1 and testData2.a = 22",
-      /** The following should fail because after reordering there are cartesian products */
-      "select * from (A join B on (A.key = B.key)) join D on (A.key=D.a) join C",
-      "select * from ((A join B on (A.key = B.key)) join C) join D on (A.key = D.a)",
-      /** Cartesian product involving C, which is not involved in a CROSS join */
-      "select * from ((A join B on (A.key = B.key)) cross join D) join C on (A.key = D.a)");
-
-    def checkCartesianDetection(query: String): Unit = {
-      val e = intercept[Exception] {
-        checkAnswer(sql(query), Nil);
+    withTempView("A", "B", "C", "D") {
+      testData.createOrReplaceTempView("A")
+      testData.createOrReplaceTempView("B")
+      testData2.createOrReplaceTempView("C")
+      testData3.createOrReplaceTempView("D")
+      upperCaseData.where('N >= 3).createOrReplaceTempView("`right`")
+      val cartesianQueries = Seq(
+        /** The following should error out since there is no explicit cross join */
+        "SELECT * FROM testData inner join testData2",
+        "SELECT * FROM testData left outer join testData2",
+        "SELECT * FROM testData right outer join testData2",
+        "SELECT * FROM testData full outer join testData2",
+        "SELECT * FROM testData, testData2",
+        "SELECT * FROM testData, testData2 where testData.key = 1 and testData2.a = 22",
+        /** The following should fail because after reordering there are cartesian products */
+        "select * from (A join B on (A.key = B.key)) join D on (A.key=D.a) join C",
+        "select * from ((A join B on (A.key = B.key)) join C) join D on (A.key = D.a)",
+        /** Cartesian product involving C, which is not involved in a CROSS join */
+        "select * from ((A join B on (A.key = B.key)) cross join D) join C on (A.key = D.a)");
+
+      def checkCartesianDetection(query: String): Unit = {
+        val e = intercept[Exception] {
+          checkAnswer(sql(query), Nil);
+        }
+        assert(e.getMessage.contains("Detected implicit cartesian product"))
       }
-      assert(e.getMessage.contains("Detected implicit cartesian product"))
-    }
 
-    withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
-      cartesianQueries.foreach(checkCartesianDetection)
-    }
+      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
+        cartesianQueries.foreach(checkCartesianDetection)
+      }
 
-    // Check that left_semi, left_anti, existence joins without conditions do not throw
-    // an exception if cross joins are disabled
-    withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
-      checkAnswer(
-        sql("SELECT * FROM testData3 LEFT SEMI JOIN testData2"),
-        Row(1, null) :: Row (2, 2) :: Nil)
-      checkAnswer(
-        sql("SELECT * FROM testData3 LEFT ANTI JOIN testData2"),
-        Nil)
-      checkAnswer(
-        sql(
-          """
-            |SELECT a FROM testData3
-            |WHERE
-            |  EXISTS (SELECT * FROM testData)
-            |OR
-            |  EXISTS (SELECT * FROM testData2)""".stripMargin),
-        Row(1) :: Row(2) :: Nil)
-      checkAnswer(
-        sql(
-          """
-            |SELECT key FROM testData
-            |WHERE
-            |  key IN (SELECT a FROM testData2)
-            |OR
-            |  key IN (SELECT a FROM testData3)""".stripMargin),
-        Row(1) :: Row(2) :: Row(3) :: Nil)
+      // Check that left_semi, left_anti, existence joins without conditions do not throw
+      // an exception if cross joins are disabled
+      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
+        checkAnswer(
+          sql("SELECT * FROM testData3 LEFT SEMI JOIN testData2"),
+          Row(1, null) :: Row (2, 2) :: Nil)
+        checkAnswer(
+          sql("SELECT * FROM testData3 LEFT ANTI JOIN testData2"),
+          Nil)
+        checkAnswer(
+          sql(
+            """
+              |SELECT a FROM testData3
+              |WHERE
+              |  EXISTS (SELECT * FROM testData)
+              |OR
+              |  EXISTS (SELECT * FROM testData2)""".stripMargin),
+          Row(1) :: Row(2) :: Nil)
+        checkAnswer(
+          sql(
+            """
+              |SELECT key FROM testData
+              |WHERE
+              |  key IN (SELECT a FROM testData2)
+              |OR
+              |  key IN (SELECT a FROM testData3)""".stripMargin),
+          Row(1) :: Row(2) :: Row(3) :: Nil)
+      }
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
index fd1e9e309558e..5e3931cecf1c5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
@@ -429,57 +429,69 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("from_json - array of arrays") {
-    val jsonDF = Seq("[[1], [2, 3], [4, 5, 6]]").toDF("a")
-    val schema = new ArrayType(ArrayType(IntegerType, false), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("[[1], [2, 3], [4, 5, 6]]").toDF("a")
+      val schema = new ArrayType(ArrayType(IntegerType, false), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select json[0][0], json[1][1], json[2][2] from jsonTable"),
-      Seq(Row(1, 3, 6)))
+      checkAnswer(
+        sql("select json[0][0], json[1][1], json[2][2] from jsonTable"),
+        Seq(Row(1, 3, 6)))
+    }
   }
 
   test("from_json - array of arrays - malformed row") {
-    val jsonDF = Seq("[[1], [2, 3], 4, 5, 6]]").toDF("a")
-    val schema = new ArrayType(ArrayType(IntegerType, false), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("[[1], [2, 3], 4, 5, 6]]").toDF("a")
+      val schema = new ArrayType(ArrayType(IntegerType, false), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(sql("select json[0] from jsonTable"), Seq(Row(null)))
+      checkAnswer(sql("select json[0] from jsonTable"), Seq(Row(null)))
+    }
   }
 
   test("from_json - array of structs") {
-    val jsonDF = Seq("""[{"a":1}, {"a":2}, {"a":3}]""").toDF("a")
-    val schema = new ArrayType(new StructType().add("a", IntegerType), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("""[{"a":1}, {"a":2}, {"a":3}]""").toDF("a")
+      val schema = new ArrayType(new StructType().add("a", IntegerType), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select json[0], json[1], json[2] from jsonTable"),
-      Seq(Row(Row(1), Row(2), Row(3))))
+      checkAnswer(
+        sql("select json[0], json[1], json[2] from jsonTable"),
+        Seq(Row(Row(1), Row(2), Row(3))))
+    }
   }
 
   test("from_json - array of structs - malformed row") {
-    val jsonDF = Seq("""[{"a":1}, {"a:2}, {"a":3}]""").toDF("a")
-    val schema = new ArrayType(new StructType().add("a", IntegerType), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("""[{"a":1}, {"a:2}, {"a":3}]""").toDF("a")
+      val schema = new ArrayType(new StructType().add("a", IntegerType), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(sql("select json[0], json[1]from jsonTable"), Seq(Row(null, null)))
+      checkAnswer(sql("select json[0], json[1]from jsonTable"), Seq(Row(null, null)))
+    }
   }
 
   test("from_json - array of maps") {
-    val jsonDF = Seq("""[{"a":1}, {"b":2}]""").toDF("a")
-    val schema = new ArrayType(MapType(StringType, IntegerType, false), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("""[{"a":1}, {"b":2}]""").toDF("a")
+      val schema = new ArrayType(MapType(StringType, IntegerType, false), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("""select json[0], json[1] from jsonTable"""),
-      Seq(Row(Map("a" -> 1), Map("b" -> 2))))
+      checkAnswer(
+        sql("""select json[0], json[1] from jsonTable"""),
+        Seq(Row(Map("a" -> 1), Map("b" -> 2))))
+    }
   }
 
   test("from_json - array of maps - malformed row") {
-    val jsonDF = Seq("""[{"a":1} "b":2}]""").toDF("a")
-    val schema = new ArrayType(MapType(StringType, IntegerType, false), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("""[{"a":1} "b":2}]""").toDF("a")
+      val schema = new ArrayType(MapType(StringType, IntegerType, false), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(sql("""select json[0] from jsonTable"""), Seq(Row(null)))
+      checkAnswer(sql("""select json[0] from jsonTable"""), Seq(Row(null)))
+    }
   }
 
   test("to_json - array of primitive types") {
@@ -653,4 +665,65 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
       assert(json_tuple_result === len)
     }
   }
+
+  test("SPARK-31065: schema_of_json - null and empty strings as strings") {
+    Seq("""{"id": null}""", """{"id": ""}""").foreach { input =>
+      checkAnswer(
+        spark.range(1).select(schema_of_json(input)),
+        Seq(Row("struct<id:string>")))
+    }
+  }
+
+  test("SPARK-31065: schema_of_json - 'dropFieldIfAllNull' option") {
+    val options = Map("dropFieldIfAllNull" -> "true")
+    // Structs
+    checkAnswer(
+      spark.range(1).select(
+        schema_of_json(
+          lit("""{"id": "a", "drop": {"drop": null}}"""),
+          options.asJava)),
+      Seq(Row("struct<id:string>")))
+
+    // Array of structs
+    checkAnswer(
+      spark.range(1).select(
+        schema_of_json(
+          lit("""[{"id": "a", "drop": {"drop": null}}]"""),
+          options.asJava)),
+      Seq(Row("array<struct<id:string>>")))
+
+    // Other types are not affected.
+    checkAnswer(
+      spark.range(1).select(
+        schema_of_json(
+          lit("""null"""),
+          options.asJava)),
+      Seq(Row("string")))
+  }
+
+  test("optional datetime parser does not affect json time formatting") {
+    val s = "2015-08-26 12:34:46"
+    def toDF(p: String): DataFrame = sql(
+      s"""
+         |SELECT
+         | to_json(
+         |   named_struct('time', timestamp'$s'), map('timestampFormat', "$p")
+         | )
+         | """.stripMargin)
+    checkAnswer(toDF("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), toDF("yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]"))
+  }
+
+  test("SPARK-33134: return partial results only for root JSON objects") {
+    val st = new StructType()
+      .add("c1", LongType)
+      .add("c2", ArrayType(new StructType().add("c3", LongType).add("c4", StringType)))
+    val df1 = Seq("""{"c2": [19], "c1": 123456}""").toDF("c0")
+    checkAnswer(df1.select(from_json($"c0", st)), Row(Row(123456, null)))
+    val df2 = Seq("""{"data": {"c2": [19], "c1": 123456}}""").toDF("c0")
+    checkAnswer(df2.select(from_json($"c0", new StructType().add("data", st))), Row(Row(null)))
+    val df3 = Seq("""[{"c2": [19], "c1": 123456}]""").toDF("c0")
+    checkAnswer(df3.select(from_json($"c0", ArrayType(st))), Row(null))
+    val df4 = Seq("""{"c2": [19]}""").toDF("c0")
+    checkAnswer(df4.select(from_json($"c0", MapType(StringType, st))), Row(null))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala
index 5ab06b1ebebf6..16edf35bb99f9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala
@@ -34,9 +34,11 @@ class MiscFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("version") {
+    val df = sql("SELECT version()")
     checkAnswer(
-      Seq("").toDF("a").selectExpr("version()"),
+      df,
       Row(SPARK_VERSION_SHORT + " " + SPARK_REVISION))
+    assert(df.schema.fieldNames === Seq("version()"))
   }
 }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
index 4a21ae9242039..e52d2262a6bf8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql
 
-import java.util.{Locale, TimeZone}
+import java.util.TimeZone
 
 import scala.collection.JavaConverters._
 
@@ -35,11 +35,6 @@ abstract class QueryTest extends PlanTest {
 
   protected def spark: SparkSession
 
-  // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-  TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-  // Add Locale setting
-  Locale.setDefault(Locale.US)
-
   /**
    * Runs the plan and makes sure the answer contains all of the keywords.
    */
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala
index aab2ae4afc7f5..a1799829932b8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala
@@ -24,14 +24,32 @@ import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{BooleanType, StringType, StructField, StructType}
 
+@deprecated("This suite is deprecated to silent compiler deprecation warnings", "2.0.0")
 class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
 
   object DummyRule extends Rule[LogicalPlan] {
     def apply(p: LogicalPlan): LogicalPlan = p
   }
 
+  test("getOrCreate instantiates SQLContext") {
+    val sqlContext = SQLContext.getOrCreate(sc)
+    assert(sqlContext != null, "SQLContext.getOrCreate returned null")
+    assert(SQLContext.getOrCreate(sc).eq(sqlContext),
+      "SQLContext created by SQLContext.getOrCreate not returned by SQLContext.getOrCreate")
+  }
+
+  test("getOrCreate return the original SQLContext") {
+    val sqlContext = SQLContext.getOrCreate(sc)
+    val newSession = sqlContext.newSession()
+    assert(SQLContext.getOrCreate(sc).eq(sqlContext),
+      "SQLContext.getOrCreate after explicitly created SQLContext did not return the context")
+    SparkSession.setActiveSession(newSession.sparkSession)
+    assert(SQLContext.getOrCreate(sc).eq(newSession),
+      "SQLContext.getOrCreate after explicitly setActive() did not return the active context")
+  }
+
   test("Sessions of SQLContext") {
-    val sqlContext = SparkSession.builder().sparkContext(sc).getOrCreate().sqlContext
+    val sqlContext = SQLContext.getOrCreate(sc)
     val session1 = sqlContext.newSession()
     val session2 = sqlContext.newSession()
 
@@ -59,13 +77,13 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
   }
 
   test("Catalyst optimization passes are modifiable at runtime") {
-    val sqlContext = SparkSession.builder().sparkContext(sc).getOrCreate().sqlContext
+    val sqlContext = SQLContext.getOrCreate(sc)
     sqlContext.experimental.extraOptimizations = Seq(DummyRule)
     assert(sqlContext.sessionState.optimizer.batches.flatMap(_.rules).contains(DummyRule))
   }
 
   test("get all tables") {
-    val sqlContext = SparkSession.builder().sparkContext(sc).getOrCreate().sqlContext
+    val sqlContext = SQLContext.getOrCreate(sc)
     val df = sqlContext.range(10)
     df.createOrReplaceTempView("listtablessuitetable")
     assert(
@@ -82,7 +100,7 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
   }
 
   test("getting all tables with a database name has no impact on returned table names") {
-    val sqlContext = SparkSession.builder().sparkContext(sc).getOrCreate().sqlContext
+    val sqlContext = SQLContext.getOrCreate(sc)
     val df = sqlContext.range(10)
     df.createOrReplaceTempView("listtablessuitetable")
     assert(
@@ -99,7 +117,7 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
   }
 
   test("query the returned DataFrame of tables") {
-    val sqlContext = SparkSession.builder().sparkContext(sc).getOrCreate().sqlContext
+    val sqlContext = SQLContext.getOrCreate(sc)
     val df = sqlContext.range(10)
     df.createOrReplaceTempView("listtablessuitetable")
 
@@ -109,7 +127,7 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
         StructField("isTemporary", BooleanType, false) :: Nil)
 
     Seq(sqlContext.tables(), sqlContext.sql("SHOW TABLes")).foreach {
-      tableDF =>
+      case tableDF =>
         assert(expectedSchema === tableDF.schema)
 
         tableDF.createOrReplaceTempView("tables")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index a6dae9a269740..8039c9b6f04b5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -22,15 +22,13 @@ import java.net.{MalformedURLException, URL}
 import java.sql.{Date, Timestamp}
 import java.util.concurrent.atomic.AtomicBoolean
 
-import scala.collection.parallel.immutable.ParVector
-
 import org.apache.spark.{AccumulatorSuite, SparkException}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
 import org.apache.spark.sql.catalyst.expressions.GenericRow
 import org.apache.spark.sql.catalyst.expressions.aggregate.{Complete, Partial}
-import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
+import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, NestedColumnAliasingSuite}
+import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.catalyst.util.StringUtils
-import org.apache.spark.sql.execution.HiveResult.hiveResultString
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
@@ -52,13 +50,15 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   setupTestData()
 
   test("SPARK-8010: promote numeric to string") {
-    val df = Seq((1, 1)).toDF("key", "value")
-    df.createOrReplaceTempView("src")
-    val queryCaseWhen = sql("select case when true then 1.0 else '1' end from src ")
-    val queryCoalesce = sql("select coalesce(null, 1, '1') from src ")
+    withTempView("src") {
+      val df = Seq((1, 1)).toDF("key", "value")
+      df.createOrReplaceTempView("src")
+      val queryCaseWhen = sql("select case when true then 1.0 else '1' end from src ")
+      val queryCoalesce = sql("select coalesce(null, 1, '1') from src ")
 
-    checkAnswer(queryCaseWhen, Row("1.0") :: Nil)
-    checkAnswer(queryCoalesce, Row("1") :: Nil)
+      checkAnswer(queryCaseWhen, Row("1.0") :: Nil)
+      checkAnswer(queryCoalesce, Row("1") :: Nil)
+    }
   }
 
   test("show functions") {
@@ -125,107 +125,36 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     }
   }
 
-  test("using _FUNC_ instead of function names in examples") {
-    val exampleRe = "(>.*;)".r
-    val setStmtRe = "(?i)^(>\\s+set\\s+).+".r
-    val ignoreSet = Set(
-      // Examples for CaseWhen show simpler syntax:
-      // `CASE WHEN ... THEN ... WHEN ... THEN ... END`
-      "org.apache.spark.sql.catalyst.expressions.CaseWhen",
-      // _FUNC_ is replaced by `locate` but `locate(... IN ...)` is not supported
-      "org.apache.spark.sql.catalyst.expressions.StringLocate",
-      // _FUNC_ is replaced by `%` which causes a parsing error on `SELECT %(2, 1.8)`
-      "org.apache.spark.sql.catalyst.expressions.Remainder",
-      // Examples demonstrate alternative names, see SPARK-20749
-      "org.apache.spark.sql.catalyst.expressions.Length")
-    spark.sessionState.functionRegistry.listFunction().foreach { funcId =>
-      val info = spark.sessionState.catalog.lookupFunctionInfo(funcId)
-      val className = info.getClassName
-      withClue(s"Expression class '$className'") {
-        val exprExamples = info.getOriginalExamples
-        if (!exprExamples.isEmpty && !ignoreSet.contains(className)) {
-          assert(exampleRe.findAllIn(exprExamples).toIterable
-            .filter(setStmtRe.findFirstIn(_).isEmpty) // Ignore SET commands
-            .forall(_.contains("_FUNC_")))
-        }
-      }
-    }
-  }
-
-  test("check outputs of expression examples") {
-    def unindentAndTrim(s: String): String = {
-      s.replaceAll("\n\\s+", "\n").trim
-    }
-    val beginSqlStmtRe = "  > ".r
-    val endSqlStmtRe = ";\n".r
-    def checkExampleSyntax(example: String): Unit = {
-      val beginStmtNum = beginSqlStmtRe.findAllIn(example).length
-      val endStmtNum = endSqlStmtRe.findAllIn(example).length
-      assert(beginStmtNum === endStmtNum,
-        "The number of ` > ` does not match to the number of `;`")
-    }
-    val exampleRe = """^(.+);\n(?s)(.+)$""".r
-    val ignoreSet = Set(
-      // One of examples shows getting the current timestamp
-      "org.apache.spark.sql.catalyst.expressions.UnixTimestamp",
-      // Random output without a seed
-      "org.apache.spark.sql.catalyst.expressions.Rand",
-      "org.apache.spark.sql.catalyst.expressions.Randn",
-      "org.apache.spark.sql.catalyst.expressions.Shuffle",
-      "org.apache.spark.sql.catalyst.expressions.Uuid",
-      // The example calls methods that return unstable results.
-      "org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection")
-
-    val parFuncs = new ParVector(spark.sessionState.functionRegistry.listFunction().toVector)
-    parFuncs.foreach { funcId =>
-      // Examples can change settings. We clone the session to prevent tests clashing.
-      val clonedSpark = spark.cloneSession()
-      val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId)
-      val className = info.getClassName
-      if (!ignoreSet.contains(className)) {
-        withClue(s"Function '${info.getName}', Expression class '$className'") {
-          val example = info.getExamples
-          checkExampleSyntax(example)
-          example.split("  > ").toList.foreach(_ match {
-            case exampleRe(sql, output) =>
-              val df = clonedSpark.sql(sql)
-              val actual = unindentAndTrim(
-                hiveResultString(df.queryExecution.executedPlan).mkString("\n"))
-              val expected = unindentAndTrim(output)
-              assert(actual === expected)
-            case _ =>
-          })
-        }
-      }
-    }
-  }
-
   test("SPARK-6743: no columns from cache") {
-    Seq(
-      (83, 0, 38),
-      (26, 0, 79),
-      (43, 81, 24)
-    ).toDF("a", "b", "c").createOrReplaceTempView("cachedData")
+    withTempView("cachedData") {
+      Seq(
+        (83, 0, 38),
+        (26, 0, 79),
+        (43, 81, 24)
+      ).toDF("a", "b", "c").createOrReplaceTempView("cachedData")
 
-    spark.catalog.cacheTable("cachedData")
-    withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
-      checkAnswer(
-        sql("SELECT t1.b FROM cachedData, cachedData t1 GROUP BY t1.b"),
-        Row(0) :: Row(81) :: Nil)
+      spark.catalog.cacheTable("cachedData")
+      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
+        checkAnswer(
+          sql("SELECT t1.b FROM cachedData, cachedData t1 GROUP BY t1.b"),
+          Row(0) :: Row(81) :: Nil)
+      }
     }
   }
 
   test("self join with aliases") {
-    Seq(1, 2, 3).map(i => (i, i.toString)).toDF("int", "str").createOrReplaceTempView("df")
+    withTempView("df") {
+      Seq(1, 2, 3).map(i => (i, i.toString)).toDF("int", "str").createOrReplaceTempView("df")
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT x.str, COUNT(*)
-          |FROM df x JOIN df y ON x.str = y.str
-          |GROUP BY x.str
+      checkAnswer(
+        sql(
+          """
+            |SELECT x.str, COUNT(*)
+            |FROM df x JOIN df y ON x.str = y.str
+            |GROUP BY x.str
         """.stripMargin),
-      Row("1", 1) :: Row("2", 1) :: Row("3", 1) :: Nil)
+        Row("1", 1) :: Row("2", 1) :: Row("3", 1) :: Nil)
+    }
   }
 
   test("support table.star") {
@@ -239,6 +168,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("self join with alias in agg") {
+    withTempView("df") {
       Seq(1, 2, 3)
         .map(i => (i, i.toString))
         .toDF("int", "str")
@@ -246,14 +176,15 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
         .agg($"str", count("str").as("strCount"))
         .createOrReplaceTempView("df")
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT x.str, SUM(x.strCount)
-          |FROM df x JOIN df y ON x.str = y.str
-          |GROUP BY x.str
+      checkAnswer(
+        sql(
+          """
+            |SELECT x.str, SUM(x.strCount)
+            |FROM df x JOIN df y ON x.str = y.str
+            |GROUP BY x.str
         """.stripMargin),
-      Row("1", 1) :: Row("2", 1) :: Row("3", 1) :: Nil)
+        Row("1", 1) :: Row("2", 1) :: Row("3", 1) :: Nil)
+    }
   }
 
   test("SPARK-8668 expr function") {
@@ -300,41 +231,47 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("grouping on nested fields") {
-    spark.read
-      .json(Seq("""{"nested": {"attribute": 1}, "value": 2}""").toDS())
-     .createOrReplaceTempView("rows")
+    withTempView("rows") {
+      spark.read
+        .json(Seq("""{"nested": {"attribute": 1}, "value": 2}""").toDS())
+       .createOrReplaceTempView("rows")
 
-    checkAnswer(
-      sql(
-        """
-          |select attribute, sum(cnt)
-          |from (
-          |  select nested.attribute, count(*) as cnt
-          |  from rows
-          |  group by nested.attribute) a
-          |group by attribute
+      checkAnswer(
+        sql(
+          """
+            |select attribute, sum(cnt)
+            |from (
+            |  select nested.attribute, count(*) as cnt
+            |  from rows
+            |  group by nested.attribute) a
+            |group by attribute
         """.stripMargin),
-      Row(1, 1) :: Nil)
+        Row(1, 1) :: Nil)
+    }
   }
 
   test("SPARK-6201 IN type conversion") {
-    spark.read
-      .json(Seq("{\"a\": \"1\"}}", "{\"a\": \"2\"}}", "{\"a\": \"3\"}}").toDS())
-      .createOrReplaceTempView("d")
+    withTempView("d") {
+      spark.read
+        .json(Seq("{\"a\": \"1\"}}", "{\"a\": \"2\"}}", "{\"a\": \"3\"}}").toDS())
+        .createOrReplaceTempView("d")
 
-    checkAnswer(
-      sql("select * from d where d.a in (1,2)"),
-      Seq(Row("1"), Row("2")))
+      checkAnswer(
+        sql("select * from d where d.a in (1,2)"),
+        Seq(Row("1"), Row("2")))
+    }
   }
 
   test("SPARK-11226 Skip empty line in json file") {
-    spark.read
-      .json(Seq("{\"a\": \"1\"}}", "{\"a\": \"2\"}}", "{\"a\": \"3\"}}", "").toDS())
-      .createOrReplaceTempView("d")
+    withTempView("d") {
+      spark.read
+        .json(Seq("{\"a\": \"1\"}}", "{\"a\": \"2\"}}", "{\"a\": \"3\"}}", "").toDS())
+        .createOrReplaceTempView("d")
 
-    checkAnswer(
-      sql("select count(1) from d"),
-      Seq(Row(3)))
+      checkAnswer(
+        sql("select count(1) from d"),
+        Seq(Row(3)))
+    }
   }
 
   test("SPARK-8828 sum should return null if all input values are null") {
@@ -496,40 +433,42 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("SPARK-3173 Timestamp support in the parser") {
-    (0 to 3).map(i => Tuple1(new Timestamp(i))).toDF("time").createOrReplaceTempView("timestamps")
+    withTempView("timestamps") {
+      (0 to 3).map(i => Tuple1(new Timestamp(i))).toDF("time").createOrReplaceTempView("timestamps")
 
-    checkAnswer(sql(
-      "SELECT time FROM timestamps WHERE time='1969-12-31 16:00:00.0'"),
-      Row(Timestamp.valueOf("1969-12-31 16:00:00")))
+      checkAnswer(sql(
+        "SELECT time FROM timestamps WHERE time='1969-12-31 16:00:00.0'"),
+        Row(Timestamp.valueOf("1969-12-31 16:00:00")))
 
-    checkAnswer(sql(
-      "SELECT time FROM timestamps WHERE time=CAST('1969-12-31 16:00:00.001' AS TIMESTAMP)"),
-      Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
+      checkAnswer(sql(
+        "SELECT time FROM timestamps WHERE time=CAST('1969-12-31 16:00:00.001' AS TIMESTAMP)"),
+        Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
 
-    checkAnswer(sql(
-      "SELECT time FROM timestamps WHERE time='1969-12-31 16:00:00.001'"),
-      Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
+      checkAnswer(sql(
+        "SELECT time FROM timestamps WHERE time='1969-12-31 16:00:00.001'"),
+        Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
 
-    checkAnswer(sql(
-      "SELECT time FROM timestamps WHERE '1969-12-31 16:00:00.001'=time"),
-      Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
+      checkAnswer(sql(
+        "SELECT time FROM timestamps WHERE '1969-12-31 16:00:00.001'=time"),
+        Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
 
-    checkAnswer(sql(
-      """SELECT time FROM timestamps WHERE time<'1969-12-31 16:00:00.003'
+      checkAnswer(sql(
+        """SELECT time FROM timestamps WHERE time<'1969-12-31 16:00:00.003'
           AND time>'1969-12-31 16:00:00.001'"""),
-      Row(Timestamp.valueOf("1969-12-31 16:00:00.002")))
+        Row(Timestamp.valueOf("1969-12-31 16:00:00.002")))
 
-    checkAnswer(sql(
-      """
-        |SELECT time FROM timestamps
-        |WHERE time IN ('1969-12-31 16:00:00.001','1969-12-31 16:00:00.002')
+      checkAnswer(sql(
+        """
+          |SELECT time FROM timestamps
+          |WHERE time IN ('1969-12-31 16:00:00.001','1969-12-31 16:00:00.002')
       """.stripMargin),
-      Seq(Row(Timestamp.valueOf("1969-12-31 16:00:00.001")),
-        Row(Timestamp.valueOf("1969-12-31 16:00:00.002"))))
+        Seq(Row(Timestamp.valueOf("1969-12-31 16:00:00.001")),
+          Row(Timestamp.valueOf("1969-12-31 16:00:00.002"))))
 
-    checkAnswer(sql(
-      "SELECT time FROM timestamps WHERE time='123'"),
-      Nil)
+      checkAnswer(sql(
+        "SELECT time FROM timestamps WHERE time='123'"),
+        Nil)
+    }
   }
 
   test("left semi greater than predicate") {
@@ -858,20 +797,22 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("SPARK-3349 partitioning after limit") {
-    sql("SELECT DISTINCT n FROM lowerCaseData ORDER BY n DESC")
-      .limit(2)
-      .createOrReplaceTempView("subset1")
-    sql("SELECT DISTINCT n FROM lowerCaseData ORDER BY n ASC")
-      .limit(2)
-      .createOrReplaceTempView("subset2")
-    checkAnswer(
-      sql("SELECT * FROM lowerCaseData INNER JOIN subset1 ON subset1.n = lowerCaseData.n"),
-      Row(3, "c", 3) ::
-      Row(4, "d", 4) :: Nil)
-    checkAnswer(
-      sql("SELECT * FROM lowerCaseData INNER JOIN subset2 ON subset2.n = lowerCaseData.n"),
-      Row(1, "a", 1) ::
-      Row(2, "b", 2) :: Nil)
+    withTempView("subset1", "subset2") {
+      sql("SELECT DISTINCT n FROM lowerCaseData ORDER BY n DESC")
+        .limit(2)
+        .createOrReplaceTempView("subset1")
+      sql("SELECT DISTINCT n FROM lowerCaseData ORDER BY n ASC")
+        .limit(2)
+        .createOrReplaceTempView("subset2")
+      checkAnswer(
+        sql("SELECT * FROM lowerCaseData INNER JOIN subset1 ON subset1.n = lowerCaseData.n"),
+        Row(3, "c", 3) ::
+        Row(4, "d", 4) :: Nil)
+      checkAnswer(
+        sql("SELECT * FROM lowerCaseData INNER JOIN subset2 ON subset2.n = lowerCaseData.n"),
+        Row(1, "a", 1) ::
+        Row(2, "b", 2) :: Nil)
+    }
   }
 
   test("mixed-case keywords") {
@@ -1114,84 +1055,87 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("apply schema") {
-    val schema1 = StructType(
-      StructField("f1", IntegerType, false) ::
-      StructField("f2", StringType, false) ::
-      StructField("f3", BooleanType, false) ::
-      StructField("f4", IntegerType, true) :: Nil)
-
-    val rowRDD1 = unparsedStrings.map { r =>
-      val values = r.split(",").map(_.trim)
-      val v4 = try values(3).toInt catch {
-        case _: NumberFormatException => null
+    withTempView("applySchema1", "applySchema2", "applySchema3") {
+      val schema1 = StructType(
+        StructField("f1", IntegerType, false) ::
+        StructField("f2", StringType, false) ::
+        StructField("f3", BooleanType, false) ::
+        StructField("f4", IntegerType, true) :: Nil)
+
+      val rowRDD1 = unparsedStrings.map { r =>
+        val values = r.split(",").map(_.trim)
+        val v4 = try values(3).toInt catch {
+          case _: NumberFormatException => null
+        }
+        Row(values(0).toInt, values(1), values(2).toBoolean, v4)
       }
-      Row(values(0).toInt, values(1), values(2).toBoolean, v4)
-    }
 
-    val df1 = spark.createDataFrame(rowRDD1, schema1)
-    df1.createOrReplaceTempView("applySchema1")
-    checkAnswer(
-      sql("SELECT * FROM applySchema1"),
-      Row(1, "A1", true, null) ::
-      Row(2, "B2", false, null) ::
-      Row(3, "C3", true, null) ::
-      Row(4, "D4", true, 2147483644) :: Nil)
-
-    checkAnswer(
-      sql("SELECT f1, f4 FROM applySchema1"),
-      Row(1, null) ::
-      Row(2, null) ::
-      Row(3, null) ::
-      Row(4, 2147483644) :: Nil)
-
-    val schema2 = StructType(
-      StructField("f1", StructType(
-        StructField("f11", IntegerType, false) ::
-        StructField("f12", BooleanType, false) :: Nil), false) ::
-      StructField("f2", MapType(StringType, IntegerType, true), false) :: Nil)
+      val df1 = spark.createDataFrame(rowRDD1, schema1)
+      df1.createOrReplaceTempView("applySchema1")
+      checkAnswer(
+        sql("SELECT * FROM applySchema1"),
+        Row(1, "A1", true, null) ::
+        Row(2, "B2", false, null) ::
+        Row(3, "C3", true, null) ::
+        Row(4, "D4", true, 2147483644) :: Nil)
 
-    val rowRDD2 = unparsedStrings.map { r =>
-      val values = r.split(",").map(_.trim)
-      val v4 = try values(3).toInt catch {
-        case _: NumberFormatException => null
+      checkAnswer(
+        sql("SELECT f1, f4 FROM applySchema1"),
+        Row(1, null) ::
+        Row(2, null) ::
+        Row(3, null) ::
+        Row(4, 2147483644) :: Nil)
+
+      val schema2 = StructType(
+        StructField("f1", StructType(
+          StructField("f11", IntegerType, false) ::
+          StructField("f12", BooleanType, false) :: Nil), false) ::
+        StructField("f2", MapType(StringType, IntegerType, true), false) :: Nil)
+
+      val rowRDD2 = unparsedStrings.map { r =>
+        val values = r.split(",").map(_.trim)
+        val v4 = try values(3).toInt catch {
+          case _: NumberFormatException => null
+        }
+        Row(Row(values(0).toInt, values(2).toBoolean), Map(values(1) -> v4))
       }
-      Row(Row(values(0).toInt, values(2).toBoolean), Map(values(1) -> v4))
-    }
-
-    val df2 = spark.createDataFrame(rowRDD2, schema2)
-    df2.createOrReplaceTempView("applySchema2")
-    checkAnswer(
-      sql("SELECT * FROM applySchema2"),
-      Row(Row(1, true), Map("A1" -> null)) ::
-      Row(Row(2, false), Map("B2" -> null)) ::
-      Row(Row(3, true), Map("C3" -> null)) ::
-      Row(Row(4, true), Map("D4" -> 2147483644)) :: Nil)
 
-    checkAnswer(
-      sql("SELECT f1.f11, f2['D4'] FROM applySchema2"),
-      Row(1, null) ::
-      Row(2, null) ::
-      Row(3, null) ::
-      Row(4, 2147483644) :: Nil)
+      val df2 = spark.createDataFrame(rowRDD2, schema2)
+      df2.createOrReplaceTempView("applySchema2")
+      checkAnswer(
+        sql("SELECT * FROM applySchema2"),
+        Row(Row(1, true), Map("A1" -> null)) ::
+        Row(Row(2, false), Map("B2" -> null)) ::
+        Row(Row(3, true), Map("C3" -> null)) ::
+        Row(Row(4, true), Map("D4" -> 2147483644)) :: Nil)
 
-    // The value of a MapType column can be a mutable map.
-    val rowRDD3 = unparsedStrings.map { r =>
-      val values = r.split(",").map(_.trim)
-      val v4 = try values(3).toInt catch {
-        case _: NumberFormatException => null
+      checkAnswer(
+        sql("SELECT f1.f11, f2['D4'] FROM applySchema2"),
+        Row(1, null) ::
+        Row(2, null) ::
+        Row(3, null) ::
+        Row(4, 2147483644) :: Nil)
+
+      // The value of a MapType column can be a mutable map.
+      val rowRDD3 = unparsedStrings.map { r =>
+        val values = r.split(",").map(_.trim)
+        val v4 = try values(3).toInt catch {
+          case _: NumberFormatException => null
+        }
+        Row(Row(values(0).toInt, values(2).toBoolean),
+          scala.collection.mutable.Map(values(1) -> v4))
       }
-      Row(Row(values(0).toInt, values(2).toBoolean), scala.collection.mutable.Map(values(1) -> v4))
-    }
 
-    val df3 = spark.createDataFrame(rowRDD3, schema2)
-    df3.createOrReplaceTempView("applySchema3")
+      val df3 = spark.createDataFrame(rowRDD3, schema2)
+      df3.createOrReplaceTempView("applySchema3")
 
-    checkAnswer(
-      sql("SELECT f1.f11, f2['D4'] FROM applySchema3"),
-      Row(1, null) ::
-      Row(2, null) ::
-      Row(3, null) ::
-      Row(4, 2147483644) :: Nil)
+      checkAnswer(
+        sql("SELECT f1.f11, f2['D4'] FROM applySchema3"),
+        Row(1, null) ::
+        Row(2, null) ::
+        Row(3, null) ::
+        Row(4, 2147483644) :: Nil)
+    }
   }
 
   test("SPARK-3423 BETWEEN") {
@@ -1243,28 +1187,30 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("metadata is propagated correctly") {
-    val person: DataFrame = sql("SELECT * FROM person")
-    val schema = person.schema
-    val docKey = "doc"
-    val docValue = "first name"
-    val metadata = new MetadataBuilder()
-      .putString(docKey, docValue)
-      .build()
-    val schemaWithMeta = new StructType(Array(
-      schema("id"), schema("name").copy(metadata = metadata), schema("age")))
-    val personWithMeta = spark.createDataFrame(person.rdd, schemaWithMeta)
-    def validateMetadata(rdd: DataFrame): Unit = {
-      assert(rdd.schema("name").metadata.getString(docKey) == docValue)
-    }
-    personWithMeta.createOrReplaceTempView("personWithMeta")
-    validateMetadata(personWithMeta.select($"name"))
-    validateMetadata(personWithMeta.select($"name"))
-    validateMetadata(personWithMeta.select($"id", $"name"))
-    validateMetadata(sql("SELECT * FROM personWithMeta"))
-    validateMetadata(sql("SELECT id, name FROM personWithMeta"))
-    validateMetadata(sql("SELECT * FROM personWithMeta JOIN salary ON id = personId"))
-    validateMetadata(sql(
-      "SELECT name, salary FROM personWithMeta JOIN salary ON id = personId"))
+    withTempView("personWithMeta") {
+      val person: DataFrame = sql("SELECT * FROM person")
+      val schema = person.schema
+      val docKey = "doc"
+      val docValue = "first name"
+      val metadata = new MetadataBuilder()
+        .putString(docKey, docValue)
+        .build()
+      val schemaWithMeta = new StructType(Array(
+        schema("id"), schema("name").copy(metadata = metadata), schema("age")))
+      val personWithMeta = spark.createDataFrame(person.rdd, schemaWithMeta)
+      def validateMetadata(rdd: DataFrame): Unit = {
+        assert(rdd.schema("name").metadata.getString(docKey) == docValue)
+      }
+      personWithMeta.createOrReplaceTempView("personWithMeta")
+      validateMetadata(personWithMeta.select($"name"))
+      validateMetadata(personWithMeta.select($"name"))
+      validateMetadata(personWithMeta.select($"id", $"name"))
+      validateMetadata(sql("SELECT * FROM personWithMeta"))
+      validateMetadata(sql("SELECT id, name FROM personWithMeta"))
+      validateMetadata(sql("SELECT * FROM personWithMeta JOIN salary ON id = personId"))
+      validateMetadata(sql(
+        "SELECT name, salary FROM personWithMeta JOIN salary ON id = personId"))
+    }
   }
 
   test("SPARK-3371 Renaming a function expression with group by gives error") {
@@ -1306,10 +1252,12 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("SPARK-3483 Special chars in column names") {
-    val data = Seq("""{"key?number1": "value1", "key.number2": "value2"}""").toDS()
-    spark.read.json(data).createOrReplaceTempView("records")
-    withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
-      sql("SELECT `key?number1`, `key.number2` FROM records")
+    withTempView("records") {
+      val data = Seq("""{"key?number1": "value1", "key.number2": "value2"}""").toDS()
+      spark.read.json(data).createOrReplaceTempView("records")
+      withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
+        sql("SELECT `key?number1`, `key.number2` FROM records")
+      }
     }
   }
 
@@ -1376,138 +1324,152 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("Supporting relational operator '<=>' in Spark SQL") {
-    val nullCheckData1 = TestData(1, "1") :: TestData(2, null) :: Nil
-    val rdd1 = sparkContext.parallelize((0 to 1).map(i => nullCheckData1(i)))
-    rdd1.toDF().createOrReplaceTempView("nulldata1")
-    val nullCheckData2 = TestData(1, "1") :: TestData(2, null) :: Nil
-    val rdd2 = sparkContext.parallelize((0 to 1).map(i => nullCheckData2(i)))
-    rdd2.toDF().createOrReplaceTempView("nulldata2")
-    checkAnswer(sql("SELECT nulldata1.key FROM nulldata1 join " +
-      "nulldata2 on nulldata1.value <=> nulldata2.value"),
+    withTempView("nulldata1", "nulldata2") {
+      val nullCheckData1 = TestData(1, "1") :: TestData(2, null) :: Nil
+      val rdd1 = sparkContext.parallelize((0 to 1).map(i => nullCheckData1(i)))
+      rdd1.toDF().createOrReplaceTempView("nulldata1")
+      val nullCheckData2 = TestData(1, "1") :: TestData(2, null) :: Nil
+      val rdd2 = sparkContext.parallelize((0 to 1).map(i => nullCheckData2(i)))
+      rdd2.toDF().createOrReplaceTempView("nulldata2")
+      checkAnswer(sql("SELECT nulldata1.key FROM nulldata1 join " +
+        "nulldata2 on nulldata1.value <=> nulldata2.value"),
         (1 to 2).map(i => Row(i)))
+    }
   }
 
   test("Multi-column COUNT(DISTINCT ...)") {
-    val data = TestData(1, "val_1") :: TestData(2, "val_2") :: Nil
-    val rdd = sparkContext.parallelize((0 to 1).map(i => data(i)))
-    rdd.toDF().createOrReplaceTempView("distinctData")
-    checkAnswer(sql("SELECT COUNT(DISTINCT key,value) FROM distinctData"), Row(2))
+    withTempView("distinctData") {
+      val data = TestData(1, "val_1") :: TestData(2, "val_2") :: Nil
+      val rdd = sparkContext.parallelize((0 to 1).map(i => data(i)))
+      rdd.toDF().createOrReplaceTempView("distinctData")
+      checkAnswer(sql("SELECT COUNT(DISTINCT key,value) FROM distinctData"), Row(2))
+    }
   }
 
   test("SPARK-4699 case sensitivity SQL query") {
-    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
-      val data = TestData(1, "val_1") :: TestData(2, "val_2") :: Nil
-      val rdd = sparkContext.parallelize((0 to 1).map(i => data(i)))
-      rdd.toDF().createOrReplaceTempView("testTable1")
-      checkAnswer(sql("SELECT VALUE FROM TESTTABLE1 where KEY = 1"), Row("val_1"))
+    withTempView("testTable1") {
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        val data = TestData(1, "val_1") :: TestData(2, "val_2") :: Nil
+        val rdd = sparkContext.parallelize((0 to 1).map(i => data(i)))
+        rdd.toDF().createOrReplaceTempView("testTable1")
+        checkAnswer(sql("SELECT VALUE FROM TESTTABLE1 where KEY = 1"), Row("val_1"))
+      }
     }
   }
 
   test("SPARK-6145: ORDER BY test for nested fields") {
-    spark.read
-      .json(Seq("""{"a": {"b": 1, "a": {"a": 1}}, "c": [{"d": 1}]}""").toDS())
-      .createOrReplaceTempView("nestedOrder")
+    withTempView("nestedOrder") {
+      spark.read
+        .json(Seq("""{"a": {"b": 1, "a": {"a": 1}}, "c": [{"d": 1}]}""").toDS())
+        .createOrReplaceTempView("nestedOrder")
 
-    checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY a.b"), Row(1))
-    checkAnswer(sql("SELECT a.b FROM nestedOrder ORDER BY a.b"), Row(1))
-    checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY a.a.a"), Row(1))
-    checkAnswer(sql("SELECT a.a.a FROM nestedOrder ORDER BY a.a.a"), Row(1))
-    checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY c[0].d"), Row(1))
-    checkAnswer(sql("SELECT c[0].d FROM nestedOrder ORDER BY c[0].d"), Row(1))
+      checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY a.b"), Row(1))
+      checkAnswer(sql("SELECT a.b FROM nestedOrder ORDER BY a.b"), Row(1))
+      checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY a.a.a"), Row(1))
+      checkAnswer(sql("SELECT a.a.a FROM nestedOrder ORDER BY a.a.a"), Row(1))
+      checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY c[0].d"), Row(1))
+      checkAnswer(sql("SELECT c[0].d FROM nestedOrder ORDER BY c[0].d"), Row(1))
+    }
   }
 
   test("SPARK-6145: special cases") {
-    spark.read
-      .json(Seq("""{"a": {"b": [1]}, "b": [{"a": 1}], "_c0": {"a": 1}}""").toDS())
-      .createOrReplaceTempView("t")
+    withTempView("t") {
+      spark.read
+        .json(Seq("""{"a": {"b": [1]}, "b": [{"a": 1}], "_c0": {"a": 1}}""").toDS())
+        .createOrReplaceTempView("t")
 
-    checkAnswer(sql("SELECT a.b[0] FROM t ORDER BY _c0.a"), Row(1))
-    checkAnswer(sql("SELECT b[0].a FROM t ORDER BY _c0.a"), Row(1))
+      checkAnswer(sql("SELECT a.b[0] FROM t ORDER BY _c0.a"), Row(1))
+      checkAnswer(sql("SELECT b[0].a FROM t ORDER BY _c0.a"), Row(1))
+    }
   }
 
   test("SPARK-6898: complete support for special chars in column names") {
-    spark.read
-      .json(Seq("""{"a": {"c.b": 1}, "b.$q": [{"a@!.q": 1}], "q.w": {"w.i&": [1]}}""").toDS())
-      .createOrReplaceTempView("t")
+    withTempView("t") {
+      spark.read
+        .json(Seq("""{"a": {"c.b": 1}, "b.$q": [{"a@!.q": 1}], "q.w": {"w.i&": [1]}}""").toDS())
+        .createOrReplaceTempView("t")
 
-    withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
-      checkAnswer(sql("SELECT a.`c.b`, `b.$q`[0].`a@!.q`, `q.w`.`w.i&`[0] FROM t"), Row(1, 1, 1))
+      withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
+        checkAnswer(sql("SELECT a.`c.b`, `b.$q`[0].`a@!.q`, `q.w`.`w.i&`[0] FROM t"), Row(1, 1, 1))
+      }
     }
   }
 
   test("SPARK-6583 order by aggregated function") {
-    Seq("1" -> 3, "1" -> 4, "2" -> 7, "2" -> 8, "3" -> 5, "3" -> 6, "4" -> 1, "4" -> 2)
-      .toDF("a", "b").createOrReplaceTempView("orderByData")
+    withTempView("orderByData") {
+      Seq("1" -> 3, "1" -> 4, "2" -> 7, "2" -> 8, "3" -> 5, "3" -> 6, "4" -> 1, "4" -> 2)
+        .toDF("a", "b").createOrReplaceTempView("orderByData")
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT a
-          |FROM orderByData
-          |GROUP BY a
-          |ORDER BY sum(b)
+      checkAnswer(
+        sql(
+          """
+            |SELECT a
+            |FROM orderByData
+            |GROUP BY a
+            |ORDER BY sum(b)
         """.stripMargin),
-      Row("4") :: Row("1") :: Row("3") :: Row("2") :: Nil)
+        Row("4") :: Row("1") :: Row("3") :: Row("2") :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT sum(b)
-          |FROM orderByData
-          |GROUP BY a
-          |ORDER BY sum(b)
+      checkAnswer(
+        sql(
+          """
+            |SELECT sum(b)
+            |FROM orderByData
+            |GROUP BY a
+            |ORDER BY sum(b)
         """.stripMargin),
-      Row(3) :: Row(7) :: Row(11) :: Row(15) :: Nil)
+        Row(3) :: Row(7) :: Row(11) :: Row(15) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT sum(b)
-          |FROM orderByData
-          |GROUP BY a
-          |ORDER BY sum(b), max(b)
+      checkAnswer(
+        sql(
+          """
+            |SELECT sum(b)
+            |FROM orderByData
+            |GROUP BY a
+            |ORDER BY sum(b), max(b)
         """.stripMargin),
-      Row(3) :: Row(7) :: Row(11) :: Row(15) :: Nil)
+        Row(3) :: Row(7) :: Row(11) :: Row(15) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT a, sum(b)
-          |FROM orderByData
-          |GROUP BY a
-          |ORDER BY sum(b)
+      checkAnswer(
+        sql(
+          """
+            |SELECT a, sum(b)
+            |FROM orderByData
+            |GROUP BY a
+            |ORDER BY sum(b)
         """.stripMargin),
-      Row("4", 3) :: Row("1", 7) :: Row("3", 11) :: Row("2", 15) :: Nil)
+        Row("4", 3) :: Row("1", 7) :: Row("3", 11) :: Row("2", 15) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
+      checkAnswer(
+        sql(
+          """
             |SELECT a, sum(b)
             |FROM orderByData
             |GROUP BY a
             |ORDER BY sum(b) + 1
           """.stripMargin),
-      Row("4", 3) :: Row("1", 7) :: Row("3", 11) :: Row("2", 15) :: Nil)
+        Row("4", 3) :: Row("1", 7) :: Row("3", 11) :: Row("2", 15) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
+      checkAnswer(
+        sql(
+          """
             |SELECT count(*)
             |FROM orderByData
             |GROUP BY a
             |ORDER BY count(*)
           """.stripMargin),
-      Row(2) :: Row(2) :: Row(2) :: Row(2) :: Nil)
+        Row(2) :: Row(2) :: Row(2) :: Row(2) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
+      checkAnswer(
+        sql(
+          """
             |SELECT a
             |FROM orderByData
             |GROUP BY a
             |ORDER BY a, count(*), sum(b)
           """.stripMargin),
-      Row("1") :: Row("2") :: Row("3") :: Row("4") :: Nil)
+        Row("1") :: Row("2") :: Row("3") :: Row("4") :: Nil)
+    }
   }
 
   test("SPARK-7952: fix the equality check between boolean and numeric types") {
@@ -1819,137 +1781,141 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("Struct Star Expansion") {
-    val structDf = testData2.select("a", "b").as("record")
+    withTempView("structTable", "nestedStructTable", "specialCharacterTable", "nameConflict") {
+      val structDf = testData2.select("a", "b").as("record")
 
-    checkAnswer(
-      structDf.select($"record.a", $"record.b"),
-      Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
+      checkAnswer(
+        structDf.select($"record.a", $"record.b"),
+        Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
 
-    checkAnswer(
-      structDf.select($"record.*"),
-      Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
+      checkAnswer(
+        structDf.select($"record.*"),
+        Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
 
-    checkAnswer(
-      structDf.select($"record.*", $"record.*"),
-      Row(1, 1, 1, 1) :: Row(1, 2, 1, 2) :: Row(2, 1, 2, 1) :: Row(2, 2, 2, 2) ::
-        Row(3, 1, 3, 1) :: Row(3, 2, 3, 2) :: Nil)
+      checkAnswer(
+        structDf.select($"record.*", $"record.*"),
+        Row(1, 1, 1, 1) :: Row(1, 2, 1, 2) :: Row(2, 1, 2, 1) :: Row(2, 2, 2, 2) ::
+          Row(3, 1, 3, 1) :: Row(3, 2, 3, 2) :: Nil)
 
-    checkAnswer(
-      sql("select struct(a, b) as r1, struct(b, a) as r2 from testData2").select($"r1.*", $"r2.*"),
-      Row(1, 1, 1, 1) :: Row(1, 2, 2, 1) :: Row(2, 1, 1, 2) :: Row(2, 2, 2, 2) ::
-        Row(3, 1, 1, 3) :: Row(3, 2, 2, 3) :: Nil)
+      checkAnswer(
+        sql("select struct(a, b) as r1, struct(b, a) as r2 from testData2")
+          .select($"r1.*", $"r2.*"),
+        Row(1, 1, 1, 1) :: Row(1, 2, 2, 1) :: Row(2, 1, 1, 2) :: Row(2, 2, 2, 2) ::
+          Row(3, 1, 1, 3) :: Row(3, 2, 2, 3) :: Nil)
 
-    // Try with a temporary view
-    sql("select struct(a, b) as record from testData2").createOrReplaceTempView("structTable")
-    checkAnswer(
-      sql("SELECT record.* FROM structTable"),
-      Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
+      // Try with a temporary view
+      sql("select struct(a, b) as record from testData2").createOrReplaceTempView("structTable")
+      checkAnswer(
+        sql("SELECT record.* FROM structTable"),
+        Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
 
-    checkAnswer(sql(
-      """
-        | SELECT min(struct(record.*)) FROM
-        |   (select struct(a,b) as record from testData2) tmp
+      checkAnswer(sql(
+        """
+          | SELECT min(struct(record.*)) FROM
+          |   (select struct(a,b) as record from testData2) tmp
       """.stripMargin),
-      Row(Row(1, 1)) :: Nil)
+        Row(Row(1, 1)) :: Nil)
 
-    // Try with an alias on the select list
-    checkAnswer(sql(
-      """
-        | SELECT max(struct(record.*)) as r FROM
-        |   (select struct(a,b) as record from testData2) tmp
+      // Try with an alias on the select list
+      checkAnswer(sql(
+        """
+          | SELECT max(struct(record.*)) as r FROM
+          |   (select struct(a,b) as record from testData2) tmp
       """.stripMargin).select($"r.*"),
-      Row(3, 2) :: Nil)
+        Row(3, 2) :: Nil)
 
-    // With GROUP BY
-    checkAnswer(sql(
-      """
-        | SELECT min(struct(record.*)) FROM
-        |   (select a as a, struct(a,b) as record from testData2) tmp
-        | GROUP BY a
+      // With GROUP BY
+      checkAnswer(sql(
+        """
+          | SELECT min(struct(record.*)) FROM
+          |   (select a as a, struct(a,b) as record from testData2) tmp
+          | GROUP BY a
       """.stripMargin),
-      Row(Row(1, 1)) :: Row(Row(2, 1)) :: Row(Row(3, 1)) :: Nil)
+        Row(Row(1, 1)) :: Row(Row(2, 1)) :: Row(Row(3, 1)) :: Nil)
 
-    // With GROUP BY and alias
-    checkAnswer(sql(
-      """
-        | SELECT max(struct(record.*)) as r FROM
-        |   (select a as a, struct(a,b) as record from testData2) tmp
-        | GROUP BY a
+      // With GROUP BY and alias
+      checkAnswer(sql(
+        """
+          | SELECT max(struct(record.*)) as r FROM
+          |   (select a as a, struct(a,b) as record from testData2) tmp
+          | GROUP BY a
       """.stripMargin).select($"r.*"),
-      Row(1, 2) :: Row(2, 2) :: Row(3, 2) :: Nil)
+        Row(1, 2) :: Row(2, 2) :: Row(3, 2) :: Nil)
 
-    // With GROUP BY and alias and additional fields in the struct
-    checkAnswer(sql(
-      """
-        | SELECT max(struct(a, record.*, b)) as r FROM
-        |   (select a as a, b as b, struct(a,b) as record from testData2) tmp
-        | GROUP BY a
+      // With GROUP BY and alias and additional fields in the struct
+      checkAnswer(sql(
+        """
+          | SELECT max(struct(a, record.*, b)) as r FROM
+          |   (select a as a, b as b, struct(a,b) as record from testData2) tmp
+          | GROUP BY a
       """.stripMargin).select($"r.*"),
-      Row(1, 1, 2, 2) :: Row(2, 2, 2, 2) :: Row(3, 3, 2, 2) :: Nil)
+        Row(1, 1, 2, 2) :: Row(2, 2, 2, 2) :: Row(3, 3, 2, 2) :: Nil)
 
-    // Create a data set that contains nested structs.
-    val nestedStructData = sql(
-      """
-        | SELECT struct(r1, r2) as record FROM
-        |   (SELECT struct(a, b) as r1, struct(b, a) as r2 FROM testData2) tmp
+      // Create a data set that contains nested structs.
+      val nestedStructData = sql(
+        """
+          | SELECT struct(r1, r2) as record FROM
+          |   (SELECT struct(a, b) as r1, struct(b, a) as r2 FROM testData2) tmp
       """.stripMargin)
 
-    checkAnswer(nestedStructData.select($"record.*"),
-      Row(Row(1, 1), Row(1, 1)) :: Row(Row(1, 2), Row(2, 1)) :: Row(Row(2, 1), Row(1, 2)) ::
-        Row(Row(2, 2), Row(2, 2)) :: Row(Row(3, 1), Row(1, 3)) :: Row(Row(3, 2), Row(2, 3)) :: Nil)
-    checkAnswer(nestedStructData.select($"record.r1"),
-      Row(Row(1, 1)) :: Row(Row(1, 2)) :: Row(Row(2, 1)) :: Row(Row(2, 2)) ::
-        Row(Row(3, 1)) :: Row(Row(3, 2)) :: Nil)
-    checkAnswer(
-      nestedStructData.select($"record.r1.*"),
-      Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
-
-    // Try with a temporary view
-    withTempView("nestedStructTable") {
-      nestedStructData.createOrReplaceTempView("nestedStructTable")
-      checkAnswer(
-        sql("SELECT record.* FROM nestedStructTable"),
-        nestedStructData.select($"record.*"))
+      checkAnswer(nestedStructData.select($"record.*"),
+        Row(Row(1, 1), Row(1, 1)) :: Row(Row(1, 2), Row(2, 1)) :: Row(Row(2, 1), Row(1, 2)) ::
+          Row(Row(2, 2), Row(2, 2)) :: Row(Row(3, 1), Row(1, 3)) :: Row(Row(3, 2), Row(2, 3)) ::
+          Nil)
+      checkAnswer(nestedStructData.select($"record.r1"),
+        Row(Row(1, 1)) :: Row(Row(1, 2)) :: Row(Row(2, 1)) :: Row(Row(2, 2)) ::
+          Row(Row(3, 1)) :: Row(Row(3, 2)) :: Nil)
       checkAnswer(
-        sql("SELECT record.r1 FROM nestedStructTable"),
-        nestedStructData.select($"record.r1"))
-      checkAnswer(
-        sql("SELECT record.r1.* FROM nestedStructTable"),
-        nestedStructData.select($"record.r1.*"))
-
-      // Try resolving something not there.
-      assert(intercept[AnalysisException](sql("SELECT abc.* FROM nestedStructTable"))
-        .getMessage.contains("cannot resolve"))
-    }
+        nestedStructData.select($"record.r1.*"),
+        Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
 
-    // Create paths with unusual characters
-    withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
-      val specialCharacterPath = sql(
-        """
-        | SELECT struct(`col$.a_`, `a.b.c.`) as `r&&b.c` FROM
-        |   (SELECT struct(a, b) as `col$.a_`, struct(b, a) as `a.b.c.` FROM testData2) tmp
-      """.stripMargin)
-      withTempView("specialCharacterTable") {
-        specialCharacterPath.createOrReplaceTempView("specialCharacterTable")
+      // Try with a temporary view
+      withTempView("nestedStructTable") {
+        nestedStructData.createOrReplaceTempView("nestedStructTable")
         checkAnswer(
-          specialCharacterPath.select($"`r&&b.c`.*"),
+          sql("SELECT record.* FROM nestedStructTable"),
           nestedStructData.select($"record.*"))
         checkAnswer(
-        sql(
-          "SELECT `r&&b.c`.`col$.a_` FROM specialCharacterTable"),
-        nestedStructData.select($"record.r1"))
-        checkAnswer(
-          sql("SELECT `r&&b.c`.`a.b.c.` FROM specialCharacterTable"),
-          nestedStructData.select($"record.r2"))
+          sql("SELECT record.r1 FROM nestedStructTable"),
+          nestedStructData.select($"record.r1"))
         checkAnswer(
-          sql("SELECT `r&&b.c`.`col$.a_`.* FROM specialCharacterTable"),
+          sql("SELECT record.r1.* FROM nestedStructTable"),
           nestedStructData.select($"record.r1.*"))
+
+        // Try resolving something not there.
+        assert(intercept[AnalysisException](sql("SELECT abc.* FROM nestedStructTable"))
+          .getMessage.contains("cannot resolve"))
       }
-    }
 
-    // Try star expanding a scalar. This should fail.
-    assert(intercept[AnalysisException](sql("select a.* from testData2")).getMessage.contains(
-      "Can only star expand struct data types."))
+      // Create paths with unusual characters
+      withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
+        val specialCharacterPath = sql(
+          """
+          | SELECT struct(`col$.a_`, `a.b.c.`) as `r&&b.c` FROM
+          |   (SELECT struct(a, b) as `col$.a_`, struct(b, a) as `a.b.c.` FROM testData2) tmp
+      """.stripMargin)
+        withTempView("specialCharacterTable") {
+          specialCharacterPath.createOrReplaceTempView("specialCharacterTable")
+          checkAnswer(
+            specialCharacterPath.select($"`r&&b.c`.*"),
+            nestedStructData.select($"record.*"))
+          checkAnswer(
+          sql(
+            "SELECT `r&&b.c`.`col$.a_` FROM specialCharacterTable"),
+          nestedStructData.select($"record.r1"))
+          checkAnswer(
+            sql("SELECT `r&&b.c`.`a.b.c.` FROM specialCharacterTable"),
+            nestedStructData.select($"record.r2"))
+          checkAnswer(
+            sql("SELECT `r&&b.c`.`col$.a_`.* FROM specialCharacterTable"),
+            nestedStructData.select($"record.r1.*"))
+        }
+      }
+
+      // Try star expanding a scalar. This should fail.
+      assert(intercept[AnalysisException](sql("select a.* from testData2")).getMessage.contains(
+        "Can only star expand struct data types."))
+    }
   }
 
   test("Struct Star Expansion - Name conflict") {
@@ -2121,6 +2087,26 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     }
   }
 
+  test("SPARK-27619: Throw analysis exception when hash and xxhash64 is used on MapType") {
+    Seq("hash", "xxhash64").foreach {
+      case hashExpression =>
+        intercept[AnalysisException] {
+          spark.createDataset(Map(1 -> 10, 2 -> 20) :: Nil).selectExpr(s"$hashExpression(*)")
+        }
+    }
+  }
+
+  test(s"SPARK-27619: When ${SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE.key} is true, hash can be " +
+    "used on Maptype") {
+    Seq("hash", "xxhash64").foreach {
+      case hashExpression =>
+        withSQLConf(SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE.key -> "true") {
+          val df = spark.createDataset(Map() :: Nil)
+          checkAnswer(df.selectExpr(s"$hashExpression(*)"), sql(s"SELECT $hashExpression(map())"))
+        }
+    }
+  }
+
   test("xxhash64 function") {
     val df = Seq(1 -> "a", 2 -> "b").toDF("i", "j")
     withTempView("tbl") {
@@ -2776,7 +2762,9 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
           sql("SELECT * FROM t, S WHERE c = C")
         }.message
         assert(
-          m.contains("cannot resolve '(default.t.`c` = default.S.`C`)' due to data type mismatch"))
+          m.contains(
+            "cannot resolve '(spark_catalog.default.t.`c` = spark_catalog.default.S.`C`)' " +
+            "due to data type mismatch"))
       }
     }
   }
@@ -3184,38 +3172,40 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("string date comparison") {
-    spark.range(1).selectExpr("date '2000-01-01' as d").createOrReplaceTempView("t1")
-    val result = Date.valueOf("2000-01-01")
-    checkAnswer(sql("select * from t1 where d < '2000'"), Nil)
-    checkAnswer(sql("select * from t1 where d < '2001'"), Row(result))
-    checkAnswer(sql("select * from t1 where d < '2000-01'"), Nil)
-    checkAnswer(sql("select * from t1 where d < '2000-01-01'"), Nil)
-    checkAnswer(sql("select * from t1 where d < '2000-1-1'"), Nil)
-    checkAnswer(sql("select * from t1 where d <= '2000-1-1'"), Row(result))
-    checkAnswer(sql("select * from t1 where d <= '1999-12-30'"), Nil)
-    checkAnswer(sql("select * from t1 where d = '2000-1-1'"), Row(result))
-    checkAnswer(sql("select * from t1 where d = '2000-01-01'"), Row(result))
-    checkAnswer(sql("select * from t1 where d = '2000-1-02'"), Nil)
-    checkAnswer(sql("select * from t1 where d > '2000-01-01'"), Nil)
-    checkAnswer(sql("select * from t1 where d > '1999'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-1'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-1-1'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-1-01'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-01-1'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-01-01'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-01-02'"), Nil)
-    checkAnswer(sql("select * from t1 where '2000' >= d"), Row(result))
-    checkAnswer(sql("select * from t1 where d > '2000-13'"), Nil)
-
-    withSQLConf(SQLConf.LEGACY_CAST_DATETIME_TO_STRING.key -> "true") {
+    withTempView("t1") {
+      spark.range(1).selectExpr("date '2000-01-01' as d").createOrReplaceTempView("t1")
+      val result = Date.valueOf("2000-01-01")
       checkAnswer(sql("select * from t1 where d < '2000'"), Nil)
       checkAnswer(sql("select * from t1 where d < '2001'"), Row(result))
-      checkAnswer(sql("select * from t1 where d < '2000-1-1'"), Row(result))
-      checkAnswer(sql("select * from t1 where d <= '1999'"), Nil)
+      checkAnswer(sql("select * from t1 where d < '2000-01'"), Nil)
+      checkAnswer(sql("select * from t1 where d < '2000-01-01'"), Nil)
+      checkAnswer(sql("select * from t1 where d < '2000-1-1'"), Nil)
+      checkAnswer(sql("select * from t1 where d <= '2000-1-1'"), Row(result))
+      checkAnswer(sql("select * from t1 where d <= '1999-12-30'"), Nil)
+      checkAnswer(sql("select * from t1 where d = '2000-1-1'"), Row(result))
+      checkAnswer(sql("select * from t1 where d = '2000-01-01'"), Row(result))
+      checkAnswer(sql("select * from t1 where d = '2000-1-02'"), Nil)
+      checkAnswer(sql("select * from t1 where d > '2000-01-01'"), Nil)
+      checkAnswer(sql("select * from t1 where d > '1999'"), Row(result))
       checkAnswer(sql("select * from t1 where d >= '2000'"), Row(result))
-      checkAnswer(sql("select * from t1 where d > '1999-13'"), Row(result))
-      checkAnswer(sql("select to_date('2000-01-01') > '1'"), Row(true))
+      checkAnswer(sql("select * from t1 where d >= '2000-1'"), Row(result))
+      checkAnswer(sql("select * from t1 where d >= '2000-1-1'"), Row(result))
+      checkAnswer(sql("select * from t1 where d >= '2000-1-01'"), Row(result))
+      checkAnswer(sql("select * from t1 where d >= '2000-01-1'"), Row(result))
+      checkAnswer(sql("select * from t1 where d >= '2000-01-01'"), Row(result))
+      checkAnswer(sql("select * from t1 where d >= '2000-01-02'"), Nil)
+      checkAnswer(sql("select * from t1 where '2000' >= d"), Row(result))
+      checkAnswer(sql("select * from t1 where d > '2000-13'"), Nil)
+
+      withSQLConf(SQLConf.LEGACY_CAST_DATETIME_TO_STRING.key -> "true") {
+        checkAnswer(sql("select * from t1 where d < '2000'"), Nil)
+        checkAnswer(sql("select * from t1 where d < '2001'"), Row(result))
+        checkAnswer(sql("select * from t1 where d < '2000-1-1'"), Row(result))
+        checkAnswer(sql("select * from t1 where d <= '1999'"), Nil)
+        checkAnswer(sql("select * from t1 where d >= '2000'"), Row(result))
+        checkAnswer(sql("select * from t1 where d > '1999-13'"), Row(result))
+        checkAnswer(sql("select to_date('2000-01-01') > '1'"), Row(true))
+      }
     }
   }
 
@@ -3265,28 +3255,30 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   test("SPARK-28156: self-join should not miss cached view") {
     withTable("table1") {
       withView("table1_vw") {
-        val df = Seq.tabulate(5) { x => (x, x + 1, x + 2, x + 3) }.toDF("a", "b", "c", "d")
-        df.write.mode("overwrite").format("orc").saveAsTable("table1")
-        sql("drop view if exists table1_vw")
-        sql("create view table1_vw as select * from table1")
-
-        val cachedView = sql("select a, b, c, d from table1_vw")
-
-        cachedView.createOrReplaceTempView("cachedview")
-        cachedView.persist()
-
-        val queryDf = sql(
-          s"""select leftside.a, leftside.b
-             |from cachedview leftside
-             |join cachedview rightside
-             |on leftside.a = rightside.a
+        withTempView("cachedview") {
+          val df = Seq.tabulate(5) { x => (x, x + 1, x + 2, x + 3) }.toDF("a", "b", "c", "d")
+          df.write.mode("overwrite").format("orc").saveAsTable("table1")
+          sql("drop view if exists table1_vw")
+          sql("create view table1_vw as select * from table1")
+
+          val cachedView = sql("select a, b, c, d from table1_vw")
+
+          cachedView.createOrReplaceTempView("cachedview")
+          cachedView.persist()
+
+          val queryDf = sql(
+            s"""select leftside.a, leftside.b
+               |from cachedview leftside
+               |join cachedview rightside
+               |on leftside.a = rightside.a
            """.stripMargin)
 
-        val inMemoryTableScan = collect(queryDf.queryExecution.executedPlan) {
-          case i: InMemoryTableScanExec => i
+          val inMemoryTableScan = collect(queryDf.queryExecution.executedPlan) {
+            case i: InMemoryTableScanExec => i
+          }
+          assert(inMemoryTableScan.size == 2)
+          checkAnswer(queryDf, Row(0, 1) :: Row(1, 2) :: Row(2, 3) :: Row(3, 4) :: Row(4, 5) :: Nil)
         }
-        assert(inMemoryTableScan.size == 2)
-        checkAnswer(queryDf, Row(0, 1) :: Row(1, 2) :: Row(2, 3) :: Row(3, 4) :: Row(4, 5) :: Nil)
       }
     }
 
@@ -3383,6 +3375,215 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       checkAnswer(df, Row(1))
     }
   }
+
+  test("SPARK-26218: Fix the corner case when casting float to Integer") {
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      intercept[ArithmeticException](
+        sql("SELECT CAST(CAST(2147483648 as FLOAT) as Integer)").collect()
+      )
+      intercept[ArithmeticException](
+        sql("SELECT CAST(CAST(2147483648 as DOUBLE) as Integer)").collect()
+      )
+    }
+  }
+
+  test("SPARK-30870: Column pruning shouldn't alias a nested column for the whole structure") {
+    withTable("t") {
+      val df = sql(
+        """
+          |SELECT value
+          |FROM VALUES array(named_struct('field', named_struct('a', 1, 'b', 2))) AS (value)
+        """.stripMargin)
+      df.write.format("parquet").saveAsTable("t")
+
+      val df2 = spark.table("t")
+        .limit(100)
+        .select(size(col("value.field")))
+      val projects = df2.queryExecution.optimizedPlan.collect {
+        case p: Project => p
+      }
+      assert(projects.length == 1)
+      val aliases = NestedColumnAliasingSuite.collectGeneratedAliases(projects(0))
+      assert(aliases.length == 0)
+    }
+  }
+
+  test("SPARK-30955: Exclude Generate output when aliasing in nested column pruning") {
+    val df1 = sql(
+      """
+        |SELECT explodedvalue.*
+        |FROM VALUES array(named_struct('nested', named_struct('a', 1, 'b', 2))) AS (value)
+        |LATERAL VIEW explode(value) AS explodedvalue
+      """.stripMargin)
+    checkAnswer(df1, Row(Row(1, 2)) :: Nil)
+
+    val df2 = sql(
+      """
+        |SELECT explodedvalue.nested.a
+        |FROM VALUES array(named_struct('nested', named_struct('a', 1, 'b', 2))) AS (value)
+        |LATERAL VIEW explode(value) AS explodedvalue
+      """.stripMargin)
+    checkAnswer(df2, Row(1) :: Nil)
+  }
+
+  test("SPARK-31166: UNION map<null, null> and other maps should not fail") {
+    checkAnswer(
+      sql("(SELECT map()) UNION ALL (SELECT map(1, 2))"),
+      Seq(Row(Map[Int, Int]()), Row(Map(1 -> 2))))
+  }
+
+  test("SPARK-31242: clone SparkSession should respect sessionInitWithConfigDefaults") {
+    // Note, only the conf explicitly set in SparkConf(e.g. in SharedSparkSessionBase) would cause
+    // problem before the fix.
+    withSQLConf(SQLConf.CODEGEN_FALLBACK.key -> "true") {
+      val cloned = spark.cloneSession()
+      SparkSession.setActiveSession(cloned)
+      assert(SQLConf.get.getConf(SQLConf.CODEGEN_FALLBACK) === true)
+    }
+  }
+
+  test("SPARK-31761: test byte, short, integer overflow for (Divide) integral type") {
+    checkAnswer(sql("Select -2147483648 DIV -1"), Seq(Row(Integer.MIN_VALUE.toLong * -1)))
+    checkAnswer(sql("select CAST(-128 as Byte) DIV CAST (-1 as Byte)"),
+      Seq(Row(Byte.MinValue.toLong * -1)))
+    checkAnswer(sql("select CAST(-32768 as short) DIV CAST (-1 as short)"),
+      Seq(Row(Short.MinValue.toLong * -1)))
+  }
+
+  test("normalize special floating numbers in subquery") {
+    withTempView("v1", "v2", "v3") {
+      Seq(-0.0).toDF("d").createTempView("v1")
+      Seq(0.0).toDF("d").createTempView("v2")
+      spark.range(2).createTempView("v3")
+
+      // non-correlated subquery
+      checkAnswer(sql("SELECT (SELECT v1.d FROM v1 JOIN v2 ON v1.d = v2.d)"), Row(-0.0))
+      // correlated subquery
+      checkAnswer(
+        sql(
+          """
+            |SELECT id FROM v3 WHERE EXISTS
+            |  (SELECT v1.d FROM v1 JOIN v2 ON v1.d = v2.d WHERE id > 0)
+            |""".stripMargin), Row(1))
+    }
+  }
+
+  test("SPARK-32237: Hint in CTE") {
+    withTable("t") {
+      sql("CREATE TABLE t USING PARQUET AS SELECT 1 AS id")
+      checkAnswer(
+        sql("""
+              |WITH cte AS (SELECT /*+ REPARTITION(3) */ * FROM t)
+              |SELECT * FROM cte
+            """.stripMargin),
+        Row(1) :: Nil)
+    }
+  }
+
+  test("SPARK-32372: ResolveReferences.dedupRight should only rewrite attributes for ancestor " +
+    "plans of the conflict plan") {
+    sql("SELECT name, avg(age) as avg_age FROM person GROUP BY name")
+      .createOrReplaceTempView("person_a")
+    sql("SELECT p1.name, p2.avg_age FROM person p1 JOIN person_a p2 ON p1.name = p2.name")
+      .createOrReplaceTempView("person_b")
+    sql("SELECT * FROM person_a UNION SELECT * FROM person_b")
+      .createOrReplaceTempView("person_c")
+    checkAnswer(
+      sql("SELECT p1.name, p2.avg_age FROM person_c p1 JOIN person_c p2 ON p1.name = p2.name"),
+      Row("jim", 20.0) :: Row("mike", 30.0) :: Nil)
+  }
+
+  test("SPARK-32280: Avoid duplicate rewrite attributes when there're multiple JOINs") {
+    sql("SELECT 1 AS id").createOrReplaceTempView("A")
+    sql("SELECT id, 'foo' AS kind FROM A").createOrReplaceTempView("B")
+    sql("SELECT l.id as id FROM B AS l LEFT SEMI JOIN B AS r ON l.kind = r.kind")
+      .createOrReplaceTempView("C")
+    checkAnswer(sql("SELECT 0 FROM ( SELECT * FROM B JOIN C USING (id)) " +
+      "JOIN ( SELECT * FROM B JOIN C USING (id)) USING (id)"), Row(0))
+  }
+
+  test("SPARK-32788: non-partitioned table scan should not have partition filter") {
+    withTable("t") {
+      spark.range(1).write.saveAsTable("t")
+      checkAnswer(sql("SELECT id FROM t WHERE (SELECT true)"), Row(0L))
+    }
+  }
+
+  test("SPARK-33306: Timezone is needed when cast Date to String") {
+    withTempView("t1", "t2") {
+      spark.sql("select to_date(concat('2000-01-0', id)) as d from range(1, 2)")
+        .createOrReplaceTempView("t1")
+      spark.sql("select concat('2000-01-0', id) as d from range(1, 2)")
+        .createOrReplaceTempView("t2")
+      val result = Date.valueOf("2000-01-01")
+
+      checkAnswer(sql("select t1.d from t1 join t2 on t1.d = t2.d"), Row(result))
+      withSQLConf(SQLConf.LEGACY_CAST_DATETIME_TO_STRING.key -> "true") {
+        checkAnswer(sql("select t1.d from t1 join t2 on t1.d = t2.d"), Row(result))
+      }
+    }
+  }
+
+  test("SPARK-33338: GROUP BY using literal map should not fail") {
+    withTempDir { dir =>
+      sql(s"CREATE TABLE t USING ORC LOCATION '${dir.toURI}' AS SELECT map('k1', 'v1') m, 'k1' k")
+      Seq(
+        "SELECT map('k1', 'v1')[k] FROM t GROUP BY 1",
+        "SELECT map('k1', 'v1')[k] FROM t GROUP BY map('k1', 'v1')[k]",
+        "SELECT map('k1', 'v1')[k] a FROM t GROUP BY a").foreach { statement =>
+        checkAnswer(sql(statement), Row("v1"))
+      }
+    }
+  }
+
+  test("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") {
+    withTempView("df") {
+      Seq("m@ca").toDF("s").createOrReplaceTempView("df")
+
+      val e = intercept[AnalysisException] {
+        sql("SELECT s LIKE 'm%@ca' ESCAPE '%' FROM df").collect()
+      }
+      assert(e.message.contains("the pattern 'm%@ca' is invalid, " +
+        "the escape character is not allowed to precede '@'"))
+
+      checkAnswer(sql("SELECT s LIKE 'm@@ca' ESCAPE '@' FROM df"), Row(true))
+    }
+  }
+
+  test("SPARK-33593: Vector reader got incorrect data with binary partition value") {
+    Seq("false", "true").foreach(value => {
+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> value) {
+        withTable("t1") {
+          sql(
+            """CREATE TABLE t1(name STRING, id BINARY, part BINARY)
+              |USING PARQUET PARTITIONED BY (part)""".stripMargin)
+          sql("INSERT INTO t1 PARTITION(part = 'Spark SQL') VALUES('a', X'537061726B2053514C')")
+          checkAnswer(sql("SELECT name, cast(id as string), cast(part as string) FROM t1"),
+            Row("a", "Spark SQL", "Spark SQL"))
+        }
+      }
+
+      withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> value) {
+        withTable("t2") {
+          sql(
+            """CREATE TABLE t2(name STRING, id BINARY, part BINARY)
+              |USING ORC PARTITIONED BY (part)""".stripMargin)
+          sql("INSERT INTO t2 PARTITION(part = 'Spark SQL') VALUES('a', X'537061726B2053514C')")
+          checkAnswer(sql("SELECT name, cast(id as string), cast(part as string) FROM t2"),
+            Row("a", "Spark SQL", "Spark SQL"))
+        }
+      }
+    })
+  }
+
+  test("SPARK-33591: null as a partition value") {
+    val t = "part_table"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (col1 INT, p1 STRING) USING PARQUET PARTITIONED BY (p1)")
+      sql(s"INSERT INTO TABLE $t PARTITION (p1 = null) SELECT 0")
+      checkAnswer(sql(s"SELECT * FROM $t"), Row(0, null))
+    }
+  }
 }
 
 case class Foo(bar: Option[String])
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 2e5a9e0b4d45d..f43b838c79f24 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -18,12 +18,14 @@
 package org.apache.spark.sql
 
 import java.io.File
-import java.util.{Locale, TimeZone}
+import java.util.Locale
 
+import scala.collection.mutable.ArrayBuffer
 import scala.util.control.NonFatal
 
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile}
@@ -62,7 +64,12 @@ import org.apache.spark.tags.ExtendedSQLTest
  * }}}
  *
  * The format for input files is simple:
- *  1. A list of SQL queries separated by semicolon.
+ *  1. A list of SQL queries separated by semicolons by default. If the semicolon cannot effectively
+ *     separate the SQL queries in the test file(e.g. bracketed comments), please use
+ *     --QUERY-DELIMITER-START and --QUERY-DELIMITER-END. Lines starting with
+ *     --QUERY-DELIMITER-START and --QUERY-DELIMITER-END represent the beginning and end of a query,
+ *     respectively. Code that is not surrounded by lines that begin with --QUERY-DELIMITER-START
+ *     and --QUERY-DELIMITER-END is still separated by semicolons.
  *  2. Lines starting with -- are treated as comments and ignored.
  *  3. Lines starting with --SET are used to specify the configs when running this testing file. You
  *     can set multiple configs in one --SET, using comma to separate them. Or you can use multiple
@@ -115,7 +122,7 @@ import org.apache.spark.tags.ExtendedSQLTest
  * different types of UDFs. See 'udf/udf-inner-join.sql' as an example.
  */
 @ExtendedSQLTest
-class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
+class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper {
 
   import IntegratedUDFTestUtils._
 
@@ -125,12 +132,6 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     // We use a path based on Spark home for 2 reasons:
     //   1. Maven can't get correct resource directory when resources in other jars.
     //   2. We test subclasses in the hive-thriftserver module.
-    val sparkHome = {
-      assert(sys.props.contains("spark.test.home") ||
-        sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-      sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-    }
-
     java.nio.file.Paths.get(sparkHome,
       "sql", "core", "src", "test", "resources", "sql-tests").toFile
   }
@@ -246,9 +247,18 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
 
   /** Run a test case. */
   protected def runTest(testCase: TestCase): Unit = {
+    def splitWithSemicolon(seq: Seq[String]) = {
+      seq.mkString("\n").split("(?<=[^\\\\]);")
+    }
+
+    def splitCommentsAndCodes(input: String) = input.split("\n").partition { line =>
+      val newLine = line.trim
+      newLine.startsWith("--") && !newLine.startsWith("--QUERY-DELIMITER")
+    }
+
     val input = fileToString(new File(testCase.inputFile))
 
-    val (comments, code) = input.split("\n").partition(_.trim.startsWith("--"))
+    val (comments, code) = splitCommentsAndCodes(input)
 
     // If `--IMPORT` found, load code from another test case file, then insert them
     // into the head in this test.
@@ -256,15 +266,43 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     val importedCode = importedTestCaseName.flatMap { testCaseName =>
       listTestCases.find(_.name == testCaseName).map { testCase =>
         val input = fileToString(new File(testCase.inputFile))
-        val (_, code) = input.split("\n").partition(_.trim.startsWith("--"))
+        val (_, code) = splitCommentsAndCodes(input)
         code
       }
     }.flatten
 
+    val allCode = importedCode ++ code
+    val tempQueries = if (allCode.exists(_.trim.startsWith("--QUERY-DELIMITER"))) {
+      // Although the loop is heavy, only used for bracketed comments test.
+      val querys = new ArrayBuffer[String]
+      val otherCodes = new ArrayBuffer[String]
+      var tempStr = ""
+      var start = false
+      for (c <- allCode) {
+        if (c.trim.startsWith("--QUERY-DELIMITER-START")) {
+          start = true
+          querys ++= splitWithSemicolon(otherCodes.toSeq)
+          otherCodes.clear()
+        } else if (c.trim.startsWith("--QUERY-DELIMITER-END")) {
+          start = false
+          querys += s"\n${tempStr.stripSuffix(";")}"
+          tempStr = ""
+        } else if (start) {
+          tempStr += s"\n$c"
+        } else {
+          otherCodes += c
+        }
+      }
+      if (otherCodes.nonEmpty) {
+        querys ++= splitWithSemicolon(otherCodes.toSeq)
+      }
+      querys.toSeq
+    } else {
+      splitWithSemicolon(allCode).toSeq
+    }
+
     // List of SQL queries to run
-    // note: this is not a robust way to split queries using semicolon, but works for now.
-    val queries = (importedCode ++ code).mkString("\n").split("(?<=[^\\\\]);")
-      .map(_.trim).filter(_ != "").toSeq
+    val queries = tempQueries.map(_.trim).filter(_ != "").toSeq
       // Fix misplacement when comment is at the end of the query.
       .map(_.split("\n").filterNot(_.startsWith("--")).mkString("\n")).map(_.trim).filter(_ != "")
 
@@ -317,7 +355,6 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     // Create a local SparkSession to have stronger isolation between different test cases.
     // This does not isolate catalog changes.
     val localSparkSession = spark.newSession()
-    loadTestData(localSparkSession)
 
     testCase match {
       case udfTestCase: UDFTest =>
@@ -469,7 +506,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     val df = session.sql(sql)
     val schema = df.schema.catalogString
     // Get answer, but also get rid of the #1234 expression ids that show up in explain plans
-    val answer = SQLExecution.withNewExecutionId(session, df.queryExecution, Some(sql)) {
+    val answer = SQLExecution.withNewExecutionId(df.queryExecution, Some(sql)) {
       hiveResultString(df.queryExecution.executedPlan).map(replaceNotIncludedMsg)
     }
 
@@ -526,14 +563,20 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
   }
 
   /** Load built-in test tables into the SparkSession. */
-  private def loadTestData(session: SparkSession): Unit = {
+  private def createTestTables(session: SparkSession): Unit = {
     import session.implicits._
 
-    (1 to 100).map(i => (i, i.toString)).toDF("key", "value").createOrReplaceTempView("testdata")
+    (1 to 100).map(i => (i, i.toString)).toDF("key", "value")
+      .repartition(1)
+      .write
+      .format("parquet")
+      .saveAsTable("testdata")
 
     ((Seq(1, 2, 3), Seq(Seq(1, 2, 3))) :: (Seq(2, 3, 4), Seq(Seq(2, 3, 4))) :: Nil)
       .toDF("arraycol", "nestedarraycol")
-      .createOrReplaceTempView("arraydata")
+      .write
+      .format("parquet")
+      .saveAsTable("arraydata")
 
     (Tuple1(Map(1 -> "a1", 2 -> "b1", 3 -> "c1", 4 -> "d1", 5 -> "e1")) ::
       Tuple1(Map(1 -> "a2", 2 -> "b2", 3 -> "c2", 4 -> "d2")) ::
@@ -541,7 +584,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
       Tuple1(Map(1 -> "a4", 2 -> "b4")) ::
       Tuple1(Map(1 -> "a5")) :: Nil)
       .toDF("mapcol")
-      .createOrReplaceTempView("mapdata")
+      .write
+      .format("parquet")
+      .saveAsTable("mapdata")
 
     session
       .read
@@ -549,7 +594,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
       .options(Map("delimiter" -> "\t", "header" -> "false"))
       .schema("a int, b float")
       .load(testFile("test-data/postgresql/agg.data"))
-      .createOrReplaceTempView("aggtest")
+      .write
+      .format("parquet")
+      .saveAsTable("aggtest")
 
     session
       .read
@@ -575,7 +622,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
           |string4 string
         """.stripMargin)
       .load(testFile("test-data/postgresql/onek.data"))
-      .createOrReplaceTempView("onek")
+      .write
+      .format("parquet")
+      .saveAsTable("onek")
 
     session
       .read
@@ -601,25 +650,29 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
           |string4 string
         """.stripMargin)
       .load(testFile("test-data/postgresql/tenk.data"))
-      .createOrReplaceTempView("tenk1")
+      .write
+      .format("parquet")
+      .saveAsTable("tenk1")
   }
 
-  private val originalTimeZone = TimeZone.getDefault
-  private val originalLocale = Locale.getDefault
+  private def removeTestTables(session: SparkSession): Unit = {
+    session.sql("DROP TABLE IF EXISTS testdata")
+    session.sql("DROP TABLE IF EXISTS arraydata")
+    session.sql("DROP TABLE IF EXISTS mapdata")
+    session.sql("DROP TABLE IF EXISTS aggtest")
+    session.sql("DROP TABLE IF EXISTS onek")
+    session.sql("DROP TABLE IF EXISTS tenk1")
+  }
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    // Add Locale setting
-    Locale.setDefault(Locale.US)
+    createTestTables(spark)
     RuleExecutor.resetMetrics()
   }
 
   override def afterAll(): Unit = {
     try {
-      TimeZone.setDefault(originalTimeZone)
-      Locale.setDefault(originalLocale)
+      removeTestTables(spark)
 
       // For debugging dump some statistics about how much time was spent in various optimizer rules
       logWarning(RuleExecutor.dumpTimeSpent())
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
index 7e305e0504729..66c6fbeabbf55 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
@@ -74,72 +74,103 @@ case class ComplexReflectData(
     mapFieldContainsNull: Map[Int, Option[Long]],
     dataField: Data)
 
+case class InvalidInJava(`abstract`: Int)
+
 class ScalaReflectionRelationSuite extends SparkFunSuite with SharedSparkSession {
   import testImplicits._
 
+  // To avoid syntax error thrown by genjavadoc, make this case class non-top level and private.
+  private case class InvalidInJava2(`0`: Int)
+
   test("query case class RDD") {
-    val data = ReflectData("a", 1, 1L, 1.toFloat, 1.toDouble, 1.toShort, 1.toByte, true,
-      new java.math.BigDecimal(1), Date.valueOf("1970-01-01"), new Timestamp(12345), Seq(1, 2, 3),
-      new java.math.BigInteger("1"), scala.math.BigInt(1))
-    Seq(data).toDF().createOrReplaceTempView("reflectData")
-
-    assert(sql("SELECT * FROM reflectData").collect().head ===
-      Row("a", 1, 1L, 1.toFloat, 1.toDouble, 1.toShort, 1.toByte, true,
-        new java.math.BigDecimal(1), Date.valueOf("1970-01-01"),
-        new Timestamp(12345), Seq(1, 2, 3), new java.math.BigDecimal(1),
-        new java.math.BigDecimal(1)))
+    withTempView("reflectData") {
+      val data = ReflectData("a", 1, 1L, 1.toFloat, 1.toDouble, 1.toShort, 1.toByte, true,
+        new java.math.BigDecimal(1), Date.valueOf("1970-01-01"), new Timestamp(12345), Seq(1, 2, 3),
+        new java.math.BigInteger("1"), scala.math.BigInt(1))
+      Seq(data).toDF().createOrReplaceTempView("reflectData")
+
+      assert(sql("SELECT * FROM reflectData").collect().head ===
+        Row("a", 1, 1L, 1.toFloat, 1.toDouble, 1.toShort, 1.toByte, true,
+          new java.math.BigDecimal(1), Date.valueOf("1970-01-01"),
+          new Timestamp(12345), Seq(1, 2, 3), new java.math.BigDecimal(1),
+          new java.math.BigDecimal(1)))
+    }
   }
 
   test("query case class RDD with nulls") {
-    val data = NullReflectData(null, null, null, null, null, null, null)
-    Seq(data).toDF().createOrReplaceTempView("reflectNullData")
+    withTempView("reflectNullData") {
+      val data = NullReflectData(null, null, null, null, null, null, null)
+      Seq(data).toDF().createOrReplaceTempView("reflectNullData")
 
-    assert(sql("SELECT * FROM reflectNullData").collect().head ===
-      Row.fromSeq(Seq.fill(7)(null)))
+      assert(sql("SELECT * FROM reflectNullData").collect().head ===
+        Row.fromSeq(Seq.fill(7)(null)))
+    }
   }
 
   test("query case class RDD with Nones") {
-    val data = OptionalReflectData(None, None, None, None, None, None, None)
-    Seq(data).toDF().createOrReplaceTempView("reflectOptionalData")
+    withTempView("reflectOptionalData") {
+      val data = OptionalReflectData(None, None, None, None, None, None, None)
+      Seq(data).toDF().createOrReplaceTempView("reflectOptionalData")
 
-    assert(sql("SELECT * FROM reflectOptionalData").collect().head ===
-      Row.fromSeq(Seq.fill(7)(null)))
+      assert(sql("SELECT * FROM reflectOptionalData").collect().head ===
+        Row.fromSeq(Seq.fill(7)(null)))
+    }
   }
 
   // Equality is broken for Arrays, so we test that separately.
   test("query binary data") {
-    Seq(ReflectBinary(Array[Byte](1))).toDF().createOrReplaceTempView("reflectBinary")
+    withTempView("reflectBinary") {
+      Seq(ReflectBinary(Array[Byte](1))).toDF().createOrReplaceTempView("reflectBinary")
 
-    val result = sql("SELECT data FROM reflectBinary")
-      .collect().head(0).asInstanceOf[Array[Byte]]
-    assert(result.toSeq === Seq[Byte](1))
+      val result = sql("SELECT data FROM reflectBinary")
+        .collect().head(0).asInstanceOf[Array[Byte]]
+      assert(result.toSeq === Seq[Byte](1))
+    }
   }
 
   test("query complex data") {
-    val data = ComplexReflectData(
-      Seq(1, 2, 3),
-      Seq(Some(1), Some(2), None),
-      Map(1 -> 10L, 2 -> 20L),
-      Map(1 -> Some(10L), 2 -> Some(20L), 3 -> None),
-      Data(
-        Seq(10, 20, 30),
-        Seq(Some(10), Some(20), None),
-        Map(10 -> 100L, 20 -> 200L),
-        Map(10 -> Some(100L), 20 -> Some(200L), 30 -> None),
-        Nested(None, "abc")))
-
-    Seq(data).toDF().createOrReplaceTempView("reflectComplexData")
-    assert(sql("SELECT * FROM reflectComplexData").collect().head ===
-      Row(
+    withTempView("reflectComplexData") {
+      val data = ComplexReflectData(
         Seq(1, 2, 3),
-        Seq(1, 2, null),
+        Seq(Some(1), Some(2), None),
         Map(1 -> 10L, 2 -> 20L),
-        Map(1 -> 10L, 2 -> 20L, 3 -> null),
-        Row(
+        Map(1 -> Some(10L), 2 -> Some(20L), 3 -> None),
+        Data(
           Seq(10, 20, 30),
-          Seq(10, 20, null),
+          Seq(Some(10), Some(20), None),
           Map(10 -> 100L, 20 -> 200L),
-          Map(10 -> 100L, 20 -> 200L, 30 -> null),
-          Row(null, "abc"))))
+          Map(10 -> Some(100L), 20 -> Some(200L), 30 -> None),
+          Nested(None, "abc")))
+
+      Seq(data).toDF().createOrReplaceTempView("reflectComplexData")
+      assert(sql("SELECT * FROM reflectComplexData").collect().head ===
+        Row(
+          Seq(1, 2, 3),
+          Seq(1, 2, null),
+          Map(1 -> 10L, 2 -> 20L),
+          Map(1 -> 10L, 2 -> 20L, 3 -> null),
+          Row(
+            Seq(10, 20, 30),
+            Seq(10, 20, null),
+            Map(10 -> 100L, 20 -> 200L),
+            Map(10 -> 100L, 20 -> 200L, 30 -> null),
+            Row(null, "abc"))))
+    }
+  }
+
+  test("better error message when use java reserved keyword as field name") {
+    val e = intercept[UnsupportedOperationException] {
+      Seq(InvalidInJava(1)).toDS()
+    }
+    assert(e.getMessage.contains(
+      "`abstract` is not a valid identifier of Java and cannot be used as field name"))
+  }
+
+  test("better error message when use invalid java identifier as field name") {
+    val e1 = intercept[UnsupportedOperationException] {
+      Seq(InvalidInJava2(1)).toDS()
+    }
+    assert(e1.getMessage.contains(
+      "`0` is not a valid identifier of Java and cannot be used as field name"))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
index 31957a99e15af..003f5bc835d5f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
@@ -135,7 +135,7 @@ class SessionStateSuite extends SparkFunSuite {
   test("fork new session and inherit listener manager") {
     class CommandCollector extends QueryExecutionListener {
       val commands: ArrayBuffer[String] = ArrayBuffer.empty[String]
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable) : Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, ex: Exception) : Unit = {}
       override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
         commands += funcName
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
index b3b94f8be0d17..6253626856bcd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
@@ -173,7 +173,7 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils {
       val createTable = "CREATE TABLE `t1` (`a` STRUCT<`b`: STRING>)"
       sql(s"$createTable USING json")
       val shownDDL = getShowDDL("SHOW CREATE TABLE t1")
-      assert(shownDDL == createTable)
+      assert(shownDDL == "CREATE TABLE `default`.`t1` (`a` STRUCT<`b`: STRING>)")
 
       checkCreateTable("t1")
     }
@@ -188,18 +188,26 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils {
     if (result.length > 1) result(0) + result(1) else result.head
   }
 
-  protected def checkCreateTable(table: String): Unit = {
-    checkCreateTableOrView(TableIdentifier(table, Some("default")), "TABLE")
+  protected def checkCreateTable(table: String, serde: Boolean = false): Unit = {
+    checkCreateTableOrView(TableIdentifier(table, Some("default")), "TABLE", serde)
   }
 
-  protected def checkCreateView(table: String): Unit = {
-    checkCreateTableOrView(TableIdentifier(table, Some("default")), "VIEW")
+  protected def checkCreateView(table: String, serde: Boolean = false): Unit = {
+    checkCreateTableOrView(TableIdentifier(table, Some("default")), "VIEW", serde)
   }
 
-  private def checkCreateTableOrView(table: TableIdentifier, checkType: String): Unit = {
+  protected def checkCreateTableOrView(
+      table: TableIdentifier,
+      checkType: String,
+      serde: Boolean): Unit = {
     val db = table.database.getOrElse("default")
     val expected = spark.sharedState.externalCatalog.getTable(db, table.table)
-    val shownDDL = sql(s"SHOW CREATE TABLE ${table.quotedString}").head().getString(0)
+    val shownDDL = if (serde) {
+      sql(s"SHOW CREATE TABLE ${table.quotedString} AS SERDE").head().getString(0)
+    } else {
+      sql(s"SHOW CREATE TABLE ${table.quotedString}").head().getString(0)
+    }
+
     sql(s"DROP $checkType ${table.quotedString}")
 
     try {
@@ -212,29 +220,6 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils {
   }
 
   protected def checkCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = {
-    def normalize(table: CatalogTable): CatalogTable = {
-      val nondeterministicProps = Set(
-        "CreateTime",
-        "transient_lastDdlTime",
-        "grantTime",
-        "lastUpdateTime",
-        "last_modified_by",
-        "last_modified_time",
-        "Owner:",
-        // The following are hive specific schema parameters which we do not need to match exactly.
-        "totalNumberFiles",
-        "maxFileSize",
-        "minFileSize"
-      )
-
-      table.copy(
-        createTime = 0L,
-        lastAccessTime = 0L,
-        properties = table.properties.filterKeys(!nondeterministicProps.contains(_)),
-        stats = None,
-        ignoredProperties = Map.empty
-      )
-    }
-    assert(normalize(actual) == normalize(expected))
+    assert(CatalogTable.normalize(actual) == CatalogTable.normalize(expected))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
index 10b17571d2aaa..159d2c02188be 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
@@ -19,9 +19,11 @@ package org.apache.spark.sql
 
 import org.scalatest.BeforeAndAfterEach
 
-import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
+import org.apache.spark.{SparkConf, SparkContext, SparkException, SparkFunSuite}
+import org.apache.spark.internal.config.EXECUTOR_ALLOW_SPARK_CONTEXT
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.StaticSQLConf._
 
 /**
  * Test cases for the builder pattern of [[SparkSession]].
@@ -152,4 +154,114 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
       session.sparkContext.hadoopConfiguration.unset(mySpecialKey)
     }
   }
+
+  test("SPARK-31234: RESET command will not change static sql configs and " +
+    "spark context conf values in SessionState") {
+    val session = SparkSession.builder()
+      .master("local")
+      .config(GLOBAL_TEMP_DATABASE.key, value = "globalTempDB-SPARK-31234")
+      .config("spark.app.name", "test-app-SPARK-31234")
+      .getOrCreate()
+
+    assert(session.sessionState.conf.getConfString("spark.app.name") === "test-app-SPARK-31234")
+    assert(session.sessionState.conf.getConf(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31234")
+    session.sql("RESET")
+    assert(session.sessionState.conf.getConfString("spark.app.name") === "test-app-SPARK-31234")
+    assert(session.sessionState.conf.getConf(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31234")
+  }
+
+  test("SPARK-31354: SparkContext only register one SparkSession ApplicationEnd listener") {
+    val conf = new SparkConf()
+      .setMaster("local")
+      .setAppName("test-app-SPARK-31354-1")
+    val context = new SparkContext(conf)
+    SparkSession
+      .builder()
+      .sparkContext(context)
+      .master("local")
+      .getOrCreate()
+    val postFirstCreation = context.listenerBus.listeners.size()
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+
+    SparkSession
+      .builder()
+      .sparkContext(context)
+      .master("local")
+      .getOrCreate()
+    val postSecondCreation = context.listenerBus.listeners.size()
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+    assert(postFirstCreation == postSecondCreation)
+  }
+
+  test("SPARK-31532: should not propagate static sql configs to the existing" +
+    " active/default SparkSession") {
+    val session = SparkSession.builder()
+      .master("local")
+      .config(GLOBAL_TEMP_DATABASE.key, value = "globalTempDB-SPARK-31532")
+      .config("spark.app.name", "test-app-SPARK-31532")
+      .getOrCreate()
+    // do not propagate static sql configs to the existing active session
+    val session1 = SparkSession
+      .builder()
+      .config(GLOBAL_TEMP_DATABASE.key, "globalTempDB-SPARK-31532-1")
+      .getOrCreate()
+    assert(session.conf.get(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31532")
+    assert(session1.conf.get(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31532")
+
+    // do not propagate static sql configs to the existing default session
+    SparkSession.clearActiveSession()
+    val session2 = SparkSession
+      .builder()
+      .config(WAREHOUSE_PATH.key, "SPARK-31532-db")
+      .config(GLOBAL_TEMP_DATABASE.key, value = "globalTempDB-SPARK-31532-2")
+      .getOrCreate()
+
+    assert(!session.conf.get(WAREHOUSE_PATH).contains("SPARK-31532-db"))
+    assert(session.conf.get(WAREHOUSE_PATH) === session2.conf.get(WAREHOUSE_PATH))
+    assert(session2.conf.get(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31532")
+  }
+
+  test("SPARK-31532: propagate static sql configs if no existing SparkSession") {
+    val conf = new SparkConf()
+      .setMaster("local")
+      .setAppName("test-app-SPARK-31532-2")
+      .set(GLOBAL_TEMP_DATABASE.key, "globaltempdb-spark-31532")
+      .set(WAREHOUSE_PATH.key, "SPARK-31532-db")
+    SparkContext.getOrCreate(conf)
+
+    // propagate static sql configs if no existing session
+    val session = SparkSession
+      .builder()
+      .config(GLOBAL_TEMP_DATABASE.key, "globalTempDB-SPARK-31532-2")
+      .config(WAREHOUSE_PATH.key, "SPARK-31532-db-2")
+      .getOrCreate()
+    assert(session.conf.get("spark.app.name") === "test-app-SPARK-31532-2")
+    assert(session.conf.get(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31532-2")
+    assert(session.conf.get(WAREHOUSE_PATH) === "SPARK-31532-db-2")
+  }
+
+  test("SPARK-32160: Disallow to create SparkSession in executors if the config is set") {
+    val session = SparkSession.builder().master("local-cluster[3, 1, 1024]").getOrCreate()
+
+    val error = intercept[SparkException] {
+      session.range(1).foreach { v =>
+        SparkSession.builder.master("local")
+          .config(EXECUTOR_ALLOW_SPARK_CONTEXT.key, false).getOrCreate()
+        ()
+      }
+    }.getMessage()
+
+    assert(error.contains("SparkSession should only be created and accessed on the driver."))
+  }
+
+  test("SPARK-32160: Allow to create SparkSession in executors") {
+    val session = SparkSession.builder().master("local-cluster[3, 1, 1024]").getOrCreate()
+
+    session.range(1).foreach { v =>
+      SparkSession.builder.master("local").getOrCreate().stop()
+      ()
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
index 99ea95089d71c..e5e8bc6917799 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
@@ -16,17 +16,24 @@
  */
 package org.apache.spark.sql
 
-import java.util.Locale
+import java.util.{Locale, UUID}
 
-import org.apache.spark.{SparkFunSuite, TaskContext}
+import scala.concurrent.Future
+
+import org.apache.spark.{MapOutputStatistics, SparkFunSuite, TaskContext}
+import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow, TableIdentifier}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParserInterface}
-import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, UnresolvedHint}
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Statistics, UnresolvedHint}
+import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, BroadcastExchangeLike, ShuffleExchangeExec, ShuffleExchangeLike}
 import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.COLUMN_BATCH_SIZE
@@ -145,33 +152,83 @@ class SparkSessionExtensionSuite extends SparkFunSuite {
     }
   }
 
-  test("inject columnar") {
+  test("inject adaptive query prep rule") {
     val extensions = create { extensions =>
+      // inject rule that will run during AQE query stage preparation and will add custom tags
+      // to the plan
+      extensions.injectQueryStagePrepRule(session => MyQueryStagePrepRule())
+      // inject rule that will run during AQE query stage optimization and will verify that the
+      // custom tags were written in the preparation phase
       extensions.injectColumnar(session =>
-        MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))
+        MyColumarRule(MyNewQueryStageRule(), MyNewQueryStageRule()))
     }
     withSession(extensions) { session =>
-      // The ApplyColumnarRulesAndInsertTransitions rule is not applied when enable AQE
-      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
+      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, true)
+      assert(session.sessionState.queryStagePrepRules.contains(MyQueryStagePrepRule()))
       assert(session.sessionState.columnarRules.contains(
-        MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
+        MyColumarRule(MyNewQueryStageRule(), MyNewQueryStageRule())))
       import session.sqlContext.implicits._
-      // repartitioning avoids having the add operation pushed up into the LocalTableScan
       val data = Seq((100L), (200L), (300L)).toDF("vals").repartition(1)
       val df = data.selectExpr("vals + 1")
-      // Verify that both pre and post processing of the plan worked.
-      val found = df.queryExecution.executedPlan.collect {
-        case rep: ReplacedRowToColumnarExec => 1
-        case proj: ColumnarProjectExec => 10
-        case c2r: ColumnarToRowExec => 100
-      }.sum
-      assert(found == 111)
+      df.collect()
+    }
+  }
+
+  test("inject columnar AQE on") {
+    testInjectColumnar(true)
+  }
+
+  test("inject columnar AQE off") {
+    testInjectColumnar(false)
+  }
+
+  private def testInjectColumnar(enableAQE: Boolean): Unit = {
+    def collectPlanSteps(plan: SparkPlan): Seq[Int] = plan match {
+      case a: AdaptiveSparkPlanExec =>
+        assert(a.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true"))
+        collectPlanSteps(a.executedPlan)
+      case _ => plan.collect {
+        case _: ReplacedRowToColumnarExec => 1
+        case _: ColumnarProjectExec => 10
+        case _: ColumnarToRowExec => 100
+        case s: QueryStageExec => collectPlanSteps(s.plan).sum
+        case _: MyShuffleExchangeExec => 1000
+        case _: MyBroadcastExchangeExec => 10000
+      }
+    }
+
+    val extensions = create { extensions =>
+      extensions.injectColumnar(session =>
+        MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))
+    }
+    withSession(extensions) { session =>
+      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, enableAQE)
+      assert(session.sessionState.columnarRules.contains(
+        MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
+      import session.sqlContext.implicits._
+      // perform a join to inject a broadcast exchange
+      val left = Seq((1, 50L), (2, 100L), (3, 150L)).toDF("l1", "l2")
+      val right = Seq((1, 50L), (2, 100L), (3, 150L)).toDF("r1", "r2")
+      val data = left.join(right, $"l1" === $"r1")
+        // repartitioning avoids having the add operation pushed up into the LocalTableScan
+        .repartition(1)
+      val df = data.selectExpr("l2 + r2")
+      // execute the plan so that the final adaptive plan is available when AQE is on
+      df.collect()
+      val found = collectPlanSteps(df.queryExecution.executedPlan).sum
+      // 1 MyBroadcastExchangeExec
+      // 1 MyShuffleExchangeExec
+      // 1 ColumnarToRowExec
+      // 2 ColumnarProjectExec
+      // 1 ReplacedRowToColumnarExec
+      // so 11121 is expected.
+      assert(found == 11121)
 
       // Verify that we get back the expected, wrong, result
       val result = df.collect()
-      assert(result(0).getLong(0) == 102L) // Check that broken columnar Add was used.
-      assert(result(1).getLong(0) == 202L)
-      assert(result(2).getLong(0) == 302L)
+      assert(result(0).getLong(0) == 101L) // Check that broken columnar Add was used.
+      assert(result(1).getLong(0) == 201L)
+      assert(result(2).getLong(0) == 301L)
     }
   }
 
@@ -327,6 +384,9 @@ case class MyParser(spark: SparkSession, delegate: ParserInterface) extends Pars
 
   override def parseDataType(sqlText: String): DataType =
     delegate.parseDataType(sqlText)
+
+  override def parseRawDataType(sqlText: String): DataType =
+    delegate.parseRawDataType(sqlText)
 }
 
 object MyExtensions {
@@ -342,6 +402,7 @@ object MyExtensions {
       """
        note
       """,
+      "",
       "3.0.0",
       """
        deprecated
@@ -667,6 +728,16 @@ case class PreRuleReplaceAddWithBrokenVersion() extends Rule[SparkPlan] {
   def replaceWithColumnarPlan(plan: SparkPlan): SparkPlan =
     try {
       plan match {
+        case e: ShuffleExchangeExec =>
+          // note that this is not actually columnar but demonstrates that exchanges can
+          // be replaced.
+          val replaced = e.withNewChildren(e.children.map(replaceWithColumnarPlan))
+          MyShuffleExchangeExec(replaced.asInstanceOf[ShuffleExchangeExec])
+        case e: BroadcastExchangeExec =>
+          // note that this is not actually columnar but demonstrates that exchanges can
+          // be replaced.
+          val replaced = e.withNewChildren(e.children.map(replaceWithColumnarPlan))
+          MyBroadcastExchangeExec(replaced.asInstanceOf[BroadcastExchangeExec])
         case plan: ProjectExec =>
           new ColumnarProjectExec(plan.projectList.map((exp) =>
             replaceWithColumnarExpression(exp).asInstanceOf[NamedExpression]),
@@ -685,6 +756,41 @@ case class PreRuleReplaceAddWithBrokenVersion() extends Rule[SparkPlan] {
   override def apply(plan: SparkPlan): SparkPlan = replaceWithColumnarPlan(plan)
 }
 
+/**
+ * Custom Exchange used in tests to demonstrate that shuffles can be replaced regardless of
+ * whether AQE is enabled.
+ */
+case class MyShuffleExchangeExec(delegate: ShuffleExchangeExec) extends ShuffleExchangeLike {
+  override def numMappers: Int = delegate.numMappers
+  override def numPartitions: Int = delegate.numPartitions
+  override def canChangeNumPartitions: Boolean = delegate.canChangeNumPartitions
+  override def mapOutputStatisticsFuture: Future[MapOutputStatistics] =
+    delegate.mapOutputStatisticsFuture
+  override def getShuffleRDD(partitionSpecs: Array[ShufflePartitionSpec]): RDD[_] =
+    delegate.getShuffleRDD(partitionSpecs)
+  override def runtimeStatistics: Statistics = delegate.runtimeStatistics
+  override def child: SparkPlan = delegate.child
+  override protected def doExecute(): RDD[InternalRow] = delegate.execute()
+  override def outputPartitioning: Partitioning = delegate.outputPartitioning
+}
+
+/**
+ * Custom Exchange used in tests to demonstrate that broadcasts can be replaced regardless of
+ * whether AQE is enabled.
+ */
+case class MyBroadcastExchangeExec(delegate: BroadcastExchangeExec) extends BroadcastExchangeLike {
+  override def runId: UUID = delegate.runId
+  override def relationFuture: java.util.concurrent.Future[Broadcast[Any]] =
+    delegate.relationFuture
+  override def completionFuture: Future[Broadcast[Any]] = delegate.completionFuture
+  override def runtimeStatistics: Statistics = delegate.runtimeStatistics
+  override def child: SparkPlan = delegate.child
+  override protected def doPrepare(): Unit = delegate.prepare()
+  override protected def doExecute(): RDD[InternalRow] = delegate.execute()
+  override def doExecuteBroadcast[T](): Broadcast[T] = delegate.executeBroadcast()
+  override def outputPartitioning: Partitioning = delegate.outputPartitioning
+}
+
 class ReplacedRowToColumnarExec(override val child: SparkPlan)
   extends RowToColumnarExec(child) {
 
@@ -727,6 +833,31 @@ class MyExtensions extends (SparkSessionExtensions => Unit) {
   }
 }
 
+object QueryPrepRuleHelper {
+  val myPrepTag: TreeNodeTag[String] = TreeNodeTag[String]("myPrepTag")
+  val myPrepTagValue: String = "myPrepTagValue"
+}
+
+// this rule will run during AQE query preparation and will write custom tags to each node
+case class MyQueryStagePrepRule() extends Rule[SparkPlan] {
+  override def apply(plan: SparkPlan): SparkPlan = plan.transformDown {
+    case plan =>
+      plan.setTagValue(QueryPrepRuleHelper.myPrepTag, QueryPrepRuleHelper.myPrepTagValue)
+      plan
+  }
+}
+
+// this rule will run during AQE query stage optimization and will verify custom tags were
+// already written during query preparation phase
+case class MyNewQueryStageRule() extends Rule[SparkPlan] {
+  override def apply(plan: SparkPlan): SparkPlan = plan.transformDown {
+    case plan if !plan.isInstanceOf[AdaptiveSparkPlanExec] =>
+      assert(plan.getTagValue(QueryPrepRuleHelper.myPrepTag).get ==
+          QueryPrepRuleHelper.myPrepTagValue)
+      plan
+  }
+}
+
 case class MyRule2(spark: SparkSession) extends Rule[LogicalPlan] {
   override def apply(plan: LogicalPlan): LogicalPlan = plan
 }
@@ -752,6 +883,7 @@ object MyExtensions2 {
       """
        note
       """,
+      "",
       "3.0.0",
       """
        deprecated
@@ -784,6 +916,7 @@ object MyExtensions2Duplicate {
       """
        note
       """,
+      "",
       "3.0.0",
       """
        deprecated
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
index e9ceab6724659..708b98e8fe15a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
@@ -481,7 +481,8 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
       }
     }
 
-    DateTimeTestUtils.outstandingTimezones.foreach { timeZone =>
+    DateTimeTestUtils.outstandingZoneIds.foreach { zid =>
+      val timeZone = TimeZone.getTimeZone(zid)
       checkTimestampStats(DateType, DateTimeUtils.TimeZoneUTC, timeZone) { stats =>
         assert(stats.min.get.asInstanceOf[Int] == TimeUnit.SECONDS.toDays(start))
         assert(stats.max.get.asInstanceOf[Int] == TimeUnit.SECONDS.toDays(end - 1))
@@ -650,4 +651,21 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
       }
     }
   }
+
+  Seq(true, false).foreach { caseSensitive =>
+    test(s"SPARK-30903: Fail fast on duplicate columns when analyze columns " +
+      s"- caseSensitive=$caseSensitive") {
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+        val table = "test_table"
+        withTable(table) {
+          sql(s"CREATE TABLE $table (value string, name string) USING PARQUET")
+          val dupCol = if (caseSensitive) "value" else "VaLuE"
+          val errorMsg = intercept[AnalysisException] {
+            sql(s"ANALYZE TABLE $table COMPUTE STATISTICS FOR COLUMNS value, name, $dupCol")
+          }.getMessage
+          assert(errorMsg.contains("Found duplicate column(s)"))
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala
index fde8ddf491bd1..b6ea26ab95549 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql
 import java.{lang => jl}
 import java.io.File
 import java.sql.{Date, Timestamp}
-import java.util.concurrent.TimeUnit
 
 import scala.collection.mutable
 import scala.util.Random
@@ -30,6 +29,7 @@ import org.apache.spark.sql.catalyst.catalog.{CatalogColumnStat, CatalogStatisti
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, Histogram, HistogramBin, HistogramSerializer, LogicalPlan}
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.sql.test.SQLTestUtils
@@ -51,10 +51,10 @@ abstract class StatisticsCollectionTestBase extends QueryTest with SQLTestUtils
   private val d2 = Date.valueOf(d2Str)
   private val t1Str = "2016-05-08 00:00:01.000000"
   private val t1Internal = date(2016, 5, 8, 0, 0, 1)
-  private val t1 = new Timestamp(TimeUnit.MICROSECONDS.toMillis(t1Internal))
+  private val t1 = new Timestamp(DateTimeUtils.toMillis(t1Internal))
   private val t2Str = "2016-05-09 00:00:02.000000"
   private val t2Internal = date(2016, 5, 9, 0, 0, 2)
-  private val t2 = new Timestamp(TimeUnit.MICROSECONDS.toMillis(t2Internal))
+  private val t2 = new Timestamp(DateTimeUtils.toMillis(t2Internal))
 
   /**
    * Define a very simple 3 row table used for testing column serialization.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
index ff8f94c68c5ee..141f1279a6ee5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -22,7 +22,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
 import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan, Sort}
 import org.apache.spark.sql.execution.{ColumnarToRowExec, ExecSubqueryExpression, FileSourceScanExec, InputAdapter, ReusedSubqueryExec, ScalarSubquery, SubqueryExec, WholeStageCodegenExec}
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution}
 import org.apache.spark.sql.execution.datasources.FileScanRDD
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -153,29 +153,31 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("uncorrelated scalar subquery on a DataFrame generated query") {
-    val df = Seq((1, "one"), (2, "two"), (3, "three")).toDF("key", "value")
-    df.createOrReplaceTempView("subqueryData")
+    withTempView("subqueryData") {
+      val df = Seq((1, "one"), (2, "two"), (3, "three")).toDF("key", "value")
+      df.createOrReplaceTempView("subqueryData")
 
-    checkAnswer(
-      sql("select (select key from subqueryData where key > 2 order by key limit 1) + 1"),
-      Array(Row(4))
-    )
+      checkAnswer(
+        sql("select (select key from subqueryData where key > 2 order by key limit 1) + 1"),
+        Array(Row(4))
+      )
 
-    checkAnswer(
-      sql("select -(select max(key) from subqueryData)"),
-      Array(Row(-3))
-    )
+      checkAnswer(
+        sql("select -(select max(key) from subqueryData)"),
+        Array(Row(-3))
+      )
 
-    checkAnswer(
-      sql("select (select value from subqueryData limit 0)"),
-      Array(Row(null))
-    )
+      checkAnswer(
+        sql("select (select value from subqueryData limit 0)"),
+        Array(Row(null))
+      )
 
-    checkAnswer(
-      sql("select (select min(value) from subqueryData" +
-        " where key = (select max(key) from subqueryData) - 1)"),
-      Array(Row("two"))
-    )
+      checkAnswer(
+        sql("select (select min(value) from subqueryData" +
+          " where key = (select max(key) from subqueryData) - 1)"),
+        Array(Row("two"))
+      )
+    }
   }
 
   test("SPARK-15677: Queries against local relations with scalar subquery in Select list") {
@@ -1357,11 +1359,9 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     }
   }
 
-  test("SPARK-27279: Reuse Subquery") {
+  test("SPARK-27279: Reuse Subquery", DisableAdaptiveExecution("reuse is dynamic in AQE")) {
     Seq(true, false).foreach { reuse =>
-      withSQLConf(SQLConf.SUBQUERY_REUSE_ENABLED.key -> reuse.toString,
-        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-        // when enable AQE, the reusedExchange is inserted when executed.
+      withSQLConf(SQLConf.SUBQUERY_REUSE_ENABLED.key -> reuse.toString) {
         val df = sql(
           """
             |SELECT (SELECT avg(key) FROM testData) + (SELECT avg(key) FROM testData)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
index aacb625d7921f..f4c6fb9b75ff8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql
 
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.util.resourceToString
 import org.apache.spark.sql.internal.SQLConf
 
@@ -46,11 +48,13 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
     "q81", "q82", "q83", "q84", "q85", "q86", "q87", "q88", "q89", "q90",
     "q91", "q92", "q93", "q94", "q95", "q96", "q97", "q98", "q99")
 
+  val sqlConfgs: Seq[(String, String)] = Nil
+
   tpcdsQueries.foreach { name =>
     val queryString = resourceToString(s"tpcds/$name.sql",
       classLoader = Thread.currentThread().getContextClassLoader)
     test(name) {
-      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
+      withSQLConf(sqlConfgs: _*) {
         // check the plans can be properly generated
         val plan = sql(queryString).queryExecution.executedPlan
         checkGeneratedCode(plan)
@@ -69,7 +73,7 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
     val queryString = resourceToString(s"tpcds-v2.7.0/$name.sql",
       classLoader = Thread.currentThread().getContextClassLoader)
     test(s"$name-v2.7") {
-      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
+      withSQLConf(sqlConfgs: _*) {
         // check the plans can be properly generated
         val plan = sql(queryString).queryExecution.executedPlan
         checkGeneratedCode(plan)
@@ -98,3 +102,28 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
     }
   }
 }
+
+class TPCDSQueryWithStatsSuite extends TPCDSQuerySuite {
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    for (tableName <- tableNames) {
+      // To simulate plan generation on actual TPCDS data, injects data stats here
+      spark.sessionState.catalog.alterTableStats(
+        TableIdentifier(tableName), Some(TPCDSTableStats.sf100TableStats(tableName)))
+    }
+  }
+
+  // Sets configurations for enabling the optimization rules that
+  // exploit data statistics.
+  override val sqlConfgs = Seq(
+    SQLConf.CBO_ENABLED.key -> "true",
+    SQLConf.PLAN_STATS_ENABLED.key -> "true",
+    SQLConf.JOIN_REORDER_ENABLED.key -> "true"
+  )
+}
+
+class TPCDSQueryANSISuite extends TPCDSQuerySuite {
+  override protected def sparkConf: SparkConf =
+    super.sparkConf.set(SQLConf.ANSI_ENABLED, true)
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala
new file mode 100644
index 0000000000000..f39b4b8b56c2e
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala
@@ -0,0 +1,503 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.catalyst.catalog.{CatalogColumnStat, CatalogStatistics}
+
+object TPCDSTableStats {
+
+  // These data statistics are extracted from generated TPCDS data with SF=100
+
+  // scalastyle:off line.size.limit
+  val sf100TableStats = Map(
+    "customer" -> CatalogStatistics(500000000L, Some(2000000L), Map(
+      "c_birth_country" -> CatalogColumnStat(Some(196L), None, None, Some(69626), Some(9), Some(20), None, CatalogColumnStat.VERSION),
+      "c_current_cdemo_sk" -> CatalogColumnStat(Some(1184426L), Some("1"), Some("1920798"), Some(69943), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_customer_id" -> CatalogColumnStat(Some(2000000L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "c_birth_day" -> CatalogColumnStat(Some(32L), Some("1"), Some("31"), Some(70166), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_last_name" -> CatalogColumnStat(Some(5250L), None, None, Some(70098), Some(7), Some(13), None, CatalogColumnStat.VERSION),
+      "c_login" -> CatalogColumnStat(Some(0L), None, None, Some(2000000), Some(20), Some(20), None, CatalogColumnStat.VERSION),
+      "c_first_shipto_date_sk" -> CatalogColumnStat(Some(3755L), Some("2449028"), Some("2452678"), Some(70080), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_first_name" -> CatalogColumnStat(Some(5146L), None, None, Some(69769), Some(6), Some(11), None, CatalogColumnStat.VERSION),
+      "c_birth_month" -> CatalogColumnStat(Some(12L), Some("1"), Some("12"), Some(69896), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_email_address" -> CatalogColumnStat(Some(1929800L), None, None, Some(70200), Some(28), Some(47), None, CatalogColumnStat.VERSION),
+      "c_current_addr_sk" -> CatalogColumnStat(Some(824389L), Some("1"), Some("1000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_salutation" -> CatalogColumnStat(Some(6L), None, None, Some(69840), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_current_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(69657), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_birth_year" -> CatalogColumnStat(Some(67L), Some("1924"), Some("1992"), Some(69986), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_last_review_date" -> CatalogColumnStat(Some(357L), None, None, Some(70102), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "c_preferred_cust_flag" -> CatalogColumnStat(Some(2L), None, None, Some(69778), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "c_first_sales_date_sk" -> CatalogColumnStat(Some(3758L), Some("2448998"), Some("2452648"), Some(69950), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "store_sales" -> CatalogStatistics(42623559552L, Some(287997024L), Map(
+      "ss_ext_sales_price" -> CatalogColumnStat(Some(712602L), Some("0.00"), Some("19878.00"), Some(12955462), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_coupon_amt" -> CatalogColumnStat(Some(925847L), Some("0.00"), Some("19225.00"), Some(12958053), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_store_sk" -> CatalogColumnStat(Some(199L), Some("1"), Some("400"), Some(12950651), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(12953654), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_ext_discount_amt" -> CatalogColumnStat(Some(925847L), Some("0.00"), Some("19225.00"), Some(12958053), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(12957139), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_ext_tax" -> CatalogColumnStat(Some(135620L), Some("0.00"), Some("1762.38"), Some(12957705), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_sold_time_sk" -> CatalogColumnStat(Some(47827L), Some("28800"), Some("75599"), Some(12953300), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_promo_sk" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(12954088), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_net_paid_inc_tax" -> CatalogColumnStat(Some(1404501L), Some("0.00"), Some("21344.38"), Some(12958941), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_wholesale_cost" -> CatalogColumnStat(Some(9503L), Some("1.00"), Some("100.00"), Some(12958327), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_cdemo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(12955252), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_sold_date_sk" -> CatalogColumnStat(Some(1781L), Some("2450816"), Some("2452642"), Some(12955025), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_list_price" -> CatalogColumnStat(Some(19079L), Some("1.00"), Some("200.00"), Some(12952108), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_ext_list_price" -> CatalogColumnStat(Some(752016L), Some("1.00"), Some("20000.00"), Some(12955040), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_net_profit" -> CatalogColumnStat(Some(1388760L), Some("-10000.00"), Some("9889.00"), Some(12955156), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_ticket_number" -> CatalogColumnStat(Some(24596280L), Some("1"), Some("24000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_net_paid" -> CatalogColumnStat(Some(1058635L), Some("0.00"), Some("19878.00"), Some(12954554), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_ext_wholesale_cost" -> CatalogColumnStat(Some(382964L), Some("1.00"), Some("10000.00"), Some(12960060), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(12952082), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(12956686), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_sales_price" -> CatalogColumnStat(Some(19261L), Some("0.00"), Some("200.00"), Some(12958587), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "web_sales" -> CatalogStatistics(14688252348L, Some(72001237L), Map(
+      "ws_sold_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(17931), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_wholesale_cost" -> CatalogColumnStat(Some(382964L), Some("1.00"), Some("10000.00"), Some(17814), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_bill_cdemo_sk" -> CatalogColumnStat(Some(1707373L), Some("1"), Some("1920800"), Some(17833), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid_inc_ship" -> CatalogColumnStat(Some(1629659L), Some("0.00"), Some("43468.92"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_sales_price" -> CatalogColumnStat(Some(29143L), Some("0.00"), Some("300.00"), Some(18005), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(17823), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ship_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(17833), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_order_number" -> CatalogColumnStat(Some(6073146L), Some("1"), Some("6000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_sold_date_sk" -> CatalogColumnStat(Some(1781L), Some("2450816"), Some("2452642"), Some(17922), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_web_site_sk" -> CatalogColumnStat(Some(25L), Some("1"), Some("24"), Some(18030), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_bill_customer_sk" -> CatalogColumnStat(Some(1800267L), Some("1"), Some("2000000"), Some(17882), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_coupon_amt" -> CatalogColumnStat(Some(863374L), Some("0.00"), Some("27591.16"), Some(18027), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(17931), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid_inc_tax" -> CatalogColumnStat(Some(1695042L), Some("0.00"), Some("32492.90"), Some(18102), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(18014), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_bill_addr_sk" -> CatalogColumnStat(Some(940457L), Some("1"), Some("1000000"), Some(17801), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid" -> CatalogColumnStat(Some(1247989L), Some("0.00"), Some("29810.00"), Some(17968), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ext_discount_amt" -> CatalogColumnStat(Some(934889L), Some("0.00"), Some("29982.00"), Some(17890), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_tax" -> CatalogColumnStat(Some(168876L), Some("0.00"), Some("2682.90"), Some(17800), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_wholesale_cost" -> CatalogColumnStat(Some(9503L), Some("1.00"), Some("100.00"), Some(17850), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_customer_sk" -> CatalogColumnStat(Some(1811873L), Some("1"), Some("2000000"), Some(17886), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid_inc_ship_tax" -> CatalogColumnStat(Some(2186741L), Some("0.00"), Some("44479.52"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_web_page_sk" -> CatalogColumnStat(Some(2108L), Some("1"), Some("2040"), Some(17920), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_sales_price" -> CatalogColumnStat(Some(908147L), Some("0.00"), Some("29810.00"), Some(17843), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ext_ship_cost" -> CatalogColumnStat(Some(482711L), Some("0.00"), Some("14927.00"), Some(17923), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_bill_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(18011), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ship_date_sk" -> CatalogColumnStat(Some(1898L), Some("2450817"), Some("2452762"), Some(17883), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_list_price" -> CatalogColumnStat(Some(1071558L), Some("1.02"), Some("29997.00"), Some(18001), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_cdemo_sk" -> CatalogColumnStat(Some(1755797L), Some("1"), Some("1920800"), Some(17903), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(17812), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_list_price" -> CatalogColumnStat(Some(29447L), Some("1.00"), Some("300.00"), Some(17824), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_promo_sk" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(18116), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_profit" -> CatalogColumnStat(Some(1482074L), Some("-9997.00"), Some("19840.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "date_dim" -> CatalogStatistics(19284936L, Some(73049L), Map(
+      "d_dom" -> CatalogColumnStat(Some(32L), Some("1"), Some("31"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_quarter" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_fy_week_seq" -> CatalogColumnStat(Some(10010L), Some("1"), Some("10436"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_holiday" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_current_day" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_week_seq" -> CatalogColumnStat(Some(10010L), Some("1"), Some("10436"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_month" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_moy" -> CatalogColumnStat(Some(12L), Some("1"), Some("12"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_date" -> CatalogColumnStat(Some(73049L), None, None, Some(0), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "d_same_day_lq" -> CatalogColumnStat(Some(73049L), Some("2414930"), Some("2487978"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_weekend" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_dow" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_year" -> CatalogColumnStat(Some(202L), Some("1900"), Some("2100"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_week" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_quarter_name" -> CatalogColumnStat(Some(774L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "d_month_seq" -> CatalogColumnStat(Some(2431L), Some("0"), Some("2400"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_fy_year" -> CatalogColumnStat(Some(202L), Some("1900"), Some("2100"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_following_holiday" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_same_day_ly" -> CatalogColumnStat(Some(73049L), Some("2414657"), Some("2487705"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_day_name" -> CatalogColumnStat(Some(7L), None, None, Some(0), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "d_qoy" -> CatalogColumnStat(Some(4L), Some("1"), Some("4"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_date_sk" -> CatalogColumnStat(Some(73049L), Some("2415022"), Some("2488070"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_fy_quarter_seq" -> CatalogColumnStat(Some(787L), Some("1"), Some("801"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_year" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_date_id" -> CatalogColumnStat(Some(73049L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "d_quarter_seq" -> CatalogColumnStat(Some(787L), Some("1"), Some("801"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_last_dom" -> CatalogColumnStat(Some(2386L), Some("2415020"), Some("2488372"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_first_dom" -> CatalogColumnStat(Some(2329L), Some("2415021"), Some("2488070"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "household_demographics" -> CatalogStatistics(316800L, Some(7200L), Map(
+      "hd_buy_potential" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(8), Some(10), None, CatalogColumnStat.VERSION),
+      "hd_income_band_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "hd_demo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "hd_vehicle_count" -> CatalogColumnStat(Some(6L), Some("-1"), Some("4"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "hd_dep_count" -> CatalogColumnStat(Some(10L), Some("0"), Some("9"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "call_center" -> CatalogStatistics(16020L, Some(30L), Map(
+      "cc_rec_start_date" -> CatalogColumnStat(Some(4L), Some("1998-01-01"), Some("2002-01-01"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_mkt_desc" -> CatalogColumnStat(Some(21L), None, None, Some(0), Some(59), Some(93), None, CatalogColumnStat.VERSION),
+      "cc_hours" -> CatalogColumnStat(Some(3L), None, None, Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cc_street_name" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(8), Some(16), None, CatalogColumnStat.VERSION),
+      "cc_manager" -> CatalogColumnStat(Some(22L), None, None, Some(0), Some(13), Some(17), None, CatalogColumnStat.VERSION),
+      "cc_city" -> CatalogColumnStat(Some(12L), None, None, Some(0), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "cc_class" -> CatalogColumnStat(Some(3L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "cc_country" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "cc_open_date_sk" -> CatalogColumnStat(Some(15L), Some("2450794"), Some("2451146"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_market_manager" -> CatalogColumnStat(Some(25L), None, None, Some(0), Some(13), Some(17), None, CatalogColumnStat.VERSION),
+      "cc_street_number" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "cc_call_center_sk" -> CatalogColumnStat(Some(30L), Some("1"), Some("30"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_name" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(14), Some(19), None, CatalogColumnStat.VERSION),
+      "cc_suite_number" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "cc_mkt_id" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_gmt_offset" -> CatalogColumnStat(Some(2L), Some("-6.00"), Some("-5.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cc_company_name" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(4), Some(5), None, CatalogColumnStat.VERSION),
+      "cc_division" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_closed_date_sk" -> CatalogColumnStat(Some(0L), None, None, Some(30), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_state" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "cc_street_type" -> CatalogColumnStat(Some(9L), None, None, Some(0), Some(5), Some(9), None, CatalogColumnStat.VERSION),
+      "cc_call_center_id" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "cc_sq_ft" -> CatalogColumnStat(Some(22L), Some("1670015"), Some("31896816"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_mkt_class" -> CatalogColumnStat(Some(26L), None, None, Some(0), Some(35), Some(50), None, CatalogColumnStat.VERSION),
+      "cc_tax_percentage" -> CatalogColumnStat(Some(10L), Some("0.00"), Some("0.12"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cc_division_name" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "cc_zip" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "cc_rec_end_date" -> CatalogColumnStat(Some(3L), Some("2000-01-01"), Some("2001-12-31"), Some(15), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_employees" -> CatalogColumnStat(Some(22L), Some("2935"), Some("69020"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_county" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(15), Some(17), None, CatalogColumnStat.VERSION),
+      "cc_company" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "catalog_page" -> CatalogStatistics(3774000L, Some(20400L), Map(
+      "cp_catalog_page_id" -> CatalogColumnStat(Some(20400L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "cp_department" -> CatalogColumnStat(Some(1L), None, None, Some(205), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "cp_description" -> CatalogColumnStat(Some(19304L), None, None, Some(190), Some(75), Some(99), None, CatalogColumnStat.VERSION),
+      "cp_start_date_sk" -> CatalogColumnStat(Some(88L), Some("2450815"), Some("2453005"), Some(196), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_end_date_sk" -> CatalogColumnStat(Some(101L), Some("2450844"), Some("2453186"), Some(206), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_catalog_page_number" -> CatalogColumnStat(Some(186L), Some("1"), Some("188"), Some(208), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_type" -> CatalogColumnStat(Some(3L), None, None, Some(197), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "cp_catalog_page_sk" -> CatalogColumnStat(Some(18915L), Some("1"), Some("20400"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_catalog_number" -> CatalogColumnStat(Some(110L), Some("1"), Some("109"), Some(197), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "catalog_returns" -> CatalogStatistics(2189464848L, Some(14404374L), Map(
+      "cr_return_amount" -> CatalogColumnStat(Some(563788L), Some("0.00"), Some("28778.31"), Some(288408), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_returning_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(288369), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_refunded_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(288053), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_catalog_page_sk" -> CatalogColumnStat(Some(11224L), Some("1"), Some("17108"), Some(288041), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_reason_sk" -> CatalogColumnStat(Some(55L), Some("1"), Some("55"), Some(287890), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_returning_cdemo_sk" -> CatalogColumnStat(Some(1839372L), Some("1"), Some("1920800"), Some(288128), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_fee" -> CatalogColumnStat(Some(9571L), Some("0.50"), Some("100.00"), Some(288038), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_refunded_cash" -> CatalogColumnStat(Some(592855L), Some("0.00"), Some("24544.84"), Some(287638), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_refunded_cdemo_sk" -> CatalogColumnStat(Some(1842661L), Some("1"), Some("1920800"), Some(287556), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_order_number" -> CatalogColumnStat(Some(8816832L), Some("2"), Some("16000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(287768), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_refunded_customer_sk" -> CatalogColumnStat(Some(1894309L), Some("1"), Some("2000000"), Some(287207), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_amt_inc_tax" -> CatalogColumnStat(Some(855338L), Some("0.00"), Some("29353.87"), Some(288246), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_returning_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(287581), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_returned_date_sk" -> CatalogColumnStat(Some(2027L), Some("2450821"), Some("2452921"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_ship_cost" -> CatalogColumnStat(Some(347391L), Some("0.00"), Some("14130.96"), Some(287952), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_returning_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(288203), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(287844), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_store_credit" -> CatalogColumnStat(Some(415989L), Some("0.00"), Some("22167.49"), Some(288118), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_tax" -> CatalogColumnStat(Some(101557L), Some("0.00"), Some("2390.75"), Some(288599), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_refunded_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(287752), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_call_center_sk" -> CatalogColumnStat(Some(31L), Some("1"), Some("30"), Some(288179), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_returned_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_net_loss" -> CatalogColumnStat(Some(550908L), Some("0.50"), Some("15781.83"), Some(287954), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_reversed_charge" -> CatalogColumnStat(Some(410432L), Some("0.00"), Some("23801.24"), Some(288476), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(288581), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "catalog_sales" -> CatalogStatistics(29375401260L, Some(143997065L), Map(
+      "cs_ship_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(720450), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_bill_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(718886), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_net_paid_inc_ship_tax" -> CatalogColumnStat(Some(2570239L), Some("0.00"), Some("45460.80"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ext_sales_price" -> CatalogColumnStat(Some(977146L), Some("0.00"), Some("29808.00"), Some(719228), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_sold_date_sk" -> CatalogColumnStat(Some(1772L), Some("2450815"), Some("2452654"), Some(719581), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_tax" -> CatalogColumnStat(Some(189612L), Some("0.00"), Some("2619.36"), Some(719627), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_sold_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(720917), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_net_paid_inc_ship" -> CatalogColumnStat(Some(1870941L), Some("0.00"), Some("43725.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(719624), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_list_price" -> CatalogColumnStat(Some(1101138L), Some("1.00"), Some("29997.00"), Some(719642), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(720147), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_discount_amt" -> CatalogColumnStat(Some(965247L), Some("0.00"), Some("29765.00"), Some(719820), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_net_profit" -> CatalogColumnStat(Some(1640663L), Some("-10000.00"), Some("19840.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_bill_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(719849), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_ship_cost" -> CatalogColumnStat(Some(508914L), Some("0.00"), Some("14896.00"), Some(719848), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ship_cdemo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(720292), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_wholesale_cost" -> CatalogColumnStat(Some(9503L), Some("1.00"), Some("100.00"), Some(721114), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_date_sk" -> CatalogColumnStat(Some(1887L), Some("2450817"), Some("2452744"), Some(719625), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_wholesale_cost" -> CatalogColumnStat(Some(382964L), Some("1.00"), Some("10000.00"), Some(719924), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_bill_cdemo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(720208), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_sales_price" -> CatalogColumnStat(Some(29282L), Some("0.00"), Some("300.00"), Some(719781), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_net_paid" -> CatalogColumnStat(Some(1330351L), Some("0.00"), Some("29760.00"), Some(719706), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_promo_sk" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(720194), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_call_center_sk" -> CatalogColumnStat(Some(31L), Some("1"), Some("30"), Some(719767), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_catalog_page_sk" -> CatalogColumnStat(Some(11224L), Some("1"), Some("17108"), Some(719180), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_bill_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(719473), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_list_price" -> CatalogColumnStat(Some(29447L), Some("1.00"), Some("300.00"), Some(720328), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(720582), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_coupon_amt" -> CatalogColumnStat(Some(982009L), Some("0.00"), Some("28422.94"), Some(719631), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(718680), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_order_number" -> CatalogColumnStat(Some(15603123L), Some("1"), Some("16000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_net_paid_inc_tax" -> CatalogColumnStat(Some(1807594L), Some("0.00"), Some("31745.52"), Some(719354), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(720146), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "customer_address" -> CatalogStatistics(245000000L, Some(1000000L), Map(
+      "ca_country" -> CatalogColumnStat(Some(1L), None, None, Some(30097), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "ca_address_id" -> CatalogColumnStat(Some(1000000L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "ca_address_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ca_county" -> CatalogColumnStat(Some(1957L), None, None, Some(30028), Some(14), Some(28), None, CatalogColumnStat.VERSION),
+      "ca_gmt_offset" -> CatalogColumnStat(Some(6L), Some("-10.00"), Some("-5.00"), Some(30131), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ca_street_type" -> CatalogColumnStat(Some(20L), None, None, Some(30124), Some(5), Some(9), None, CatalogColumnStat.VERSION),
+      "ca_street_name" -> CatalogColumnStat(Some(8292L), None, None, Some(30178), Some(9), Some(21), None, CatalogColumnStat.VERSION),
+      "ca_city" -> CatalogColumnStat(Some(977L), None, None, Some(30183), Some(9), Some(20), None, CatalogColumnStat.VERSION),
+      "ca_location_type" -> CatalogColumnStat(Some(3L), None, None, Some(30172), Some(9), Some(13), None, CatalogColumnStat.VERSION),
+      "ca_suite_number" -> CatalogColumnStat(Some(76L), None, None, Some(30047), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "ca_zip" -> CatalogColumnStat(Some(8311L), None, None, Some(30370), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "ca_state" -> CatalogColumnStat(Some(54L), None, None, Some(30124), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "ca_street_number" -> CatalogColumnStat(Some(1034L), None, None, Some(30226), Some(3), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "customer_demographics" -> CatalogStatistics(182476000L, Some(1920800L), Map(
+      "cd_dep_employed_count" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_dep_count" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_education_status" -> CatalogColumnStat(Some(7L), None, None, Some(0), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "cd_marital_status" -> CatalogColumnStat(Some(5L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "cd_gender" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "cd_purchase_estimate" -> CatalogColumnStat(Some(20L), Some("500"), Some("10000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_demo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_credit_rating" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(7), Some(9), None, CatalogColumnStat.VERSION),
+      "cd_dep_college_count" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "income_band" -> CatalogStatistics(400L, Some(20L), Map(
+      "ib_income_band_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ib_lower_bound" -> CatalogColumnStat(Some(19L), Some("0"), Some("190001"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ib_upper_bound" -> CatalogColumnStat(Some(20L), Some("10000"), Some("200000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "inventory" -> CatalogStatistics(9583920000L, Some(399330000L), Map(
+      "inv_date_sk" -> CatalogColumnStat(Some(267L), Some("2450815"), Some("2452635"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "inv_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "inv_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "inv_quantity_on_hand" -> CatalogColumnStat(Some(1026L), Some("0"), Some("1000"), Some(19969395), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "item" -> CatalogStatistics(94248000L, Some(204000L), Map(
+      "i_container" -> CatalogColumnStat(Some(1L), None, None, Some(510), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "i_formulation" -> CatalogColumnStat(Some(158873L), None, None, Some(530), Some(20), Some(20), None, CatalogColumnStat.VERSION),
+      "i_brand_id" -> CatalogColumnStat(Some(862L), Some("1001001"), Some("10016017"), Some(516), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_manager_id" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(506), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_rec_end_date" -> CatalogColumnStat(Some(3L), None, None, Some(102000), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "i_class" -> CatalogColumnStat(Some(100L), None, None, Some(499), Some(8), Some(15), None, CatalogColumnStat.VERSION),
+      "i_wholesale_cost" -> CatalogColumnStat(Some(6297L), Some("0.02"), Some("88.91"), Some(489), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "i_item_id" -> CatalogColumnStat(Some(104042L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "i_manufact_id" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(498), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_item_desc" -> CatalogColumnStat(Some(158754L), None, None, Some(508), Some(101), Some(200), None, CatalogColumnStat.VERSION),
+      "i_rec_start_date" -> CatalogColumnStat(Some(4L), None, None, Some(522), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "i_color" -> CatalogColumnStat(Some(88L), None, None, Some(524), Some(6), Some(10), None, CatalogColumnStat.VERSION),
+      "i_product_name" -> CatalogColumnStat(Some(203486L), None, None, Some(514), Some(23), Some(30), None, CatalogColumnStat.VERSION),
+      "i_category" -> CatalogColumnStat(Some(10L), None, None, Some(482), Some(6), Some(11), None, CatalogColumnStat.VERSION),
+      "i_class_id" -> CatalogColumnStat(Some(16L), Some("1"), Some("16"), Some(491), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_item_sk" -> CatalogColumnStat(Some(204000L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_manufact" -> CatalogColumnStat(Some(997L), None, None, Some(544), Some(12), Some(15), None, CatalogColumnStat.VERSION),
+      "i_brand" -> CatalogColumnStat(Some(671L), None, None, Some(510), Some(17), Some(22), None, CatalogColumnStat.VERSION),
+      "i_current_price" -> CatalogColumnStat(Some(8736L), Some("0.09"), Some("99.99"), Some(518), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "i_category_id" -> CatalogColumnStat(Some(10L), Some("1"), Some("10"), Some(515), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_size" -> CatalogColumnStat(Some(7L), None, None, Some(515), Some(5), Some(11), None, CatalogColumnStat.VERSION),
+      "i_units" -> CatalogColumnStat(Some(21L), None, None, Some(503), Some(5), Some(7), None, CatalogColumnStat.VERSION)
+    )),
+    "promotion" -> CatalogStatistics(268000L, Some(1000L), Map(
+      "p_promo_name" -> CatalogColumnStat(Some(10L), None, None, Some(18), Some(4), Some(5), None, CatalogColumnStat.VERSION),
+      "p_end_date_sk" -> CatalogColumnStat(Some(564L), Some("2450116"), Some("2450967"), Some(12), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_channel_radio" -> CatalogColumnStat(Some(1L), None, None, Some(13), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_demo" -> CatalogColumnStat(Some(1L), None, None, Some(16), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_item_sk" -> CatalogColumnStat(Some(986L), Some("280"), Some("203966"), Some(14), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_channel_dmail" -> CatalogColumnStat(Some(2L), None, None, Some(13), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_discount_active" -> CatalogColumnStat(Some(1L), None, None, Some(19), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_promo_id" -> CatalogColumnStat(Some(1000L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "p_channel_tv" -> CatalogColumnStat(Some(1L), None, None, Some(14), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_cost" -> CatalogColumnStat(Some(1L), Some("1000.00"), Some("1000.00"), Some(14), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "p_channel_email" -> CatalogColumnStat(Some(1L), None, None, Some(13), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_event" -> CatalogColumnStat(Some(1L), None, None, Some(14), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_catalog" -> CatalogColumnStat(Some(1L), None, None, Some(14), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_promo_sk" -> CatalogColumnStat(Some(1000L), Some("1"), Some("1000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_purpose" -> CatalogColumnStat(Some(1L), None, None, Some(13), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "p_start_date_sk" -> CatalogColumnStat(Some(577L), Some("2450100"), Some("2450915"), Some(17), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_channel_press" -> CatalogColumnStat(Some(1L), None, None, Some(15), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_details" -> CatalogColumnStat(Some(990L), None, None, Some(10), Some(40), Some(60), None, CatalogColumnStat.VERSION),
+      "p_response_target" -> CatalogColumnStat(Some(1L), Some("1"), Some("1"), Some(19), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "reason" -> CatalogStatistics(3630L, Some(55L), Map(
+      "r_reason_sk" -> CatalogColumnStat(Some(55L), Some("1"), Some("55"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "r_reason_id" -> CatalogColumnStat(Some(55L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "r_reason_desc" -> CatalogColumnStat(Some(55L), None, None, Some(0), Some(14), Some(43), None, CatalogColumnStat.VERSION)
+    )),
+    "ship_mode" -> CatalogStatistics(2420L, Some(20L), Map(
+      "sm_carrier" -> CatalogColumnStat(Some(19L), None, None, Some(0), Some(7), Some(14), None, CatalogColumnStat.VERSION),
+      "sm_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "sm_code" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(5), Some(7), None, CatalogColumnStat.VERSION),
+      "sm_type" -> CatalogColumnStat(Some(5L), None, None, Some(0), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "sm_contract" -> CatalogColumnStat(Some(18L), None, None, Some(0), Some(13), Some(20), None, CatalogColumnStat.VERSION),
+      "sm_ship_mode_id" -> CatalogColumnStat(Some(20L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION)
+    )),
+    "store" -> CatalogStatistics(207432L, Some(402L), Map(
+      "s_rec_end_date" -> CatalogColumnStat(Some(3L), None, None, Some(201), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "s_state" -> CatalogColumnStat(Some(9L), None, None, Some(2), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "s_street_type" -> CatalogColumnStat(Some(20L), None, None, Some(6), Some(5), Some(9), None, CatalogColumnStat.VERSION),
+      "s_floor_space" -> CatalogColumnStat(Some(300L), Some("5004767"), Some("9997773"), Some(3), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_division_name" -> CatalogColumnStat(Some(1L), None, None, Some(5), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "s_market_desc" -> CatalogColumnStat(Some(318L), None, None, Some(5), Some(59), Some(100), None, CatalogColumnStat.VERSION),
+      "s_street_name" -> CatalogColumnStat(Some(255L), None, None, Some(6), Some(9), Some(16), None, CatalogColumnStat.VERSION),
+      "s_county" -> CatalogColumnStat(Some(9L), None, None, Some(4), Some(15), Some(17), None, CatalogColumnStat.VERSION),
+      "s_rec_start_date" -> CatalogColumnStat(Some(4L), None, None, Some(4), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "s_company_id" -> CatalogColumnStat(Some(1L), Some("1"), Some("1"), Some(4), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_market_manager" -> CatalogColumnStat(Some(275L), None, None, Some(7), Some(13), Some(20), None, CatalogColumnStat.VERSION),
+      "s_number_employees" -> CatalogColumnStat(Some(94L), Some("200"), Some("300"), Some(5), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_manager" -> CatalogColumnStat(Some(300L), None, None, Some(5), Some(13), Some(20), None, CatalogColumnStat.VERSION),
+      "s_country" -> CatalogColumnStat(Some(1L), None, None, Some(4), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "s_store_name" -> CatalogColumnStat(Some(10L), None, None, Some(2), Some(4), Some(5), None, CatalogColumnStat.VERSION),
+      "s_division_id" -> CatalogColumnStat(Some(1L), Some("1"), Some("1"), Some(2), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_street_number" -> CatalogColumnStat(Some(274L), None, None, Some(5), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "s_company_name" -> CatalogColumnStat(Some(1L), None, None, Some(3), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "s_gmt_offset" -> CatalogColumnStat(Some(2L), Some("-6.00"), Some("-5.00"), Some(4), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "s_store_sk" -> CatalogColumnStat(Some(399L), Some("1"), Some("402"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_city" -> CatalogColumnStat(Some(18L), None, None, Some(5), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "s_zip" -> CatalogColumnStat(Some(99L), None, None, Some(6), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "s_market_id" -> CatalogColumnStat(Some(10L), Some("1"), Some("10"), Some(6), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_hours" -> CatalogColumnStat(Some(3L), None, None, Some(4), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "s_suite_number" -> CatalogColumnStat(Some(75L), None, None, Some(2), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "s_closed_date_sk" -> CatalogColumnStat(Some(70L), Some("2450823"), Some("2451313"), Some(296), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_store_id" -> CatalogColumnStat(Some(210L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "s_geography_class" -> CatalogColumnStat(Some(1L), None, None, Some(3), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "s_tax_precentage" -> CatalogColumnStat(Some(12L), Some("0.00"), Some("0.11"), Some(5), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "store_returns" -> CatalogStatistics(4837573440L, Some(28795080L), Map(
+      "sr_item_sk" -> CatalogColumnStat(Some(197284L), Some("1"), Some("204000"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_reversed_charge" -> CatalogColumnStat(Some(423824L), Some("0.00"), Some("16099.52"), Some(1009035), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_amt_inc_tax" -> CatalogColumnStat(Some(807709L), Some("0.00"), Some("20002.89"), Some(1006919), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_store_sk" -> CatalogColumnStat(Some(199L), Some("1"), Some("400"), Some(1007164), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_quantity" -> CatalogColumnStat(Some(103L), Some("1"), Some("100"), Some(1007948), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_fee" -> CatalogColumnStat(Some(9571L), Some("0.50"), Some("100.00"), Some(1008291), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_refunded_cash" -> CatalogColumnStat(Some(559685L), Some("0.00"), Some("17556.95"), Some(1008003), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_time_sk" -> CatalogColumnStat(Some(31932L), Some("28799"), Some("61199"), Some(1009330), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_addr_sk" -> CatalogColumnStat(Some(925738L), Some("1"), Some("1000000"), Some(1008253), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_amt" -> CatalogColumnStat(Some(492928L), Some("0.00"), Some("18973.20"), Some(1007419), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_returned_date_sk" -> CatalogColumnStat(Some(2010L), Some("2450820"), Some("2452822"), Some(1007464), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_tax" -> CatalogColumnStat(Some(86618L), Some("0.00"), Some("1611.71"), Some(1008618), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_ship_cost" -> CatalogColumnStat(Some(290195L), Some("0.00"), Some("9578.25"), Some(1007846), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_reason_sk" -> CatalogColumnStat(Some(57L), Some("1"), Some("55"), Some(1008299), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_net_loss" -> CatalogColumnStat(Some(487073L), Some("0.50"), Some("10447.72"), Some(1007153), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_cdemo_sk" -> CatalogColumnStat(Some(1807132L), Some("1"), Some("1920800"), Some(1006835), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_hdemo_sk" -> CatalogColumnStat(Some(6609L), Some("1"), Some("7200"), Some(1008547), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_store_credit" -> CatalogColumnStat(Some(410133L), Some("0.00"), Some("15642.11"), Some(1007102), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_customer_sk" -> CatalogColumnStat(Some(1860981L), Some("1"), Some("2000000"), Some(1008429), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_ticket_number" -> CatalogColumnStat(Some(15853105L), Some("1"), Some("23999996"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "time_dim" -> CatalogStatistics(10886400L, Some(86400L), Map(
+      "t_sub_shift" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(7), Some(9), None, CatalogColumnStat.VERSION),
+      "t_time" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_second" -> CatalogColumnStat(Some(60L), Some("0"), Some("59"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_hour" -> CatalogColumnStat(Some(25L), Some("0"), Some("23"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_am_pm" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "t_minute" -> CatalogColumnStat(Some(60L), Some("0"), Some("59"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_shift" -> CatalogColumnStat(Some(3L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "t_time_id" -> CatalogColumnStat(Some(80197L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "t_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_meal_time" -> CatalogColumnStat(Some(3L), None, None, Some(50400), Some(7), Some(9), None, CatalogColumnStat.VERSION)
+    )),
+    "warehouse" -> CatalogStatistics(3855L, Some(15L), Map(
+      "w_state" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "w_street_name" -> CatalogColumnStat(Some(14L), None, None, Some(1), Some(10), Some(14), None, CatalogColumnStat.VERSION),
+      "w_warehouse_name" -> CatalogColumnStat(Some(14L), None, None, Some(1), Some(17), Some(20), None, CatalogColumnStat.VERSION),
+      "w_county" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(14), Some(16), None, CatalogColumnStat.VERSION),
+      "w_street_number" -> CatalogColumnStat(Some(14L), None, None, Some(1), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "w_warehouse_sq_ft" -> CatalogColumnStat(Some(14L), Some("73065"), Some("977787"), Some(1), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "w_city" -> CatalogColumnStat(Some(11L), None, None, Some(0), Some(8), Some(13), None, CatalogColumnStat.VERSION),
+      "w_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "w_suite_number" -> CatalogColumnStat(Some(13L), None, None, Some(1), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "w_zip" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "w_gmt_offset" -> CatalogColumnStat(Some(2L), Some("-6.00"), Some("-5.00"), Some(1), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "w_street_type" -> CatalogColumnStat(Some(10L), None, None, Some(1), Some(5), Some(7), None, CatalogColumnStat.VERSION),
+      "w_warehouse_id" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "w_country" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(13), Some(13), None, CatalogColumnStat.VERSION)
+    )),
+    "web_page" -> CatalogStatistics(281520L, Some(2040L), Map(
+      "wp_type" -> CatalogColumnStat(Some(7L), None, None, Some(19), Some(7), Some(9), None, CatalogColumnStat.VERSION),
+      "wp_autogen_flag" -> CatalogColumnStat(Some(2L), None, None, Some(25), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "wp_url" -> CatalogColumnStat(Some(1L), None, None, Some(25), Some(18), Some(18), None, CatalogColumnStat.VERSION),
+      "wp_image_count" -> CatalogColumnStat(Some(7L), Some("1"), Some("7"), Some(20), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_max_ad_count" -> CatalogColumnStat(Some(5L), Some("0"), Some("4"), Some(21), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_customer_sk" -> CatalogColumnStat(Some(486L), Some("711"), Some("1996257"), Some(1471), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_rec_end_date" -> CatalogColumnStat(Some(3L), Some("1999-09-03"), Some("2001-09-02"), Some(1020), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_creation_date_sk" -> CatalogColumnStat(Some(121L), Some("2450672"), Some("2450815"), Some(20), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_link_count" -> CatalogColumnStat(Some(25L), Some("2"), Some("25"), Some(16), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_web_page_id" -> CatalogColumnStat(Some(1004L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "wp_web_page_sk" -> CatalogColumnStat(Some(2040L), Some("1"), Some("2040"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_rec_start_date" -> CatalogColumnStat(Some(4L), Some("1997-09-03"), Some("2001-09-03"), Some(21), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_access_date_sk" -> CatalogColumnStat(Some(100L), Some("2452548"), Some("2452648"), Some(19), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_char_count" -> CatalogColumnStat(Some(1493L), Some("303"), Some("8523"), Some(25), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "web_returns" -> CatalogStatistics(1439534000L, Some(7197670L), Map(
+      "wr_returning_addr_sk" -> CatalogColumnStat(Some(925738L), Some("1"), Some("1000000"), Some(323850), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_reversed_charge" -> CatalogColumnStat(Some(346909L), Some("0.00"), Some("22972.36"), Some(323810), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returned_date_sk" -> CatalogColumnStat(Some(2189L), Some("2450820"), Some("2453002"), Some(324185), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_order_number" -> CatalogColumnStat(Some(4098425L), Some("1"), Some("5999999"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_fee" -> CatalogColumnStat(Some(9571L), Some("0.50"), Some("100.00"), Some(324065), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_account_credit" -> CatalogColumnStat(Some(334119L), Some("0.00"), Some("23028.27"), Some(324422), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_customer_sk" -> CatalogColumnStat(Some(1808850L), Some("1"), Some("2000000"), Some(324191), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returning_customer_sk" -> CatalogColumnStat(Some(1813094L), Some("1"), Some("2000000"), Some(324024), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_tax" -> CatalogColumnStat(Some(88469L), Some("0.00"), Some("2551.16"), Some(323621), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_web_page_sk" -> CatalogColumnStat(Some(1994L), Some("1"), Some("2040"), Some(324900), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_item_sk" -> CatalogColumnStat(Some(197284L), Some("1"), Some("204000"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_reason_sk" -> CatalogColumnStat(Some(57L), Some("1"), Some("55"), Some(323666), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_amt_inc_tax" -> CatalogColumnStat(Some(683544L), Some("0.00"), Some("29493.38"), Some(323171), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_net_loss" -> CatalogColumnStat(Some(494707L), Some("0.50"), Some("15068.96"), Some(324438), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_cdemo_sk" -> CatalogColumnStat(Some(1755065L), Some("1"), Some("1920800"), Some(323863), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returned_time_sk" -> CatalogColumnStat(Some(84232L), Some("0"), Some("86399"), Some(323677), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_amt" -> CatalogColumnStat(Some(453015L), Some("0.00"), Some("28346.31"), Some(323473), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returning_cdemo_sk" -> CatalogColumnStat(Some(1755065L), Some("1"), Some("1920800"), Some(323899), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returning_hdemo_sk" -> CatalogColumnStat(Some(6609L), Some("1"), Some("7200"), Some(323999), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_cash" -> CatalogColumnStat(Some(484316L), Some("0.00"), Some("26466.56"), Some(324693), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_hdemo_sk" -> CatalogColumnStat(Some(6609L), Some("1"), Some("7200"), Some(324230), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_quantity" -> CatalogColumnStat(Some(103L), Some("1"), Some("100"), Some(323764), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_ship_cost" -> CatalogColumnStat(Some(302038L), Some("0.00"), Some("13602.60"), Some(323341), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_addr_sk" -> CatalogColumnStat(Some(925738L), Some("1"), Some("1000000"), Some(324482), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "web_site" -> CatalogStatistics(11760L, Some(24L), Map(
+      "web_rec_end_date" -> CatalogColumnStat(Some(3L), Some("1999-08-16"), Some("2001-08-15"), Some(12), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_market_manager" -> CatalogColumnStat(Some(21L), None, None, Some(0), Some(13), Some(15), None, CatalogColumnStat.VERSION),
+      "web_country" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "web_street_name" -> CatalogColumnStat(Some(24L), None, None, Some(0), Some(10), Some(14), None, CatalogColumnStat.VERSION),
+      "web_city" -> CatalogColumnStat(Some(11L), None, None, Some(0), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "web_mkt_id" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_close_date_sk" -> CatalogColumnStat(Some(8L), Some("2443328"), Some("2447131"), Some(4), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_street_number" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "web_gmt_offset" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "web_rec_start_date" -> CatalogColumnStat(Some(4L), Some("1997-08-16"), Some("2001-08-16"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_mkt_class" -> CatalogColumnStat(Some(17L), None, None, Some(0), Some(32), Some(47), None, CatalogColumnStat.VERSION),
+      "web_county" -> CatalogColumnStat(Some(9L), None, None, Some(0), Some(14), Some(17), None, CatalogColumnStat.VERSION),
+      "web_class" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "web_site_sk" -> CatalogColumnStat(Some(24L), Some("1"), Some("24"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_manager" -> CatalogColumnStat(Some(19L), None, None, Some(0), Some(13), Some(16), None, CatalogColumnStat.VERSION),
+      "web_suite_number" -> CatalogColumnStat(Some(20L), None, None, Some(0), Some(9), Some(9), None, CatalogColumnStat.VERSION),
+      "web_site_id" -> CatalogColumnStat(Some(12L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "web_company_name" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "web_state" -> CatalogColumnStat(Some(9L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "web_mkt_desc" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(66), Some(92), None, CatalogColumnStat.VERSION),
+      "web_name" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "web_company_id" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_street_type" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(4), Some(9), None, CatalogColumnStat.VERSION),
+      "web_open_date_sk" -> CatalogColumnStat(Some(12L), Some("2450628"), Some("2450807"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_tax_percentage" -> CatalogColumnStat(Some(8L), Some("0.00"), Some("0.12"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "web_zip" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION)
+    ))
+  )
+  // scalastyle:on line.size.limit
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala b/sql/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala
index fd6bc9662bfad..d2a6358ee822b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala
@@ -28,7 +28,7 @@ class TestQueryExecutionListener extends QueryExecutionListener {
     OnSuccessCall.isOnSuccessCalled.set(true)
   }
 
-  override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = { }
+  override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = { }
 }
 
 /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index cc3995516dcc2..91e9f1d9afb80 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -19,8 +19,9 @@ package org.apache.spark.sql
 
 import java.math.BigDecimal
 
+import org.apache.spark.SparkException
 import org.apache.spark.sql.api.java._
-import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.encoders.OuterScopes
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.execution.{QueryExecution, SimpleMode}
 import org.apache.spark.sql.execution.columnar.InMemoryRelation
@@ -33,7 +34,6 @@ import org.apache.spark.sql.test.SQLTestData._
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.QueryExecutionListener
 
-
 private case class FunctionResult(f1: String, f2: String)
 
 class UDFSuite extends QueryTest with SharedSparkSession {
@@ -134,10 +134,12 @@ class UDFSuite extends QueryTest with SharedSparkSession {
     assert(df1.logicalPlan.asInstanceOf[Project].projectList.forall(!_.deterministic))
     assert(df1.head().getDouble(0) >= 0.0)
 
-    val bar = udf(() => Math.random(), DataTypes.DoubleType).asNondeterministic()
-    val df2 = testData.select(bar())
-    assert(df2.logicalPlan.asInstanceOf[Project].projectList.forall(!_.deterministic))
-    assert(df2.head().getDouble(0) >= 0.0)
+    withSQLConf(SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF.key -> "true") {
+      val bar = udf(() => Math.random(), DataTypes.DoubleType).asNondeterministic()
+      val df2 = testData.select(bar())
+      assert(df2.logicalPlan.asInstanceOf[Project].projectList.forall(!_.deterministic))
+      assert(df2.head().getDouble(0) >= 0.0)
+    }
 
     val javaUdf = udf(new UDF0[Double] {
       override def call(): Double = Math.random()
@@ -339,7 +341,7 @@ class UDFSuite extends QueryTest with SharedSparkSession {
       withTempPath { path =>
         var numTotalCachedHit = 0
         val listener = new QueryExecutionListener {
-          override def onFailure(f: String, qe: QueryExecution, e: Throwable): Unit = {}
+          override def onFailure(f: String, qe: QueryExecution, e: Exception): Unit = {}
 
           override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
             qe.withCachedData match {
@@ -441,16 +443,23 @@ class UDFSuite extends QueryTest with SharedSparkSession {
   }
 
   test("SPARK-25044 Verify null input handling for primitive types - with udf(Any, DataType)") {
-    val f = udf((x: Int) => x, IntegerType)
-    checkAnswer(
-      Seq(Integer.valueOf(1), null).toDF("x").select(f($"x")),
-      Row(1) :: Row(0) :: Nil)
+    withSQLConf(SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF.key -> "true") {
+      val f = udf((x: Int) => x, IntegerType)
+      checkAnswer(
+        Seq(Integer.valueOf(1), null).toDF("x").select(f($"x")),
+        Row(1) :: Row(0) :: Nil)
+
+      val f2 = udf((x: Double) => x, DoubleType)
+      checkAnswer(
+        Seq(java.lang.Double.valueOf(1.1), null).toDF("x").select(f2($"x")),
+        Row(1.1) :: Row(0.0) :: Nil)
+    }
 
-    val f2 = udf((x: Double) => x, DoubleType)
-    checkAnswer(
-      Seq(java.lang.Double.valueOf(1.1), null).toDF("x").select(f2($"x")),
-      Row(1.1) :: Row(0.0) :: Nil)
+  }
 
+  test("use untyped Scala UDF should fail by default") {
+    val e = intercept[AnalysisException](udf((x: Int) => x, IntegerType))
+    assert(e.getMessage.contains("You're using untyped Scala UDF"))
   }
 
   test("SPARK-26308: udf with decimal") {
@@ -525,21 +534,79 @@ class UDFSuite extends QueryTest with SharedSparkSession {
     assert(spark.range(2).select(nonDeterministicJavaUDF()).distinct().count() == 2)
   }
 
-  test("Replace _FUNC_ in UDF ExpressionInfo") {
-    val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("upper"))
-    assert(info.getName === "upper")
-    assert(info.getClassName === "org.apache.spark.sql.catalyst.expressions.Upper")
-    assert(info.getUsage === "upper(str) - Returns `str` with all characters changed to uppercase.")
-    assert(info.getExamples.contains("> SELECT upper('SparkSql');"))
-    assert(info.getSince === "1.0.1")
-    assert(info.getNote === "")
-    assert(info.getExtended.contains("> SELECT upper('SparkSql');"))
-  }
-
   test("SPARK-28521 error message for CAST(parameter types contains DataType)") {
     val e = intercept[AnalysisException] {
       spark.sql("SELECT CAST(1)")
     }
     assert(e.getMessage.contains("Invalid arguments for function cast"))
   }
+
+  test("only one case class parameter") {
+    val f = (d: TestData) => d.key * d.value.toInt
+    val myUdf = udf(f)
+    val df = Seq(("data", TestData(50, "2"))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col2"))), Row(100) :: Nil)
+  }
+
+  test("one case class with primitive parameter") {
+    val f = (i: Int, p: TestData) => p.key * i
+    val myUdf = udf(f)
+    val df = Seq((2, TestData(50, "data"))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col1"), Column("col2"))), Row(100) :: Nil)
+  }
+
+  test("multiple case class parameters") {
+    val f = (d1: TestData, d2: TestData) => d1.key * d2.key
+    val myUdf = udf(f)
+    val df = Seq((TestData(10, "d1"), TestData(50, "d2"))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col1"), Column("col2"))), Row(500) :: Nil)
+  }
+
+  test("input case class parameter and return case class") {
+    val f = (d: TestData) => TestData(d.key * 2, "copy")
+    val myUdf = udf(f)
+    val df = Seq(("data", TestData(50, "d2"))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col2"))), Row(Row(100, "copy")) :: Nil)
+  }
+
+  test("any and case class parameter") {
+    val f = (any: Any, d: TestData) => s"${any.toString}, ${d.value}"
+    val myUdf = udf(f)
+    val df = Seq(("Hello", TestData(50, "World"))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col1"), Column("col2"))), Row("Hello, World") :: Nil)
+  }
+
+  test("nested case class parameter") {
+    val f = (y: Int, training: TrainingSales) => training.sales.year + y
+    val myUdf = udf(f)
+    val df = Seq((20, TrainingSales("training", CourseSales("course", 2000, 3.14))))
+      .toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col1"), Column("col2"))), Row(2020) :: Nil)
+  }
+
+  object MalformedClassObject extends Serializable {
+    class MalformedNonPrimitiveFunction extends (String => Int) with Serializable {
+      override def apply(v1: String): Int = v1.toInt / 0
+    }
+
+    class MalformedPrimitiveFunction extends (Int => Int) with Serializable {
+      override def apply(v1: Int): Int = v1 / 0
+    }
+  }
+
+  test("SPARK-32238: Use Utils.getSimpleName to avoid hitting Malformed class name") {
+    OuterScopes.addOuterScope(MalformedClassObject)
+    val f1 = new MalformedClassObject.MalformedNonPrimitiveFunction()
+    val f2 = new MalformedClassObject.MalformedPrimitiveFunction()
+
+    val e1 = intercept[SparkException] {
+      Seq("20").toDF("col").select(udf(f1).apply(Column("col"))).collect()
+    }
+    assert(e1.getMessage.contains("UDFSuite$MalformedClassObject$MalformedNonPrimitiveFunction"))
+
+    val e2 = intercept[SparkException] {
+      Seq(20).toDF("col").select(udf(f2).apply(Column("col"))).collect()
+    }
+    assert(e2.getMessage.contains("UDFSuite$MalformedClassObject$MalformedPrimitiveFunction"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index ffc2018d2132d..7c126f5e9a0b5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql
 
+import java.time.{LocalDateTime, ZoneOffset}
 import java.util.Arrays
 
 import org.apache.spark.rdd.RDD
@@ -103,6 +104,24 @@ private[spark] class ExampleSubTypeUDT extends UserDefinedType[IExampleSubType]
   override def userClass: Class[IExampleSubType] = classOf[IExampleSubType]
 }
 
+private[sql] case class FooWithDate(date: LocalDateTime, s: String, i: Int)
+
+private[sql] class LocalDateTimeUDT extends UserDefinedType[LocalDateTime] {
+  override def sqlType: DataType = LongType
+
+  override def serialize(obj: LocalDateTime): Long = {
+    obj.toEpochSecond(ZoneOffset.UTC)
+  }
+
+  def deserialize(datum: Any): LocalDateTime = datum match {
+    case value: Long => LocalDateTime.ofEpochSecond(value, 0, ZoneOffset.UTC)
+  }
+
+  override def userClass: Class[LocalDateTime] = classOf[LocalDateTime]
+
+  private[spark] override def asNullable: LocalDateTimeUDT = this
+}
+
 class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with ParquetTest
     with ExpressionEvalHelper {
   import testImplicits._
@@ -115,6 +134,24 @@ class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with Parque
     MyLabeledPoint(1.0, new TestUDT.MyDenseVector(Array(0.1, 1.0))),
     MyLabeledPoint(0.0, new TestUDT.MyDenseVector(Array(0.3, 3.0)))).toDF()
 
+
+  test("SPARK-32090: equal") {
+    val udt1 = new ExampleBaseTypeUDT
+    val udt2 = new ExampleSubTypeUDT
+    val udt3 = new ExampleSubTypeUDT
+    assert(udt1 !== udt2)
+    assert(udt2 !== udt1)
+    assert(udt2 === udt3)
+    assert(udt3 === udt2)
+  }
+
+  test("SPARK-32090: acceptsType") {
+    val udt1 = new ExampleBaseTypeUDT
+    val udt2 = new ExampleSubTypeUDT
+    assert(udt1.acceptsType(udt2))
+    assert(!udt2.acceptsType(udt1))
+  }
+
   test("register user type: MyDenseVector for MyLabeledPoint") {
     val labels: RDD[Double] = pointsRDD.select('label).rdd.map { case Row(v: Double) => v }
     val labelsArrays: Array[Double] = labels.collect()
@@ -131,12 +168,14 @@ class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with Parque
   }
 
   test("UDTs and UDFs") {
-    spark.udf.register("testType",
-      (d: TestUDT.MyDenseVector) => d.isInstanceOf[TestUDT.MyDenseVector])
-    pointsRDD.createOrReplaceTempView("points")
-    checkAnswer(
-      sql("SELECT testType(features) from points"),
-      Seq(Row(true), Row(true)))
+    withTempView("points") {
+      spark.udf.register("testType",
+        (d: TestUDT.MyDenseVector) => d.isInstanceOf[TestUDT.MyDenseVector])
+      pointsRDD.createOrReplaceTempView("points")
+      checkAnswer(
+        sql("SELECT testType(features) from points"),
+        Seq(Row(true), Row(true)))
+    }
   }
 
   testStandardAndLegacyModes("UDTs with Parquet") {
@@ -287,4 +326,22 @@ class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with Parque
     checkAnswer(spark.createDataFrame(data, schema).selectExpr("typeof(a)"),
       Seq(Row("array<double>")))
   }
+
+  test("SPARK-30993: UserDefinedType matched to fixed length SQL type shouldn't be corrupted") {
+    def concatFoo(a: FooWithDate, b: FooWithDate): FooWithDate = {
+      FooWithDate(b.date, a.s + b.s, a.i)
+    }
+
+    UDTRegistration.register(classOf[LocalDateTime].getName, classOf[LocalDateTimeUDT].getName)
+
+    // remove sub-millisecond part as we only use millis based timestamp while serde
+    val date = LocalDateTime.ofEpochSecond(LocalDateTime.now().toEpochSecond(ZoneOffset.UTC),
+      0, ZoneOffset.UTC)
+    val inputDS = List(FooWithDate(date, "Foo", 1), FooWithDate(date, "Foo", 3),
+      FooWithDate(date, "Foo", 3)).toDS()
+    val agg = inputDS.groupByKey(x => x.i).mapGroups((_, iter) => iter.reduce(concatFoo))
+    val result = agg.collect()
+
+    assert(result.toSet === Set(FooWithDate(date, "FooFoo", 3), FooWithDate(date, "Foo", 1)))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala
new file mode 100644
index 0000000000000..524fc8a9b0dea
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.python
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
+
+class PythonSQLUtilsSuite extends SparkFunSuite {
+
+  test("listing sql configurations contains runtime ones only") {
+    val configs = PythonSQLUtils.listRuntimeSQLConfigs()
+
+    // static sql configurations
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key),
+      "listSQLConfigs should contain public static sql configuration")
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.DEBUG_MODE.key),
+      "listSQLConfigs should not contain internal static sql configuration")
+
+    // dynamic sql configurations
+    assert(configs.exists(entry => entry._1 == SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key),
+      "listSQLConfigs should contain public dynamic sql configuration")
+    assert(!configs.exists(entry => entry._1 == SQLConf.ANALYZER_MAX_ITERATIONS.key),
+      "listSQLConfigs should not contain internal dynamic sql configuration")
+
+    // spark core configurations
+    assert(!configs.exists(entry => entry._1 == "spark.master"),
+      "listSQLConfigs should not contain core configuration")
+  }
+
+  test("listing static sql configurations contains public static ones only") {
+    val configs = PythonSQLUtils.listStaticSQLConfigs()
+
+    // static sql configurations
+    assert(configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key),
+      "listStaticSQLConfigs should contain public static sql configuration")
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.DEBUG_MODE.key),
+      "listStaticSQLConfigs should not contain internal static sql configuration")
+
+    // dynamic sql configurations
+    assert(!configs.exists(entry => entry._1 == SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key),
+      "listStaticSQLConfigs should not contain dynamic sql configuration")
+    assert(!configs.exists(entry => entry._1 == SQLConf.ANALYZER_MAX_ITERATIONS.key),
+      "listStaticSQLConfigs should not contain internal dynamic sql configuration")
+
+    // spark core configurations
+    assert(!configs.exists(entry => entry._1 == "spark.master"),
+      "listStaticSQLConfigs should not contain core configuration")
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
index 3cdac59c20fc9..436862920f53f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
@@ -67,9 +67,10 @@ trait AlterTableTests extends SharedSparkSession {
       assert(exc.getMessage.contains("Unsupported table change"))
       assert(exc.getMessage.contains("Cannot drop all fields")) // from the implementation
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("id", IntegerType))
     }
   }
@@ -80,9 +81,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN data string")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("id", IntegerType).add("data", StringType))
     }
   }
@@ -93,9 +95,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN data string NOT NULL")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === StructType(Seq(
         StructField("id", IntegerType),
         StructField("data", StringType, nullable = false))))
@@ -108,9 +111,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN data string COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === StructType(Seq(
         StructField("id", IntegerType),
         StructField("data", StringType).withComment("doc"))))
@@ -136,12 +140,13 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (point struct<x: int>) USING $v2Format")
 
       sql(s"ALTER TABLE $t ADD COLUMN a string FIRST")
-      assert(getTableMetadata(t).schema == new StructType()
+      val tableName = fullTableName(t)
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", StringType)
         .add("point", new StructType().add("x", IntegerType)))
 
       sql(s"ALTER TABLE $t ADD COLUMN b string AFTER point")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", StringType)
         .add("point", new StructType().add("x", IntegerType))
         .add("b", StringType))
@@ -151,7 +156,7 @@ trait AlterTableTests extends SharedSparkSession {
       assert(e1.getMessage().contains("Couldn't find the reference column"))
 
       sql(s"ALTER TABLE $t ADD COLUMN point.y int FIRST")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", StringType)
         .add("point", new StructType()
           .add("y", IntegerType)
@@ -159,7 +164,7 @@ trait AlterTableTests extends SharedSparkSession {
         .add("b", StringType))
 
       sql(s"ALTER TABLE $t ADD COLUMN point.z int AFTER x")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", StringType)
         .add("point", new StructType()
           .add("y", IntegerType)
@@ -173,15 +178,53 @@ trait AlterTableTests extends SharedSparkSession {
     }
   }
 
+  test("SPARK-30814: add column with position referencing new columns being added") {
+    val t = s"${catalogAndNamespace}table_name"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (a string, b int, point struct<x: double, y: double>) USING $v2Format")
+      sql(s"ALTER TABLE $t ADD COLUMNS (x int AFTER a, y int AFTER x, z int AFTER y)")
+
+      val tableName = fullTableName(t)
+      assert(getTableMetadata(tableName).schema === new StructType()
+        .add("a", StringType)
+        .add("x", IntegerType)
+        .add("y", IntegerType)
+        .add("z", IntegerType)
+        .add("b", IntegerType)
+        .add("point", new StructType()
+          .add("x", DoubleType)
+          .add("y", DoubleType)))
+
+      sql(s"ALTER TABLE $t ADD COLUMNS (point.z double AFTER x, point.zz double AFTER z)")
+      assert(getTableMetadata(tableName).schema === new StructType()
+        .add("a", StringType)
+        .add("x", IntegerType)
+        .add("y", IntegerType)
+        .add("z", IntegerType)
+        .add("b", IntegerType)
+        .add("point", new StructType()
+          .add("x", DoubleType)
+          .add("z", DoubleType)
+          .add("zz", DoubleType)
+          .add("y", DoubleType)))
+
+      // The new column being referenced should come before being referenced.
+      val e = intercept[AnalysisException](
+        sql(s"ALTER TABLE $t ADD COLUMNS (yy int AFTER xx, xx int)"))
+      assert(e.getMessage().contains("Couldn't find the reference column for AFTER xx at root"))
+    }
+  }
+
   test("AlterTable: add multiple columns") {
     val t = s"${catalogAndNamespace}table_name"
     withTable(t) {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMNS data string COMMENT 'doc', ts timestamp")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === StructType(Seq(
         StructField("id", IntegerType),
         StructField("data", StringType).withComment("doc"),
@@ -195,9 +238,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, point struct<x: double, y: double>) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN point.z double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -214,9 +258,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN points.key.z double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StructType(Seq(
@@ -233,9 +278,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN points.value.z double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StringType, StructType(Seq(
@@ -251,9 +297,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: double, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN points.element.z double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -269,9 +316,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN points array<struct<x: double, y: double>>")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -286,9 +334,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: double, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN points.element.z double COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -342,8 +391,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN id TYPE bigint")
 
-      val table = getTableMetadata(t)
-      assert(table.name === fullTableName(t))
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
+
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("id", LongType))
     }
   }
@@ -363,13 +414,14 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id bigint NOT NULL) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN id SET NOT NULL")
 
-      val table = getTableMetadata(t)
-      assert(table.name === fullTableName(t))
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("id", LongType, nullable = false))
 
       sql(s"ALTER TABLE $t ALTER COLUMN id DROP NOT NULL")
-      val table2 = getTableMetadata(t)
-      assert(table2.name === fullTableName(t))
+      val table2 = getTableMetadata(tableName)
+      assert(table2.name === tableName)
       assert(table2.schema === new StructType().add("id", LongType))
 
       val e = intercept[AnalysisException] {
@@ -385,8 +437,9 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, point struct<x: float, y: double>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN point.x TYPE double")
 
-      val table = getTableMetadata(t)
-      assert(table.name === fullTableName(t))
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -407,9 +460,10 @@ trait AlterTableTests extends SharedSparkSession {
       assert(exc.getMessage.contains("point"))
       assert(exc.getMessage.contains("update a struct by updating its fields"))
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -429,9 +483,10 @@ trait AlterTableTests extends SharedSparkSession {
 
       assert(exc.getMessage.contains("update the element by updating points.element"))
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(IntegerType)))
@@ -444,9 +499,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<int>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.element TYPE long")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(LongType)))
@@ -464,9 +520,10 @@ trait AlterTableTests extends SharedSparkSession {
 
       assert(exc.getMessage.contains("update a map by updating m.key or m.value"))
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("m", MapType(StringType, IntegerType)))
@@ -479,9 +536,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, m map<string, int>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN m.value TYPE long")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("m", MapType(StringType, LongType)))
@@ -495,9 +553,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.key.x TYPE double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StructType(Seq(
@@ -513,9 +572,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.value.x TYPE double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StringType, StructType(Seq(
@@ -530,9 +590,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: float, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.element.x TYPE double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -589,9 +650,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN id COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === StructType(Seq(StructField("id", IntegerType).withComment("doc"))))
     }
   }
@@ -602,7 +664,8 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (a int, b int, point struct<x: int, y: int, z: int>) USING $v2Format")
 
       sql(s"ALTER TABLE $t ALTER COLUMN b FIRST")
-      assert(getTableMetadata(t).schema == new StructType()
+      val tableName = fullTableName(t)
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("b", IntegerType)
         .add("a", IntegerType)
         .add("point", new StructType()
@@ -611,7 +674,7 @@ trait AlterTableTests extends SharedSparkSession {
           .add("z", IntegerType)))
 
       sql(s"ALTER TABLE $t ALTER COLUMN b AFTER point")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", IntegerType)
         .add("point", new StructType()
           .add("x", IntegerType)
@@ -624,7 +687,7 @@ trait AlterTableTests extends SharedSparkSession {
       assert(e1.getMessage.contains("Couldn't resolve positional argument"))
 
       sql(s"ALTER TABLE $t ALTER COLUMN point.y FIRST")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", IntegerType)
         .add("point", new StructType()
           .add("y", IntegerType)
@@ -633,7 +696,7 @@ trait AlterTableTests extends SharedSparkSession {
         .add("b", IntegerType))
 
       sql(s"ALTER TABLE $t ALTER COLUMN point.y AFTER z")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", IntegerType)
         .add("point", new StructType()
           .add("x", IntegerType)
@@ -651,28 +714,16 @@ trait AlterTableTests extends SharedSparkSession {
     }
   }
 
-  test("AlterTable: update column type and comment") {
-    val t = s"${catalogAndNamespace}table_name"
-    withTable(t) {
-      sql(s"CREATE TABLE $t (id int) USING $v2Format")
-      sql(s"ALTER TABLE $t ALTER COLUMN id TYPE bigint COMMENT 'doc'")
-
-      val table = getTableMetadata(t)
-
-      assert(table.name === fullTableName(t))
-      assert(table.schema === StructType(Seq(StructField("id", LongType).withComment("doc"))))
-    }
-  }
-
   test("AlterTable: update nested column comment") {
     val t = s"${catalogAndNamespace}table_name"
     withTable(t) {
       sql(s"CREATE TABLE $t (id int, point struct<x: double, y: double>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN point.y COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -688,9 +739,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.key.y COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StructType(Seq(
@@ -706,9 +758,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.value.y COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StringType, StructType(Seq(
@@ -723,9 +776,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: double, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.element.y COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -768,9 +822,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t RENAME COLUMN id TO user_id")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("user_id", IntegerType))
     }
   }
@@ -781,9 +836,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, point struct<x: double, y: double>) USING $v2Format")
       sql(s"ALTER TABLE $t RENAME COLUMN point.y TO t")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -799,9 +855,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t RENAME COLUMN point.key.y TO t")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", MapType(StructType(Seq(
@@ -817,9 +874,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t RENAME COLUMN points.value.y TO t")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StringType, StructType(Seq(
@@ -834,9 +892,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: double, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t RENAME COLUMN points.element.y TO t")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -910,9 +969,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, data string) USING $v2Format")
       sql(s"ALTER TABLE $t DROP COLUMN data")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("id", IntegerType))
     }
   }
@@ -924,9 +984,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t DROP COLUMN point.t")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -942,9 +1003,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t DROP COLUMN point.key.y")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", MapType(StructType(Seq(
@@ -959,9 +1021,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t DROP COLUMN points.value.y")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StringType, StructType(Seq(
@@ -975,9 +1038,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: double, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t DROP COLUMN points.element.y")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -1019,9 +1083,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t SET LOCATION 's3://bucket/path'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.properties ===
         withDefaultOwnership(Map("provider" -> v2Format, "location" -> "s3://bucket/path")).asJava)
     }
@@ -1046,9 +1111,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t SET TBLPROPERTIES ('test'='34')")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.properties ===
         withDefaultOwnership(Map("provider" -> v2Format, "test" -> "34")).asJava)
     }
@@ -1059,17 +1125,18 @@ trait AlterTableTests extends SharedSparkSession {
     withTable(t) {
       sql(s"CREATE TABLE $t (id int) USING $v2Format TBLPROPERTIES('test' = '34')")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.properties ===
         withDefaultOwnership(Map("provider" -> v2Format, "test" -> "34")).asJava)
 
       sql(s"ALTER TABLE $t UNSET TBLPROPERTIES ('test')")
 
-      val updated = getTableMetadata(t)
+      val updated = getTableMetadata(tableName)
 
-      assert(updated.name === fullTableName(t))
+      assert(updated.name === tableName)
       assert(updated.properties === withDefaultOwnership(Map("provider" -> v2Format)).asJava)
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
index 4c67888cbdc48..6b25d7c61663c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
@@ -85,7 +85,7 @@ class DataSourceV2DataFrameSessionCatalogSuite
     withTable(t1) {
       spark.range(20).write.format(v2Format).option("path", "abc").saveAsTable(t1)
       val cat = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog]
-      val tableInfo = cat.loadTable(Identifier.of(Array.empty, t1))
+      val tableInfo = cat.loadTable(Identifier.of(Array("default"), t1))
       assert(tableInfo.properties().get("location") === "abc")
       assert(tableInfo.properties().get("provider") === v2Format)
     }
@@ -101,9 +101,15 @@ class InMemoryTableSessionCatalog extends TestV2SessionCatalogBase[InMemoryTable
     new InMemoryTable(name, schema, partitions, properties)
   }
 
+  override def loadTable(ident: Identifier): Table = {
+    val identToUse = Option(InMemoryTableSessionCatalog.customIdentifierResolution)
+      .map(_(ident))
+      .getOrElse(ident)
+    super.loadTable(identToUse)
+  }
+
   override def alterTable(ident: Identifier, changes: TableChange*): Table = {
-    val fullIdent = fullIdentifier(ident)
-    Option(tables.get(fullIdent)) match {
+    Option(tables.get(ident)) match {
       case Some(table) =>
         val properties = CatalogV2Util.applyPropertiesChanges(table.properties, changes)
         val schema = CatalogV2Util.applySchemaChanges(table.schema, changes)
@@ -116,7 +122,7 @@ class InMemoryTableSessionCatalog extends TestV2SessionCatalogBase[InMemoryTable
         val newTable = new InMemoryTable(table.name, schema, table.partitioning, properties)
           .withData(table.data)
 
-        tables.put(fullIdent, newTable)
+        tables.put(ident, newTable)
 
         newTable
       case _ =>
@@ -125,6 +131,21 @@ class InMemoryTableSessionCatalog extends TestV2SessionCatalogBase[InMemoryTable
   }
 }
 
+object InMemoryTableSessionCatalog {
+  private var customIdentifierResolution: Identifier => Identifier = _
+
+  def withCustomIdentifierResolver(
+      resolver: Identifier => Identifier)(
+      f: => Unit): Unit = {
+    try {
+      customIdentifierResolution = resolver
+      f
+    } finally {
+      customIdentifierResolution = null
+    }
+  }
+}
+
 private [connector] trait SessionCatalogTest[T <: Table, Catalog <: TestV2SessionCatalogBase[T]]
   extends QueryTest
   with SharedSparkSession
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
index 0a6897b829994..7c7afa9cfbd41 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
@@ -141,7 +141,7 @@ class DataSourceV2DataFrameSuite
       override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {
         plan = qe.analyzed
       }
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
     }
 
     try {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
index 27725bcadbcd5..cf00b3b5e4410 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector
 
-import org.apache.spark.sql.{DataFrame, SaveMode}
+import org.apache.spark.sql.{DataFrame, Row, SaveMode}
 import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog}
 
 class DataSourceV2SQLSessionCatalogSuite
@@ -47,6 +47,36 @@ class DataSourceV2SQLSessionCatalogSuite
     val v2Catalog = spark.sessionState.catalogManager.currentCatalog
     val nameParts = spark.sessionState.sqlParser.parseMultipartIdentifier(tableName)
     v2Catalog.asInstanceOf[TableCatalog]
-      .loadTable(Identifier.of(Array.empty, nameParts.last))
+      .loadTable(Identifier.of(nameParts.init.toArray, nameParts.last))
+  }
+
+  test("SPARK-30697: catalog.isView doesn't throw an error for specialized identifiers") {
+    val t1 = "tbl"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format")
+
+      def idResolver(id: Identifier): Identifier = Identifier.of(Array("default"), id.name())
+
+      InMemoryTableSessionCatalog.withCustomIdentifierResolver(idResolver) {
+        // The following should not throw AnalysisException.
+        sql(s"DESCRIBE TABLE ignored.$t1")
+      }
+    }
+  }
+
+  test("SPARK-31624: SHOW TBLPROPERTIES working with V2 tables and the session catalog") {
+    val t1 = "tbl"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format TBLPROPERTIES " +
+        "(key='v', key2='v2')")
+
+      checkAnswer(sql(s"SHOW TBLPROPERTIES $t1"), Seq(Row("key", "v"), Row("key2", "v2")))
+
+      checkAnswer(sql(s"SHOW TBLPROPERTIES $t1('key')"), Row("key", "v"))
+
+      checkAnswer(
+        sql(s"SHOW TBLPROPERTIES $t1('keyX')"),
+        Row("keyX", s"Table default.$t1 does not have property: keyX"))
+    }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 2c8349a0e6a75..df71a859899b7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -17,6 +17,9 @@
 
 package org.apache.spark.sql.connector
 
+import java.sql.Timestamp
+import java.time.LocalDate
+
 import scala.collection.JavaConverters._
 
 import org.apache.spark.SparkException
@@ -27,7 +30,7 @@ import org.apache.spark.sql.connector.catalog._
 import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
 import org.apache.spark.sql.connector.catalog.CatalogV2Util.withDefaultOwnership
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
-import org.apache.spark.sql.internal.SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION
+import org.apache.spark.sql.internal.SQLConf.{PARTITION_OVERWRITE_MODE, PartitionOverwriteMode, V2_SESSION_CATALOG_IMPLEMENTATION}
 import org.apache.spark.sql.internal.connector.SimpleTableProvider
 import org.apache.spark.sql.sources.SimpleScanSource
 import org.apache.spark.sql.types.{BooleanType, LongType, StringType, StructField, StructType}
@@ -172,7 +175,7 @@ class DataSourceV2SQLSuite
     spark.sql(s"CREATE TABLE table_name (id bigint, data string) USING $v2Source")
 
     val testCatalog = catalog(SESSION_CATALOG_NAME).asTableCatalog
-    val table = testCatalog.loadTable(Identifier.of(Array(), "table_name"))
+    val table = testCatalog.loadTable(Identifier.of(Array("default"), "table_name"))
 
     assert(table.name == "default.table_name")
     assert(table.partitioning.isEmpty)
@@ -256,6 +259,24 @@ class DataSourceV2SQLSuite
     checkAnswer(spark.internalCreateDataFrame(rdd, table.schema), Seq.empty)
   }
 
+  // TODO: ignored by SPARK-31707, restore the test after create table syntax unification
+  ignore("CreateTable: without USING clause") {
+    // unset this config to use the default v2 session catalog.
+    spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+    val testCatalog = catalog("testcat").asTableCatalog
+
+    sql("CREATE TABLE testcat.t1 (id int)")
+    val t1 = testCatalog.loadTable(Identifier.of(Array(), "t1"))
+    // Spark shouldn't set the default provider for catalog plugins.
+    assert(!t1.properties.containsKey(TableCatalog.PROP_PROVIDER))
+
+    sql("CREATE TABLE t2 (id int)")
+    val t2 = spark.sessionState.catalogManager.v2SessionCatalog.asTableCatalog
+      .loadTable(Identifier.of(Array("default"), "t2")).asInstanceOf[V1Table]
+    // Spark should set the default provider as DEFAULT_DATA_SOURCE_NAME for the session catalog.
+    assert(t2.v1Table.provider == Some(conf.defaultDataSourceName))
+  }
+
   test("CreateTable/RepalceTable: invalid schema if has interval type") {
     Seq("CREATE", "REPLACE").foreach { action =>
       val e1 = intercept[AnalysisException](
@@ -302,6 +323,37 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("CreateTableAsSelect: do not double execute on collect(), take() and other queries") {
+    val basicCatalog = catalog("testcat").asTableCatalog
+    val atomicCatalog = catalog("testcat_atomic").asTableCatalog
+    val basicIdentifier = "testcat.table_name"
+    val atomicIdentifier = "testcat_atomic.table_name"
+
+    Seq((basicCatalog, basicIdentifier), (atomicCatalog, atomicIdentifier)).foreach {
+      case (catalog, identifier) =>
+        val df = spark.sql(s"CREATE TABLE $identifier USING foo AS SELECT id, data FROM source")
+
+        df.collect()
+        df.take(5)
+        df.tail(5)
+        df.where("true").collect()
+        df.where("true").take(5)
+        df.where("true").tail(5)
+
+        val table = catalog.loadTable(Identifier.of(Array(), "table_name"))
+
+        assert(table.name == identifier)
+        assert(table.partitioning.isEmpty)
+        assert(table.properties == withDefaultOwnership(Map("provider" -> "foo")).asJava)
+        assert(table.schema == new StructType()
+          .add("id", LongType)
+          .add("data", StringType))
+
+        val rdd = spark.sparkContext.parallelize(table.asInstanceOf[InMemoryTable].rows)
+        checkAnswer(spark.internalCreateDataFrame(rdd, table.schema), spark.table("source"))
+    }
+  }
+
   test("ReplaceTableAsSelect: basic v2 implementation.") {
     val basicCatalog = catalog("testcat").asTableCatalog
     val atomicCatalog = catalog("testcat_atomic").asTableCatalog
@@ -329,6 +381,43 @@ class DataSourceV2SQLSuite
     }
   }
 
+  Seq("REPLACE", "CREATE OR REPLACE").foreach { cmd =>
+    test(s"ReplaceTableAsSelect: do not double execute $cmd on collect()") {
+      val basicCatalog = catalog("testcat").asTableCatalog
+      val atomicCatalog = catalog("testcat_atomic").asTableCatalog
+      val basicIdentifier = "testcat.table_name"
+      val atomicIdentifier = "testcat_atomic.table_name"
+
+      Seq((basicCatalog, basicIdentifier), (atomicCatalog, atomicIdentifier)).foreach {
+        case (catalog, identifier) =>
+          spark.sql(s"CREATE TABLE $identifier USING foo AS SELECT id, data FROM source")
+          val originalTable = catalog.loadTable(Identifier.of(Array(), "table_name"))
+
+          val df = spark.sql(s"$cmd TABLE $identifier USING foo AS SELECT id FROM source")
+
+          df.collect()
+          df.take(5)
+          df.tail(5)
+          df.where("true").collect()
+          df.where("true").take(5)
+          df.where("true").tail(5)
+
+          val replacedTable = catalog.loadTable(Identifier.of(Array(), "table_name"))
+
+          assert(replacedTable != originalTable, "Table should have been replaced.")
+          assert(replacedTable.name == identifier)
+          assert(replacedTable.partitioning.isEmpty)
+          assert(replacedTable.properties == withDefaultOwnership(Map("provider" -> "foo")).asJava)
+          assert(replacedTable.schema == new StructType().add("id", LongType))
+
+          val rdd = spark.sparkContext.parallelize(replacedTable.asInstanceOf[InMemoryTable].rows)
+          checkAnswer(
+            spark.internalCreateDataFrame(rdd, replacedTable.schema),
+            spark.table("source").select("id"))
+      }
+    }
+  }
+
   test("ReplaceTableAsSelect: Non-atomic catalog drops the table if the write fails.") {
     spark.sql("CREATE TABLE testcat.table_name USING foo AS SELECT id, data FROM source")
     val testCatalog = catalog("testcat").asTableCatalog
@@ -453,7 +542,7 @@ class DataSourceV2SQLSuite
     spark.sql(s"CREATE TABLE table_name USING $v2Source AS SELECT id, data FROM source")
 
     val testCatalog = catalog(SESSION_CATALOG_NAME).asTableCatalog
-    val table = testCatalog.loadTable(Identifier.of(Array(), "table_name"))
+    val table = testCatalog.loadTable(Identifier.of(Array("default"), "table_name"))
 
     assert(table.name == "default.table_name")
     assert(table.partitioning.isEmpty)
@@ -565,7 +654,7 @@ class DataSourceV2SQLSuite
     // The fact that the following line doesn't throw an exception means, the session catalog
     // can load the table.
     val t = catalog(SESSION_CATALOG_NAME).asTableCatalog
-      .loadTable(Identifier.of(Array.empty, "table_name"))
+      .loadTable(Identifier.of(Array("default"), "table_name"))
     assert(t.isInstanceOf[V1Table], "V1 table wasn't returned as an unresolved table")
   }
 
@@ -595,6 +684,24 @@ class DataSourceV2SQLSuite
     }
   }
 
+  // TODO: ignored by SPARK-31707, restore the test after create table syntax unification
+  ignore("CreateTableAsSelect: without USING clause") {
+    // unset this config to use the default v2 session catalog.
+    spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+    val testCatalog = catalog("testcat").asTableCatalog
+
+    sql("CREATE TABLE testcat.t1 AS SELECT 1 i")
+    val t1 = testCatalog.loadTable(Identifier.of(Array(), "t1"))
+    // Spark shouldn't set the default provider for catalog plugins.
+    assert(!t1.properties.containsKey(TableCatalog.PROP_PROVIDER))
+
+    sql("CREATE TABLE t2 AS SELECT 1 i")
+    val t2 = spark.sessionState.catalogManager.v2SessionCatalog.asTableCatalog
+      .loadTable(Identifier.of(Array("default"), "t2")).asInstanceOf[V1Table]
+    // Spark should set the default provider as DEFAULT_DATA_SOURCE_NAME for the session catalog.
+    assert(t2.v1Table.provider == Some(conf.defaultDataSourceName))
+  }
+
   test("DropTable: basic") {
     val tableName = "testcat.ns1.ns2.tbl"
     val ident = Identifier.of(Array("ns1", "ns2"), "tbl")
@@ -605,10 +712,10 @@ class DataSourceV2SQLSuite
   }
 
   test("DropTable: table qualified with the session catalog name") {
-    val ident = Identifier.of(Array(), "tbl")
+    val ident = Identifier.of(Array("default"), "tbl")
     sql("CREATE TABLE tbl USING json AS SELECT 1 AS i")
     assert(catalog("spark_catalog").asTableCatalog.tableExists(ident) === true)
-    sql("DROP TABLE spark_catalog.tbl")
+    sql("DROP TABLE spark_catalog.default.tbl")
     assert(catalog("spark_catalog").asTableCatalog.tableExists(ident) === false)
   }
 
@@ -679,6 +786,59 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("qualified column names for v2 tables") {
+    val t = "testcat.ns1.ns2.tbl"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (id bigint, point struct<x: bigint, y: bigint>) USING foo")
+      sql(s"INSERT INTO $t VALUES (1, (10, 20))")
+
+      def check(tbl: String): Unit = {
+        checkAnswer(
+          sql(s"SELECT testcat.ns1.ns2.tbl.id, testcat.ns1.ns2.tbl.point.x FROM $tbl"),
+          Row(1, 10))
+        checkAnswer(sql(s"SELECT ns1.ns2.tbl.id, ns1.ns2.tbl.point.x FROM $tbl"), Row(1, 10))
+        checkAnswer(sql(s"SELECT ns2.tbl.id, ns2.tbl.point.x FROM $tbl"), Row(1, 10))
+        checkAnswer(sql(s"SELECT tbl.id, tbl.point.x FROM $tbl"), Row(1, 10))
+      }
+
+      // Test with qualified table name "testcat.ns1.ns2.tbl".
+      check(t)
+
+      // Test if current catalog and namespace is respected in column resolution.
+      sql("USE testcat.ns1.ns2")
+      check("tbl")
+
+      val ex = intercept[AnalysisException] {
+        sql(s"SELECT ns1.ns2.ns3.tbl.id from $t")
+      }
+      assert(ex.getMessage.contains("cannot resolve '`ns1.ns2.ns3.tbl.id`"))
+    }
+  }
+
+  test("qualified column names for v1 tables") {
+    Seq(true, false).foreach { useV1Table =>
+      val format = if (useV1Table) "json" else v2Format
+      if (useV1Table) {
+        // unset this config to use the default v2 session catalog.
+        spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+      } else {
+        spark.conf.set(
+          V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[InMemoryTableSessionCatalog].getName)
+      }
+
+      withTable("t") {
+        sql(s"CREATE TABLE t USING $format AS SELECT 1 AS i")
+        checkAnswer(sql("select i from t"), Row(1))
+        checkAnswer(sql("select t.i from t"), Row(1))
+        checkAnswer(sql("select default.t.i from t"), Row(1))
+        checkAnswer(sql("select spark_catalog.default.t.i from t"), Row(1))
+        checkAnswer(sql("select t.i from spark_catalog.default.t"), Row(1))
+        checkAnswer(sql("select default.t.i from spark_catalog.default.t"), Row(1))
+        checkAnswer(sql("select spark_catalog.default.t.i from spark_catalog.default.t"), Row(1))
+      }
+    }
+  }
+
   test("InsertInto: append - across catalog") {
     val t1 = "testcat.ns1.ns2.tbl"
     val t2 = "testcat2.db.tbl"
@@ -752,6 +912,23 @@ class DataSourceV2SQLSuite
     assert(exception.getMessage.contains("The database name is not valid: a.b"))
   }
 
+  test("ShowViews: using v1 catalog, db name with multipartIdentifier ('a.b') is not allowed.") {
+    val exception = intercept[AnalysisException] {
+      sql("SHOW TABLES FROM a.b")
+    }
+
+    assert(exception.getMessage.contains("The database name is not valid: a.b"))
+  }
+
+  test("ShowViews: using v2 catalog, command not supported.") {
+    val exception = intercept[AnalysisException] {
+      sql("SHOW VIEWS FROM testcat")
+    }
+
+    assert(exception.getMessage.contains("Catalog testcat doesn't support SHOW VIEWS," +
+      " only SessionCatalog supports this command."))
+  }
+
   test("ShowTables: using v2 catalog with empty namespace") {
     spark.sql("CREATE TABLE testcat.table (id bigint, data string) USING foo")
     runShowTablesSql("SHOW TABLES FROM testcat", Seq(Row("", "table")))
@@ -1340,7 +1517,12 @@ class DataSourceV2SQLSuite
     val sessionCatalog = catalog(SESSION_CATALOG_NAME).asTableCatalog
 
     def checkPartitioning(cat: TableCatalog, partition: String): Unit = {
-      val table = cat.loadTable(Identifier.of(Array.empty, "tbl"))
+      val namespace = if (cat.name == SESSION_CATALOG_NAME) {
+        Array("default")
+      } else {
+        Array[String]()
+      }
+      val table = cat.loadTable(Identifier.of(namespace, "tbl"))
       val partitions = table.partitioning().map(_.references())
       assert(partitions.length === 1)
       val fieldNames = partitions.flatMap(_.map(_.fieldNames()))
@@ -1376,48 +1558,48 @@ class DataSourceV2SQLSuite
   }
 
   test("tableCreation: duplicate column names in the table definition") {
-    val errorMsg = "Found duplicate column(s) in the table definition of t"
+    val errorMsg = "Found duplicate column(s) in the table definition of"
     Seq((true, ("a", "a")), (false, ("aA", "Aa"))).foreach { case (caseSensitive, (c0, c1)) =>
       withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
         assertAnalysisError(
           s"CREATE TABLE t ($c0 INT, $c1 INT) USING $v2Source",
-          errorMsg
+          s"$errorMsg default.t"
         )
         assertAnalysisError(
           s"CREATE TABLE testcat.t ($c0 INT, $c1 INT) USING $v2Source",
-          errorMsg
+          s"$errorMsg t"
         )
         assertAnalysisError(
           s"CREATE OR REPLACE TABLE t ($c0 INT, $c1 INT) USING $v2Source",
-          errorMsg
+          s"$errorMsg default.t"
         )
         assertAnalysisError(
           s"CREATE OR REPLACE TABLE testcat.t ($c0 INT, $c1 INT) USING $v2Source",
-          errorMsg
+          s"$errorMsg t"
         )
       }
     }
   }
 
   test("tableCreation: duplicate nested column names in the table definition") {
-    val errorMsg = "Found duplicate column(s) in the table definition of t"
+    val errorMsg = "Found duplicate column(s) in the table definition of"
     Seq((true, ("a", "a")), (false, ("aA", "Aa"))).foreach { case (caseSensitive, (c0, c1)) =>
       withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
         assertAnalysisError(
           s"CREATE TABLE t (d struct<$c0: INT, $c1: INT>) USING $v2Source",
-          errorMsg
+          s"$errorMsg default.t"
         )
         assertAnalysisError(
           s"CREATE TABLE testcat.t (d struct<$c0: INT, $c1: INT>) USING $v2Source",
-          errorMsg
+          s"$errorMsg t"
         )
         assertAnalysisError(
           s"CREATE OR REPLACE TABLE t (d struct<$c0: INT, $c1: INT>) USING $v2Source",
-          errorMsg
+          s"$errorMsg default.t"
         )
         assertAnalysisError(
           s"CREATE OR REPLACE TABLE testcat.t (d struct<$c0: INT, $c1: INT>) USING $v2Source",
-          errorMsg
+          s"$errorMsg t"
         )
       }
     }
@@ -1451,7 +1633,6 @@ class DataSourceV2SQLSuite
         """
           |CREATE TABLE testcat.t (id int, `a.b` string) USING foo
           |CLUSTERED BY (`a.b`) INTO 4 BUCKETS
-          |OPTIONS ('allow-unsupported-transforms'=true)
         """.stripMargin)
 
       val testCatalog = catalog("testcat").asTableCatalog.asInstanceOf[InMemoryTableCatalog]
@@ -1530,6 +1711,20 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("SPARK-33435: REFRESH TABLE should invalidate all caches referencing the table") {
+    val tblName = "testcat.ns.t"
+    withTable(tblName) {
+      withTempView("t") {
+        sql(s"CREATE TABLE $tblName (id bigint) USING foo")
+        sql(s"CACHE TABLE t AS SELECT id FROM $tblName")
+
+        assert(spark.sharedState.cacheManager.lookupCachedData(spark.table("t")).isDefined)
+        sql(s"REFRESH TABLE $tblName")
+        assert(spark.sharedState.cacheManager.lookupCachedData(spark.table("t")).isEmpty)
+      }
+    }
+  }
+
   test("REPLACE TABLE: v1 table") {
     val e = intercept[AnalysisException] {
       sql(s"CREATE OR REPLACE TABLE tbl (a int) USING ${classOf[SimpleScanSource].getName}")
@@ -1748,7 +1943,7 @@ class DataSourceV2SQLSuite
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
       testV1Command("ANALYZE TABLE", s"$t COMPUTE STATISTICS")
-      testV1Command("ANALYZE TABLE", s"$t COMPUTE STATISTICS FOR ALL COLUMNS")
+      testV1CommandSupportingTempView("ANALYZE TABLE", s"$t COMPUTE STATISTICS FOR ALL COLUMNS")
     }
   }
 
@@ -1812,7 +2007,7 @@ class DataSourceV2SQLSuite
     val t = "testcat.ns1.ns2.tbl"
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
-      testV1Command("SHOW CREATE TABLE", t)
+      testV1CommandSupportingTempView("SHOW CREATE TABLE", t)
     }
   }
 
@@ -1821,12 +2016,12 @@ class DataSourceV2SQLSuite
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
 
-      testV1Command("CACHE TABLE", t)
+      testV1CommandSupportingTempView("CACHE TABLE", t)
 
       val e = intercept[AnalysisException] {
         sql(s"CACHE LAZY TABLE $t")
       }
-      assert(e.message.contains("CACHE TABLE is only supported with v1 tables"))
+      assert(e.message.contains("CACHE TABLE is only supported with temp views or v1 tables"))
     }
   }
 
@@ -1835,8 +2030,8 @@ class DataSourceV2SQLSuite
     withTable(t) {
       sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
 
-      testV1Command("UNCACHE TABLE", t)
-      testV1Command("UNCACHE TABLE", s"IF EXISTS $t")
+      testV1CommandSupportingTempView("UNCACHE TABLE", t)
+      testV1CommandSupportingTempView("UNCACHE TABLE", s"IF EXISTS $t")
     }
   }
 
@@ -1845,8 +2040,8 @@ class DataSourceV2SQLSuite
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
 
-      testV1Command("SHOW COLUMNS", s"FROM $t")
-      testV1Command("SHOW COLUMNS", s"IN $t")
+      testV1CommandSupportingTempView("SHOW COLUMNS", s"FROM $t")
+      testV1CommandSupportingTempView("SHOW COLUMNS", s"IN $t")
 
       val e3 = intercept[AnalysisException] {
         sql(s"SHOW COLUMNS FROM tbl IN testcat.ns1.ns2")
@@ -1916,7 +2111,7 @@ class DataSourceV2SQLSuite
     val e = intercept[AnalysisException] {
       sql(s"ALTER VIEW $v AS SELECT 1")
     }
-    assert(e.message.contains("ALTER VIEW QUERY is only supported with v1 tables"))
+    assert(e.message.contains("ALTER VIEW QUERY is only supported with temp views or v1 tables"))
   }
 
   test("CREATE VIEW") {
@@ -1943,8 +2138,6 @@ class DataSourceV2SQLSuite
         .add("value", StringType, nullable = false)
 
       val expected = Seq(
-        Row(TableCatalog.PROP_OWNER, defaultUser),
-        Row("provider", provider),
         Row("status", status),
         Row("user", user))
 
@@ -1994,7 +2187,8 @@ class DataSourceV2SQLSuite
     val e1 = intercept[AnalysisException] {
       sql("DESCRIBE FUNCTION default.ns1.ns2.fun")
     }
-    assert(e1.message.contains("Unsupported function name 'default.ns1.ns2.fun'"))
+    assert(e1.message.contains(
+      "The namespace in session catalog must have exactly one name part: default.ns1.ns2.fun"))
   }
 
   test("SHOW FUNCTIONS not valid v1 namespace") {
@@ -2013,9 +2207,10 @@ class DataSourceV2SQLSuite
     assert(e.message.contains("DROP FUNCTION is only supported in v1 catalog"))
 
     val e1 = intercept[AnalysisException] {
-      sql("DESCRIBE FUNCTION default.ns1.ns2.fun")
+      sql("DROP FUNCTION default.ns1.ns2.fun")
     }
-    assert(e1.message.contains("Unsupported function name 'default.ns1.ns2.fun'"))
+    assert(e1.message.contains(
+      "The namespace in session catalog must have exactly one name part: default.ns1.ns2.fun"))
   }
 
   test("CREATE FUNCTION: only support session catalog") {
@@ -2027,7 +2222,8 @@ class DataSourceV2SQLSuite
     val e1 = intercept[AnalysisException] {
       sql("CREATE FUNCTION default.ns1.ns2.fun as 'f'")
     }
-    assert(e1.message.contains("Unsupported function name 'default.ns1.ns2.fun'"))
+    assert(e1.message.contains(
+      "The namespace in session catalog must have exactly one name part: default.ns1.ns2.fun"))
   }
 
   test("global temp view should not be masked by v2 catalog") {
@@ -2064,7 +2260,8 @@ class DataSourceV2SQLSuite
       // the session catalog, not the `gloabl_temp` v2 catalog.
       sql(s"CREATE TABLE $globalTempDB.ns1.ns2.tbl (id bigint, data string) USING json")
     }
-    assert(e.message.contains("global_temp.ns1.ns2.tbl is not a valid TableIdentifier"))
+    assert(e.message.contains(
+      "The namespace in session catalog must have exactly one name part: global_temp.ns1.ns2.tbl"))
   }
 
   test("table name same as catalog can be used") {
@@ -2083,23 +2280,60 @@ class DataSourceV2SQLSuite
     withTable("t") {
       sql("CREATE TABLE t USING json AS SELECT 1 AS i")
       checkAnswer(sql("select * from t"), Row(1))
-      checkAnswer(sql("select * from spark_catalog.t"), Row(1))
       checkAnswer(sql("select * from spark_catalog.default.t"), Row(1))
     }
   }
 
+  test("SPARK-30885: v1 table name should be fully qualified") {
+    def assertWrongTableIdent(): Unit = {
+      withTable("t") {
+        sql("CREATE TABLE t USING json AS SELECT 1 AS i")
+
+        val t = "spark_catalog.t"
+        def verify(sql: String): Unit = {
+          val e = intercept[AnalysisException](spark.sql(sql))
+          assert(e.message.contains(
+            s"The namespace in session catalog must have exactly one name part: $t"))
+        }
+
+        verify(s"select * from $t")
+        // Verify V1 commands that bypass table lookups.
+        verify(s"REFRESH TABLE $t")
+        verify(s"DESCRIBE $t i")
+        verify(s"DROP TABLE $t")
+        verify(s"DROP VIEW $t")
+        verify(s"ANALYZE TABLE $t COMPUTE STATISTICS")
+        verify(s"ANALYZE TABLE $t COMPUTE STATISTICS FOR ALL COLUMNS")
+        verify(s"MSCK REPAIR TABLE $t")
+        verify(s"LOAD DATA INPATH 'filepath' INTO TABLE $t")
+        verify(s"SHOW CREATE TABLE $t")
+        verify(s"SHOW CREATE TABLE $t AS SERDE")
+        verify(s"CACHE TABLE $t")
+        verify(s"UNCACHE TABLE $t")
+        verify(s"TRUNCATE TABLE $t")
+        verify(s"SHOW PARTITIONS $t")
+        verify(s"SHOW COLUMNS FROM $t")
+      }
+    }
+
+    assertWrongTableIdent()
+    // unset this config to use the default v2 session catalog.
+    spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+    assertWrongTableIdent()
+  }
+
   test("SPARK-30259: session catalog can be specified in CREATE TABLE AS SELECT command") {
     withTable("tbl") {
-      val ident = Identifier.of(Array(), "tbl")
-      sql("CREATE TABLE spark_catalog.tbl USING json AS SELECT 1 AS i")
+      val ident = Identifier.of(Array("default"), "tbl")
+      sql("CREATE TABLE spark_catalog.default.tbl USING json AS SELECT 1 AS i")
       assert(catalog("spark_catalog").asTableCatalog.tableExists(ident) === true)
     }
   }
 
   test("SPARK-30259: session catalog can be specified in CREATE TABLE command") {
     withTable("tbl") {
-      val ident = Identifier.of(Array(), "tbl")
-      sql("CREATE TABLE spark_catalog.tbl (col string) USING json")
+      val ident = Identifier.of(Array("default"), "tbl")
+      sql("CREATE TABLE spark_catalog.default.tbl (col string) USING json")
       assert(catalog("spark_catalog").asTableCatalog.tableExists(ident) === true)
     }
   }
@@ -2108,7 +2342,7 @@ class DataSourceV2SQLSuite
     // unset this config to use the default v2 session catalog.
     spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
 
-    withTable("spark_catalog.t", "testcat.ns.t") {
+    withTable("spark_catalog.default.t", "testcat.ns.t") {
       sql("CREATE TABLE t USING parquet AS SELECT 1")
       sql("CREATE TABLE testcat.ns.t USING parquet AS SELECT 2")
 
@@ -2130,17 +2364,18 @@ class DataSourceV2SQLSuite
 
     withTempView("t") {
       spark.range(10).createTempView("t")
-      withView(s"$sessionCatalogName.v") {
+      withView(s"$sessionCatalogName.default.v") {
         val e = intercept[AnalysisException] {
-          sql(s"CREATE VIEW $sessionCatalogName.v AS SELECT * FROM t")
+          sql(s"CREATE VIEW $sessionCatalogName.default.v AS SELECT * FROM t")
         }
         assert(e.message.contains("referencing a temporary view"))
       }
     }
 
     withTempView("t") {
-      withView(s"$sessionCatalogName.v") {
-        sql(s"CREATE VIEW $sessionCatalogName.v AS SELECT t1.col FROM t t1 JOIN ns1.ns2.t t2")
+      withView(s"$sessionCatalogName.default.v") {
+        sql(s"CREATE VIEW $sessionCatalogName.default.v " +
+          "AS SELECT t1.col FROM t t1 JOIN ns1.ns2.t t2")
         sql(s"USE $sessionCatalogName")
         // The view should read data from table `testcat.ns1.ns2.t` not the temp view.
         spark.range(10).createTempView("t")
@@ -2214,6 +2449,81 @@ class DataSourceV2SQLSuite
       .head().getString(1) === expectedComment)
   }
 
+  test("SPARK-30799: temp view name can't contain catalog name") {
+    val sessionCatalogName = CatalogManager.SESSION_CATALOG_NAME
+    withTempView("v") {
+      spark.range(10).createTempView("v")
+      val e1 = intercept[AnalysisException](
+        sql(s"CACHE TABLE $sessionCatalogName.v")
+      )
+      assert(e1.message.contains(
+        "The namespace in session catalog must have exactly one name part: spark_catalog.v"))
+    }
+    val e2 = intercept[AnalysisException] {
+      sql(s"CREATE TEMP VIEW $sessionCatalogName.v AS SELECT 1")
+    }
+    assert(e2.message.contains("It is not allowed to add database prefix"))
+  }
+
+  test("SPARK-31015: star expression should work for qualified column names for v2 tables") {
+    val t = "testcat.ns1.ns2.tbl"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (id bigint, name string) USING foo")
+      sql(s"INSERT INTO $t VALUES (1, 'hello')")
+
+      def check(tbl: String): Unit = {
+        checkAnswer(sql(s"SELECT testcat.ns1.ns2.tbl.* FROM $tbl"), Row(1, "hello"))
+        checkAnswer(sql(s"SELECT ns1.ns2.tbl.* FROM $tbl"), Row(1, "hello"))
+        checkAnswer(sql(s"SELECT ns2.tbl.* FROM $tbl"), Row(1, "hello"))
+        checkAnswer(sql(s"SELECT tbl.* FROM $tbl"), Row(1, "hello"))
+      }
+
+      // Test with qualified table name "testcat.ns1.ns2.tbl".
+      check(t)
+
+      // Test if current catalog and namespace is respected in column resolution.
+      sql("USE testcat.ns1.ns2")
+      check("tbl")
+
+      val ex = intercept[AnalysisException] {
+        sql(s"SELECT ns1.ns2.ns3.tbl.* from $t")
+      }
+      assert(ex.getMessage.contains("cannot resolve 'ns1.ns2.ns3.tbl.*"))
+    }
+  }
+
+  test("SPARK-32168: INSERT OVERWRITE - hidden days partition - dynamic mode") {
+    def testTimestamp(daysOffset: Int): Timestamp = {
+      Timestamp.valueOf(LocalDate.of(2020, 1, 1 + daysOffset).atStartOfDay())
+    }
+
+    withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString) {
+      val t1 = s"${catalogAndNamespace}tbl"
+      withTable(t1) {
+        val df = spark.createDataFrame(Seq(
+          (testTimestamp(1), "a"),
+          (testTimestamp(2), "b"),
+          (testTimestamp(3), "c"))).toDF("ts", "data")
+        df.createOrReplaceTempView("source_view")
+
+        sql(s"CREATE TABLE $t1 (ts timestamp, data string) " +
+            s"USING $v2Format PARTITIONED BY (days(ts))")
+        sql(s"INSERT INTO $t1 VALUES " +
+            s"(CAST(date_add('2020-01-01', 2) AS timestamp), 'dummy'), " +
+            s"(CAST(date_add('2020-01-01', 4) AS timestamp), 'keep')")
+        sql(s"INSERT OVERWRITE TABLE $t1 SELECT ts, data FROM source_view")
+
+        val expected = spark.createDataFrame(Seq(
+          (testTimestamp(1), "a"),
+          (testTimestamp(2), "b"),
+          (testTimestamp(3), "c"),
+          (testTimestamp(4), "keep"))).toDF("ts", "data")
+
+        verifyTable(t1, expected)
+      }
+    }
+  }
+
   private def testV1Command(sqlCommand: String, sqlParams: String): Unit = {
     val e = intercept[AnalysisException] {
       sql(s"$sqlCommand $sqlParams")
@@ -2221,6 +2531,13 @@ class DataSourceV2SQLSuite
     assert(e.message.contains(s"$sqlCommand is only supported with v1 tables"))
   }
 
+  private def testV1CommandSupportingTempView(sqlCommand: String, sqlParams: String): Unit = {
+    val e = intercept[AnalysisException] {
+      sql(s"$sqlCommand $sqlParams")
+    }
+    assert(e.message.contains(s"$sqlCommand is only supported with temp views or v1 tables"))
+  }
+
   private def assertAnalysisError(sqlStatement: String, expectedError: String): Unit = {
     val errMsg = intercept[AnalysisException] {
       sql(sqlStatement)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
index 2d8761f872da7..c2edcce8ce2ce 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
@@ -394,6 +394,35 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
       checkAnswer(df, (0 until 3).map(i => Row(i)))
     }
   }
+
+  test("SPARK-32609: DataSourceV2 with different pushedfilters should be different") {
+    def getScanExec(query: DataFrame): BatchScanExec = {
+      query.queryExecution.executedPlan.collect {
+        case d: BatchScanExec => d
+      }.head
+    }
+
+    Seq(classOf[AdvancedDataSourceV2], classOf[JavaAdvancedDataSourceV2]).foreach { cls =>
+      withClue(cls.getName) {
+        val df = spark.read.format(cls.getName).load()
+        val q1 = df.select('i).filter('i > 6)
+        val q2 = df.select('i).filter('i > 5)
+        val scan1 = getScanExec(q1)
+        val scan2 = getScanExec(q2)
+        assert(!scan1.equals(scan2))
+      }
+    }
+  }
+
+  test("SPARK-33267: push down with condition 'in (..., null)' should not throw NPE") {
+    Seq(classOf[AdvancedDataSourceV2], classOf[JavaAdvancedDataSourceV2]).foreach { cls =>
+      withClue(cls.getName) {
+        val df = spark.read.format(cls.getName).load()
+        // before SPARK-33267 below query just threw NPE
+        df.select('i).where("i in (1, null)").collect()
+      }
+    }
+  }
 }
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala
index 01fcced5b12a8..a58bab276a41b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala
@@ -37,8 +37,8 @@ class DataSourceV2UtilsSuite extends SparkFunSuite {
     val source = new DataSourceV2WithSessionConfig
     val confs = DataSourceV2Utils.extractSessionConfigs(source, conf)
     assert(confs.size == 2)
-    assert(confs.keySet.filter(_.startsWith("spark.datasource")).size == 0)
-    assert(confs.keySet.filter(_.startsWith("not.exist.prefix")).size == 0)
+    assert(!confs.keySet.exists(_.startsWith("spark.datasource")))
+    assert(!confs.keySet.exists(_.startsWith("not.exist.prefix")))
     assert(confs.keySet.contains("foo.bar"))
     assert(confs.keySet.contains("whateverConfigName"))
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
index b0da2eb697f36..51d734279414a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
@@ -157,13 +157,13 @@ class FileDataSourceV2FallBackSuite extends QueryTest with SharedSparkSession {
     Seq("parquet", classOf[ParquetDataSourceV2].getCanonicalName).foreach { format =>
       withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> format) {
         val commands = ArrayBuffer.empty[(String, LogicalPlan)]
-        val errors = ArrayBuffer.empty[(String, Throwable)]
+        val exceptions = ArrayBuffer.empty[(String, Exception)]
         val listener = new QueryExecutionListener {
           override def onFailure(
               funcName: String,
               qe: QueryExecution,
-              error: Throwable): Unit = {
-            errors += funcName -> error
+              exception: Exception): Unit = {
+            exceptions += funcName -> exception
           }
 
           override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
index 0fd6cf1b6746c..2cc7a1f994645 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
@@ -446,21 +446,35 @@ trait InsertIntoSQLOnlyTests
       }
     }
 
-    test("InsertInto: overwrite - multiple static partitions - dynamic mode") {
-      // Since all partitions are provided statically, this should be supported by everyone
-      withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString) {
-        val t1 = s"${catalogAndNamespace}tbl"
-        withTableAndData(t1) { view =>
-          sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " +
-            s"USING $v2Format PARTITIONED BY (id, p)")
-          sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'keep', 2)")
-          sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id = 2, p = 2) SELECT data FROM $view")
-          verifyTable(t1, Seq(
-            (2, "a", 2),
-            (2, "b", 2),
-            (2, "c", 2),
-            (4, "keep", 2)).toDF("id", "data", "p"))
-        }
+    dynamicOverwriteTest("InsertInto: overwrite - multiple static partitions - dynamic mode") {
+      val t1 = s"${catalogAndNamespace}tbl"
+      withTableAndData(t1) { view =>
+        sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " +
+          s"USING $v2Format PARTITIONED BY (id, p)")
+        sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'keep', 2)")
+        sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id = 2, p = 2) SELECT data FROM $view")
+        verifyTable(t1, Seq(
+          (2, "a", 2),
+          (2, "b", 2),
+          (2, "c", 2),
+          (4, "keep", 2)).toDF("id", "data", "p"))
+      }
+    }
+
+    test("do not double insert on INSERT INTO collect()") {
+      val t1 = s"${catalogAndNamespace}tbl"
+      withTableAndData(t1) { view =>
+        sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format")
+        val df = sql(s"INSERT INTO TABLE $t1 SELECT * FROM $view")
+
+        df.collect()
+        df.take(5)
+        df.tail(5)
+        df.where("true").collect()
+        df.where("true").take(5)
+        df.where("true").tail(5)
+
+        verifyTable(t1, spark.table(view))
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
index 7bff955b18360..550bec7505422 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
@@ -75,7 +75,12 @@ class SupportsCatalogOptionsSuite extends QueryTest with SharedSparkSession with
     withCatalogOption.foreach(cName => dfw.option("catalog", cName))
     dfw.partitionBy(partitionBy: _*).save()
 
-    val table = catalog(withCatalogOption.getOrElse(SESSION_CATALOG_NAME)).loadTable("t1")
+    val ident = if (withCatalogOption.isEmpty) {
+      Identifier.of(Array("default"), "t1")
+    } else {
+      Identifier.of(Array(), "t1")
+    }
+    val table = catalog(withCatalogOption.getOrElse(SESSION_CATALOG_NAME)).loadTable(ident)
     val namespace = withCatalogOption.getOrElse("default")
     assert(table.name() === s"$namespace.t1", "Table identifier was wrong")
     assert(table.partitioning().length === partitionBy.length, "Partitioning did not match")
@@ -134,7 +139,7 @@ class SupportsCatalogOptionsSuite extends QueryTest with SharedSparkSession with
     val dfw = df.write.format(format).mode(SaveMode.Ignore).option("name", "t1")
     dfw.save()
 
-    val table = catalog(SESSION_CATALOG_NAME).loadTable("t1")
+    val table = catalog(SESSION_CATALOG_NAME).loadTable(Identifier.of(Array("default"), "t1"))
     assert(table.partitioning().isEmpty, "Partitioning should be empty")
     assert(table.schema() === new StructType().add("id", LongType), "Schema did not match")
     assert(load("t1", None).count() === 0)
@@ -211,7 +216,7 @@ class SupportsCatalogOptionsSuite extends QueryTest with SharedSparkSession with
       override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {
         plan = qe.analyzed
       }
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
     }
 
     spark.listenerManager.register(listener)
@@ -279,7 +284,12 @@ class CatalogSupportingInMemoryTableProvider
   override def extractIdentifier(options: CaseInsensitiveStringMap): Identifier = {
     val name = options.get("name")
     assert(name != null, "The name should be provided for this table")
-    Identifier.of(Array.empty, name)
+    val namespace = if (options.containsKey("catalog")) {
+      Array[String]()
+    } else {
+      Array("default")
+    }
+    Identifier.of(namespace, name)
   }
 
   override def extractCatalog(options: CaseInsensitiveStringMap): String = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala
index 3f6ac0b7f8d3c..637cf2fd16515 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
 
 import scala.collection.JavaConverters._
 
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
 import org.apache.spark.sql.connector.catalog.{DelegatingCatalogExtension, Identifier, Table}
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.types.StructType
@@ -41,23 +42,14 @@ private[connector] trait TestV2SessionCatalogBase[T <: Table] extends Delegating
       partitions: Array[Transform],
       properties: util.Map[String, String]): T
 
-  protected def fullIdentifier(ident: Identifier): Identifier = {
-    if (ident.namespace().isEmpty) {
-      Identifier.of(Array("default"), ident.name())
-    } else {
-      ident
-    }
-  }
-
   override def loadTable(ident: Identifier): Table = {
-    val fullIdent = fullIdentifier(ident)
-    if (tables.containsKey(fullIdent)) {
-      tables.get(fullIdent)
+    if (tables.containsKey(ident)) {
+      tables.get(ident)
     } else {
       // Table was created through the built-in catalog
-      val t = super.loadTable(fullIdent)
+      val t = super.loadTable(ident)
       val table = newTable(t.name(), t.schema(), t.partitioning(), t.properties())
-      tables.put(fullIdent, table)
+      tables.put(ident, table)
       table
     }
   }
@@ -69,13 +61,12 @@ private[connector] trait TestV2SessionCatalogBase[T <: Table] extends Delegating
       properties: util.Map[String, String]): Table = {
     val created = super.createTable(ident, schema, partitions, properties)
     val t = newTable(created.name(), schema, partitions, properties)
-    val fullIdent = fullIdentifier(ident)
-    tables.put(fullIdent, t)
+    tables.put(ident, t)
     t
   }
 
   override def dropTable(ident: Identifier): Boolean = {
-    tables.remove(fullIdentifier(ident))
+    tables.remove(ident)
     super.dropTable(ident)
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala
index 10ed2048dbf61..4b52a4cbf4116 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala
@@ -25,10 +25,14 @@ import scala.collection.mutable
 import org.scalatest.BeforeAndAfter
 
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode, SparkSession, SQLContext}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability}
 import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform}
 import org.apache.spark.sql.connector.write.{LogicalWriteInfo, LogicalWriteInfoImpl, SupportsOverwrite, SupportsTruncate, V1WriteBuilder, WriteBuilder}
 import org.apache.spark.sql.execution.datasources.DataSourceUtils
+import org.apache.spark.sql.internal.SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION
 import org.apache.spark.sql.internal.connector.SimpleTableProvider
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.test.SharedSparkSession
@@ -124,6 +128,23 @@ class V1WriteFallbackSuite extends QueryTest with SharedSparkSession with Before
     }
     assert(e3.getMessage.contains("schema"))
   }
+
+  test("fallback writes should only analyze plan once") {
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+    try {
+      val session = SparkSession.builder()
+        .master("local[1]")
+        .withExtensions(_.injectPostHocResolutionRule(_ => OnlyOnceRule))
+        .config(V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[V1FallbackTableCatalog].getName)
+        .getOrCreate()
+      val df = session.createDataFrame(Seq((1, "x"), (2, "y"), (3, "z")))
+      df.write.mode("append").option("name", "t1").format(v2Format).saveAsTable("test")
+    } finally {
+      SparkSession.setActiveSession(spark)
+      SparkSession.setDefaultSession(spark)
+    }
+  }
 }
 
 class V1WriteFallbackSessionCatalogSuite
@@ -318,3 +339,24 @@ class InMemoryTableWithV1Fallback(
     }
   }
 }
+
+/** A rule that fails if a query plan is analyzed twice. */
+object OnlyOnceRule extends Rule[LogicalPlan] {
+  private val tag = TreeNodeTag[String]("test")
+  private val counts = new mutable.HashMap[LogicalPlan, Int]()
+
+  override def apply(plan: LogicalPlan): LogicalPlan = {
+    if (plan.getTagValue(tag).isEmpty) {
+      plan.setTagValue(tag, "abc")
+      plan
+    } else {
+      val cnt = counts.getOrElseUpdate(plan, 0) + 1
+      // This rule will be run as injectPostHocResolutionRule, and is supposed to be run only twice.
+      // Once during planning and once during checkBatchIdempotence
+      assert(cnt <= 1, "This rule shouldn't have been called again")
+      counts.put(plan, cnt)
+      plan
+    }
+
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
index 289f9dc427795..dd95ceb59bdc4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
@@ -151,6 +151,17 @@ class V2CommandsCaseSensitivitySuite extends SharedSparkSession with AnalysisTes
     }
   }
 
+  test("AlterTable: add column resolution - column position referencing new column") {
+    alterTableTest(
+      Seq(
+        TableChange.addColumn(
+          Array("x"), LongType, true, null, ColumnPosition.after("id")),
+        TableChange.addColumn(
+          Array("y"), LongType, true, null, ColumnPosition.after("X"))),
+      Seq("Couldn't find the reference column for AFTER X at root")
+    )
+  }
+
   test("AlterTable: add column resolution - nested positional") {
     Seq("X", "Y").foreach { ref =>
       alterTableTest(
@@ -161,6 +172,17 @@ class V2CommandsCaseSensitivitySuite extends SharedSparkSession with AnalysisTes
     }
   }
 
+  test("AlterTable: add column resolution - column position referencing new nested column") {
+    alterTableTest(
+      Seq(
+        TableChange.addColumn(
+          Array("point", "z"), LongType, true, null),
+        TableChange.addColumn(
+          Array("point", "zz"), LongType, true, null, ColumnPosition.after("Z"))),
+      Seq("Couldn't find the reference column for AFTER Z at point")
+    )
+  }
+
   test("AlterTable: drop column resolution") {
     Seq(Array("ID"), Array("point", "X"), Array("POINT", "X"), Array("POINT", "x")).foreach { ref =>
       alterTableTest(
@@ -207,13 +229,17 @@ class V2CommandsCaseSensitivitySuite extends SharedSparkSession with AnalysisTes
   }
 
   private def alterTableTest(change: TableChange, error: Seq[String]): Unit = {
+    alterTableTest(Seq(change), error)
+  }
+
+  private def alterTableTest(changes: Seq[TableChange], error: Seq[String]): Unit = {
     Seq(true, false).foreach { caseSensitive =>
       withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
         val plan = AlterTable(
           catalog,
           Identifier.of(Array(), "table_name"),
           TestRelation2,
-          Seq(change)
+          changes
         )
 
         if (caseSensitive) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
similarity index 60%
rename from sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala
rename to sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
index 04b4d4f29f850..9e77f618eded6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
@@ -19,16 +19,17 @@ package org.apache.spark.sql.execution
 
 import org.scalatest.BeforeAndAfterAll
 
-import org.apache.spark.{MapOutputStatistics, SparkConf, SparkFunSuite}
+import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql._
 import org.apache.spark.sql.execution.adaptive._
-import org.apache.spark.sql.execution.adaptive.{CoalescedShuffleReaderExec, ReduceNumShufflePartitions}
+import org.apache.spark.sql.execution.adaptive.CoalesceShufflePartitions.COALESCED_SHUFFLE_READER_DESCRIPTION
+import org.apache.spark.sql.execution.adaptive.CustomShuffleReaderExec
 import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 
-class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAll {
+class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAll {
 
   private var originalActiveSparkSession: Option[SparkSession] = _
   private var originalInstantiatedSparkSession: Option[SparkSession] = _
@@ -52,212 +53,6 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
     }
   }
 
-  private def checkEstimation(
-      rule: ReduceNumShufflePartitions,
-      bytesByPartitionIdArray: Array[Array[Long]],
-      expectedPartitionStartIndices: Array[Int]): Unit = {
-    val mapOutputStatistics = bytesByPartitionIdArray.zipWithIndex.map {
-      case (bytesByPartitionId, index) =>
-        new MapOutputStatistics(index, bytesByPartitionId)
-    }
-    val estimatedPartitionStartIndices =
-      rule.estimatePartitionStartAndEndIndices(mapOutputStatistics).map(_._1)
-    assert(estimatedPartitionStartIndices === expectedPartitionStartIndices)
-  }
-
-  private def createReduceNumShufflePartitionsRule(
-      advisoryTargetPostShuffleInputSize: Long,
-      minNumPostShufflePartitions: Int = 1): ReduceNumShufflePartitions = {
-    val conf = new SQLConf().copy(
-      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE -> advisoryTargetPostShuffleInputSize,
-      SQLConf.SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS -> minNumPostShufflePartitions)
-    ReduceNumShufflePartitions(conf)
-  }
-
-  test("test estimatePartitionStartIndices - 1 Exchange") {
-    val rule = createReduceNumShufflePartitionsRule(100L)
-
-    {
-      // All bytes per partition are 0.
-      val bytesByPartitionId = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-
-    {
-      // Some bytes per partition are 0 and total size is less than the target size.
-      // 1 post-shuffle partition is needed.
-      val bytesByPartitionId = Array[Long](10, 0, 20, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-
-    {
-      // 2 post-shuffle partitions are needed.
-      val bytesByPartitionId = Array[Long](10, 0, 90, 20, 0)
-      val expectedPartitionStartIndices = Array[Int](0, 3)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-
-    {
-      // There are a few large pre-shuffle partitions.
-      val bytesByPartitionId = Array[Long](110, 10, 100, 110, 0)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-
-    {
-      // All pre-shuffle partitions are larger than the targeted size.
-      val bytesByPartitionId = Array[Long](100, 110, 100, 110, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-
-    {
-      // The last pre-shuffle partition is in a single post-shuffle partition.
-      val bytesByPartitionId = Array[Long](30, 30, 0, 40, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 4)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-  }
-
-  test("test estimatePartitionStartIndices - 2 Exchanges") {
-    val rule = createReduceNumShufflePartitionsRule(100L)
-
-    {
-      // If there are multiple values of the number of pre-shuffle partitions,
-      // we should see an assertion error.
-      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
-      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0, 0)
-      val mapOutputStatistics =
-        Array(
-          new MapOutputStatistics(0, bytesByPartitionId1),
-          new MapOutputStatistics(1, bytesByPartitionId2))
-      intercept[AssertionError](rule.estimatePartitionStartAndEndIndices(
-        mapOutputStatistics))
-    }
-
-    {
-      // All bytes per partition are 0.
-      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
-      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // Some bytes per partition are 0.
-      // 1 post-shuffle partition is needed.
-      val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 20, 0, 20)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // 2 post-shuffle partition are needed.
-      val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 2, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // 4 post-shuffle partition are needed.
-      val bytesByPartitionId1 = Array[Long](0, 99, 0, 20, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // 2 post-shuffle partition are needed.
-      val bytesByPartitionId1 = Array[Long](0, 100, 0, 30, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // There are a few large pre-shuffle partitions.
-      val bytesByPartitionId1 = Array[Long](0, 100, 40, 30, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 60, 0, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // All pairs of pre-shuffle partitions are larger than the targeted size.
-      val bytesByPartitionId1 = Array[Long](100, 100, 40, 30, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 60, 70, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-  }
-
-  test("test estimatePartitionStartIndices and enforce minimal number of reducers") {
-    val rule = createReduceNumShufflePartitionsRule(100L, 2)
-
-    {
-      // The minimal number of post-shuffle partitions is not enforced because
-      // the size of data is 0.
-      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
-      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // The minimal number of post-shuffle partitions is enforced.
-      val bytesByPartitionId1 = Array[Long](10, 5, 5, 0, 20)
-      val bytesByPartitionId2 = Array[Long](5, 10, 0, 10, 5)
-      val expectedPartitionStartIndices = Array[Int](0, 3)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // The number of post-shuffle partitions is determined by the coordinator.
-      val bytesByPartitionId1 = Array[Long](10, 50, 20, 80, 20)
-      val bytesByPartitionId2 = Array[Long](40, 10, 0, 10, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 3, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Query tests
-  ///////////////////////////////////////////////////////////////////////////
-
   val numInputPartitions: Int = 10
 
   def withSparkSession(
@@ -270,17 +65,17 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         .setAppName("test")
         .set(UI_ENABLED, false)
         .set(SQLConf.SHUFFLE_PARTITIONS.key, "5")
-        .set(SQLConf.SHUFFLE_MAX_NUM_POSTSHUFFLE_PARTITIONS.key, "5")
+        .set(SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key, "5")
         .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")
         .set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1")
         .set(
-          SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key,
+          SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key,
           targetPostShuffleInputSize.toString)
     minNumPostShufflePartitions match {
       case Some(numPartitions) =>
-        sparkConf.set(SQLConf.SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS.key, numPartitions.toString)
+        sparkConf.set(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key, numPartitions.toString)
       case None =>
-        sparkConf.set(SQLConf.SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS.key, "1")
+        sparkConf.set(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key, "1")
     }
 
     val spark = SparkSession.builder()
@@ -313,7 +108,7 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         val finalPlan = agg.queryExecution.executedPlan
           .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
         val shuffleReaders = finalPlan.collect {
-          case reader: CoalescedShuffleReaderExec => reader
+          case r @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => r
         }
         assert(shuffleReaders.length === 1)
         minNumPostShufflePartitions match {
@@ -360,7 +155,7 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         val finalPlan = join.queryExecution.executedPlan
           .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
         val shuffleReaders = finalPlan.collect {
-          case reader: CoalescedShuffleReaderExec => reader
+          case r @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => r
         }
         assert(shuffleReaders.length === 2)
         minNumPostShufflePartitions match {
@@ -412,7 +207,7 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         val finalPlan = join.queryExecution.executedPlan
           .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
         val shuffleReaders = finalPlan.collect {
-          case reader: CoalescedShuffleReaderExec => reader
+          case r @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => r
         }
         assert(shuffleReaders.length === 2)
         minNumPostShufflePartitions match {
@@ -464,7 +259,7 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         val finalPlan = join.queryExecution.executedPlan
           .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
         val shuffleReaders = finalPlan.collect {
-          case reader: CoalescedShuffleReaderExec => reader
+          case r @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => r
         }
         assert(shuffleReaders.length === 2)
         minNumPostShufflePartitions match {
@@ -507,7 +302,7 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
           val finalPlan = join.queryExecution.executedPlan
             .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
           val shuffleReaders = finalPlan.collect {
-            case reader: CoalescedShuffleReaderExec => reader
+            case r @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => r
           }
           assert(shuffleReaders.length === 0)
         } finally {
@@ -535,7 +330,10 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
       assert(finalPlan.collect {
         case ShuffleQueryStageExec(_, r: ReusedExchangeExec) => r
       }.length == 2)
-      assert(finalPlan.collect { case p: CoalescedShuffleReaderExec => p }.length == 3)
+      assert(
+        finalPlan.collect {
+          case p @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => p
+        }.length == 3)
 
 
       // test case 2: a query stage has 2 parent stages.
@@ -583,7 +381,10 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         Seq(0, 1, 2).map(i => Row(i)))
       val finalPlan = resultDf.queryExecution.executedPlan
         .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
-      assert(finalPlan.collect { case p: CoalescedShuffleReaderExec => p }.length == 0)
+      assert(
+        finalPlan.collect {
+          case p @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => p
+        }.isEmpty)
     }
     withSparkSession(test, 200, None)
   }
@@ -601,7 +402,10 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
       // As the pre-shuffle partition number are different, we will skip reducing
       // the shuffle partition numbers.
-      assert(finalPlan.collect { case p: CoalescedShuffleReaderExec => p }.length == 0)
+      assert(
+        finalPlan.collect {
+          case p @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => p
+        }.isEmpty)
     }
     withSparkSession(test, 100, None)
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala
index 1e90754ad7721..b27a940c364a4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala
@@ -18,30 +18,27 @@
 package org.apache.spark.sql.execution
 
 import org.apache.spark.sql.QueryTest
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
 import org.apache.spark.sql.expressions.scalalang.typed
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
+// Disable AQE because the WholeStageCodegenExec is added when running QueryStageExec
 @deprecated("This test suite will be removed.", "3.0.0")
 class DeprecatedWholeStageCodegenSuite extends QueryTest
   with SharedSparkSession
-  with AdaptiveSparkPlanHelper {
+  with DisableAdaptiveExecutionSuite {
 
   test("simple typed UDAF should be included in WholeStageCodegen") {
-    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-      // With enable AQE, the WholeStageCodegenExec rule is applied when running QueryStageExec.
-      import testImplicits._
+    import testImplicits._
 
-      val ds = Seq(("a", 10), ("b", 1), ("b", 2), ("c", 1)).toDS()
-        .groupByKey(_._1).agg(typed.sum(_._2))
+    val ds = Seq(("a", 10), ("b", 1), ("b", 2), ("c", 1)).toDS()
+      .groupByKey(_._1).agg(typed.sum(_._2))
 
-      val plan = ds.queryExecution.executedPlan
-      assert(find(plan)(p =>
-        p.isInstanceOf[WholeStageCodegenExec] &&
-          p.asInstanceOf[WholeStageCodegenExec].child.isInstanceOf[HashAggregateExec]).isDefined)
-      assert(ds.collect() === Array(("a", 10.0), ("b", 3.0), ("c", 1.0)))
-    }
+    val plan = ds.queryExecution.executedPlan
+    assert(plan.find(p =>
+      p.isInstanceOf[WholeStageCodegenExec] &&
+        p.asInstanceOf[WholeStageCodegenExec].child.isInstanceOf[HashAggregateExec]).isDefined)
+    assert(ds.collect() === Array(("a", 10.0), ("b", 3.0), ("c", 1.0)))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala
index b29de9c4adbaa..98aba3ba25f17 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala
@@ -27,32 +27,29 @@ import org.apache.spark.sql.catalyst.expressions.UnsafeRow
 
 class ExternalAppendOnlyUnsafeRowArraySuite extends SparkFunSuite with LocalSparkContext {
   private val random = new java.util.Random()
-  private var taskContext: TaskContext = _
-
-  override def afterAll(): Unit = try {
-    TaskContext.unset()
-  } finally {
-    super.afterAll()
-  }
 
   private def withExternalArray(inMemoryThreshold: Int, spillThreshold: Int)
                                (f: ExternalAppendOnlyUnsafeRowArray => Unit): Unit = {
     sc = new SparkContext("local", "test", new SparkConf(false))
 
-    taskContext = MemoryTestingUtils.fakeTaskContext(SparkEnv.get)
+    val taskContext = MemoryTestingUtils.fakeTaskContext(SparkEnv.get)
     TaskContext.setTaskContext(taskContext)
 
-    val array = new ExternalAppendOnlyUnsafeRowArray(
-      taskContext.taskMemoryManager(),
-      SparkEnv.get.blockManager,
-      SparkEnv.get.serializerManager,
-      taskContext,
-      1024,
-      SparkEnv.get.memoryManager.pageSizeBytes,
-      inMemoryThreshold,
-      spillThreshold)
-    try f(array) finally {
-      array.clear()
+    try {
+      val array = new ExternalAppendOnlyUnsafeRowArray(
+        taskContext.taskMemoryManager(),
+        SparkEnv.get.blockManager,
+        SparkEnv.get.serializerManager,
+        taskContext,
+        1024,
+        SparkEnv.get.memoryManager.pageSizeBytes,
+        inMemoryThreshold,
+        spillThreshold)
+      try f(array) finally {
+        array.clear()
+      }
+    } finally {
+      TaskContext.unset()
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala
index 7fbfa73623c85..28e82aa14e0d0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala
@@ -20,7 +20,8 @@ package org.apache.spark.sql.execution
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalog.Table
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
+import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, HintInfo, Join, JoinHint}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.StructType
 
@@ -170,4 +171,25 @@ class GlobalTempViewSuite extends QueryTest with SharedSparkSession {
         isTemporary = true).toString)
     }
   }
+
+  test("broadcast hint on global temp view") {
+    withGlobalTempView("v1") {
+      spark.range(10).createGlobalTempView("v1")
+      withTempView("v2") {
+        spark.range(10).createTempView("v2")
+
+        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+          Seq(
+            "SELECT /*+ MAPJOIN(v1) */ * FROM global_temp.v1, v2 WHERE v1.id = v2.id",
+            "SELECT /*+ MAPJOIN(global_temp.v1) */ * FROM global_temp.v1, v2 WHERE v1.id = v2.id"
+          ).foreach { statement =>
+            sql(statement).queryExecution.optimizedPlan match {
+              case Join(_, _, _, _, JoinHint(Some(HintInfo(Some(BROADCAST))), None)) =>
+              case _ => fail("broadcast hint not found in a left-side table")
+            }
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala
index 80340b5552c6d..4b2a2b439c89e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala
@@ -28,14 +28,16 @@ class GroupedIteratorSuite extends SparkFunSuite {
   test("basic") {
     val schema = new StructType().add("i", IntegerType).add("s", StringType)
     val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = encoder.createSerializer()
+    val fromRow = encoder.createDeserializer()
     val input = Seq(Row(1, "a"), Row(1, "b"), Row(2, "c"))
-    val grouped = GroupedIterator(input.iterator.map(encoder.toRow),
+    val grouped = GroupedIterator(input.iterator.map(toRow),
       Seq('i.int.at(0)), schema.toAttributes)
 
     val result = grouped.map {
       case (key, data) =>
         assert(key.numFields == 1)
-        key.getInt(0) -> data.map(encoder.fromRow).toSeq
+        key.getInt(0) -> data.map(fromRow).toSeq
     }.toSeq
 
     assert(result ==
@@ -46,6 +48,8 @@ class GroupedIteratorSuite extends SparkFunSuite {
   test("group by 2 columns") {
     val schema = new StructType().add("i", IntegerType).add("l", LongType).add("s", StringType)
     val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = encoder.createSerializer()
+    val fromRow = encoder.createDeserializer()
 
     val input = Seq(
       Row(1, 2L, "a"),
@@ -54,13 +58,13 @@ class GroupedIteratorSuite extends SparkFunSuite {
       Row(2, 1L, "d"),
       Row(3, 2L, "e"))
 
-    val grouped = GroupedIterator(input.iterator.map(encoder.toRow),
+    val grouped = GroupedIterator(input.iterator.map(toRow),
       Seq('i.int.at(0), 'l.long.at(1)), schema.toAttributes)
 
     val result = grouped.map {
       case (key, data) =>
         assert(key.numFields == 2)
-        (key.getInt(0), key.getLong(1), data.map(encoder.fromRow).toSeq)
+        (key.getInt(0), key.getLong(1), data.map(fromRow).toSeq)
     }.toSeq
 
     assert(result ==
@@ -73,8 +77,9 @@ class GroupedIteratorSuite extends SparkFunSuite {
   test("do nothing to the value iterator") {
     val schema = new StructType().add("i", IntegerType).add("s", StringType)
     val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = encoder.createSerializer()
     val input = Seq(Row(1, "a"), Row(1, "b"), Row(2, "c"))
-    val grouped = GroupedIterator(input.iterator.map(encoder.toRow),
+    val grouped = GroupedIterator(input.iterator.map(toRow),
       Seq('i.int.at(0)), schema.toAttributes)
 
     assert(grouped.length == 2)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
index bb59b12e6f350..a0b212d2cf6fd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
@@ -17,27 +17,34 @@
 
 package org.apache.spark.sql.execution
 
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
+import org.apache.spark.sql.connector.InMemoryTableCatalog
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession}
 
 class HiveResultSuite extends SharedSparkSession {
   import testImplicits._
 
   test("date formatting in hive result") {
-    val dates = Seq("2018-12-28", "1582-10-13", "1582-10-14", "1582-10-15")
-    val df = dates.toDF("a").selectExpr("cast(a as date) as b")
-    val executedPlan1 = df.queryExecution.executedPlan
-    val result = HiveResult.hiveResultString(executedPlan1)
-    assert(result == dates)
-    val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
-    val result2 = HiveResult.hiveResultString(executedPlan2)
-    assert(result2 == dates.map(x => s"[$x]"))
+    DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zoneId) {
+        val dates = Seq("2018-12-28", "1582-10-03", "1582-10-04", "1582-10-15")
+        val df = dates.toDF("a").selectExpr("cast(a as date) as b")
+        val executedPlan1 = df.queryExecution.executedPlan
+        val result = HiveResult.hiveResultString(executedPlan1)
+        assert(result == dates)
+        val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
+        val result2 = HiveResult.hiveResultString(executedPlan2)
+        assert(result2 == dates.map(x => s"[$x]"))
+      }
+    }
   }
 
   test("timestamp formatting in hive result") {
     val timestamps = Seq(
       "2018-12-28 01:02:03",
-      "1582-10-13 01:02:03",
-      "1582-10-14 01:02:03",
+      "1582-10-03 01:02:03",
+      "1582-10-04 01:02:03",
       "1582-10-15 01:02:03")
     val df = timestamps.toDF("a").selectExpr("cast(a as timestamp) as b")
     val executedPlan1 = df.queryExecution.executedPlan
@@ -68,4 +75,35 @@ class HiveResultSuite extends SharedSparkSession {
     val result = HiveResult.hiveResultString(executedPlan)
     assert(result.head === "0.00000000")
   }
+
+  test("SHOW TABLES in hive result") {
+    withSQLConf("spark.sql.catalog.testcat" -> classOf[InMemoryTableCatalog].getName) {
+      Seq(("testcat.ns", "tbl", "foo"), ("spark_catalog.default", "tbl", "csv")).foreach {
+        case (ns, tbl, source) =>
+          withTable(s"$ns.$tbl") {
+            spark.sql(s"CREATE TABLE $ns.$tbl (id bigint) USING $source")
+            val df = spark.sql(s"SHOW TABLES FROM $ns")
+            val executedPlan = df.queryExecution.executedPlan
+            assert(HiveResult.hiveResultString(executedPlan).head == tbl)
+          }
+      }
+    }
+  }
+
+  test("DESCRIBE TABLE in hive result") {
+    withSQLConf("spark.sql.catalog.testcat" -> classOf[InMemoryTableCatalog].getName) {
+      Seq(("testcat.ns", "tbl", "foo"), ("spark_catalog.default", "tbl", "csv")).foreach {
+        case (ns, tbl, source) =>
+          withTable(s"$ns.$tbl") {
+            spark.sql(s"CREATE TABLE $ns.$tbl (id bigint COMMENT 'col1') USING $source")
+            val df = spark.sql(s"DESCRIBE $ns.$tbl")
+            val executedPlan = df.queryExecution.executedPlan
+            val expected = "id                  " +
+              "\tbigint              " +
+              "\tcol1                "
+            assert(HiveResult.hiveResultString(executedPlan).head == expected)
+          }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
index 311f84c07a955..5bcec9b1e517c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
@@ -22,6 +22,7 @@ import scala.reflect.ClassTag
 import org.apache.spark.sql.TPCDSQuerySuite
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete, Final}
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Generate, Join, LocalRelation, LogicalPlan, Range, Sample, Union, Window}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
 import org.apache.spark.sql.execution.columnar.{InMemoryRelation, InMemoryTableScanExec}
 import org.apache.spark.sql.execution.datasources.LogicalRelation
@@ -29,22 +30,9 @@ import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec}
 import org.apache.spark.sql.execution.joins._
 import org.apache.spark.sql.execution.window.WindowExec
-import org.apache.spark.sql.internal.SQLConf
 
-class LogicalPlanTagInSparkPlanSuite extends TPCDSQuerySuite {
-
-  var originalValue: String = _
-  // when enable AQE, the 'AdaptiveSparkPlanExec' node does not have a logical plan link
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
+// Disable AQE because AdaptiveSparkPlanExec does not have a logical plan link
+class LogicalPlanTagInSparkPlanSuite extends TPCDSQuerySuite with DisableAdaptiveExecutionSuite {
 
   override protected def checkGeneratedCode(
       plan: SparkPlan, checkMethodCodeSize: Boolean = true): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala
index afb438e0bbc72..68691e2f7fdac 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala
@@ -103,6 +103,20 @@ class OptimizeMetadataOnlyQuerySuite extends QueryTest with SharedSparkSession {
     "select partcol2, min(partcol1) from srcpart where partcol1 = 0 group by partcol2",
     "select max(c1) from (select partcol1 + 1 as c1 from srcpart where partcol1 = 0) t")
 
+  testMetadataOnly(
+    "SPARK-31590 Metadata-only queries should not include subquery in partition filters",
+    """
+      |SELECT partcol1, MAX(partcol2) AS partcol2
+      |FROM srcpart
+      |WHERE partcol1 = (
+      |  SELECT MAX(partcol1)
+      |  FROM srcpart
+      |)
+      |AND partcol2 = 'even'
+      |GROUP BY partcol1
+      |""".stripMargin
+  )
+
   testNotMetadataOnly(
     "Don't optimize metadata only query for non-partition columns",
     "select col1 from srcpart group by col1",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
index 0c5e2e3c7d1d4..b7be0f1320e61 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Range, Repartition, Sort, Union}
 import org.apache.spark.sql.catalyst.plans.physical._
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution}
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
 import org.apache.spark.sql.execution.columnar.{InMemoryRelation, InMemoryTableScanExec}
 import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReusedExchangeExec, ReuseExchange, ShuffleExchangeExec}
@@ -234,19 +234,6 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
     }
   }
 
-  test("SPARK-23375: Cached sorted data doesn't need to be re-sorted") {
-    val query = testData.select('key, 'value).sort('key.desc).cache()
-    assert(query.queryExecution.optimizedPlan.isInstanceOf[InMemoryRelation])
-    val resorted = query.sort('key.desc)
-    assert(resorted.queryExecution.optimizedPlan.collect { case s: Sort => s}.isEmpty)
-    assert(resorted.select('key).collect().map(_.getInt(0)).toSeq ==
-      (1 to 100).reverse)
-    // with a different order, the sort is needed
-    val sortedAsc = query.sort('key)
-    assert(sortedAsc.queryExecution.optimizedPlan.collect { case s: Sort => s}.size == 1)
-    assert(sortedAsc.select('key).collect().map(_.getInt(0)).toSeq == (1 to 100))
-  }
-
   test("PartitioningCollection") {
     withTempView("normal", "small", "tiny") {
       testData.createOrReplaceTempView("normal")
@@ -752,7 +739,8 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
   }
 
   test("SPARK-24556: always rewrite output partitioning in ReusedExchangeExec " +
-    "and InMemoryTableScanExec") {
+    "and InMemoryTableScanExec",
+    DisableAdaptiveExecution("Reuse is dynamic in AQE")) {
     def checkOutputPartitioningRewrite(
         plans: Seq[SparkPlan],
         expectedPartitioningClass: Class[_]): Unit = {
@@ -782,8 +770,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       checkOutputPartitioningRewrite(inMemoryScan, expectedPartitioningClass)
     }
     // when enable AQE, the reusedExchange is inserted when executed.
-    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
       // ReusedExchange is HashPartitioning
       val df1 = Seq(1 -> "a").toDF("i", "j").repartition($"i")
       val df2 = Seq(1 -> "a").toDF("i", "j").repartition($"i")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala
index 987338cf6cbbf..5ff459513e848 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala
@@ -58,4 +58,13 @@ class QueryPlanningTrackerEndToEndSuite extends StreamTest {
       StopStream)
   }
 
+  test("The start times should be in order: parsing <= analysis <= optimization <= planning") {
+    val df = spark.sql("select count(*) from range(1)")
+    df.queryExecution.executedPlan
+    val phases = df.queryExecution.tracker.phases
+    assert(phases("parsing").startTimeMs <= phases("analysis").startTimeMs)
+    assert(phases("analysis").startTimeMs <= phases("optimization").startTimeMs)
+    assert(phases("optimization").startTimeMs <= phases("planning").startTimeMs)
+  }
+
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
new file mode 100644
index 0000000000000..9cea81239f37d
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.{DataFrame, QueryTest}
+import org.apache.spark.sql.catalyst.plans.physical.{RangePartitioning, UnknownPartitioning}
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
+import org.apache.spark.sql.execution.joins.SortMergeJoinExec
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+
+abstract class RemoveRedundantSortsSuiteBase
+    extends QueryTest
+    with SharedSparkSession
+    with AdaptiveSparkPlanHelper {
+  import testImplicits._
+
+  private def checkNumSorts(df: DataFrame, count: Int): Unit = {
+    val plan = df.queryExecution.executedPlan
+    assert(collectWithSubqueries(plan) { case s: SortExec => s }.length == count)
+  }
+
+  private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
+    withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "true") {
+      val df = sql(query)
+      checkNumSorts(df, enabledCount)
+      val result = df.collect()
+      withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "false") {
+        val df = sql(query)
+        checkNumSorts(df, disabledCount)
+        checkAnswer(df, result)
+      }
+    }
+  }
+
+  test("remove redundant sorts with limit") {
+    withTempView("t") {
+      spark.range(100).select('id as "key").createOrReplaceTempView("t")
+      val query =
+        """
+          |SELECT key FROM
+          | (SELECT key FROM t WHERE key > 10 ORDER BY key DESC LIMIT 10)
+          |ORDER BY key DESC
+          |""".stripMargin
+      checkSorts(query, 0, 1)
+    }
+  }
+
+  test("remove redundant sorts with sort merge join") {
+    withTempView("t1", "t2") {
+      spark.range(1000).select('id as "key").createOrReplaceTempView("t1")
+      spark.range(1000).select('id as "key").createOrReplaceTempView("t2")
+      val query = """
+        |SELECT /*+ MERGE(t1) */ t1.key FROM
+        | (SELECT key FROM t1 WHERE key > 10 ORDER BY key DESC LIMIT 10) t1
+        |JOIN
+        | (SELECT key FROM t2 WHERE key > 50 ORDER BY key DESC LIMIT 100) t2
+        |ON t1.key = t2.key
+        |ORDER BY t1.key
+      """.stripMargin
+
+      val queryAsc = query + " ASC"
+      checkSorts(queryAsc, 2, 3)
+
+      // The top level sort should not be removed since the child output ordering is ASC and
+      // the required ordering is DESC.
+      val queryDesc = query + " DESC"
+      checkSorts(queryDesc, 3, 3)
+    }
+  }
+
+  test("cached sorted data doesn't need to be re-sorted") {
+    withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "true") {
+      val df = spark.range(1000).select('id as "key").sort('key.desc).cache()
+      val resorted = df.sort('key.desc)
+      val sortedAsc = df.sort('key.asc)
+      checkNumSorts(df, 0)
+      checkNumSorts(resorted, 0)
+      checkNumSorts(sortedAsc, 1)
+      val result = resorted.collect()
+      withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "false") {
+        val resorted = df.sort('key.desc)
+        checkNumSorts(resorted, 1)
+        checkAnswer(resorted, result)
+      }
+    }
+  }
+
+  test("SPARK-33472: shuffled join with different left and right side partition numbers") {
+    withTempView("t1", "t2") {
+      spark.range(0, 100, 1, 2).select('id as "key").createOrReplaceTempView("t1")
+      (0 to 100).toDF("key").createOrReplaceTempView("t2")
+
+      val query = """
+        |SELECT /*+ MERGE(t1) */ t1.key
+        |FROM t1 JOIN t2 ON t1.key = t2.key
+        |WHERE t1.key > 10 AND t2.key < 50
+        |ORDER BY t1.key ASC
+      """.stripMargin
+
+      val df = sql(query)
+      val sparkPlan = df.queryExecution.sparkPlan
+      val join = sparkPlan.collect { case j: SortMergeJoinExec => j }.head
+      val leftPartitioning = join.left.outputPartitioning
+      assert(leftPartitioning.isInstanceOf[RangePartitioning])
+      assert(leftPartitioning.numPartitions == 2)
+      assert(join.right.outputPartitioning == UnknownPartitioning(0))
+      checkSorts(query, 3, 3)
+    }
+  }
+}
+
+class RemoveRedundantSortsSuite extends RemoveRedundantSortsSuiteBase
+  with DisableAdaptiveExecutionSuite
+
+class RemoveRedundantSortsSuiteAE extends RemoveRedundantSortsSuiteBase
+  with EnableAdaptiveExecutionSuite
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala
index 8bf7fe62cd49b..81e692076b432 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala
@@ -17,11 +17,17 @@
 
 package org.apache.spark.sql.execution
 
+import java.util.concurrent.Executors
+
 import scala.collection.parallel.immutable.ParRange
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
 
 import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.{Row, SparkSession}
+import org.apache.spark.sql.types._
+import org.apache.spark.util.ThreadUtils
 
 class SQLExecutionSuite extends SparkFunSuite {
 
@@ -119,6 +125,38 @@ class SQLExecutionSuite extends SparkFunSuite {
 
     spark.stop()
   }
+
+  test("SPARK-32813: Table scan should work in different thread") {
+    val executor1 = Executors.newSingleThreadExecutor()
+    val executor2 = Executors.newSingleThreadExecutor()
+    var session: SparkSession = null
+    SparkSession.cleanupAnyExistingSession()
+
+    withTempDir { tempDir =>
+      try {
+        val tablePath = tempDir.toString + "/table"
+        val df = ThreadUtils.awaitResult(Future {
+          session = SparkSession.builder().appName("test").master("local[*]").getOrCreate()
+
+          session.createDataFrame(
+            session.sparkContext.parallelize(Row(Array(1, 2, 3)) :: Nil),
+            StructType(Seq(
+              StructField("a", ArrayType(IntegerType, containsNull = false), nullable = false))))
+            .write.parquet(tablePath)
+
+          session.read.parquet(tablePath)
+        }(ExecutionContext.fromExecutorService(executor1)), 1.minute)
+
+        ThreadUtils.awaitResult(Future {
+          assert(df.rdd.collect()(0) === Row(Seq(1, 2, 3)))
+        }(ExecutionContext.fromExecutorService(executor2)), 1.minute)
+      } finally {
+        executor1.shutdown()
+        executor2.shutdown()
+        session.stop()
+      }
+    }
+  }
 }
 
 object SQLExecutionSuite {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
index 9a393f19ce9bb..575efec364812 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
@@ -79,7 +79,7 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
           var e = intercept[AnalysisException] {
             sql("CREATE VIEW jtv1 AS SELECT * FROM temp_jtv1 WHERE id < 6")
           }.getMessage
-          assert(e.contains("Not allowed to create a permanent view `jtv1` by " +
+          assert(e.contains("Not allowed to create a permanent view `default`.`jtv1` by " +
             "referencing a temporary view temp_jtv1. " +
             "Please create a temp view instead by CREATE TEMP VIEW"))
 
@@ -88,8 +88,8 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
           e = intercept[AnalysisException] {
             sql(s"CREATE VIEW jtv1 AS SELECT * FROM $globalTempDB.global_temp_jtv1 WHERE id < 6")
           }.getMessage
-          assert(e.contains(s"Not allowed to create a permanent view `jtv1` by referencing " +
-            s"a temporary view global_temp.global_temp_jtv1"))
+          assert(e.contains("Not allowed to create a permanent view `default`.`jtv1` by " +
+            "referencing a temporary view global_temp.global_temp_jtv1"))
         }
       }
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
index 7aabf4d039f08..67ec1028f1998 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
@@ -41,67 +41,70 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 10)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product), sum(sum(product)) over (partition by area)
-          |from windowData group by month, area
-        """.stripMargin),
-      Seq(
-        ("a", 5, 11),
-        ("a", 6, 11),
-        ("b", 7, 15),
-        ("b", 8, 15),
-        ("c", 9, 19),
-        ("c", 10, 19)
-      ).map(i => Row(i._1, i._2, i._3)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product) - 1, sum(sum(product)) over (partition by area)
-          |from windowData group by month, area
-        """.stripMargin),
-      Seq(
-        ("a", 4, 11),
-        ("a", 5, 11),
-        ("b", 6, 15),
-        ("b", 7, 15),
-        ("c", 8, 19),
-        ("c", 9, 19)
-      ).map(i => Row(i._1, i._2, i._3)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product), sum(product) / sum(sum(product)) over (partition by area)
-          |from windowData group by month, area
-        """.stripMargin),
-      Seq(
-        ("a", 5, 5d/11),
-        ("a", 6, 6d/11),
-        ("b", 7, 7d/15),
-        ("b", 8, 8d/15),
-        ("c", 10, 10d/19),
-        ("c", 9, 9d/19)
-      ).map(i => Row(i._1, i._2, i._3)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product), sum(product) / sum(sum(product) - 1) over (partition by area)
-          |from windowData group by month, area
-        """.stripMargin),
-      Seq(
-        ("a", 5, 5d/9),
-        ("a", 6, 6d/9),
-        ("b", 7, 7d/13),
-        ("b", 8, 8d/13),
-        ("c", 10, 10d/17),
-        ("c", 9, 9d/17)
-      ).map(i => Row(i._1, i._2, i._3)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product), sum(sum(product)) over (partition by area)
+            |from windowData group by month, area
+          """.stripMargin),
+        Seq(
+          ("a", 5, 11),
+          ("a", 6, 11),
+          ("b", 7, 15),
+          ("b", 8, 15),
+          ("c", 9, 19),
+          ("c", 10, 19)
+        ).map(i => Row(i._1, i._2, i._3)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product) - 1, sum(sum(product)) over (partition by area)
+            |from windowData group by month, area
+          """.stripMargin),
+        Seq(
+          ("a", 4, 11),
+          ("a", 5, 11),
+          ("b", 6, 15),
+          ("b", 7, 15),
+          ("c", 8, 19),
+          ("c", 9, 19)
+        ).map(i => Row(i._1, i._2, i._3)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product), sum(product) / sum(sum(product)) over (partition by area)
+            |from windowData group by month, area
+          """.stripMargin),
+        Seq(
+          ("a", 5, 5d/11),
+          ("a", 6, 6d/11),
+          ("b", 7, 7d/15),
+          ("b", 8, 8d/15),
+          ("c", 10, 10d/19),
+          ("c", 9, 9d/19)
+        ).map(i => Row(i._1, i._2, i._3)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product), sum(product) / sum(sum(product) - 1) over
+            |(partition by area)
+            |from windowData group by month, area
+          """.stripMargin),
+        Seq(
+          ("a", 5, 5d/9),
+          ("a", 6, 6d/9),
+          ("b", 7, 7d/13),
+          ("b", 8, 8d/13),
+          ("c", 10, 10d/17),
+          ("c", 9, 9d/17)
+        ).map(i => Row(i._1, i._2, i._3)))
+    }
   }
 
   test("window function: refer column in inner select block") {
@@ -113,22 +116,24 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 10)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql(
-        """
-          |select area, rank() over (partition by area order by tmp.month) + tmp.tmp1 as c1
-          |from (select month, area, product, 1 as tmp1 from windowData) tmp
-        """.stripMargin),
-      Seq(
-        ("a", 2),
-        ("a", 3),
-        ("b", 2),
-        ("b", 3),
-        ("c", 2),
-        ("c", 3)
-      ).map(i => Row(i._1, i._2)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql(
+          """
+            |select area, rank() over (partition by area order by tmp.month) + tmp.tmp1 as c1
+            |from (select month, area, product, 1 as tmp1 from windowData) tmp
+          """.stripMargin),
+        Seq(
+          ("a", 2),
+          ("a", 3),
+          ("b", 2),
+          ("b", 3),
+          ("c", 2),
+          ("c", 3)
+        ).map(i => Row(i._1, i._2)))
+    }
   }
 
   test("window function: partition and order expressions") {
@@ -140,38 +145,40 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 10)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql(
-        """
-          |select month, area, product, sum(product + 1) over (partition by 1 order by 2)
-          |from windowData
-        """.stripMargin),
-      Seq(
-        (1, "a", 5, 51),
-        (2, "a", 6, 51),
-        (3, "b", 7, 51),
-        (4, "b", 8, 51),
-        (5, "c", 9, 51),
-        (6, "c", 10, 51)
-      ).map(i => Row(i._1, i._2, i._3, i._4)))
-
-    checkAnswer(
-      sql(
-        """
-          |select month, area, product, sum(product)
-          |over (partition by month % 2 order by 10 - product)
-          |from windowData
-        """.stripMargin),
-      Seq(
-        (1, "a", 5, 21),
-        (2, "a", 6, 24),
-        (3, "b", 7, 16),
-        (4, "b", 8, 18),
-        (5, "c", 9, 9),
-        (6, "c", 10, 10)
-      ).map(i => Row(i._1, i._2, i._3, i._4)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql(
+          """
+            |select month, area, product, sum(product + 1) over (partition by 1 order by 2)
+            |from windowData
+          """.stripMargin),
+        Seq(
+          (1, "a", 5, 51),
+          (2, "a", 6, 51),
+          (3, "b", 7, 51),
+          (4, "b", 8, 51),
+          (5, "c", 9, 51),
+          (6, "c", 10, 51)
+        ).map(i => Row(i._1, i._2, i._3, i._4)))
+
+      checkAnswer(
+        sql(
+          """
+            |select month, area, product, sum(product)
+            |over (partition by month % 2 order by 10 - product)
+            |from windowData
+          """.stripMargin),
+        Seq(
+          (1, "a", 5, 21),
+          (2, "a", 6, 24),
+          (3, "b", 7, 16),
+          (4, "b", 8, 18),
+          (5, "c", 9, 9),
+          (6, "c", 10, 10)
+        ).map(i => Row(i._1, i._2, i._3, i._4)))
+    }
   }
 
   test("window function: distinct should not be silently ignored") {
@@ -183,16 +190,18 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 10)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    val e = intercept[AnalysisException] {
-      sql(
-        """
-          |select month, area, product, sum(distinct product + 1) over (partition by 1 order by 2)
-          |from windowData
-        """.stripMargin)
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      val e = intercept[AnalysisException] {
+        sql(
+          """
+            |select month, area, product, sum(distinct product + 1) over (partition by 1 order by 2)
+            |from windowData
+          """.stripMargin)
+      }
+      assert(e.getMessage.contains("Distinct window functions are not supported"))
     }
-    assert(e.getMessage.contains("Distinct window functions are not supported"))
   }
 
   test("window function: expressions in arguments of a window functions") {
@@ -204,23 +213,25 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 10)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql(
-        """
-          |select month, area, month % 2,
-          |lag(product, 1 + 1, product) over (partition by month % 2 order by area)
-          |from windowData
-        """.stripMargin),
-      Seq(
-        (1, "a", 1, 5),
-        (2, "a", 0, 6),
-        (3, "b", 1, 7),
-        (4, "b", 0, 8),
-        (5, "c", 1, 5),
-        (6, "c", 0, 6)
-      ).map(i => Row(i._1, i._2, i._3, i._4)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql(
+          """
+            |select month, area, month % 2,
+            |lag(product, 1 + 1, product) over (partition by month % 2 order by area)
+            |from windowData
+          """.stripMargin),
+        Seq(
+          (1, "a", 1, 5),
+          (2, "a", 0, 6),
+          (3, "b", 1, 7),
+          (4, "b", 0, 8),
+          (5, "c", 1, 5),
+          (6, "c", 0, 6)
+        ).map(i => Row(i._1, i._2, i._3, i._4)))
+    }
   }
 
 
@@ -233,63 +244,65 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 11)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql("select month, product, sum(product + 1) over() from windowData order by area"),
-      Seq(
-        (2, 6, 57),
-        (3, 7, 57),
-        (4, 8, 57),
-        (5, 9, 57),
-        (6, 11, 57),
-        (1, 10, 57)
-      ).map(i => Row(i._1, i._2, i._3)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, rank() over (partition by area order by tmp.month) + tmp.tmp1 as c1
-          |from (select month, area, product as p, 1 as tmp1 from windowData) tmp order by p
-        """.stripMargin),
-      Seq(
-        ("a", 2),
-        ("b", 2),
-        ("b", 3),
-        ("c", 2),
-        ("d", 2),
-        ("c", 3)
-      ).map(i => Row(i._1, i._2)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, rank() over (partition by area order by month) as c1
-          |from windowData group by product, area, month order by product, area
-        """.stripMargin),
-      Seq(
-        ("a", 1),
-        ("b", 1),
-        ("b", 2),
-        ("c", 1),
-        ("d", 1),
-        ("c", 2)
-      ).map(i => Row(i._1, i._2)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product) / sum(sum(product)) over (partition by area) as c1
-          |from windowData group by area, month order by month, c1
-        """.stripMargin),
-      Seq(
-        ("d", 1.0),
-        ("a", 1.0),
-        ("b", 0.4666666666666667),
-        ("b", 0.5333333333333333),
-        ("c", 0.45),
-        ("c", 0.55)
-      ).map(i => Row(i._1, i._2)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql("select month, product, sum(product + 1) over() from windowData order by area"),
+        Seq(
+          (2, 6, 57),
+          (3, 7, 57),
+          (4, 8, 57),
+          (5, 9, 57),
+          (6, 11, 57),
+          (1, 10, 57)
+        ).map(i => Row(i._1, i._2, i._3)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, rank() over (partition by area order by tmp.month) + tmp.tmp1 as c1
+            |from (select month, area, product as p, 1 as tmp1 from windowData) tmp order by p
+          """.stripMargin),
+        Seq(
+          ("a", 2),
+          ("b", 2),
+          ("b", 3),
+          ("c", 2),
+          ("d", 2),
+          ("c", 3)
+        ).map(i => Row(i._1, i._2)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, rank() over (partition by area order by month) as c1
+            |from windowData group by product, area, month order by product, area
+          """.stripMargin),
+        Seq(
+          ("a", 1),
+          ("b", 1),
+          ("b", 2),
+          ("c", 1),
+          ("d", 1),
+          ("c", 2)
+        ).map(i => Row(i._1, i._2)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product) / sum(sum(product)) over (partition by area) as c1
+            |from windowData group by area, month order by month, c1
+          """.stripMargin),
+        Seq(
+          ("d", 1.0),
+          ("a", 1.0),
+          ("b", 0.4666666666666667),
+          ("b", 0.5333333333333333),
+          ("c", 0.45),
+          ("c", 0.55)
+        ).map(i => Row(i._1, i._2)))
+    }
   }
 
   // todo: fix this test case by reimplementing the function ResolveAggregateFunctions
@@ -302,23 +315,25 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 11)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product) over () as c from windowData
-          |where product > 3 group by area, product
-          |having avg(month) > 0 order by avg(month), product
-        """.stripMargin),
-      Seq(
-        ("a", 51),
-        ("b", 51),
-        ("b", 51),
-        ("c", 51),
-        ("c", 51),
-        ("d", 51)
-      ).map(i => Row(i._1, i._2)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product) over () as c from windowData
+            |where product > 3 group by area, product
+            |having avg(month) > 0 order by avg(month), product
+          """.stripMargin),
+        Seq(
+          ("a", 51),
+          ("b", 51),
+          ("b", 51),
+          ("c", 51),
+          ("c", 51),
+          ("d", 51)
+        ).map(i => Row(i._1, i._2)))
+    }
   }
 
   test("window function: multiple window expressions in a single expression") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
new file mode 100644
index 0000000000000..7acc33c43b19d
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.{MapOutputStatistics, SparkFunSuite}
+import org.apache.spark.sql.execution.adaptive.ShufflePartitionsUtil
+
+class ShufflePartitionsUtilSuite extends SparkFunSuite {
+
+  private def checkEstimation(
+      bytesByPartitionIdArray: Array[Array[Long]],
+      expectedPartitionStartIndices: Seq[CoalescedPartitionSpec],
+      targetSize: Long,
+      minNumPartitions: Int = 1): Unit = {
+    val mapOutputStatistics = bytesByPartitionIdArray.zipWithIndex.map {
+      case (bytesByPartitionId, index) =>
+        new MapOutputStatistics(index, bytesByPartitionId)
+    }
+    val estimatedPartitionStartIndices = ShufflePartitionsUtil.coalescePartitions(
+      mapOutputStatistics,
+      targetSize,
+      minNumPartitions)
+    assert(estimatedPartitionStartIndices === expectedPartitionStartIndices)
+  }
+
+  test("1 shuffle") {
+    val targetSize = 100
+
+    {
+      // All bytes per partition are 0.
+      val bytesByPartitionId = Array[Long](0, 0, 0, 0, 0)
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
+    }
+
+    {
+      // Some bytes per partition are 0 and total size is less than the target size.
+      // 1 coalesced partition is expected.
+      val bytesByPartitionId = Array[Long](10, 0, 20, 0, 0)
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
+    }
+
+    {
+      // 2 coalesced partitions are expected.
+      val bytesByPartitionId = Array[Long](10, 0, 90, 20, 0)
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 3), CoalescedPartitionSpec(3, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
+    }
+
+    {
+      // There are a few large shuffle partitions.
+      val bytesByPartitionId = Array[Long](110, 10, 100, 110, 0)
+      val expectedPartitionSpecs = Seq(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 3),
+        CoalescedPartitionSpec(3, 4),
+        CoalescedPartitionSpec(4, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
+    }
+
+    {
+      // All shuffle partitions are larger than the targeted size.
+      val bytesByPartitionId = Array[Long](100, 110, 100, 110, 110)
+      val expectedPartitionSpecs = Seq(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 3),
+        CoalescedPartitionSpec(3, 4),
+        CoalescedPartitionSpec(4, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
+    }
+
+    {
+      // The last shuffle partition is in a single coalesced partition.
+      val bytesByPartitionId = Array[Long](30, 30, 0, 40, 110)
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 4), CoalescedPartitionSpec(4, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
+    }
+  }
+
+  test("2 shuffles") {
+    val targetSize = 100
+
+    {
+      // If there are multiple values of the number of shuffle partitions,
+      // we should see an assertion error.
+      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
+      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0, 0)
+      intercept[AssertionError] {
+        checkEstimation(Array(bytesByPartitionId1, bytesByPartitionId2), Seq.empty, targetSize)
+      }
+    }
+
+    {
+      // All bytes per partition are 0.
+      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
+      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionSpecs,
+        targetSize)
+    }
+
+    {
+      // Some bytes per partition are 0.
+      // 1 coalesced partition is expected.
+      val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 20, 0, 20)
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionSpecs,
+        targetSize)
+    }
+
+    {
+      // 2 coalesced partition are expected.
+      val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
+      val expectedPartitionSpecs = Seq(
+        CoalescedPartitionSpec(0, 2),
+        CoalescedPartitionSpec(2, 4),
+        CoalescedPartitionSpec(4, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionSpecs,
+        targetSize)
+    }
+
+    {
+      // 4 coalesced partition are expected.
+      val bytesByPartitionId1 = Array[Long](0, 99, 0, 20, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
+      val expectedPartitionSpecs = Seq(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 4),
+        CoalescedPartitionSpec(4, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionSpecs,
+        targetSize)
+    }
+
+    {
+      // 2 coalesced partition are needed.
+      val bytesByPartitionId1 = Array[Long](0, 100, 0, 30, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
+      val expectedPartitionSpecs = Seq(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 4),
+        CoalescedPartitionSpec(4, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionSpecs,
+        targetSize)
+    }
+
+    {
+      // There are a few large shuffle partitions.
+      val bytesByPartitionId1 = Array[Long](0, 100, 40, 30, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 60, 0, 110)
+      val expectedPartitionSpecs = Seq(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 3),
+        CoalescedPartitionSpec(3, 4),
+        CoalescedPartitionSpec(4, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionSpecs,
+        targetSize)
+    }
+
+    {
+      // All pairs of shuffle partitions are larger than the targeted size.
+      val bytesByPartitionId1 = Array[Long](100, 100, 40, 30, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 60, 70, 110)
+      val expectedPartitionSpecs = Seq(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 3),
+        CoalescedPartitionSpec(3, 4),
+        CoalescedPartitionSpec(4, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionSpecs,
+        targetSize)
+    }
+  }
+
+  test("enforce minimal number of coalesced partitions") {
+    val targetSize = 100
+    val minNumPartitions = 2
+
+    {
+      // The minimal number of coalesced partitions is not enforced because
+      // the size of data is 0.
+      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
+      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionSpecs,
+        targetSize, minNumPartitions)
+    }
+
+    {
+      // The minimal number of coalesced partitions is enforced.
+      val bytesByPartitionId1 = Array[Long](10, 5, 5, 0, 20)
+      val bytesByPartitionId2 = Array[Long](5, 10, 0, 10, 5)
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 3), CoalescedPartitionSpec(3, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionSpecs,
+        targetSize, minNumPartitions)
+    }
+
+    {
+      // The number of coalesced partitions is determined by the algorithm.
+      val bytesByPartitionId1 = Array[Long](10, 50, 20, 80, 20)
+      val bytesByPartitionId2 = Array[Long](40, 10, 0, 10, 30)
+      val expectedPartitionSpecs = Seq(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 3),
+        CoalescedPartitionSpec(3, 4),
+        CoalescedPartitionSpec(4, 5))
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionSpecs,
+        targetSize, minNumPartitions)
+    }
+  }
+
+  test("splitSizeListByTargetSize") {
+    val targetSize = 100
+
+    // merge the small partitions at the beginning/end
+    val sizeList1 = Seq[Long](15, 90, 15, 15, 15, 90, 15)
+    assert(ShufflePartitionsUtil.splitSizeListByTargetSize(sizeList1, targetSize).toSeq ==
+      Seq(0, 2, 5))
+
+    // merge the small partitions in the middle
+    val sizeList2 = Seq[Long](30, 15, 90, 10, 90, 15, 30)
+    assert(ShufflePartitionsUtil.splitSizeListByTargetSize(sizeList2, targetSize).toSeq ==
+      Seq(0, 2, 4, 5))
+
+    // merge small partitions if the partition itself is smaller than
+    // targetSize * SMALL_PARTITION_FACTOR
+    val sizeList3 = Seq[Long](15, 1000, 15, 1000)
+    assert(ShufflePartitionsUtil.splitSizeListByTargetSize(sizeList3, targetSize).toSeq ==
+      Seq(0, 3))
+
+    // merge small partitions if the combined size is smaller than
+    // targetSize * MERGED_PARTITION_FACTOR
+    val sizeList4 = Seq[Long](35, 75, 90, 20, 35, 25, 35)
+    assert(ShufflePartitionsUtil.splitSizeListByTargetSize(sizeList4, targetSize).toSeq ==
+      Seq(0, 2, 3))
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala
index 7654a9d982059..6a4f3f62641f8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala
@@ -97,6 +97,19 @@ class SortSuite extends SparkPlanTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-33260: sort order is a Stream") {
+    val input = Seq(
+      ("Hello", 4, 2.0),
+      ("Hello", 1, 1.0),
+      ("World", 8, 3.0)
+    )
+    checkAnswer(
+      input.toDF("a", "b", "c"),
+      (child: SparkPlan) => SortExec(Stream('a.asc, 'b.asc, 'c.asc), global = true, child = child),
+      input.sortBy(t => (t._1, t._2, t._3)).map(Row.fromTuple),
+      sortAnswers = false)
+  }
+
   // Test sorting on different data types
   for (
     dataType <- DataTypeTestUtils.atomicTypes ++ Set(NullType);
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
index e3bc414516c04..56fff1107ae39 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
@@ -84,4 +84,8 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession {
       }
     }
   }
+
+  test("SPARK-30780 empty LocalTableScan should use RDD without partitions") {
+    assert(LocalTableScanExec(Nil, Nil).execute().getNumPartitions == 0)
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala
index b29e822add8bc..7ddf9d87a6aca 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala
@@ -23,6 +23,7 @@ import scala.util.control.NonFatal
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.{DataFrame, Row, SparkSession, SQLContext}
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.test.SQLTestUtils
 
 /**
@@ -237,7 +238,7 @@ object SparkPlanTest {
    * @param spark SqlContext used for execution of the plan
    */
   def executePlan(outputPlan: SparkPlan, spark: SQLContext): Seq[Row] = {
-    val execution = new QueryExecution(spark.sparkSession, null) {
+    val execution = new QueryExecution(spark.sparkSession, LocalRelation(Nil)) {
       override lazy val sparkPlan: SparkPlan = outputPlan transform {
         case plan: SparkPlan =>
           val inputMap = plan.children.flatMap(_.output).map(a => (a.name, a)).toMap
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
index 06574a9f8fd2c..343d3c1c13469 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
@@ -21,11 +21,12 @@ import org.apache.spark.sql.SaveMode
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedAlias, UnresolvedAttribute, UnresolvedRelation, UnresolvedStar}
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType}
-import org.apache.spark.sql.catalyst.expressions.{Ascending, Concat, SortOrder}
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, RepartitionByExpression, Sort}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, Concat, SortOrder}
+import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.datasources.{CreateTable, RefreshResource}
-import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
+import org.apache.spark.sql.execution.datasources.{CreateTable, CreateTempViewUsing, RefreshResource}
+import org.apache.spark.sql.internal.{HiveSerDe, SQLConf, StaticSQLConf}
 import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType}
 
 /**
@@ -80,6 +81,15 @@ class SparkSqlParserSuite extends AnalysisTest {
     intercept("REFRESH", "Resource paths cannot be empty in REFRESH statements")
   }
 
+  test("SPARK-33118 CREATE TMEPORARY TABLE with LOCATION") {
+    assertEqual("CREATE TEMPORARY TABLE t USING parquet OPTIONS (path '/data/tmp/testspark1')",
+      CreateTempViewUsing(TableIdentifier("t", None), None, false, false, "parquet",
+        Map("path" -> "/data/tmp/testspark1")))
+    assertEqual("CREATE TEMPORARY TABLE t USING parquet LOCATION '/data/tmp/testspark1'",
+      CreateTempViewUsing(TableIdentifier("t", None), None, false, false, "parquet",
+        Map("path" -> "/data/tmp/testspark1")))
+  }
+
   private def createTableUsing(
       table: String,
       database: Option[String] = None,
@@ -251,4 +261,44 @@ class SparkSqlParserSuite extends AnalysisTest {
     assertEqual("ADD FILE /path with space/abc.txt", AddFileCommand("/path with space/abc.txt"))
     assertEqual("ADD JAR /path with space/abc.jar", AddJarCommand("/path with space/abc.jar"))
   }
+
+  test("SPARK-32608: script transform with row format delimit") {
+    assertEqual(
+      """
+        |SELECT TRANSFORM(a, b, c)
+        |  ROW FORMAT DELIMITED
+        |  FIELDS TERMINATED BY ','
+        |  COLLECTION ITEMS TERMINATED BY '#'
+        |  MAP KEYS TERMINATED BY '@'
+        |  LINES TERMINATED BY '\n'
+        |  NULL DEFINED AS 'null'
+        |  USING 'cat' AS (a, b, c)
+        |  ROW FORMAT DELIMITED
+        |  FIELDS TERMINATED BY ','
+        |  COLLECTION ITEMS TERMINATED BY '#'
+        |  MAP KEYS TERMINATED BY '@'
+        |  LINES TERMINATED BY '\n'
+        |  NULL DEFINED AS 'NULL'
+        |FROM testData
+      """.stripMargin,
+      ScriptTransformation(
+        Seq('a, 'b, 'c),
+        "cat",
+        Seq(AttributeReference("a", StringType)(),
+          AttributeReference("b", StringType)(),
+          AttributeReference("c", StringType)()),
+        UnresolvedRelation(TableIdentifier("testData")),
+        ScriptInputOutputSchema(
+          Seq(("TOK_TABLEROWFORMATFIELD", ","),
+            ("TOK_TABLEROWFORMATCOLLITEMS", "#"),
+            ("TOK_TABLEROWFORMATMAPKEYS", "@"),
+            ("TOK_TABLEROWFORMATLINES", "\n"),
+            ("TOK_TABLEROWFORMATNULL", "null")),
+          Seq(("TOK_TABLEROWFORMATFIELD", ","),
+            ("TOK_TABLEROWFORMATCOLLITEMS", "#"),
+            ("TOK_TABLEROWFORMATMAPKEYS", "@"),
+            ("TOK_TABLEROWFORMATLINES", "\n"),
+            ("TOK_TABLEROWFORMATNULL", "NULL")), None, None,
+          List.empty, List.empty, None, None, false)))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala
index 8aa003a3dfeb0..f630cd8322c61 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala
@@ -210,23 +210,8 @@ class UnsafeKVExternalSorterSuite extends SparkFunSuite with SharedSparkSession
   test("SPARK-23376: Create UnsafeKVExternalSorter with BytesToByteMap having duplicated keys") {
     val memoryManager = new TestMemoryManager(new SparkConf())
     val taskMemoryManager = new TaskMemoryManager(memoryManager, 0)
-    val map = new BytesToBytesMap(taskMemoryManager, 64, taskMemoryManager.pageSizeBytes())
-
-    // Key/value are a unsafe rows with a single int column
+    val map = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
     val schema = new StructType().add("i", IntegerType)
-    val key = new UnsafeRow(1)
-    key.pointTo(new Array[Byte](32), 32)
-    key.setInt(0, 1)
-    val value = new UnsafeRow(1)
-    value.pointTo(new Array[Byte](32), 32)
-    value.setInt(0, 2)
-
-    for (_ <- 1 to 65) {
-      val loc = map.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes)
-      loc.append(
-        key.getBaseObject, key.getBaseOffset, key.getSizeInBytes,
-        value.getBaseObject, value.getBaseOffset, value.getSizeInBytes)
-    }
 
     // Make sure we can successfully create a UnsafeKVExternalSorter with a `BytesToBytesMap`
     // which has duplicated keys and the number of entries exceeds its capacity.
@@ -245,4 +230,82 @@ class UnsafeKVExternalSorterSuite extends SparkFunSuite with SharedSparkSession
       TaskContext.unset()
     }
   }
+
+  test("SPARK-31952: create UnsafeKVExternalSorter with existing map should count spilled memory " +
+    "size correctly") {
+    val memoryManager = new TestMemoryManager(new SparkConf())
+    val taskMemoryManager = new TaskMemoryManager(memoryManager, 0)
+    val map = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
+    val schema = new StructType().add("i", IntegerType)
+
+    try {
+      val context = new TaskContextImpl(0, 0, 0, 0, 0, taskMemoryManager, new Properties(), null)
+      TaskContext.setTaskContext(context)
+      val expectedSpillSize = map.getTotalMemoryConsumption
+      val sorter = new UnsafeKVExternalSorter(
+        schema,
+        schema,
+        sparkContext.env.blockManager,
+        sparkContext.env.serializerManager,
+        taskMemoryManager.pageSizeBytes(),
+        Int.MaxValue,
+        map)
+      assert(sorter.getSpillSize === expectedSpillSize)
+    } finally {
+      TaskContext.unset()
+    }
+  }
+
+  test("SPARK-31952: UnsafeKVExternalSorter.merge should accumulate totalSpillBytes") {
+    val memoryManager = new TestMemoryManager(new SparkConf())
+    val taskMemoryManager = new TaskMemoryManager(memoryManager, 0)
+    val map1 = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
+    val map2 = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
+    val schema = new StructType().add("i", IntegerType)
+
+    try {
+      val context = new TaskContextImpl(0, 0, 0, 0, 0, taskMemoryManager, new Properties(), null)
+      TaskContext.setTaskContext(context)
+      val expectedSpillSize = map1.getTotalMemoryConsumption + map2.getTotalMemoryConsumption
+      val sorter1 = new UnsafeKVExternalSorter(
+        schema,
+        schema,
+        sparkContext.env.blockManager,
+        sparkContext.env.serializerManager,
+        taskMemoryManager.pageSizeBytes(),
+        Int.MaxValue,
+        map1)
+      val sorter2 = new UnsafeKVExternalSorter(
+        schema,
+        schema,
+        sparkContext.env.blockManager,
+        sparkContext.env.serializerManager,
+        taskMemoryManager.pageSizeBytes(),
+        Int.MaxValue,
+        map2)
+      sorter1.merge(sorter2)
+      assert(sorter1.getSpillSize === expectedSpillSize)
+    } finally {
+      TaskContext.unset()
+    }
+  }
+
+  private def createBytesToBytesMapWithDuplicateKeys(taskMemoryManager: TaskMemoryManager)
+    : BytesToBytesMap = {
+    val map = new BytesToBytesMap(taskMemoryManager, 64, taskMemoryManager.pageSizeBytes())
+    // Key/value are a unsafe rows with a single int column
+    val key = new UnsafeRow(1)
+    key.pointTo(new Array[Byte](32), 32)
+    key.setInt(0, 1)
+    val value = new UnsafeRow(1)
+    value.pointTo(new Array[Byte](32), 32)
+    value.setInt(0, 2)
+    for (_ <- 1 to 65) {
+      val loc = map.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes)
+      loc.append(
+        key.getBaseObject, key.getBaseOffset, key.getSizeInBytes,
+        value.getBaseObject, value.getBaseOffset, value.getSizeInBytes)
+    }
+    map
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
index f6814d8ff8a3d..c5a01de911962 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution
 
 import org.scalatest.{Assertions, BeforeAndAfterEach, Matchers}
 import org.scalatest.concurrent.TimeLimits
+import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.{SparkFunSuite, TestUtils}
 import org.apache.spark.deploy.SparkSubmitSuite
@@ -50,7 +51,7 @@ class WholeStageCodegenSparkSubmitSuite extends SparkFunSuite
       "--conf", "spark.executor.extraJavaOptions=-XX:+UseCompressedOops",
       "--conf", "spark.sql.adaptive.enabled=false",
       unusedJar.toString)
-    SparkSubmitSuite.runSparkSubmit(argsForSparkSubmit, "../..")
+    SparkSubmitSuite.runSparkSubmit(argsForSparkSubmit, "../..", 3.minutes)
   }
 }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
index 06a016fac5300..f7396ee2a89c8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution
 
 import org.apache.spark.sql.{Dataset, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.catalyst.expressions.codegen.{ByteCodeStats, CodeAndComment, CodeGenerator}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
@@ -28,23 +29,12 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
 
-class WholeStageCodegenSuite extends QueryTest with SharedSparkSession {
+// Disable AQE because the WholeStageCodegenExec is added when running QueryStageExec
+class WholeStageCodegenSuite extends QueryTest with SharedSparkSession
+  with DisableAdaptiveExecutionSuite {
 
   import testImplicits._
 
-  var originalValue: String = _
-  // With on AQE, the WholeStageCodegenExec is added when running QueryStageExec.
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
-
   test("range/filter should be combined") {
     val df = spark.range(10).filter("id = 1").selectExpr("id + 1")
     val plan = df.queryExecution.executedPlan
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 78a1183664749..6d97a6bb47d0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -20,14 +20,21 @@ package org.apache.spark.sql.execution.adaptive
 import java.io.File
 import java.net.URI
 
+import org.apache.log4j.Level
+
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
-import org.apache.spark.sql.QueryTest
-import org.apache.spark.sql.execution.{ReusedSubqueryExec, SparkPlan}
-import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec}
+import org.apache.spark.sql.{QueryTest, Row, SparkSession, Strategy}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
+import org.apache.spark.sql.execution.{PartialReducerPartitionSpec, ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
+import org.apache.spark.sql.execution.adaptive.OptimizeLocalShuffleReader.LOCAL_SHUFFLE_READER_DESCRIPTION
+import org.apache.spark.sql.execution.command.DataWritingCommandExec
+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec, ShuffleExchangeExec}
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildRight, SortMergeJoinExec}
 import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{IntegerType, StructType}
 import org.apache.spark.util.Utils
 
 class AdaptiveQueryExecSuite
@@ -46,7 +53,7 @@ class AdaptiveQueryExecSuite
         event match {
           case SparkListenerSQLAdaptiveExecutionUpdate(_, _, sparkPlanInfo) =>
             if (sparkPlanInfo.simpleString.startsWith(
-              "AdaptiveSparkPlan(isFinalPlan=true)")) {
+              "AdaptiveSparkPlan isFinalPlan=true")) {
               finalPlanCnt += 1
             }
           case _ => // ignore other events
@@ -57,20 +64,23 @@ class AdaptiveQueryExecSuite
 
     val dfAdaptive = sql(query)
     val planBefore = dfAdaptive.queryExecution.executedPlan
-    assert(planBefore.toString.startsWith("AdaptiveSparkPlan(isFinalPlan=false)"))
+    assert(planBefore.toString.startsWith("AdaptiveSparkPlan isFinalPlan=false"))
     val result = dfAdaptive.collect()
     withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
       val df = sql(query)
-      QueryTest.sameRows(result.toSeq, df.collect().toSeq)
+      checkAnswer(df, result)
     }
     val planAfter = dfAdaptive.queryExecution.executedPlan
-    assert(planAfter.toString.startsWith("AdaptiveSparkPlan(isFinalPlan=true)"))
+    assert(planAfter.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true"))
+    val adaptivePlan = planAfter.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
 
     spark.sparkContext.listenerBus.waitUntilEmpty()
-    assert(finalPlanCnt == 1)
+    // AQE will post `SparkListenerSQLAdaptiveExecutionUpdate` twice in case of subqueries that
+    // exist out of query stages.
+    val expectedFinalPlanCnt = adaptivePlan.find(_.subqueries.nonEmpty).map(_ => 2).getOrElse(1)
+    assert(finalPlanCnt == expectedFinalPlanCnt)
     spark.sparkContext.removeSparkListener(listener)
 
-    val adaptivePlan = planAfter.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
     val exchanges = adaptivePlan.collect {
       case e: Exchange => e
     }
@@ -91,14 +101,14 @@ class AdaptiveQueryExecSuite
   }
 
   private def findReusedExchange(plan: SparkPlan): Seq[ReusedExchangeExec] = {
-    collectInPlanAndSubqueries(plan) {
+    collectWithSubqueries(plan) {
       case ShuffleQueryStageExec(_, e: ReusedExchangeExec) => e
       case BroadcastQueryStageExec(_, e: ReusedExchangeExec) => e
     }
   }
 
   private def findReusedSubquery(plan: SparkPlan): Seq[ReusedSubqueryExec] = {
-    collectInPlanAndSubqueries(plan) {
+    collectWithSubqueries(plan) {
       case e: ReusedSubqueryExec => e
     }
   }
@@ -110,10 +120,14 @@ class AdaptiveQueryExecSuite
     }.length
 
     val numLocalReaders = collect(plan) {
-      case reader: LocalShuffleReaderExec => reader
-    }.length
-
-    assert(numShuffles === (numLocalReaders + numShufflesWithoutLocalReader))
+      case reader @ CustomShuffleReaderExec(_, _, LOCAL_SHUFFLE_READER_DESCRIPTION) => reader
+    }
+    numLocalReaders.foreach { r =>
+      val rdd = r.execute()
+      val parts = rdd.partitions
+      assert(parts.forall(rdd.preferredLocations(_).nonEmpty))
+    }
+    assert(numShuffles === (numLocalReaders.length + numShufflesWithoutLocalReader))
   }
 
   test("Change merge join to broadcast join") {
@@ -134,7 +148,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80",
-      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "10") {
+      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "10") {
       val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(
         "SELECT * FROM testData join testData2 ON key = a where value = '1'")
       val smj = findTopLevelSortMergeJoin(plan)
@@ -142,11 +156,11 @@ class AdaptiveQueryExecSuite
       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
       assert(bhj.size == 1)
       val localReaders = collect(adaptivePlan) {
-        case reader: LocalShuffleReaderExec => reader
+        case reader @ CustomShuffleReaderExec(_, _, LOCAL_SHUFFLE_READER_DESCRIPTION) => reader
       }
       assert(localReaders.length == 2)
-      val localShuffleRDD0 = localReaders(0).execute().asInstanceOf[LocalShuffledRowRDD]
-      val localShuffleRDD1 = localReaders(1).execute().asInstanceOf[LocalShuffledRowRDD]
+      val localShuffleRDD0 = localReaders(0).execute().asInstanceOf[ShuffledRowRDD]
+      val localShuffleRDD1 = localReaders(1).execute().asInstanceOf[ShuffledRowRDD]
       // The pre-shuffle partition size is [0, 0, 0, 72, 0]
       // And the partitionStartIndices is [0, 3, 4], so advisoryParallelism = 3.
       // the final parallelism is
@@ -166,7 +180,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80",
-      SQLConf.REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key -> "false") {
+      SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "false") {
       val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(
         "SELECT * FROM testData join testData2 ON key = a where value = '1'")
       val smj = findTopLevelSortMergeJoin(plan)
@@ -174,11 +188,11 @@ class AdaptiveQueryExecSuite
       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
       assert(bhj.size == 1)
       val localReaders = collect(adaptivePlan) {
-        case reader: LocalShuffleReaderExec => reader
+        case reader @ CustomShuffleReaderExec(_, _, LOCAL_SHUFFLE_READER_DESCRIPTION) => reader
       }
       assert(localReaders.length == 2)
-      val localShuffleRDD0 = localReaders(0).execute().asInstanceOf[LocalShuffledRowRDD]
-      val localShuffleRDD1 = localReaders(1).execute().asInstanceOf[LocalShuffledRowRDD]
+      val localShuffleRDD0 = localReaders(0).execute().asInstanceOf[ShuffledRowRDD]
+      val localShuffleRDD1 = localReaders(1).execute().asInstanceOf[ShuffledRowRDD]
       // the final parallelism is math.max(1, numReduces / numMappers): math.max(1, 5/2) = 2
       // and the partitions length is 2 * numMappers = 4
       assert(localShuffleRDD0.getPartitions.length == 4)
@@ -583,171 +597,92 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key -> "100",
-      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "700") {
+      SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100",
+      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100") {
       withTempView("skewData1", "skewData2") {
         spark
           .range(0, 1000, 1, 10)
-          .selectExpr("id % 2 as key1", "id as value1")
+          .selectExpr("id % 3 as key1", "id as value1")
           .createOrReplaceTempView("skewData1")
         spark
           .range(0, 1000, 1, 10)
           .selectExpr("id % 1 as key2", "id as value2")
           .createOrReplaceTempView("skewData2")
-        val (innerPlan, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
-          "SELECT key1 FROM skewData1 join skewData2 ON key1 = key2 group by key1")
-        val innerSmj = findTopLevelSortMergeJoin(innerPlan)
-        assert(innerSmj.size == 1)
+
+        def checkSkewJoin(query: String, optimizeSkewJoin: Boolean): Unit = {
+          val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(query)
+          val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
+          assert(innerSmj.size == 1 && innerSmj.head.isSkewJoin == optimizeSkewJoin)
+        }
+
+        checkSkewJoin(
+          "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2", true)
         // Additional shuffle introduced, so disable the "OptimizeSkewedJoin" optimization
-        val innerSmjAfter = findTopLevelSortMergeJoin(innerAdaptivePlan)
-        assert(innerSmjAfter.size == 1)
+        checkSkewJoin(
+          "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 GROUP BY key1", false)
       }
     }
   }
 
   test("SPARK-29544: adaptive skew join with different join types") {
-    Seq("false", "true").foreach { reducePostShufflePartitionsEnabled =>
-      withSQLConf(
-        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-        SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-        SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key -> "100",
-        SQLConf.REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key -> reducePostShufflePartitionsEnabled,
-        SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "700") {
-        withTempView("skewData1", "skewData2") {
-          spark
-            .range(0, 1000, 1, 10)
-            .selectExpr("id % 2 as key1", "id as value1")
-            .createOrReplaceTempView("skewData1")
-          spark
-            .range(0, 1000, 1, 10)
-            .selectExpr("id % 1 as key2", "id as value2")
-            .createOrReplaceTempView("skewData2")
-          // skewed inner join optimization
-          val (innerPlan, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
-            "SELECT * FROM skewData1 join skewData2 ON key1 = key2")
-          val innerSmj = findTopLevelSortMergeJoin(innerPlan)
-          assert(innerSmj.size == 1)
-          // left stats: [3496, 0, 0, 0, 4014]
-          // right stats:[6292, 0, 0, 0, 0]
-          // the partition 0 in both left and right side are all skewed.
-          // And divide into 5 splits both in left and right (the max splits number).
-          // So there are 5 x 5 smjs for partition 0.
-          // Partition 4 in left side is skewed and is divided into 5 splits.
-          // The right side of partition 4 is not skewed.
-          // So there are 5 smjs for partition 4.
-          // So total (25 + 5 + 1) smjs.
-          // Union
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //             .
-          //             .
-          //             .
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-
-          val innerSmjAfter = findTopLevelSortMergeJoin(innerAdaptivePlan)
-          assert(innerSmjAfter.size == 31)
-
-          // skewed left outer join optimization
-          val (leftPlan, leftAdaptivePlan) = runAdaptiveAndVerifyResult(
-            "SELECT * FROM skewData1 left outer join skewData2 ON key1 = key2")
-          val leftSmj = findTopLevelSortMergeJoin(leftPlan)
-          assert(leftSmj.size == 1)
-          // left stats: [3496, 0, 0, 0, 4014]
-          // right stats:[6292, 0, 0, 0, 0]
-          // The partition 0 in both left and right are all skewed.
-          // The partition 4 in left side is skewed.
-          // But for left outer join, we don't split the right partition even skewed.
-          // So the partition 0 in left side is divided into 5 splits(the max split number).
-          // the partition 4 in left side is divided into 5 splits(the max split number).
-          // So total (5 + 5 + 1) smjs.
-          // Union
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //             .
-          //             .
-          //             .
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-
-          val leftSmjAfter = findTopLevelSortMergeJoin(leftAdaptivePlan)
-          assert(leftSmjAfter.size == 11)
-
-          // skewed right outer join optimization
-          val (rightPlan, rightAdaptivePlan) = runAdaptiveAndVerifyResult(
-            "SELECT * FROM skewData1 right outer join skewData2 ON key1 = key2")
-          val rightSmj = findTopLevelSortMergeJoin(rightPlan)
-          assert(rightSmj.size == 1)
-          // left stats: [3496, 0, 0, 0, 4014]
-          // right stats:[6292, 0, 0, 0, 0]
-          // The partition 0 in both left and right side are all skewed.
-          // And the partition 4 in left side is skewed.
-          // But for right outer join, we don't split the left partition even skewed.
-          // And divide right side into 5 splits(the max split number)
-          // So total 6 smjs.
-          // Union
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //             .
-          //             .
-          //             .
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-
-          val rightSmjAfter = findTopLevelSortMergeJoin(rightAdaptivePlan)
-          assert(rightSmjAfter.size == 6)
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+      SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1",
+      SQLConf.SHUFFLE_PARTITIONS.key -> "100",
+      SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "800",
+      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "800") {
+      withTempView("skewData1", "skewData2") {
+        spark
+          .range(0, 1000, 1, 10)
+          .select(
+            when('id < 250, 249)
+              .when('id >= 750, 1000)
+              .otherwise('id).as("key1"),
+            'id as "value1")
+          .createOrReplaceTempView("skewData1")
+        spark
+          .range(0, 1000, 1, 10)
+          .select(
+            when('id < 250, 249)
+              .otherwise('id).as("key2"),
+            'id as "value2")
+          .createOrReplaceTempView("skewData2")
+
+        def checkSkewJoin(
+            joins: Seq[SortMergeJoinExec],
+            leftSkewNum: Int,
+            rightSkewNum: Int): Unit = {
+          assert(joins.size == 1 && joins.head.isSkewJoin)
+          assert(joins.head.left.collect {
+            case r: CustomShuffleReaderExec => r
+          }.head.partitionSpecs.collect {
+            case p: PartialReducerPartitionSpec => p.reducerIndex
+          }.distinct.length == leftSkewNum)
+          assert(joins.head.right.collect {
+            case r: CustomShuffleReaderExec => r
+          }.head.partitionSpecs.collect {
+            case p: PartialReducerPartitionSpec => p.reducerIndex
+          }.distinct.length == rightSkewNum)
         }
+
+        // skewed inner join optimization
+        val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
+          "SELECT * FROM skewData1 join skewData2 ON key1 = key2")
+        val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
+        checkSkewJoin(innerSmj, 2, 1)
+
+        // skewed left outer join optimization
+        val (_, leftAdaptivePlan) = runAdaptiveAndVerifyResult(
+          "SELECT * FROM skewData1 left outer join skewData2 ON key1 = key2")
+        val leftSmj = findTopLevelSortMergeJoin(leftAdaptivePlan)
+        checkSkewJoin(leftSmj, 2, 0)
+
+        // skewed right outer join optimization
+        val (_, rightAdaptivePlan) = runAdaptiveAndVerifyResult(
+          "SELECT * FROM skewData1 right outer join skewData2 ON key1 = key2")
+        val rightSmj = findTopLevelSortMergeJoin(rightAdaptivePlan)
+        checkSkewJoin(rightSmj, 0, 1)
       }
     }
   }
@@ -768,7 +703,8 @@ class AdaptiveQueryExecSuite
         val error = intercept[Exception] {
           agged.count()
         }
-        assert(error.getCause().toString contains "Failed to materialize query stage")
+        assert(error.getCause().toString contains "Invalid bucket file")
+        assert(error.getSuppressed.size === 0)
       }
     }
   }
@@ -780,4 +716,191 @@ class AdaptiveQueryExecSuite
       )
     }
   }
+
+  test("force apply AQE") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
+      val plan = sql("SELECT * FROM testData").queryExecution.executedPlan
+      assert(plan.isInstanceOf[AdaptiveSparkPlanExec])
+    }
+  }
+
+  test("SPARK-30719: do not log warning if intentionally skip AQE") {
+    val testAppender = new LogAppender("aqe logging warning test when skip")
+    withLogAppender(testAppender) {
+      withSQLConf(
+        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+        val plan = sql("SELECT * FROM testData").queryExecution.executedPlan
+        assert(!plan.isInstanceOf[AdaptiveSparkPlanExec])
+      }
+    }
+    assert(!testAppender.loggingEvents
+      .exists(msg => msg.getRenderedMessage.contains(
+        s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} is" +
+        s" enabled but is not supported for")))
+  }
+
+  test("test log level") {
+    def verifyLog(expectedLevel: Level): Unit = {
+      val logAppender = new LogAppender("adaptive execution")
+      withLogAppender(
+        logAppender,
+        loggerName = Some(AdaptiveSparkPlanExec.getClass.getName.dropRight(1)),
+        level = Some(Level.TRACE)) {
+        withSQLConf(
+          SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+          SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
+          sql("SELECT * FROM testData join testData2 ON key = a where value = '1'").collect()
+        }
+      }
+      Seq("Plan changed", "Final plan").foreach { msg =>
+        assert(
+          logAppender.loggingEvents.exists { event =>
+            event.getRenderedMessage.contains(msg) && event.getLevel == expectedLevel
+          })
+      }
+    }
+
+    // Verify default log level
+    verifyLog(Level.DEBUG)
+
+    // Verify custom log level
+    val levels = Seq(
+      "TRACE" -> Level.TRACE,
+      "trace" -> Level.TRACE,
+      "DEBUG" -> Level.DEBUG,
+      "debug" -> Level.DEBUG,
+      "INFO" -> Level.INFO,
+      "info" -> Level.INFO,
+      "WARN" -> Level.WARN,
+      "warn" -> Level.WARN,
+      "ERROR" -> Level.ERROR,
+      "error" -> Level.ERROR,
+      "deBUG" -> Level.DEBUG)
+
+    levels.foreach { level =>
+      withSQLConf(SQLConf.ADAPTIVE_EXECUTION_LOG_LEVEL.key -> level._1) {
+        verifyLog(level._2)
+      }
+    }
+  }
+
+  test("SPARK-31384: avoid NPE in OptimizeSkewedJoin when there's 0 partition plan") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      withTempView("t2") {
+        // create DataFrame with 0 partition
+        spark.createDataFrame(sparkContext.emptyRDD[Row], new StructType().add("b", IntegerType))
+          .createOrReplaceTempView("t2")
+        // should run successfully without NPE
+        runAdaptiveAndVerifyResult("SELECT * FROM testData2 t1 left semi join t2 ON t1.a=t2.b")
+      }
+    }
+  }
+
+  test("SPARK-30953: InsertAdaptiveSparkPlan should apply AQE on child plan of write commands") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
+      withTable("t1") {
+        val plan = sql("CREATE TABLE t1 USING parquet AS SELECT 1 col").queryExecution.executedPlan
+        assert(plan.isInstanceOf[DataWritingCommandExec])
+        assert(plan.asInstanceOf[DataWritingCommandExec].child.isInstanceOf[AdaptiveSparkPlanExec])
+      }
+    }
+  }
+
+  test("AQE should set active session during execution") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      val df = spark.range(10).select(sum('id))
+      assert(df.queryExecution.executedPlan.isInstanceOf[AdaptiveSparkPlanExec])
+      SparkSession.setActiveSession(null)
+      checkAnswer(df, Seq(Row(45)))
+      SparkSession.setActiveSession(spark) // recover the active session.
+    }
+  }
+
+  test("No deadlock in UI update") {
+    object TestStrategy extends Strategy {
+      def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
+        case _: Aggregate =>
+          withSQLConf(
+            SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+            SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
+            spark.range(5).rdd
+          }
+          Nil
+        case _ => Nil
+      }
+    }
+
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
+      try {
+        spark.experimental.extraStrategies = TestStrategy :: Nil
+        val df = spark.range(10).groupBy('id).count()
+        df.collect()
+      } finally {
+        spark.experimental.extraStrategies = Nil
+      }
+    }
+  }
+
+  test("SPARK-31658: SQL UI should show write commands") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
+      withTable("t1") {
+        var checkDone = false
+        val listener = new SparkListener {
+          override def onOtherEvent(event: SparkListenerEvent): Unit = {
+            event match {
+              case SparkListenerSQLAdaptiveExecutionUpdate(_, _, planInfo) =>
+                assert(planInfo.nodeName == "Execute CreateDataSourceTableAsSelectCommand")
+                checkDone = true
+              case _ => // ignore other events
+            }
+          }
+        }
+        spark.sparkContext.addSparkListener(listener)
+        try {
+          sql("CREATE TABLE t1 USING parquet AS SELECT 1 col").collect()
+          spark.sparkContext.listenerBus.waitUntilEmpty()
+          assert(checkDone)
+        } finally {
+          spark.sparkContext.removeSparkListener(listener)
+        }
+      }
+    }
+  }
+
+  test("SPARK-31220 repartition obeys initialPartitionNum when adaptiveExecutionEnabled") {
+    Seq(true, false).foreach { enableAQE =>
+      withSQLConf(
+        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> enableAQE.toString,
+        SQLConf.SHUFFLE_PARTITIONS.key -> "6",
+        SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "7") {
+        val partitionsNum = spark.range(10).repartition($"id").rdd.collectPartitions().length
+        if (enableAQE) {
+          assert(partitionsNum === 7)
+        } else {
+          assert(partitionsNum === 6)
+        }
+      }
+    }
+  }
+
+  test("SPARK-32753: Only copy tags to node with no tags") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      withTempView("v1") {
+        spark.range(10).union(spark.range(10)).createOrReplaceTempView("v1")
+
+        val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
+          "SELECT id FROM v1 GROUP BY id DISTRIBUTE BY id")
+        assert(collect(adaptivePlan) {
+          case s: ShuffleExchangeExec => s
+        }.length == 1)
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala
new file mode 100644
index 0000000000000..48f85ae76cd8c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import java.io.{PrintWriter, StringWriter}
+
+import org.scalactic.source.Position
+import org.scalatest.Tag
+
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SQLTestUtils
+
+/**
+ * Test with this tag will be ignored if the test suite extends `EnableAdaptiveExecutionSuite`.
+ * Otherwise, it will be executed with adaptive execution disabled.
+ */
+case class DisableAdaptiveExecution(reason: String) extends Tag("DisableAdaptiveExecution")
+
+/**
+ * Helper trait that enables AQE for all tests regardless of default config values, except that
+ * tests tagged with [[DisableAdaptiveExecution]] will be skipped.
+ */
+trait EnableAdaptiveExecutionSuite extends SQLTestUtils {
+  protected val forceApply = true
+
+  override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
+      (implicit pos: Position): Unit = {
+    if (testTags.exists(_.isInstanceOf[DisableAdaptiveExecution])) {
+      // we ignore the test here but assume that another test suite which extends
+      // `DisableAdaptiveExecutionSuite` will test it anyway to ensure test coverage
+      ignore(testName + " (disabled when AQE is on)", testTags: _*)(testFun)
+    } else {
+      super.test(testName, testTags: _*) {
+        withSQLConf(
+          SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+          SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> forceApply.toString) {
+          testFun
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Helper trait that disables AQE for all tests regardless of default config values.
+ */
+trait DisableAdaptiveExecutionSuite extends SQLTestUtils {
+  override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
+      (implicit pos: Position): Unit = {
+    super.test(testName, testTags: _*) {
+      withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+        testFun
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
index fdb23d5be78a1..1e6e59456c887 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
@@ -1208,15 +1208,13 @@ class ArrowConvertersSuite extends SharedSparkSession {
     spark.conf.unset(SQLConf.ARROW_EXECUTION_MAX_RECORDS_PER_BATCH.key)
   }
 
-  testQuietly("unsupported types") {
-    def runUnsupported(block: => Unit): Unit = {
-      val msg = intercept[UnsupportedOperationException] {
-        block
-      }
-      assert(msg.getMessage.contains("is not supported"))
+  testQuietly("interval is unsupported for arrow") {
+    val e = intercept[SparkException] {
+      calenderIntervalData.toDF().toArrowBatchRdd.collect()
     }
 
-    runUnsupported { calenderIntervalData.toDF().toArrowBatchRdd.collect() }
+    assert(e.getCause.isInstanceOf[UnsupportedOperationException])
+    assert(e.getCause.getMessage.contains("Unsupported data type: interval"))
   }
 
   test("test Arrow Validator") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala
index a084bec985510..0fc43c7052d06 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala
@@ -23,7 +23,6 @@ import scala.util.Random
 
 import org.apache.spark.SparkConf
 import org.apache.spark.benchmark.Benchmark
-import org.apache.spark.internal.config.UI._
 import org.apache.spark.sql.{DataFrame, DataFrameWriter, Row, SparkSession}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.datasources.parquet.{SpecificParquetRecordReaderBase, VectorizedParquetRecordReader}
@@ -52,7 +51,6 @@ object DataSourceReadBenchmark extends SqlBasedBenchmark {
       .set("spark.master", "local[1]")
       .setIfMissing("spark.driver.memory", "3g")
       .setIfMissing("spark.executor.memory", "3g")
-      .setIfMissing(UI_ENABLED, false)
 
     val sparkSession = SparkSession.builder.config(conf).getOrCreate()
 
@@ -169,7 +167,7 @@ object DataSourceReadBenchmark extends SqlBasedBenchmark {
 
           files.map(_.asInstanceOf[String]).foreach { p =>
             val reader = new VectorizedParquetRecordReader(
-              null, enableOffHeapColumnVector, vectorizedReaderBatchSize)
+              enableOffHeapColumnVector, vectorizedReaderBatchSize)
             try {
               reader.initialize(p, ("id" :: Nil).asJava)
               val batch = reader.resultBatch()
@@ -203,7 +201,7 @@ object DataSourceReadBenchmark extends SqlBasedBenchmark {
 
           files.map(_.asInstanceOf[String]).foreach { p =>
             val reader = new VectorizedParquetRecordReader(
-              null, enableOffHeapColumnVector, vectorizedReaderBatchSize)
+              enableOffHeapColumnVector, vectorizedReaderBatchSize)
             try {
               reader.initialize(p, ("id" :: Nil).asJava)
               val batch = reader.resultBatch()
@@ -458,7 +456,7 @@ object DataSourceReadBenchmark extends SqlBasedBenchmark {
           var sum = 0
           files.map(_.asInstanceOf[String]).foreach { p =>
             val reader = new VectorizedParquetRecordReader(
-              null, enableOffHeapColumnVector, vectorizedReaderBatchSize)
+              enableOffHeapColumnVector, vectorizedReaderBatchSize)
             try {
               reader.initialize(p, ("c1" :: "c2" :: Nil).asJava)
               val batch = reader.resultBatch()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index 086583fdafe6d..c7b8737b7a753 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -17,9 +17,15 @@
 
 package org.apache.spark.sql.execution.benchmark
 
-import java.sql.Timestamp
+import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate}
 
 import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.Dataset
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
+import org.apache.spark.sql.execution.HiveResult
+import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Synthetic benchmark for date and timestamp functions.
@@ -53,96 +59,192 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
   }
 
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
-    val N = 10000000
-    runBenchmark("Extract components") {
-      run(N, "cast to timestamp", "cast(id as timestamp)")
-      run(N, "year")
-      run(N, "quarter")
-      run(N, "month")
-      run(N, "weekofyear")
-      run(N, "day")
-      run(N, "dayofyear")
-      run(N, "dayofmonth")
-      run(N, "dayofweek")
-      run(N, "weekday")
-      run(N, "hour")
-      run(N, "minute")
-      run(N, "second")
-    }
-    runBenchmark("Current date and time") {
-      run(N, "current_date", "current_date")
-      run(N, "current_timestamp", "current_timestamp")
-    }
-    runBenchmark("Date arithmetic") {
-      val dateExpr = "cast(cast(id as timestamp) as date)"
-      run(N, "cast to date", dateExpr)
-      run(N, "last_day", s"last_day($dateExpr)")
-      run(N, "next_day", s"next_day($dateExpr, 'TU')")
-      run(N, "date_add", s"date_add($dateExpr, 10)")
-      run(N, "date_sub", s"date_sub($dateExpr, 10)")
-      run(N, "add_months", s"add_months($dateExpr, 10)")
-    }
-    runBenchmark("Formatting dates") {
-      val dateExpr = "cast(cast(id as timestamp) as date)"
-      run(N, "format date", s"date_format($dateExpr, 'MMM yyyy')")
-    }
-    runBenchmark("Formatting timestamps") {
-      run(N, "from_unixtime", "from_unixtime(id, 'yyyy-MM-dd HH:mm:ss.SSSSSS')")
-    }
-    runBenchmark("Convert timestamps") {
-      val timestampExpr = "cast(id as timestamp)"
-      run(N, "from_utc_timestamp", s"from_utc_timestamp($timestampExpr, 'CET')")
-      run(N, "to_utc_timestamp", s"to_utc_timestamp($timestampExpr, 'CET')")
-    }
-    runBenchmark("Intervals") {
-      val (start, end) = ("cast(id as timestamp)", "cast((id+8640000) as timestamp)")
-      run(N, "cast interval", start, end)
-      run(N, "datediff", s"datediff($start, $end)")
-      run(N, "months_between", s"months_between($start, $end)")
-      run(1000000, "window", s"window($start, 100, 10, 1)")
-    }
-    runBenchmark("Truncation") {
-      val timestampExpr = "cast(id as timestamp)"
-      Seq("YEAR", "YYYY", "YY", "MON", "MONTH", "MM", "DAY", "DD", "HOUR", "MINUTE",
-          "SECOND", "WEEK", "QUARTER").foreach { level =>
-        run(N, s"date_trunc $level", s"date_trunc('$level', $timestampExpr)")
-      }
-      val dateExpr = "cast(cast(id as timestamp) as date)"
-      Seq("year", "yyyy", "yy", "mon", "month", "mm").foreach { level =>
-        run(N, s"trunc $level", s"trunc('$level', $dateExpr)")
-      }
-    }
-    runBenchmark("Parsing") {
-      val n = 1000000
-      val timestampStrExpr = "concat('2019-01-27 11:02:01.', cast(mod(id, 1000) as string))"
-      val pattern = "'yyyy-MM-dd HH:mm:ss.SSS'"
-      run(n, "to timestamp str", timestampStrExpr)
-      run(n, "to_timestamp", s"to_timestamp($timestampStrExpr, $pattern)")
-      run(n, "to_unix_timestamp", s"to_unix_timestamp($timestampStrExpr, $pattern)")
-      val dateStrExpr = "concat('2019-01-', cast(mod(id, 25) as string))"
-      run(n, "to date str", dateStrExpr)
-      run(n, "to_date", s"to_date($dateStrExpr, 'yyyy-MM-dd')")
-    }
-    runBenchmark("Conversion from/to external types") {
-      import spark.implicits._
-      val rowsNum = 5000000
-      val numIters = 3
-      val benchmark = new Benchmark("To/from java.sql.Timestamp", rowsNum, output = output)
-      benchmark.addCase("From java.sql.Timestamp", numIters) { _ =>
-        spark.range(rowsNum)
-          .map(millis => new Timestamp(millis))
-          .noop()
-      }
-      benchmark.addCase("Collect longs", numIters) { _ =>
-        spark.range(0, rowsNum, 1, 1)
-          .collect()
-      }
-      benchmark.addCase("Collect timestamps", numIters) { _ =>
-        spark.range(0, rowsNum, 1, 1)
-          .map(millis => new Timestamp(millis))
-          .collect()
+    withDefaultTimeZone(LA) {
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId) {
+        val N = 10000000
+        runBenchmark("datetime +/- interval") {
+          val benchmark = new Benchmark("datetime +/- interval", N, output = output)
+          val ts = "cast(id as timestamp)"
+          val dt = s"cast($ts as date)"
+          benchmark.addCase("date + interval(m)") { _ =>
+            doBenchmark(N, s"$dt + interval 1 month")
+          }
+          benchmark.addCase("date + interval(m, d)") { _ =>
+            doBenchmark(N, s"$dt + interval 1 month 2 day")
+          }
+          benchmark.addCase("date + interval(m, d, ms)") { _ =>
+            doBenchmark(N, s"$dt + interval 1 month 2 day 5 hour")
+          }
+          benchmark.addCase("date - interval(m)") { _ =>
+            doBenchmark(N, s"$dt - interval 1 month")
+          }
+          benchmark.addCase("date - interval(m, d)") { _ =>
+            doBenchmark(N, s"$dt - interval 1 month 2 day")
+          }
+          benchmark.addCase("date - interval(m, d, ms)") { _ =>
+            doBenchmark(N, s"$dt - interval 1 month 2 day 5 hour")
+          }
+          benchmark.addCase("timestamp + interval(m)") { _ =>
+            doBenchmark(N, s"$ts + interval 1 month")
+          }
+          benchmark.addCase("timestamp + interval(m, d)") { _ =>
+            doBenchmark(N, s"$ts + interval 1 month 2 day")
+          }
+          benchmark.addCase("timestamp + interval(m, d, ms)") { _ =>
+            doBenchmark(N, s"$ts + interval 1 month 2 day 5 hour")
+          }
+          benchmark.addCase("timestamp - interval(m)") { _ =>
+            doBenchmark(N, s"$ts - interval 1 month")
+          }
+          benchmark.addCase("timestamp - interval(m, d)") { _ =>
+            doBenchmark(N, s"$ts - interval 1 month 2 day")
+          }
+          benchmark.addCase("timestamp - interval(m, d, ms)") { _ =>
+            doBenchmark(N, s"$ts - interval 1 month 2 day 5 hour")
+          }
+          benchmark.run()
+        }
+        runBenchmark("Extract components") {
+          run(N, "cast to timestamp", "cast(id as timestamp)")
+          run(N, "year")
+          run(N, "quarter")
+          run(N, "month")
+          run(N, "weekofyear")
+          run(N, "day")
+          run(N, "dayofyear")
+          run(N, "dayofmonth")
+          run(N, "dayofweek")
+          run(N, "weekday")
+          run(N, "hour")
+          run(N, "minute")
+          run(N, "second")
+        }
+        runBenchmark("Current date and time") {
+          run(N, "current_date", "current_date")
+          run(N, "current_timestamp", "current_timestamp")
+        }
+        runBenchmark("Date arithmetic") {
+          val dateExpr = "cast(cast(id as timestamp) as date)"
+          run(N, "cast to date", dateExpr)
+          run(N, "last_day", s"last_day($dateExpr)")
+          run(N, "next_day", s"next_day($dateExpr, 'TU')")
+          run(N, "date_add", s"date_add($dateExpr, 10)")
+          run(N, "date_sub", s"date_sub($dateExpr, 10)")
+          run(N, "add_months", s"add_months($dateExpr, 10)")
+        }
+        runBenchmark("Formatting dates") {
+          val dateExpr = "cast(cast(id as timestamp) as date)"
+          run(N, "format date", s"date_format($dateExpr, 'MMM yyyy')")
+        }
+        runBenchmark("Formatting timestamps") {
+          run(N, "from_unixtime", "from_unixtime(id, 'yyyy-MM-dd HH:mm:ss.SSSSSS')")
+        }
+        runBenchmark("Convert timestamps") {
+          val timestampExpr = "cast(id as timestamp)"
+          run(N, "from_utc_timestamp", s"from_utc_timestamp($timestampExpr, 'CET')")
+          run(N, "to_utc_timestamp", s"to_utc_timestamp($timestampExpr, 'CET')")
+        }
+        runBenchmark("Intervals") {
+          val (start, end) = ("cast(id as timestamp)", "cast((id+8640000) as timestamp)")
+          run(N, "cast interval", start, end)
+          run(N, "datediff", s"datediff($start, $end)")
+          run(N, "months_between", s"months_between($start, $end)")
+          run(1000000, "window", s"window($start, 100, 10, 1)")
+        }
+        runBenchmark("Truncation") {
+          val timestampExpr = "cast(id as timestamp)"
+          Seq("YEAR", "YYYY", "YY", "MON", "MONTH", "MM", "DAY", "DD", "HOUR", "MINUTE",
+            "SECOND", "WEEK", "QUARTER").foreach { level =>
+            run(N, s"date_trunc $level", s"date_trunc('$level', $timestampExpr)")
+          }
+          val dateExpr = "cast(cast(id as timestamp) as date)"
+          Seq("year", "yyyy", "yy", "mon", "month", "mm").foreach { level =>
+            run(N, s"trunc $level", s"trunc('$level', $dateExpr)")
+          }
+        }
+        runBenchmark("Parsing") {
+          val n = 1000000
+          val timestampStrExpr = "concat('2019-01-27 11:02:01.', cast(mod(id, 1000) as string))"
+          val pattern = "'yyyy-MM-dd HH:mm:ss.SSS'"
+          run(n, "to timestamp str", timestampStrExpr)
+          run(n, "to_timestamp", s"to_timestamp($timestampStrExpr, $pattern)")
+          run(n, "to_unix_timestamp", s"to_unix_timestamp($timestampStrExpr, $pattern)")
+          val dateStrExpr = "concat('2019-01-', lpad(mod(id, 25), 2, '0'))"
+          run(n, "to date str", dateStrExpr)
+          run(n, "to_date", s"to_date($dateStrExpr, 'yyyy-MM-dd')")
+        }
+        runBenchmark("Conversion from/to external types") {
+          import spark.implicits._
+          val rowsNum = 5000000
+          val numIters = 3
+          val benchmark = new Benchmark("To/from Java's date-time", rowsNum, output = output)
+          benchmark.addCase("From java.sql.Date", numIters) { _ =>
+            spark.range(rowsNum).map(millis => new Date(millis)).noop()
+          }
+          benchmark.addCase("From java.time.LocalDate", numIters) { _ =>
+            spark.range(rowsNum).map(millis => LocalDate.ofEpochDay(millis / MILLIS_PER_DAY)).noop()
+          }
+          def dates = {
+            spark.range(0, rowsNum, 1, 1).map(millis => new Date(millis))
+          }
+          benchmark.addCase("Collect java.sql.Date", numIters) { _ =>
+            dates.collect()
+          }
+          def localDates = {
+            spark.range(0, rowsNum, 1, 1)
+              .map(millis => LocalDate.ofEpochDay(millis / MILLIS_PER_DAY))
+          }
+          benchmark.addCase("Collect java.time.LocalDate", numIters) { _ =>
+            withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+              localDates.collect()
+            }
+          }
+          benchmark.addCase("From java.sql.Timestamp", numIters) { _ =>
+            spark.range(rowsNum).map(millis => new Timestamp(millis)).noop()
+          }
+          benchmark.addCase("From java.time.Instant", numIters) { _ =>
+            spark.range(rowsNum).map(millis => Instant.ofEpochMilli(millis)).noop()
+          }
+          benchmark.addCase("Collect longs", numIters) { _ =>
+            spark.range(0, rowsNum, 1, 1)
+              .collect()
+          }
+          def timestamps = {
+            spark.range(0, rowsNum, 1, 1).map(millis => new Timestamp(millis))
+          }
+          benchmark.addCase("Collect java.sql.Timestamp", numIters) { _ =>
+            timestamps.collect()
+          }
+          def instants = {
+            spark.range(0, rowsNum, 1, 1).map(millis => Instant.ofEpochMilli(millis))
+          }
+          benchmark.addCase("Collect java.time.Instant", numIters) { _ =>
+            withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+              instants.collect()
+            }
+          }
+          def toHiveString(df: Dataset[_]): Unit = {
+            HiveResult.hiveResultString(df.queryExecution.executedPlan)
+          }
+          benchmark.addCase("java.sql.Date to Hive string", numIters) { _ =>
+            toHiveString(dates)
+          }
+          benchmark.addCase("java.time.LocalDate to Hive string", numIters) { _ =>
+            withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+              toHiveString(localDates)
+            }
+          }
+          benchmark.addCase("java.sql.Timestamp to Hive string", numIters) { _ =>
+            toHiveString(timestamps)
+          }
+          benchmark.addCase("java.time.Instant to Hive string", numIters) { _ =>
+            withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+              toHiveString(instants)
+            }
+          }
+          benchmark.run()
+        }
       }
-      benchmark.run()
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
new file mode 100644
index 0000000000000..7caaa5376db7f
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.benchmark
+
+import java.io.File
+import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneOffset}
+
+import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.SECONDS_PER_DAY
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.{LegacyBehaviorPolicy, ParquetOutputTimestampType}
+
+object DateTime extends Enumeration {
+  type DateTime = Value
+  val DATE, TIMESTAMP, TIMESTAMP_INT96, TIMESTAMP_MICROS, TIMESTAMP_MILLIS = Value
+}
+
+/**
+ * Synthetic benchmark for rebasing of date and timestamp in read/write.
+ * To run this benchmark:
+ * {{{
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *   2. build/sbt "sql/test:runMain <this class>"
+ *   3. generate result:
+ *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ *      Results will be written to "benchmarks/DateTimeRebaseBenchmark-results.txt".
+ * }}}
+ */
+object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
+  import spark.implicits._
+  import DateTime._
+
+  private def genTs(cardinality: Int, start: LocalDateTime, end: LocalDateTime): DataFrame = {
+    val startSec = start.toEpochSecond(ZoneOffset.UTC)
+    val endSec = end.toEpochSecond(ZoneOffset.UTC)
+    spark.range(0, cardinality, 1, 1)
+      .select((($"id" % (endSec - startSec)) + startSec).as("seconds"))
+      .select($"seconds".cast("timestamp").as("ts"))
+  }
+
+  private def genTsAfter1900(cardinality: Int): DataFrame = {
+    val start = LocalDateTime.of(1900, 1, 31, 0, 0, 0)
+    val end = LocalDateTime.of(3000, 1, 1, 0, 0, 0)
+    genTs(cardinality, start, end)
+  }
+
+  private def genTsBefore1900(cardinality: Int): DataFrame = {
+    val start = LocalDateTime.of(10, 1, 1, 0, 0, 0)
+    val end = LocalDateTime.of(1900, 1, 1, 0, 0, 0)
+    genTs(cardinality, start, end)
+  }
+
+  private def genDate(cardinality: Int, start: LocalDate, end: LocalDate): DataFrame = {
+    val startSec = LocalDateTime.of(start, LocalTime.MIDNIGHT).toEpochSecond(ZoneOffset.UTC)
+    val endSec = LocalDateTime.of(end, LocalTime.MIDNIGHT).toEpochSecond(ZoneOffset.UTC)
+    spark.range(0, cardinality * SECONDS_PER_DAY, SECONDS_PER_DAY, 1)
+      .select((($"id" % (endSec - startSec)) + startSec).as("seconds"))
+      .select($"seconds".cast("timestamp").as("ts"))
+      .select($"ts".cast("date").as("date"))
+  }
+
+  private def genDateAfter1582(cardinality: Int): DataFrame = {
+    val start = LocalDate.of(1582, 10, 31)
+    val end = LocalDate.of(3000, 1, 1)
+    genDate(cardinality, start, end)
+  }
+
+  private def genDateBefore1582(cardinality: Int): DataFrame = {
+    val start = LocalDate.of(10, 1, 1)
+    val end = LocalDate.of(1580, 10, 1)
+    genDate(cardinality, start, end)
+  }
+
+  private def genDF(cardinality: Int, dateTime: DateTime, modernDates: Boolean): DataFrame = {
+    dateTime match {
+      case DATE =>
+        if (modernDates) genDateAfter1582(cardinality) else genDateBefore1582(cardinality)
+      case TIMESTAMP | TIMESTAMP_INT96 | TIMESTAMP_MICROS | TIMESTAMP_MILLIS =>
+        if (modernDates) genTsAfter1900(cardinality) else genTsBefore1900(cardinality)
+      case _ => throw new IllegalArgumentException(
+        s"cardinality = $cardinality dateTime = $dateTime modernDates = $modernDates")
+    }
+  }
+
+  private def benchmarkInputs(benchmark: Benchmark, rowsNum: Int, dateTime: DateTime): Unit = {
+    val year = if (dateTime == DATE) 1582 else 1900
+    benchmark.addCase(s"after $year, noop", 1) { _ =>
+      genDF(rowsNum, dateTime, modernDates = true).noop()
+    }
+    benchmark.addCase(s"before $year, noop", 1) { _ =>
+      genDF(rowsNum, dateTime, modernDates = false).noop()
+    }
+  }
+
+  private def flagToStr(flag: Boolean): String = {
+    if (flag) "on" else "off"
+  }
+
+  private def caseName(
+      modernDates: Boolean,
+      dateTime: DateTime,
+      mode: Option[LegacyBehaviorPolicy.Value] = None,
+      vec: Option[Boolean] = None): String = {
+    val period = if (modernDates) "after" else "before"
+    val year = if (dateTime == DATE) 1582 else 1900
+    val vecFlag = vec.map(flagToStr).map(flag => s", vec $flag").getOrElse("")
+    val rebaseFlag = mode.map(_.toString).map(m => s", rebase $m").getOrElse("")
+    s"$period $year$vecFlag$rebaseFlag"
+  }
+
+  private def getPath(
+      basePath: File,
+      dateTime: DateTime,
+      modernDates: Boolean,
+      mode: Option[LegacyBehaviorPolicy.Value] = None): String = {
+    val period = if (modernDates) "after" else "before"
+    val year = if (dateTime == DATE) 1582 else 1900
+    val rebaseFlag = mode.map(_.toString).map(m => s"_$m").getOrElse("")
+    basePath.getAbsolutePath + s"/${dateTime}_${period}_$year$rebaseFlag"
+  }
+
+  private def getOutputType(dateTime: DateTime): String = dateTime match {
+    case TIMESTAMP_INT96 => ParquetOutputTimestampType.INT96.toString
+    case TIMESTAMP_MILLIS => ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString
+    case _ => ParquetOutputTimestampType.TIMESTAMP_MICROS.toString
+  }
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    val rowsNum = 100000000
+
+    withDefaultTimeZone(LA) {
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId) {
+        withTempPath { path =>
+          runBenchmark("Rebasing dates/timestamps in Parquet datasource") {
+            Seq(
+              DATE, TIMESTAMP_INT96, TIMESTAMP_MICROS, TIMESTAMP_MILLIS
+            ).foreach { dateTime =>
+              val benchmark = new Benchmark(
+                s"Save $dateTime to parquet",
+                rowsNum,
+                output = output)
+              benchmarkInputs(benchmark, rowsNum, dateTime)
+              Seq(true, false).foreach { modernDates =>
+                LegacyBehaviorPolicy.values
+                  .filterNot(v => !modernDates && v == LegacyBehaviorPolicy.EXCEPTION)
+                  .foreach { mode =>
+                  benchmark.addCase(caseName(modernDates, dateTime, Some(mode)), 1) { _ =>
+                    withSQLConf(
+                      SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> getOutputType(dateTime),
+                      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> mode.toString) {
+                      genDF(rowsNum, dateTime, modernDates)
+                        .write
+                        .mode("overwrite")
+                        .format("parquet")
+                        .save(getPath(path, dateTime, modernDates, Some(mode)))
+                    }
+                  }
+                }
+              }
+              benchmark.run()
+
+              val benchmark2 = new Benchmark(
+                s"Load $dateTime from parquet", rowsNum, output = output)
+              Seq(true, false).foreach { modernDates =>
+                Seq(false, true).foreach { vec =>
+                  LegacyBehaviorPolicy.values
+                    .filterNot(v => !modernDates && v == LegacyBehaviorPolicy.EXCEPTION)
+                    .foreach { mode =>
+                    val name = caseName(modernDates, dateTime, Some(mode), Some(vec))
+                    benchmark2.addCase(name, 3) { _ =>
+                      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString) {
+                        spark.read
+                          .format("parquet")
+                          .load(getPath(path, dateTime, modernDates, Some(mode)))
+                          .noop()
+                      }
+                    }
+                  }
+                }
+              }
+              benchmark2.run()
+            }
+          }
+        }
+
+        withTempPath { path =>
+          runBenchmark("Rebasing dates/timestamps in ORC datasource") {
+            Seq(DATE, TIMESTAMP).foreach { dateTime =>
+              val benchmark = new Benchmark(s"Save $dateTime to ORC", rowsNum, output = output)
+              benchmarkInputs(benchmark, rowsNum, dateTime)
+              Seq(true, false).foreach { modernDates =>
+                benchmark.addCase(caseName(modernDates, dateTime), 1) { _ =>
+                  genDF(rowsNum, dateTime, modernDates)
+                    .write
+                    .mode("overwrite")
+                    .format("orc")
+                    .save(getPath(path, dateTime, modernDates))
+                }
+              }
+              benchmark.run()
+
+              val benchmark2 = new Benchmark(
+                s"Load $dateTime from ORC",
+                rowsNum,
+                output = output)
+              Seq(true, false).foreach { modernDates =>
+                Seq(false, true).foreach { vec =>
+                  benchmark2.addCase(caseName(modernDates, dateTime, vec = Some(vec)), 3) { _ =>
+                    withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vec.toString) {
+                      spark
+                        .read
+                        .format("orc")
+                        .load(getPath(path, dateTime, modernDates))
+                        .noop()
+                    }
+                  }
+                }
+              }
+              benchmark2.run()
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
index de23132284dc8..287854dc3646c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
@@ -42,7 +42,9 @@ object ExtractBenchmark extends SqlBasedBenchmark {
       spark
         .range(sinceSecond, sinceSecond + cardinality, 1, 1)
         .selectExpr(exprs: _*)
-        .noop()
+        .queryExecution
+        .toRdd
+        .foreach(_ => ())
     }
   }
 
@@ -82,34 +84,22 @@ object ExtractBenchmark extends SqlBasedBenchmark {
     }
   }
 
-  private case class Settings(fields: Seq[String], func: Seq[String], iterNum: Long)
-
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
     val N = 10000000L
-    val datetimeFields = Seq(
-      "MILLENNIUM", "CENTURY", "DECADE", "YEAR",
-      "ISOYEAR", "QUARTER", "MONTH", "WEEK",
-      "DAY", "DAYOFWEEK", "DOW", "ISODOW",
-      "DOY", "HOUR", "MINUTE", "SECOND",
-      "MILLISECONDS", "MICROSECONDS", "EPOCH")
-    val intervalFields = Seq(
-      "MILLENNIUM", "CENTURY", "DECADE", "YEAR",
-      "QUARTER", "MONTH", "DAY",
-      "HOUR", "MINUTE", "SECOND",
-      "MILLISECONDS", "MICROSECONDS", "EPOCH")
+    val datetimeFields = Seq("YEAR", "YEAROFWEEK", "QUARTER", "MONTH", "WEEK", "DAY", "DAYOFWEEK",
+      "DOW", "DOW_ISO", "DAYOFWEEK_ISO", "DOY", "HOUR", "MINUTE", "SECOND")
+    val intervalFields = Seq("YEAR", "MONTH", "DAY", "HOUR", "MINUTE", "SECOND")
     val settings = Map(
-      "timestamp" -> Settings(datetimeFields, Seq("extract", "date_part"), N),
-      "date" -> Settings(datetimeFields, Seq("extract", "date_part"), N),
-      "interval" -> Settings(intervalFields, Seq("date_part"), N))
+      "timestamp" -> datetimeFields,
+      "date" -> datetimeFields,
+      "interval" -> intervalFields)
 
-    for {
-      (dataType, Settings(fields, funcs, iterNum)) <- settings
-      func <- funcs} {
+    for {(dataType, fields) <- settings; func <- Seq("extract", "date_part")} {
 
       val benchmark = new Benchmark(s"Invoke $func for $dataType", N, output = output)
 
-      run(benchmark, iterNum, s"cast to $dataType", castExpr(dataType))
-      fields.foreach(run(benchmark, func, iterNum, _, dataType))
+      run(benchmark, N, s"cast to $dataType", castExpr(dataType))
+      fields.foreach(run(benchmark, func, N, _, dataType))
 
       benchmark.run()
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
index 444ffa4f99697..b3f65d40ad95b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
@@ -23,7 +23,6 @@ import scala.util.Random
 
 import org.apache.spark.SparkConf
 import org.apache.spark.benchmark.Benchmark
-import org.apache.spark.internal.config.UI._
 import org.apache.spark.sql.{DataFrame, SparkSession}
 import org.apache.spark.sql.functions.monotonically_increasing_id
 import org.apache.spark.sql.internal.SQLConf
@@ -49,7 +48,6 @@ object FilterPushdownBenchmark extends SqlBasedBenchmark {
       .set("spark.master", "local[1]")
       .setIfMissing("spark.driver.memory", "3g")
       .setIfMissing("spark.executor.memory", "3g")
-      .setIfMissing(UI_ENABLED, false)
       .setIfMissing("orc.compression", "snappy")
       .setIfMissing("spark.sql.parquet.compression.codec", "snappy")
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala
index 94e763459a111..907e3f40c1911 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala
@@ -44,7 +44,9 @@ object IntervalBenchmark extends SqlBasedBenchmark {
       spark
         .range(0, cardinality, 1, 1)
         .select(exprs: _*)
-        .noop()
+        .queryExecution
+        .toRdd
+        .foreach(_ => ())
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala
new file mode 100644
index 0000000000000..d2bd962b50654
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.benchmark
+
+import org.apache.spark.SparkConf
+import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Synthetic benchmark for nested fields predicate push down performance for Parquet datasource.
+ * To run this benchmark:
+ * {{{
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *   2. build/sbt "sql/test:runMain <this class>"
+ *   3. generate result:
+ *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ *      Results will be written to "benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt".
+ * }}}
+ */
+object ParquetNestedPredicatePushDownBenchmark extends SqlBasedBenchmark {
+
+  private val N = 100 * 1024 * 1024
+  private val NUMBER_OF_ITER = 10
+
+  private val df: DataFrame = spark
+    .range(1, N, 1, 4)
+    .toDF("id")
+    .selectExpr("id", "STRUCT(id x, STRUCT(CAST(id AS STRING) z) y) nested")
+    .sort("id")
+
+  private def addCase(
+      benchmark: Benchmark,
+      inputPath: String,
+      enableNestedPD: String,
+      name: String,
+      withFilter: DataFrame => DataFrame): Unit = {
+    val loadDF = spark.read.parquet(inputPath)
+    benchmark.addCase(name) { _ =>
+      withSQLConf((SQLConf.NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST.key, enableNestedPD)) {
+        withFilter(loadDF).noop()
+      }
+    }
+  }
+
+  private def createAndRunBenchmark(name: String, withFilter: DataFrame => DataFrame): Unit = {
+    withTempPath { tempDir =>
+      val outputPath = tempDir.getCanonicalPath
+      df.write.mode(SaveMode.Overwrite).parquet(outputPath)
+      val benchmark = new Benchmark(name, N, NUMBER_OF_ITER, output = output)
+      addCase(
+        benchmark,
+        outputPath,
+        enableNestedPD = "",
+        "Without nested predicate Pushdown",
+        withFilter)
+      addCase(
+        benchmark,
+        outputPath,
+        enableNestedPD = "parquet",
+        "With nested predicate Pushdown",
+        withFilter)
+      benchmark.run()
+    }
+  }
+
+  /**
+   * Benchmark for sorted data with a filter which allows to filter out all the row groups
+   * when nested fields predicate push down enabled
+   */
+  def runLoadNoRowGroupWhenPredicatePushedDown(): Unit = {
+    createAndRunBenchmark("Can skip all row groups", _.filter("nested.x < 0"))
+  }
+
+  /**
+   * Benchmark with a filter which allows to load only some row groups
+   * when nested fields predicate push down enabled
+   */
+  def runLoadSomeRowGroupWhenPredicatePushedDown(): Unit = {
+    createAndRunBenchmark("Can skip some row groups", _.filter("nested.x = 100"))
+  }
+
+  /**
+   * Benchmark with a filter which still requires to
+   * load all the row groups on sorted data to see if we introduce too much
+   * overhead or not if enable nested predicate push down.
+   */
+  def runLoadAllRowGroupsWhenPredicatePushedDown(): Unit = {
+    createAndRunBenchmark("Can skip no row groups", _.filter(s"nested.x >= 0 and nested.x <= $N"))
+  }
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    runLoadNoRowGroupWhenPredicatePushedDown()
+    runLoadSomeRowGroupWhenPredicatePushedDown()
+    runLoadAllRowGroupsWhenPredicatePushedDown()
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala
index ee7a03e5e0542..28387dcef125b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.benchmark
 
 import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
+import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql.{Dataset, SparkSession}
 import org.apache.spark.sql.SaveMode.Overwrite
 import org.apache.spark.sql.catalyst.plans.SQLHelper
@@ -37,6 +38,7 @@ trait SqlBasedBenchmark extends BenchmarkBase with SQLHelper {
       .appName(this.getClass.getCanonicalName)
       .config(SQLConf.SHUFFLE_PARTITIONS.key, 1)
       .config(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, 1)
+      .config(UI_ENABLED.key, false)
       .getOrCreate()
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
index c93d27f02c686..43bc7c12937ec 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
@@ -31,7 +31,8 @@ import org.apache.spark.sql.execution.datasources.LogicalRelation
  * To run this:
  * {{{
  *   1. without sbt:
- *        bin/spark-submit --class <this class> <spark sql test jar> --data-location <location>
+ *        bin/spark-submit --jars <spark core test jar>,<spark catalyst test jar>
+ *          --class <this class> <spark sql test jar> --data-location <location>
  *   2. build/sbt "sql/test:runMain <this class> --data-location <TPCDS data location>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt
  *        "sql/test:runMain <this class> --data-location <location>"
@@ -81,7 +82,7 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
       val queryRelations = scala.collection.mutable.HashSet[String]()
       spark.sql(queryString).queryExecution.analyzed.foreach {
         case SubqueryAlias(alias, _: LogicalRelation) =>
-          queryRelations.add(alias.identifier)
+          queryRelations.add(alias.name)
         case LogicalRelation(_, _, Some(catalogTable), _) =>
           queryRelations.add(catalogTable.identifier.table)
         case HiveTableRelation(tableMeta, _, _, _, _) =>
@@ -97,11 +98,16 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
     }
   }
 
-  def filterQueries(
+  private def filterQueries(
       origQueries: Seq[String],
-      args: TPCDSQueryBenchmarkArguments): Seq[String] = {
-    if (args.queryFilter.nonEmpty) {
-      origQueries.filter(args.queryFilter.contains)
+      queryFilter: Set[String],
+      nameSuffix: String = ""): Seq[String] = {
+    if (queryFilter.nonEmpty) {
+      if (nameSuffix.nonEmpty) {
+        origQueries.filter { name => queryFilter.contains(s"$name$nameSuffix") }
+      } else {
+        origQueries.filter(queryFilter.contains)
+      }
     } else {
       origQueries
     }
@@ -124,6 +130,7 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
       "q91", "q92", "q93", "q94", "q95", "q96", "q97", "q98", "q99")
 
     // This list only includes TPC-DS v2.7 queries that are different from v1.4 ones
+    val nameSuffixForQueriesV2_7 = "-v2.7"
     val tpcdsQueriesV2_7 = Seq(
       "q5a", "q6", "q10a", "q11", "q12", "q14", "q14a", "q18a",
       "q20", "q22", "q22a", "q24", "q27a", "q34", "q35", "q35a", "q36a", "q47", "q49",
@@ -131,8 +138,9 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
       "q80a", "q86a", "q98")
 
     // If `--query-filter` defined, filters the queries that this option selects
-    val queriesV1_4ToRun = filterQueries(tpcdsQueries, benchmarkArgs)
-    val queriesV2_7ToRun = filterQueries(tpcdsQueriesV2_7, benchmarkArgs)
+    val queriesV1_4ToRun = filterQueries(tpcdsQueries, benchmarkArgs.queryFilter)
+    val queriesV2_7ToRun = filterQueries(tpcdsQueriesV2_7, benchmarkArgs.queryFilter,
+      nameSuffix = nameSuffixForQueriesV2_7)
 
     if ((queriesV1_4ToRun ++ queriesV2_7ToRun).isEmpty) {
       throw new RuntimeException(
@@ -142,6 +150,6 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
     val tableSizes = setupTables(benchmarkArgs.dataLocation)
     runTpcdsQueries(queryLocation = "tpcds", queries = queriesV1_4ToRun, tableSizes)
     runTpcdsQueries(queryLocation = "tpcds-v2.7.0", queries = queriesV2_7ToRun, tableSizes,
-      nameSuffix = "-v2.7")
+      nameSuffix = nameSuffixForQueriesV2_7)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
index f582d844cdc47..9b0389c6d1ea4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
@@ -40,13 +40,16 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
     UnsafeArrayData.calculateHeaderPortionInBytes(count)
   }
 
+  private lazy val intEncoder = ExpressionEncoder[Array[Int]]().resolveAndBind()
+
+  private lazy val doubleEncoder = ExpressionEncoder[Array[Double]]().resolveAndBind()
+
   def readUnsafeArray(iters: Int): Unit = {
     val count = 1024 * 1024 * 16
     val rand = new Random(42)
-
+    val intArrayToRow = intEncoder.createSerializer()
     val intPrimitiveArray = Array.fill[Int](count) { rand.nextInt }
-    val intEncoder = ExpressionEncoder[Array[Int]].resolveAndBind()
-    val intUnsafeArray = intEncoder.toRow(intPrimitiveArray).getArray(0)
+    val intUnsafeArray = intArrayToRow(intPrimitiveArray).getArray(0)
     val readIntArray = { i: Int =>
       var n = 0
       while (n < iters) {
@@ -62,8 +65,8 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
     }
 
     val doublePrimitiveArray = Array.fill[Double](count) { rand.nextDouble }
-    val doubleEncoder = ExpressionEncoder[Array[Double]].resolveAndBind()
-    val doubleUnsafeArray = doubleEncoder.toRow(doublePrimitiveArray).getArray(0)
+    val doubleArrayToRow = doubleEncoder.createSerializer()
+    val doubleUnsafeArray = doubleArrayToRow(doublePrimitiveArray).getArray(0)
     val readDoubleArray = { i: Int =>
       var n = 0
       while (n < iters) {
@@ -90,12 +93,12 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
 
     var intTotalLength: Int = 0
     val intPrimitiveArray = Array.fill[Int](count) { rand.nextInt }
-    val intEncoder = ExpressionEncoder[Array[Int]].resolveAndBind()
+    val intArrayToRow = intEncoder.createSerializer()
     val writeIntArray = { i: Int =>
       var len = 0
       var n = 0
       while (n < iters) {
-        len += intEncoder.toRow(intPrimitiveArray).getArray(0).numElements()
+        len += intArrayToRow(intPrimitiveArray).getArray(0).numElements()
         n += 1
       }
       intTotalLength = len
@@ -103,12 +106,12 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
 
     var doubleTotalLength: Int = 0
     val doublePrimitiveArray = Array.fill[Double](count) { rand.nextDouble }
-    val doubleEncoder = ExpressionEncoder[Array[Double]].resolveAndBind()
+    val doubleArrayToRow = doubleEncoder.createSerializer()
     val writeDoubleArray = { i: Int =>
       var len = 0
       var n = 0
       while (n < iters) {
-        len += doubleEncoder.toRow(doublePrimitiveArray).getArray(0).numElements()
+        len += doubleArrayToRow(doublePrimitiveArray).getArray(0).numElements()
         n += 1
       }
       doubleTotalLength = len
@@ -126,8 +129,8 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
 
     var intTotalLength: Int = 0
     val intPrimitiveArray = Array.fill[Int](count) { rand.nextInt }
-    val intEncoder = ExpressionEncoder[Array[Int]].resolveAndBind()
-    val intUnsafeArray = intEncoder.toRow(intPrimitiveArray).getArray(0)
+    val intArrayToRow = intEncoder.createSerializer()
+    val intUnsafeArray = intArrayToRow(intPrimitiveArray).getArray(0)
     val readIntArray = { i: Int =>
       var len = 0
       var n = 0
@@ -140,8 +143,8 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
 
     var doubleTotalLength: Int = 0
     val doublePrimitiveArray = Array.fill[Double](count) { rand.nextDouble }
-    val doubleEncoder = ExpressionEncoder[Array[Double]].resolveAndBind()
-    val doubleUnsafeArray = doubleEncoder.toRow(doublePrimitiveArray).getArray(0)
+    val doubleArrayToRow = doubleEncoder.createSerializer()
+    val doubleUnsafeArray = doubleArrayToRow(doublePrimitiveArray).getArray(0)
     val readDoubleArray = { i: Int =>
       var len = 0
       var n = 0
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
index 77047f329e105..18f29f7b90ad5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
@@ -126,13 +126,15 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
   }
 
   test("default size avoids broadcast") {
-    // TODO: Improve this test when we have better statistics
-    sparkContext.parallelize(1 to 10).map(i => TestData(i, i.toString))
-      .toDF().createOrReplaceTempView("sizeTst")
-    spark.catalog.cacheTable("sizeTst")
-    assert(
-      spark.table("sizeTst").queryExecution.analyzed.stats.sizeInBytes >
-        spark.conf.get(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD))
+    withTempView("sizeTst") {
+      // TODO: Improve this test when we have better statistics
+      sparkContext.parallelize(1 to 10).map(i => TestData(i, i.toString))
+        .toDF().createOrReplaceTempView("sizeTst")
+      spark.catalog.cacheTable("sizeTst")
+      assert(
+        spark.table("sizeTst").queryExecution.analyzed.stats.sizeInBytes >
+          spark.conf.get(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD))
+    }
   }
 
   test("projection") {
@@ -187,18 +189,20 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
   }
 
   test("SPARK-2729 regression: timestamp data type") {
-    val timestamps = (0 to 3).map(i => Tuple1(new Timestamp(i))).toDF("time")
-    timestamps.createOrReplaceTempView("timestamps")
+    withTempView("timestamps") {
+      val timestamps = (0 to 3).map(i => Tuple1(new Timestamp(i))).toDF("time")
+      timestamps.createOrReplaceTempView("timestamps")
 
-    checkAnswer(
-      sql("SELECT time FROM timestamps"),
-      timestamps.collect().toSeq)
+      checkAnswer(
+        sql("SELECT time FROM timestamps"),
+        timestamps.collect().toSeq)
 
-    spark.catalog.cacheTable("timestamps")
+      spark.catalog.cacheTable("timestamps")
 
-    checkAnswer(
-      sql("SELECT time FROM timestamps"),
-      timestamps.collect().toSeq)
+      checkAnswer(
+        sql("SELECT time FROM timestamps"),
+        timestamps.collect().toSeq)
+    }
   }
 
   test("SPARK-3320 regression: batched column buffer building should work with empty partitions") {
@@ -229,10 +233,12 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
 
     assert(df.schema.head.dataType === DecimalType(15, 10))
 
-    df.cache().createOrReplaceTempView("test_fixed_decimal")
-    checkAnswer(
-      sql("SELECT * FROM test_fixed_decimal"),
-      (1 to 10).map(i => Row(Decimal(i, 15, 10).toJavaBigDecimal)))
+    withTempView("test_fixed_decimal") {
+      df.cache().createOrReplaceTempView("test_fixed_decimal")
+      checkAnswer(
+        sql("SELECT * FROM test_fixed_decimal"),
+        (1 to 10).map(i => Row(Decimal(i, 15, 10).toJavaBigDecimal)))
+    }
   }
 
   test("test different data types") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala
index 192db0e910d03..111a620df8c24 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala
@@ -156,4 +156,30 @@ class BooleanBitSetSuite extends SparkFunSuite {
   test(s"$BooleanBitSet: multiple words and 1 more bit for decompression()") {
     skeletonForDecompress(BITS_PER_LONG * 2 + 1)
   }
+
+  test(s"$BooleanBitSet: Only nulls for decompression()") {
+    val builder = TestCompressibleColumnBuilder(new NoopColumnStats, BOOLEAN, BooleanBitSet)
+    val numRows = 10
+
+    val rows = Seq.fill[InternalRow](numRows)({
+      val row = new GenericInternalRow(1)
+      row.setNullAt(0)
+      row
+    })
+    rows.foreach(builder.appendFrom(_, 0))
+    val buffer = builder.build()
+
+    // Rewinds, skips column header and 4 more bytes for compression scheme ID
+    val headerSize = CompressionScheme.columnHeaderSize(buffer)
+    buffer.position(headerSize)
+    assertResult(BooleanBitSet.typeId, "Wrong compression scheme ID")(buffer.getInt())
+
+    val decoder = BooleanBitSet.decoder(buffer, BOOLEAN)
+    val columnVector = new OnHeapColumnVector(numRows, BooleanType)
+    decoder.decompress(columnVector, numRows)
+
+    (0 until numRows).foreach { rowNum =>
+      assert(columnVector.isNullAt(rowNum))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala
index f946a6779ec95..c6fe64d1058ab 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala
@@ -46,7 +46,7 @@ class PassThroughSuite extends SparkFunSuite {
 
       val builder = TestCompressibleColumnBuilder(columnStats, columnType, PassThrough)
 
-      input.map { value =>
+      input.foreach { value =>
         val row = new GenericInternalRow(1)
         columnType.setField(row, 0, value)
         builder.appendFrom(row, 0)
@@ -98,7 +98,7 @@ class PassThroughSuite extends SparkFunSuite {
       val row = new GenericInternalRow(1)
       val nullRow = new GenericInternalRow(1)
       nullRow.setNullAt(0)
-      input.map { value =>
+      input.foreach { value =>
         if (value == nullValue) {
           builder.appendFrom(nullRow, 0)
         } else {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
index 81965e4c6c353..c6a533dfae4d0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
@@ -75,12 +75,6 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
     }.head
   }
 
-  private def withCreateTableStatement(sql: String)(prediction: CreateTableStatement => Unit)
-    : Unit = {
-    val statement = parser.parsePlan(sql).asInstanceOf[CreateTableStatement]
-    prediction(statement)
-  }
-
   test("alter database - property values must be set") {
     assertUnsupported(
       sql = "ALTER DATABASE my_db SET DBPROPERTIES('key_without_value', 'key_with_value'='x')",
@@ -486,17 +480,21 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
 
   test("Test CTAS #3") {
     val s3 = """CREATE TABLE page_view AS SELECT * FROM src"""
-    val statement = parser.parsePlan(s3).asInstanceOf[CreateTableAsSelectStatement]
-    assert(statement.tableName(0) == "page_view")
-    assert(statement.asSelect == parser.parsePlan("SELECT * FROM src"))
-    assert(statement.partitioning.isEmpty)
-    assert(statement.bucketSpec.isEmpty)
-    assert(statement.properties.isEmpty)
-    assert(statement.provider == conf.defaultDataSourceName)
-    assert(statement.options.isEmpty)
-    assert(statement.location.isEmpty)
-    assert(statement.comment.isEmpty)
-    assert(!statement.ifNotExists)
+    val (desc, exists) = extractTableDesc(s3)
+    assert(exists == false)
+    assert(desc.identifier.database == None)
+    assert(desc.identifier.table == "page_view")
+    assert(desc.tableType == CatalogTableType.MANAGED)
+    assert(desc.storage.locationUri == None)
+    assert(desc.schema.isEmpty)
+    assert(desc.viewText == None) // TODO will be SQLText
+    assert(desc.viewQueryColumnNames.isEmpty)
+    assert(desc.storage.properties == Map())
+    assert(desc.storage.inputFormat == Some("org.apache.hadoop.mapred.TextInputFormat"))
+    assert(desc.storage.outputFormat ==
+      Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"))
+    assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
+    assert(desc.properties == Map())
   }
 
   test("Test CTAS #4") {
@@ -656,60 +654,67 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
 
   test("create table - basic") {
     val query = "CREATE TABLE my_table (id int, name string)"
-    withCreateTableStatement(query) { state =>
-      assert(state.tableName(0) == "my_table")
-      assert(state.tableSchema == new StructType().add("id", "int").add("name", "string"))
-      assert(state.partitioning.isEmpty)
-      assert(state.bucketSpec.isEmpty)
-      assert(state.properties.isEmpty)
-      assert(state.provider == conf.defaultDataSourceName)
-      assert(state.options.isEmpty)
-      assert(state.location.isEmpty)
-      assert(state.comment.isEmpty)
-      assert(!state.ifNotExists)
-    }
+    val (desc, allowExisting) = extractTableDesc(query)
+    assert(!allowExisting)
+    assert(desc.identifier.database.isEmpty)
+    assert(desc.identifier.table == "my_table")
+    assert(desc.tableType == CatalogTableType.MANAGED)
+    assert(desc.schema == new StructType().add("id", "int").add("name", "string"))
+    assert(desc.partitionColumnNames.isEmpty)
+    assert(desc.bucketSpec.isEmpty)
+    assert(desc.viewText.isEmpty)
+    assert(desc.viewQueryColumnNames.isEmpty)
+    assert(desc.storage.locationUri.isEmpty)
+    assert(desc.storage.inputFormat ==
+      Some("org.apache.hadoop.mapred.TextInputFormat"))
+    assert(desc.storage.outputFormat ==
+      Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"))
+    assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
+    assert(desc.storage.properties.isEmpty)
+    assert(desc.properties.isEmpty)
+    assert(desc.comment.isEmpty)
   }
 
   test("create table - with database name") {
     val query = "CREATE TABLE dbx.my_table (id int, name string)"
-    withCreateTableStatement(query) { state =>
-      assert(state.tableName(0) == "dbx")
-      assert(state.tableName(1) == "my_table")
-    }
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.identifier.database == Some("dbx"))
+    assert(desc.identifier.table == "my_table")
   }
 
   test("create table - temporary") {
     val query = "CREATE TEMPORARY TABLE tab1 (id int, name string)"
     val e = intercept[ParseException] { parser.parsePlan(query) }
-    assert(e.message.contains("CREATE TEMPORARY TABLE without a provider is not allowed."))
+    assert(e.message.contains("CREATE TEMPORARY TABLE is not supported yet"))
   }
 
   test("create table - external") {
     val query = "CREATE EXTERNAL TABLE tab1 (id int, name string) LOCATION '/path/to/nowhere'"
-    val e = intercept[ParseException] { parser.parsePlan(query) }
-    assert(e.message.contains("Operation not allowed: CREATE EXTERNAL TABLE ..."))
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.tableType == CatalogTableType.EXTERNAL)
+    assert(desc.storage.locationUri == Some(new URI("/path/to/nowhere")))
   }
 
   test("create table - if not exists") {
     val query = "CREATE TABLE IF NOT EXISTS tab1 (id int, name string)"
-    withCreateTableStatement(query) { state =>
-      assert(state.ifNotExists)
-    }
+    val (_, allowExisting) = extractTableDesc(query)
+    assert(allowExisting)
   }
 
   test("create table - comment") {
     val query = "CREATE TABLE my_table (id int, name string) COMMENT 'its hot as hell below'"
-    withCreateTableStatement(query) { state =>
-      assert(state.comment == Some("its hot as hell below"))
-    }
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.comment == Some("its hot as hell below"))
   }
 
   test("create table - partitioned columns") {
-    val query = "CREATE TABLE my_table (id int, name string) PARTITIONED BY (id)"
-    withCreateTableStatement(query) { state =>
-      val transform = IdentityTransform(FieldReference(Seq("id")))
-      assert(state.partitioning == Seq(transform))
-    }
+    val query = "CREATE TABLE my_table (id int, name string) PARTITIONED BY (month int)"
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.schema == new StructType()
+      .add("id", "int")
+      .add("name", "string")
+      .add("month", "int"))
+    assert(desc.partitionColumnNames == Seq("month"))
   }
 
   test("create table - clustered by") {
@@ -725,22 +730,20 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
        """
 
     val query1 = s"$baseQuery INTO $numBuckets BUCKETS"
-    withCreateTableStatement(query1) { state =>
-      assert(state.bucketSpec.isDefined)
-      val bucketSpec = state.bucketSpec.get
-      assert(bucketSpec.numBuckets == numBuckets)
-      assert(bucketSpec.bucketColumnNames.head.equals(bucketedColumn))
-      assert(bucketSpec.sortColumnNames.isEmpty)
-    }
+    val (desc1, _) = extractTableDesc(query1)
+    assert(desc1.bucketSpec.isDefined)
+    val bucketSpec1 = desc1.bucketSpec.get
+    assert(bucketSpec1.numBuckets == numBuckets)
+    assert(bucketSpec1.bucketColumnNames.head.equals(bucketedColumn))
+    assert(bucketSpec1.sortColumnNames.isEmpty)
 
     val query2 = s"$baseQuery SORTED BY($sortColumn) INTO $numBuckets BUCKETS"
-    withCreateTableStatement(query2) { state =>
-      assert(state.bucketSpec.isDefined)
-      val bucketSpec = state.bucketSpec.get
-      assert(bucketSpec.numBuckets == numBuckets)
-      assert(bucketSpec.bucketColumnNames.head.equals(bucketedColumn))
-      assert(bucketSpec.sortColumnNames.head.equals(sortColumn))
-    }
+    val (desc2, _) = extractTableDesc(query2)
+    assert(desc2.bucketSpec.isDefined)
+    val bucketSpec2 = desc2.bucketSpec.get
+    assert(bucketSpec2.numBuckets == numBuckets)
+    assert(bucketSpec2.bucketColumnNames.head.equals(bucketedColumn))
+    assert(bucketSpec2.sortColumnNames.head.equals(sortColumn))
   }
 
   test("create table(hive) - skewed by") {
@@ -810,9 +813,8 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
 
   test("create table - properties") {
     val query = "CREATE TABLE my_table (id int, name string) TBLPROPERTIES ('k1'='v1', 'k2'='v2')"
-    withCreateTableStatement(query) { state =>
-      assert(state.properties == Map("k1" -> "v1", "k2" -> "v2"))
-    }
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.properties == Map("k1" -> "v1", "k2" -> "v2"))
   }
 
   test("create table(hive) - everything!") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 913cd80a24c6e..5986cdc78d6b4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier}
 import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, NoSuchDatabaseException, NoSuchPartitionException, NoSuchTableException, TempTableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces.PROP_OWNER
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
@@ -188,7 +189,7 @@ class InMemoryCatalogedDDLSuite extends DDLSuite with SharedSparkSession {
     withTable("t") {
       sql("CREATE TABLE t(i INT) USING parquet")
       val e = intercept[AnalysisException] {
-        sql("ALTER TABLE t ALTER COLUMN i TYPE INT FIRST")
+        sql("ALTER TABLE t ALTER COLUMN i FIRST")
       }
       assert(e.message.contains("ALTER COLUMN ... FIRST | ALTER is only supported with v2 tables"))
     }
@@ -473,17 +474,12 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
       withEmptyDirInTablePath("tab1") { tableLoc =>
         val hiddenGarbageFile = new File(tableLoc.getCanonicalPath, ".garbage")
         hiddenGarbageFile.createNewFile()
-        val exMsg = "Can not create the managed table('`tab1`'). The associated location"
         val exMsgWithDefaultDB =
           "Can not create the managed table('`default`.`tab1`'). The associated location"
         var ex = intercept[AnalysisException] {
           sql(s"CREATE TABLE tab1 USING ${dataSource} AS SELECT 1, 'a'")
         }.getMessage
-        if (isUsingHiveMetastore) {
-          assert(ex.contains(exMsgWithDefaultDB))
-        } else {
-          assert(ex.contains(exMsg))
-        }
+        assert(ex.contains(exMsgWithDefaultDB))
 
         ex = intercept[AnalysisException] {
           sql(s"CREATE TABLE tab1 (col1 int, col2 string) USING ${dataSource}")
@@ -509,8 +505,8 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
         val ex = intercept[AnalysisException] {
           sql("ALTER TABLE tab1 RENAME TO tab2")
         }.getMessage
-        val expectedMsg = "Can not rename the managed table('`tab1`'). The associated location"
-        assert(ex.contains(expectedMsg))
+        assert(ex.contains(
+          "Can not rename the managed table('`default`.`tab1`'). The associated location"))
       }
     }
   }
@@ -640,7 +636,8 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
         val errMsg = intercept[AnalysisException] {
           sql(s"CREATE TABLE t($c0 INT, $c1 INT) USING parquet")
         }.getMessage
-        assert(errMsg.contains("Found duplicate column(s) in the table definition of `t`"))
+        assert(errMsg.contains(
+          "Found duplicate column(s) in the table definition of `default`.`t`"))
       }
     }
   }
@@ -649,7 +646,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     val e = intercept[AnalysisException] {
       sql("CREATE TABLE tbl(a int, b string) USING json PARTITIONED BY (c)")
     }
-    assert(e.message == "partition column c is not defined in table tbl, " +
+    assert(e.message == "partition column c is not defined in table default.tbl, " +
       "defined table columns are: a, b")
   }
 
@@ -657,7 +654,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     val e = intercept[AnalysisException] {
       sql("CREATE TABLE tbl(a int, b string) USING json CLUSTERED BY (c) INTO 4 BUCKETS")
     }
-    assert(e.message == "bucket column c is not defined in table tbl, " +
+    assert(e.message == "bucket column c is not defined in table default.tbl, " +
       "defined table columns are: a, b")
   }
 
@@ -1037,7 +1034,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     df.write.insertInto("students")
     spark.catalog.cacheTable("students")
     checkAnswer(spark.table("students"), df)
-    assume(spark.catalog.isCached("students"), "bad test: table was not cached in the first place")
+    assert(spark.catalog.isCached("students"), "bad test: table was not cached in the first place")
     sql("ALTER TABLE students RENAME TO teachers")
     sql("CREATE TABLE students (age INT, name STRING) USING parquet")
     // Now we have both students and teachers.
@@ -1207,14 +1204,24 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
   }
 
   test("alter table: recover partitions (sequential)") {
-    withSQLConf(RDD_PARALLEL_LISTING_THRESHOLD.key -> "10") {
+    val oldRddParallelListingThreshold = spark.sparkContext.conf.get(
+      RDD_PARALLEL_LISTING_THRESHOLD)
+    try {
+      spark.sparkContext.conf.set(RDD_PARALLEL_LISTING_THRESHOLD.key, "10")
       testRecoverPartitions()
+    } finally {
+      spark.sparkContext.conf.set(RDD_PARALLEL_LISTING_THRESHOLD, oldRddParallelListingThreshold)
     }
   }
 
   test("alter table: recover partition (parallel)") {
-    withSQLConf(RDD_PARALLEL_LISTING_THRESHOLD.key -> "0") {
+    val oldRddParallelListingThreshold = spark.sparkContext.conf.get(
+      RDD_PARALLEL_LISTING_THRESHOLD)
+    try {
+      spark.sparkContext.conf.set(RDD_PARALLEL_LISTING_THRESHOLD.key, "0")
       testRecoverPartitions()
+    } finally {
+      spark.sparkContext.conf.set(RDD_PARALLEL_LISTING_THRESHOLD, oldRddParallelListingThreshold)
     }
   }
 
@@ -1724,6 +1731,14 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     // use int literal as partition value for int type partition column
     sql("ALTER TABLE tab1 DROP PARTITION (a=9, b=9)")
     assert(catalog.listPartitions(tableIdent).isEmpty)
+
+    // null partition values
+    createTablePartition(catalog, Map("a" -> null, "b" -> null), tableIdent)
+    val nullPartValue = if (isUsingHiveMetastore) "__HIVE_DEFAULT_PARTITION__" else null
+    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet ==
+      Set(Map("a" -> nullPartValue, "b" -> nullPartValue)))
+    sql("ALTER TABLE tab1 DROP PARTITION (a = null, b = null)")
+    assert(catalog.listPartitions(tableIdent).isEmpty)
   }
 
   protected def testRenamePartitions(isDatasourceTable: Boolean): Unit = {
@@ -1786,7 +1801,8 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
       column.map(_.metadata).getOrElse(Metadata.empty)
     }
     // Ensure that change column will preserve other metadata fields.
-    sql("ALTER TABLE dbx.tab1 CHANGE COLUMN col1 TYPE INT COMMENT 'this is col1'")
+    sql("ALTER TABLE dbx.tab1 CHANGE COLUMN col1 TYPE INT")
+    sql("ALTER TABLE dbx.tab1 CHANGE COLUMN col1 COMMENT 'this is col1'")
     assert(getMetadata("col1").getString("key") == "value")
     assert(getMetadata("col1").getString("comment") == "this is col1")
   }
@@ -1962,7 +1978,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     Seq("json", "parquet").foreach { format =>
       withTable("rectangles") {
         data.write.format(format).saveAsTable("rectangles")
-        assume(spark.table("rectangles").collect().nonEmpty,
+        assert(spark.table("rectangles").collect().nonEmpty,
           "bad test; table was empty to begin with")
 
         sql("TRUNCATE TABLE rectangles")
@@ -2041,6 +2057,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
           // Set ACL to table path.
           val customAcl = new java.util.ArrayList[AclEntry]()
           customAcl.add(new AclEntry.Builder()
+            .setName("test")
             .setType(AclEntryType.USER)
             .setScope(AclEntryScope.ACCESS)
             .setPermission(FsAction.READ).build())
@@ -2060,14 +2077,53 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
           if (ignore) {
             assert(aclEntries.size() == 0)
           } else {
-            assert(aclEntries.size() == 1)
+            assert(aclEntries.size() == 4)
             assert(aclEntries.get(0) == customAcl.get(0))
+
+            // Setting ACLs will also set user/group/other permissions
+            // as ACL entries.
+            val user = new AclEntry.Builder()
+              .setType(AclEntryType.USER)
+              .setScope(AclEntryScope.ACCESS)
+              .setPermission(FsAction.ALL).build()
+            val group = new AclEntry.Builder()
+              .setType(AclEntryType.GROUP)
+              .setScope(AclEntryScope.ACCESS)
+              .setPermission(FsAction.ALL).build()
+            val other = new AclEntry.Builder()
+              .setType(AclEntryType.OTHER)
+              .setScope(AclEntryScope.ACCESS)
+              .setPermission(FsAction.ALL).build()
+            assert(aclEntries.get(1) == user)
+            assert(aclEntries.get(2) == group)
+            assert(aclEntries.get(3) == other)
           }
         }
       }
     }
   }
 
+  test("SPARK-31163: acl/permission should handle non-existed path when truncating table") {
+    withSQLConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL.key -> "false") {
+      withTable("tab1") {
+        sql("CREATE TABLE tab1 (col1 STRING, col2 INT) USING parquet PARTITIONED BY (col2)")
+        sql("INSERT INTO tab1 SELECT 'one', 1")
+        checkAnswer(spark.table("tab1"), Row("one", 1))
+        val part = spark.sessionState.catalog.listPartitions(TableIdentifier("tab1")).head
+        val path = new File(part.location.getPath)
+        sql("TRUNCATE TABLE tab1")
+        // simulate incomplete/unsuccessful truncate
+        assert(path.exists())
+        path.delete()
+        assert(!path.exists())
+        // execute without java.io.FileNotFoundException
+        sql("TRUNCATE TABLE tab1")
+        // partition path should be re-created
+        assert(path.exists())
+      }
+    }
+  }
+
   test("create temporary view with mismatched schema") {
     withTable("tab1") {
       spark.range(10).write.saveAsTable("tab1")
@@ -2966,16 +3022,16 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
-  test("Add a directory when spark.sql.legacy.addDirectory.recursive.enabled set to true") {
+  test(s"Add a directory when ${SQLConf.LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE.key} set to false") {
     val directoryToAdd = Utils.createTempDir("/tmp/spark/addDirectory/")
     val testFile = File.createTempFile("testFile", "1", directoryToAdd)
     spark.sql(s"ADD FILE $directoryToAdd")
     assert(new File(SparkFiles.get(s"${directoryToAdd.getName}/${testFile.getName}")).exists())
   }
 
-  test("Add a directory when spark.sql.legacy.addDirectory.recursive.enabled not set to true") {
+  test(s"Add a directory when ${SQLConf.LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE.key} set to true") {
     withTempDir { testDir =>
-      withSQLConf(SQLConf.LEGACY_ADD_DIRECTORY_USING_RECURSIVE.key -> "false") {
+      withSQLConf(SQLConf.LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE.key -> "true") {
         val msg = intercept[SparkException] {
           spark.sql(s"ADD FILE $testDir")
         }.getMessage
@@ -2983,6 +3039,58 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
       }
     }
   }
+
+  test("SPARK-33588: case sensitivity of partition spec in SHOW TABLE") {
+    val t = "part_table"
+    withTable(t) {
+      sql(s"""
+        |CREATE TABLE $t (price int, qty int, year int, month int)
+        |USING $dataSource
+        |PARTITIONED BY (year, month)""".stripMargin)
+      sql(s"INSERT INTO $t PARTITION(year = 2015, month = 1) SELECT 1, 1")
+      Seq(
+        true -> "PARTITION(year = 2015, month = 1)",
+        false -> "PARTITION(YEAR = 2015, Month = 1)"
+      ).foreach { case (caseSensitive, partitionSpec) =>
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+          val df = sql(s"SHOW TABLE EXTENDED LIKE '$t' $partitionSpec")
+          val information = df.select("information").first().getString(0)
+          assert(information.contains("Partition Values: [year=2015, month=1]"))
+        }
+      }
+    }
+  }
+
+  test("SPARK-33667: case sensitivity of partition spec in SHOW PARTITIONS") {
+    val t = "part_table"
+    withTable(t) {
+      sql(s"""
+        |CREATE TABLE $t (price int, qty int, year int, month int)
+        |USING $dataSource
+        |PARTITIONED BY (year, month)""".stripMargin)
+      sql(s"INSERT INTO $t PARTITION(year = 2015, month = 1) SELECT 1, 1")
+      Seq(
+        true -> "PARTITION(year = 2015, month = 1)",
+        false -> "PARTITION(YEAR = 2015, Month = 1)"
+      ).foreach { case (caseSensitive, partitionSpec) =>
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+          checkAnswer(
+            sql(s"SHOW PARTITIONS $t $partitionSpec"),
+            Row("year=2015/month=1"))
+        }
+      }
+    }
+  }
+
+  test("SPARK-33670: show partitions from a datasource table") {
+    import testImplicits._
+    val t = "part_datasrc"
+    withTable(t) {
+      val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c")
+      df.write.partitionBy("a").format("parquet").mode(SaveMode.Overwrite).saveAsTable(t)
+      assert(sql(s"SHOW TABLE EXTENDED LIKE '$t' PARTITION(a = 1)").count() === 1)
+    }
+  }
 }
 
 object FakeLocalFsFileSystem {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
index c0c3cd70fcc9e..8a3e1bb968efc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
@@ -37,11 +37,13 @@ import org.apache.spark.sql.connector.catalog.TableChange.{UpdateColumnComment,
 import org.apache.spark.sql.execution.datasources.CreateTable
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.SimpleScanSource
 import org.apache.spark.sql.types.{CharType, DoubleType, HIVE_TYPE_STRING, IntegerType, LongType, MetadataBuilder, StringType, StructField, StructType}
 
 class PlanResolutionSuite extends AnalysisTest {
   import CatalystSqlParser._
 
+  private val v1Format = classOf[SimpleScanSource].getName
   private val v2Format = classOf[FakeV2Provider].getName
 
   private val table: Table = {
@@ -61,6 +63,15 @@ class PlanResolutionSuite extends AnalysisTest {
     val t = mock(classOf[CatalogTable])
     when(t.schema).thenReturn(new StructType().add("i", "int").add("s", "string"))
     when(t.tableType).thenReturn(CatalogTableType.MANAGED)
+    when(t.provider).thenReturn(Some(v1Format))
+    V1Table(t)
+  }
+
+  private val v1HiveTable: V1Table = {
+    val t = mock(classOf[CatalogTable])
+    when(t.schema).thenReturn(new StructType().add("i", "int").add("s", "string"))
+    when(t.tableType).thenReturn(CatalogTableType.MANAGED)
+    when(t.provider).thenReturn(Some("hive"))
     V1Table(t)
   }
 
@@ -83,6 +94,7 @@ class PlanResolutionSuite extends AnalysisTest {
       invocation.getArgument[Identifier](0).name match {
         case "v1Table" => v1Table
         case "v1Table1" => v1Table
+        case "v1HiveTable" => v1HiveTable
         case "v2Table" => table
         case "v2Table1" => table
         case "v2TableWithAcceptAnySchemaCapability" => tableWithAcceptAnySchemaCapability
@@ -128,6 +140,7 @@ class PlanResolutionSuite extends AnalysisTest {
       }
     })
     when(manager.currentCatalog).thenReturn(v2SessionCatalog)
+    when(manager.currentNamespace).thenReturn(Array("default"))
     when(manager.v1SessionCatalog).thenReturn(v1SessionCatalog)
     manager
   }
@@ -145,7 +158,7 @@ class PlanResolutionSuite extends AnalysisTest {
       ResolveInlineTables(conf),
       analyzer.ResolveRelations,
       new ResolveCatalogs(catalogManager),
-      new ResolveSessionCatalog(catalogManager, conf, _ == Seq("v")),
+      new ResolveSessionCatalog(catalogManager, conf, _ == Seq("v"), _ => false),
       analyzer.ResolveTables,
       analyzer.ResolveReferences,
       analyzer.ResolveSubqueryColumnAliases,
@@ -170,7 +183,7 @@ class PlanResolutionSuite extends AnalysisTest {
         "USING parquet PARTITIONED BY (a)"
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("my_tab"),
+      identifier = TableIdentifier("my_tab", Some("default")),
       tableType = CatalogTableType.MANAGED,
       storage = CatalogStorageFormat.empty,
       schema = new StructType()
@@ -214,7 +227,7 @@ class PlanResolutionSuite extends AnalysisTest {
         "CLUSTERED BY (a) SORTED BY (b) INTO 5 BUCKETS"
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("my_tab"),
+      identifier = TableIdentifier("my_tab", Some("default")),
       tableType = CatalogTableType.MANAGED,
       storage = CatalogStorageFormat.empty,
       schema = new StructType().add("a", IntegerType).add("b", StringType),
@@ -235,7 +248,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val sql = "CREATE TABLE my_tab(a INT, b STRING) USING parquet COMMENT 'abc'"
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("my_tab"),
+      identifier = TableIdentifier("my_tab", Some("default")),
       tableType = CatalogTableType.MANAGED,
       storage = CatalogStorageFormat.empty,
       schema = new StructType().add("a", IntegerType).add("b", StringType),
@@ -255,7 +268,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val sql = "CREATE TABLE my_tab(a INT, b STRING) USING parquet TBLPROPERTIES('test' = 'test')"
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("my_tab"),
+      identifier = TableIdentifier("my_tab", Some("default")),
       tableType = CatalogTableType.MANAGED,
       storage = CatalogStorageFormat.empty,
       schema = new StructType().add("a", IntegerType).add("b", StringType),
@@ -275,7 +288,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val v1 = "CREATE TABLE my_tab(a INT, b STRING) USING parquet LOCATION '/tmp/file'"
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("my_tab"),
+      identifier = TableIdentifier("my_tab", Some("default")),
       tableType = CatalogTableType.EXTERNAL,
       storage = CatalogStorageFormat.empty.copy(locationUri = Some(new URI("/tmp/file"))),
       schema = new StructType().add("a", IntegerType).add("b", StringType),
@@ -329,7 +342,7 @@ class PlanResolutionSuite extends AnalysisTest {
       """.stripMargin
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("table_name"),
+      identifier = TableIdentifier("table_name", Some("default")),
       tableType = CatalogTableType.MANAGED,
       storage = CatalogStorageFormat.empty.copy(
         properties = Map("a" -> "1", "b" -> "0.1", "c" -> "true")
@@ -539,7 +552,7 @@ class PlanResolutionSuite extends AnalysisTest {
         assert(ctas.catalog.name == "testcat")
         assert(ctas.tableName == Identifier.of(Array("mydb"), "table_name"))
         assert(ctas.properties == expectedProperties)
-        assert(ctas.writeOptions == Map("other" -> "20"))
+        assert(ctas.writeOptions.isEmpty)
         assert(ctas.partitioning.isEmpty)
         assert(ctas.ignoreIfExists)
 
@@ -573,7 +586,7 @@ class PlanResolutionSuite extends AnalysisTest {
         assert(ctas.catalog.name == "testcat")
         assert(ctas.tableName == Identifier.of(Array("mydb"), "table_name"))
         assert(ctas.properties == expectedProperties)
-        assert(ctas.writeOptions == Map("other" -> "20"))
+        assert(ctas.writeOptions.isEmpty)
         assert(ctas.partitioning.isEmpty)
         assert(ctas.ignoreIfExists)
 
@@ -620,7 +633,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val tableName1 = "db.tab"
     val tableIdent1 = TableIdentifier("tab", Option("db"))
     val tableName2 = "tab"
-    val tableIdent2 = TableIdentifier("tab", None)
+    val tableIdent2 = TableIdentifier("tab", Some("default"))
 
     parseResolveCompare(s"DROP TABLE $tableName1",
       DropTableCommand(tableIdent1, ifExists = false, isView = false, purge = false))
@@ -656,7 +669,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val viewName1 = "db.view"
     val viewIdent1 = TableIdentifier("view", Option("db"))
     val viewName2 = "view"
-    val viewIdent2 = TableIdentifier("view")
+    val viewIdent2 = TableIdentifier("view", Option("default"))
 
     parseResolveCompare(s"DROP VIEW $viewName1",
       DropTableCommand(viewIdent1, ifExists = false, isView = true, purge = false))
@@ -687,7 +700,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val parsed2_view = parseAndResolve(sql2_view)
     val parsed3_view = parseAndResolve(sql3_view)
 
-    val tableIdent = TableIdentifier("table_name", None)
+    val tableIdent = TableIdentifier("table_name", Some("default"))
     val expected1_view = AlterTableSetPropertiesCommand(
       tableIdent, Map("test" -> "test", "comment" -> "new_comment"), isView = true)
     val expected2_view = AlterTableUnsetPropertiesCommand(
@@ -714,8 +727,8 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed2 = parseAndResolve(sql2)
         val parsed3 = parseAndResolve(sql3)
 
-        val tableIdent = TableIdentifier(tblName, None)
         if (useV1Command) {
+          val tableIdent = TableIdentifier(tblName, Some("default"))
           val expected1 = AlterTableSetPropertiesCommand(
             tableIdent, Map("test" -> "test", "comment" -> "new_comment"), isView = false)
           val expected2 = AlterTableUnsetPropertiesCommand(
@@ -780,7 +793,7 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed = parseAndResolve(sql)
         if (useV1Command) {
           val expected = AlterTableSetPropertiesCommand(
-            TableIdentifier(tblName),
+            TableIdentifier(tblName, Some("default")),
             Map("a" -> "1", "b" -> "0.1", "c" -> "true"),
             isView = false)
 
@@ -805,7 +818,7 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed = parseAndResolve(sql)
         if (useV1Command) {
           val expected = AlterTableSetLocationCommand(
-            TableIdentifier(tblName, None),
+            TableIdentifier(tblName, Some("default")),
             None,
             "new location")
           comparePlans(parsed, expected)
@@ -827,8 +840,10 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed1 = parseAndResolve(sql1)
         val parsed2 = parseAndResolve(sql2)
         if (useV1Command) {
-          val expected1 = DescribeTableCommand(TableIdentifier(tblName, None), Map.empty, false)
-          val expected2 = DescribeTableCommand(TableIdentifier(tblName, None), Map.empty, true)
+          val expected1 = DescribeTableCommand(
+            TableIdentifier(tblName, Some("default")), Map.empty, false)
+          val expected2 = DescribeTableCommand(
+            TableIdentifier(tblName, Some("default")), Map.empty, true)
 
           comparePlans(parsed1, expected1)
           comparePlans(parsed2, expected2)
@@ -850,7 +865,7 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed3 = parseAndResolve(sql3)
         if (useV1Command) {
           val expected3 = DescribeTableCommand(
-            TableIdentifier(tblName, None), Map("a" -> "1"), false)
+            TableIdentifier(tblName, Some("default")), Map("a" -> "1"), false)
           comparePlans(parsed3, expected3)
         } else {
           parsed3 match {
@@ -885,33 +900,34 @@ class PlanResolutionSuite extends AnalysisTest {
       val parsed4 = parseAndResolve(sql4)
 
       parsed1 match {
-        case DeleteFromTable(_: DataSourceV2Relation, None) =>
-        case _ => fail("Expect DeleteFromTable, bug got:\n" + parsed1.treeString)
+        case DeleteFromTable(AsDataSourceV2Relation(_), None) =>
+        case _ => fail("Expect DeleteFromTable, but got:\n" + parsed1.treeString)
       }
 
       parsed2 match {
         case DeleteFromTable(
-          _: DataSourceV2Relation,
+          AsDataSourceV2Relation(_),
           Some(EqualTo(name: UnresolvedAttribute, StringLiteral("Robert")))) =>
           assert(name.name == "name")
-        case _ => fail("Expect DeleteFromTable, bug got:\n" + parsed2.treeString)
+        case _ => fail("Expect DeleteFromTable, but got:\n" + parsed2.treeString)
       }
 
       parsed3 match {
         case DeleteFromTable(
-          SubqueryAlias(AliasIdentifier("t", None), _: DataSourceV2Relation),
+          SubqueryAlias(AliasIdentifier("t", Seq()), AsDataSourceV2Relation(_)),
           Some(EqualTo(name: UnresolvedAttribute, StringLiteral("Robert")))) =>
           assert(name.name == "t.name")
-        case _ => fail("Expect DeleteFromTable, bug got:\n" + parsed3.treeString)
+        case _ => fail("Expect DeleteFromTable, but got:\n" + parsed3.treeString)
       }
 
       parsed4 match {
-        case DeleteFromTable(SubqueryAlias(AliasIdentifier("t", None), _: DataSourceV2Relation),
+        case DeleteFromTable(
+            SubqueryAlias(AliasIdentifier("t", Seq()), AsDataSourceV2Relation(_)),
             Some(InSubquery(values, query))) =>
           assert(values.size == 1 && values.head.isInstanceOf[UnresolvedAttribute])
           assert(values.head.asInstanceOf[UnresolvedAttribute].name == "t.name")
           query match {
-            case ListQuery(Project(projects, SubqueryAlias(AliasIdentifier("s", None),
+            case ListQuery(Project(projects, SubqueryAlias(AliasIdentifier("s", Seq()),
                 UnresolvedSubqueryColumnAliases(outputColumnNames, Project(_, _: OneRowRelation)))),
                 _, _, _) =>
               assert(projects.size == 1 && projects.head.name == "s.name")
@@ -944,7 +960,7 @@ class PlanResolutionSuite extends AnalysisTest {
 
       parsed1 match {
         case UpdateTable(
-            _: DataSourceV2Relation,
+            AsDataSourceV2Relation(_),
             Seq(Assignment(name: UnresolvedAttribute, StringLiteral("Robert")),
               Assignment(age: UnresolvedAttribute, IntegerLiteral(32))),
             None) =>
@@ -956,7 +972,7 @@ class PlanResolutionSuite extends AnalysisTest {
 
       parsed2 match {
         case UpdateTable(
-            SubqueryAlias(AliasIdentifier("t", None), _: DataSourceV2Relation),
+            SubqueryAlias(AliasIdentifier("t", Seq()), AsDataSourceV2Relation(_)),
             Seq(Assignment(name: UnresolvedAttribute, StringLiteral("Robert")),
               Assignment(age: UnresolvedAttribute, IntegerLiteral(32))),
             None) =>
@@ -968,7 +984,7 @@ class PlanResolutionSuite extends AnalysisTest {
 
       parsed3 match {
         case UpdateTable(
-            SubqueryAlias(AliasIdentifier("t", None), _: DataSourceV2Relation),
+            SubqueryAlias(AliasIdentifier("t", Seq()), AsDataSourceV2Relation(_)),
             Seq(Assignment(name: UnresolvedAttribute, StringLiteral("Robert")),
               Assignment(age: UnresolvedAttribute, IntegerLiteral(32))),
             Some(EqualTo(p: UnresolvedAttribute, IntegerLiteral(1)))) =>
@@ -980,14 +996,14 @@ class PlanResolutionSuite extends AnalysisTest {
       }
 
       parsed4 match {
-        case UpdateTable(SubqueryAlias(AliasIdentifier("t", None), _: DataSourceV2Relation),
+        case UpdateTable(SubqueryAlias(AliasIdentifier("t", Seq()), AsDataSourceV2Relation(_)),
           Seq(Assignment(key: UnresolvedAttribute, IntegerLiteral(32))),
           Some(InSubquery(values, query))) =>
           assert(key.name == "t.age")
           assert(values.size == 1 && values.head.isInstanceOf[UnresolvedAttribute])
           assert(values.head.asInstanceOf[UnresolvedAttribute].name == "t.name")
           query match {
-            case ListQuery(Project(projects, SubqueryAlias(AliasIdentifier("s", None),
+            case ListQuery(Project(projects, SubqueryAlias(AliasIdentifier("s", Seq()),
                 UnresolvedSubqueryColumnAliases(outputColumnNames, Project(_, _: OneRowRelation)))),
                 _, _, _) =>
               assert(projects.size == 1 && projects.head.name == "s.name")
@@ -1012,27 +1028,29 @@ class PlanResolutionSuite extends AnalysisTest {
     Seq("v1Table" -> true, "v2Table" -> false, "testcat.tab" -> false).foreach {
       case (tblName, useV1Command) =>
         val sql1 = s"ALTER TABLE $tblName ALTER COLUMN i TYPE bigint"
-        val sql2 = s"ALTER TABLE $tblName ALTER COLUMN i TYPE bigint COMMENT 'new comment'"
-        val sql3 = s"ALTER TABLE $tblName ALTER COLUMN i COMMENT 'new comment'"
+        val sql2 = s"ALTER TABLE $tblName ALTER COLUMN i COMMENT 'new comment'"
 
         val parsed1 = parseAndResolve(sql1)
         val parsed2 = parseAndResolve(sql2)
 
-        val tableIdent = TableIdentifier(tblName, None)
         if (useV1Command) {
+          val tableIdent = TableIdentifier(tblName, Some("default"))
+          val oldColumn = StructField("i", IntegerType)
           val newColumn = StructField("i", LongType)
           val expected1 = AlterTableChangeColumnCommand(
             tableIdent, "i", newColumn)
           val expected2 = AlterTableChangeColumnCommand(
-            tableIdent, "i", newColumn.withComment("new comment"))
+            tableIdent, "i", oldColumn.withComment("new comment"))
 
           comparePlans(parsed1, expected1)
           comparePlans(parsed2, expected2)
 
+          val sql3 = s"ALTER TABLE $tblName ALTER COLUMN j COMMENT 'new comment'"
           val e1 = intercept[AnalysisException] {
             parseAndResolve(sql3)
           }
-          assert(e1.getMessage.contains("ALTER COLUMN with v1 tables must specify new data type"))
+          assert(e1.getMessage.contains(
+            "ALTER COLUMN cannot find column j in v1 table. Available: i, s"))
 
           val sql4 = s"ALTER TABLE $tblName ALTER COLUMN a.b.c TYPE bigint"
           val e2 = intercept[AnalysisException] {
@@ -1040,18 +1058,7 @@ class PlanResolutionSuite extends AnalysisTest {
           }
           assert(e2.getMessage.contains(
             "ALTER COLUMN with qualified column is only supported with v2 tables"))
-
-          val sql5 = s"ALTER TABLE $tblName ALTER COLUMN i TYPE char(1)"
-          val builder = new MetadataBuilder
-          builder.putString(HIVE_TYPE_STRING, CharType(1).catalogString)
-          val newColumnWithCleanedType = StructField("i", StringType, true, builder.build())
-          val expected5 = AlterTableChangeColumnCommand(
-            tableIdent, "i", newColumnWithCleanedType)
-          val parsed5 = parseAndResolve(sql5)
-          comparePlans(parsed5, expected5)
         } else {
-          val parsed3 = parseAndResolve(sql3)
-
           parsed1 match {
             case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
               assert(changes == Seq(
@@ -1062,18 +1069,50 @@ class PlanResolutionSuite extends AnalysisTest {
           parsed2 match {
             case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
               assert(changes == Seq(
-                TableChange.updateColumnType(Array("i"), LongType),
                 TableChange.updateColumnComment(Array("i"), "new comment")))
             case _ => fail("expect AlterTable")
           }
+        }
+    }
 
-          parsed3 match {
-            case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
-              assert(changes == Seq(
-                TableChange.updateColumnComment(Array("i"), "new comment")))
-            case _ => fail("expect AlterTable")
+    val sql = s"ALTER TABLE v1HiveTable ALTER COLUMN i TYPE char(1)"
+    val builder = new MetadataBuilder
+    builder.putString(HIVE_TYPE_STRING, CharType(1).catalogString)
+    val newColumnWithCleanedType = StructField("i", StringType, true, builder.build())
+    val expected = AlterTableChangeColumnCommand(
+      TableIdentifier("v1HiveTable", Some("default")), "i", newColumnWithCleanedType)
+    val parsed = parseAndResolve(sql)
+    comparePlans(parsed, expected)
+  }
+
+  test("alter table: alter column action is not specified") {
+    val e = intercept[AnalysisException] {
+      parseAndResolve("ALTER TABLE v1Table ALTER COLUMN i")
+    }
+    assert(e.getMessage.contains(
+      "ALTER TABLE table ALTER COLUMN requires a TYPE, a SET/DROP, a COMMENT, or a FIRST/AFTER"))
+  }
+
+  test("alter table: alter column case sensitivity for v1 table") {
+    val tblName = "v1Table"
+    Seq(true, false).foreach { caseSensitive =>
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+        val sql = s"ALTER TABLE $tblName ALTER COLUMN I COMMENT 'new comment'"
+        if (caseSensitive) {
+          val e = intercept[AnalysisException] {
+            parseAndResolve(sql)
           }
+          assert(e.getMessage.contains(
+            "ALTER COLUMN cannot find column I in v1 table. Available: i, s"))
+        } else {
+          val actual = parseAndResolve(sql)
+          val expected = AlterTableChangeColumnCommand(
+            TableIdentifier(tblName, Some("default")),
+            "I",
+            StructField("I", IntegerType).withComment("new comment"))
+          comparePlans(actual, expected)
         }
+      }
     }
   }
 
@@ -1100,7 +1139,7 @@ class PlanResolutionSuite extends AnalysisTest {
   }
 
   val DSV2ResolutionTests = {
-    val v2SessionCatalogTable = s"${CatalogManager.SESSION_CATALOG_NAME}.v2Table"
+    val v2SessionCatalogTable = s"${CatalogManager.SESSION_CATALOG_NAME}.default.v2Table"
     Seq(
       ("ALTER TABLE testcat.tab ALTER COLUMN i TYPE bigint", false),
       ("ALTER TABLE tab ALTER COLUMN i TYPE bigint", false),
@@ -1116,7 +1155,7 @@ class PlanResolutionSuite extends AnalysisTest {
       (s"SHOW TBLPROPERTIES $v2SessionCatalogTable", true),
       ("SELECT * from tab", false),
       ("SELECT * from testcat.tab", false),
-      (s"SELECT * from ${CatalogManager.SESSION_CATALOG_NAME}.v2Table", true)
+      (s"SELECT * from $v2SessionCatalogTable", true)
     )
   }
 
@@ -1129,7 +1168,7 @@ class PlanResolutionSuite extends AnalysisTest {
         case AlterTable(_, _, r: DataSourceV2Relation, _) =>
           assert(r.catalog.exists(_ == catlogIdent))
           assert(r.identifier.exists(_.name() == tableIdent))
-        case Project(_, r: DataSourceV2Relation) =>
+        case Project(_, AsDataSourceV2Relation(r)) =>
           assert(r.catalog.exists(_ == catlogIdent))
           assert(r.identifier.exists(_.name() == tableIdent))
         case InsertIntoStatement(r: DataSourceV2Relation, _, _, _, _) =>
@@ -1206,8 +1245,8 @@ class PlanResolutionSuite extends AnalysisTest {
            """.stripMargin
         parseAndResolve(sql1) match {
           case MergeIntoTable(
-              SubqueryAlias(AliasIdentifier("target", None), target: DataSourceV2Relation),
-              SubqueryAlias(AliasIdentifier("source", None), source: DataSourceV2Relation),
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), AsDataSourceV2Relation(source)),
               mergeCondition,
               Seq(DeleteAction(Some(EqualTo(dl: AttributeReference, StringLiteral("delete")))),
                 UpdateAction(Some(EqualTo(ul: AttributeReference, StringLiteral("update"))),
@@ -1232,8 +1271,8 @@ class PlanResolutionSuite extends AnalysisTest {
            """.stripMargin
         parseAndResolve(sql2) match {
           case MergeIntoTable(
-              SubqueryAlias(AliasIdentifier("target", None), target: DataSourceV2Relation),
-              SubqueryAlias(AliasIdentifier("source", None), source: DataSourceV2Relation),
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), AsDataSourceV2Relation(source)),
               mergeCondition,
               Seq(DeleteAction(Some(EqualTo(dl: AttributeReference, StringLiteral("delete")))),
                 UpdateAction(Some(EqualTo(ul: AttributeReference,
@@ -1252,16 +1291,16 @@ class PlanResolutionSuite extends AnalysisTest {
              |MERGE INTO $target AS target
              |USING $source AS source
              |ON target.i = source.i
-             |WHEN MATCHED THEN DELETE
+             |WHEN MATCHED AND (target.s='delete') THEN DELETE
              |WHEN MATCHED THEN UPDATE SET target.s = source.s
              |WHEN NOT MATCHED THEN INSERT (target.i, target.s) values (source.i, source.s)
            """.stripMargin
         parseAndResolve(sql3) match {
           case MergeIntoTable(
-              SubqueryAlias(AliasIdentifier("target", None), target: DataSourceV2Relation),
-              SubqueryAlias(AliasIdentifier("source", None), source: DataSourceV2Relation),
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), AsDataSourceV2Relation(source)),
               mergeCondition,
-              Seq(DeleteAction(None), UpdateAction(None, updateAssigns)),
+              Seq(DeleteAction(Some(_)), UpdateAction(None, updateAssigns)),
               Seq(InsertAction(None, insertAssigns))) =>
             checkResolution(target, source, mergeCondition, None, None, None,
               updateAssigns, insertAssigns)
@@ -1282,8 +1321,8 @@ class PlanResolutionSuite extends AnalysisTest {
            """.stripMargin
         parseAndResolve(sql4) match {
           case MergeIntoTable(
-              SubqueryAlias(AliasIdentifier("target", None), target: DataSourceV2Relation),
-              SubqueryAlias(AliasIdentifier("source", None), source: Project),
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), source: Project),
               mergeCondition,
               Seq(DeleteAction(Some(EqualTo(dl: AttributeReference, StringLiteral("delete")))),
                 UpdateAction(Some(EqualTo(ul: AttributeReference, StringLiteral("update"))),
@@ -1311,8 +1350,8 @@ class PlanResolutionSuite extends AnalysisTest {
            """.stripMargin
         parseAndResolve(sql5) match {
           case MergeIntoTable(
-              SubqueryAlias(AliasIdentifier("target", None), target: DataSourceV2Relation),
-              SubqueryAlias(AliasIdentifier("source", None), source: Project),
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), source: Project),
               mergeCondition,
               Seq(DeleteAction(Some(EqualTo(dl: AttributeReference, StringLiteral("delete")))),
                 UpdateAction(Some(EqualTo(ul: AttributeReference, StringLiteral("update"))),
@@ -1339,17 +1378,17 @@ class PlanResolutionSuite extends AnalysisTest {
            |MERGE INTO $target
            |USING $source
            |ON 1 = 1
-           |WHEN MATCHED THEN DELETE
+           |WHEN MATCHED AND (${target}.s='delete') THEN DELETE
            |WHEN MATCHED THEN UPDATE SET s = 1
            |WHEN NOT MATCHED AND (s = 'a') THEN INSERT (i) values (i)
          """.stripMargin
 
       parseAndResolve(sql1) match {
         case MergeIntoTable(
-            target: DataSourceV2Relation,
-            source: DataSourceV2Relation,
+            AsDataSourceV2Relation(target),
+            AsDataSourceV2Relation(source),
             _,
-            Seq(DeleteAction(None), UpdateAction(None, updateAssigns)),
+            Seq(DeleteAction(Some(_)), UpdateAction(None, updateAssigns)),
             Seq(InsertAction(
               Some(EqualTo(il: AttributeReference, StringLiteral("a"))),
               insertAssigns))) =>
@@ -1426,7 +1465,7 @@ class PlanResolutionSuite extends AnalysisTest {
          |MERGE INTO non_exist_target
          |USING non_exist_source
          |ON target.i = source.i
-         |WHEN MATCHED THEN DELETE
+         |WHEN MATCHED AND (non_exist_target.s='delete') THEN DELETE
          |WHEN MATCHED THEN UPDATE SET *
          |WHEN NOT MATCHED THEN INSERT *
        """.stripMargin
@@ -1453,8 +1492,8 @@ class PlanResolutionSuite extends AnalysisTest {
 
     parseAndResolve(sql) match {
       case MergeIntoTable(
-          SubqueryAlias(AliasIdentifier("target", None), _: DataSourceV2Relation),
-          SubqueryAlias(AliasIdentifier("source", None), _: DataSourceV2Relation),
+          SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(_)),
+          SubqueryAlias(AliasIdentifier("source", Seq()), AsDataSourceV2Relation(_)),
           EqualTo(l: UnresolvedAttribute, r: UnresolvedAttribute),
           Seq(
             DeleteAction(Some(EqualTo(dl: UnresolvedAttribute, StringLiteral("delete")))),
@@ -1479,5 +1518,48 @@ class PlanResolutionSuite extends AnalysisTest {
       case l => fail("Expected unresolved MergeIntoTable, but got:\n" + l.treeString)
     }
   }
+
+  test("SPARK-31147: forbid CHAR type in non-Hive tables") {
+    def checkFailure(t: String, provider: String): Unit = {
+      val types = Seq(
+        "CHAR(2)",
+        "ARRAY<CHAR(2)>",
+        "MAP<INT, CHAR(2)>",
+        "MAP<CHAR(2), INT>",
+        "STRUCT<s: CHAR(2)>")
+      types.foreach { tpe =>
+        intercept[AnalysisException] {
+          parseAndResolve(s"CREATE TABLE $t(col $tpe) USING $provider")
+        }
+        intercept[AnalysisException] {
+          parseAndResolve(s"REPLACE TABLE $t(col $tpe) USING $provider")
+        }
+        intercept[AnalysisException] {
+          parseAndResolve(s"CREATE OR REPLACE TABLE $t(col $tpe) USING $provider")
+        }
+        intercept[AnalysisException] {
+          parseAndResolve(s"ALTER TABLE $t ADD COLUMN col $tpe")
+        }
+        intercept[AnalysisException] {
+          parseAndResolve(s"ALTER TABLE $t ADD COLUMN col $tpe")
+        }
+        intercept[AnalysisException] {
+          parseAndResolve(s"ALTER TABLE $t ALTER COLUMN col TYPE $tpe")
+        }
+      }
+    }
+
+    checkFailure("v1Table", v1Format)
+    checkFailure("v2Table", v2Format)
+    checkFailure("testcat.tab", "foo")
+  }
+
   // TODO: add tests for more commands.
 }
+
+object AsDataSourceV2Relation {
+  def unapply(plan: LogicalPlan): Option[DataSourceV2Relation] = plan match {
+    case SubqueryAlias(_, r: DataSourceV2Relation) => Some(r)
+    case _ => None
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
index b76db70494cf8..b94918eccd46e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
@@ -22,68 +22,128 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.sources
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
 
 class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
-
-  test("translate simple expression") {
-    val attrInt = 'cint.int
-    val attrStr = 'cstr.string
-
-    testTranslateFilter(EqualTo(attrInt, 1), Some(sources.EqualTo("cint", 1)))
-    testTranslateFilter(EqualTo(1, attrInt), Some(sources.EqualTo("cint", 1)))
+  val attrInts = Seq(
+    'cint.int,
+    Symbol("c.int").int,
+    GetStructField('a.struct(StructType(
+      StructField("cstr", StringType, nullable = true) ::
+        StructField("cint", IntegerType, nullable = true) :: Nil)), 1, None),
+    GetStructField('a.struct(StructType(
+      StructField("c.int", IntegerType, nullable = true) ::
+        StructField("cstr", StringType, nullable = true) :: Nil)), 0, None),
+    GetStructField(Symbol("a.b").struct(StructType(
+      StructField("cstr1", StringType, nullable = true) ::
+        StructField("cstr2", StringType, nullable = true) ::
+        StructField("cint", IntegerType, nullable = true) :: Nil)), 2, None),
+    GetStructField(Symbol("a.b").struct(StructType(
+      StructField("c.int", IntegerType, nullable = true) :: Nil)), 0, None),
+    GetStructField(GetStructField('a.struct(StructType(
+      StructField("cstr1", StringType, nullable = true) ::
+        StructField("b", StructType(StructField("cint", IntegerType, nullable = true) ::
+          StructField("cstr2", StringType, nullable = true) :: Nil)) :: Nil)), 1, None), 0, None)
+  ).zip(Seq(
+    "cint",
+    "`c.int`", // single level field that contains `dot` in name
+    "a.cint", // two level nested field
+    "a.`c.int`", // two level nested field, and nested level contains `dot`
+    "`a.b`.cint", // two level nested field, and top level contains `dot`
+    "`a.b`.`c.int`", // two level nested field, and both levels contain `dot`
+    "a.b.cint" // three level nested field
+  ))
+
+  val attrStrs = Seq(
+    'cstr.string,
+    Symbol("c.str").string,
+    GetStructField('a.struct(StructType(
+      StructField("cint", IntegerType, nullable = true) ::
+        StructField("cstr", StringType, nullable = true) :: Nil)), 1, None),
+    GetStructField('a.struct(StructType(
+      StructField("c.str", StringType, nullable = true) ::
+        StructField("cint", IntegerType, nullable = true) :: Nil)), 0, None),
+    GetStructField(Symbol("a.b").struct(StructType(
+      StructField("cint1", IntegerType, nullable = true) ::
+        StructField("cint2", IntegerType, nullable = true) ::
+        StructField("cstr", StringType, nullable = true) :: Nil)), 2, None),
+    GetStructField(Symbol("a.b").struct(StructType(
+      StructField("c.str", StringType, nullable = true) :: Nil)), 0, None),
+    GetStructField(GetStructField('a.struct(StructType(
+      StructField("cint1", IntegerType, nullable = true) ::
+        StructField("b", StructType(StructField("cstr", StringType, nullable = true) ::
+          StructField("cint2", IntegerType, nullable = true) :: Nil)) :: Nil)), 1, None), 0, None)
+  ).zip(Seq(
+    "cstr",
+    "`c.str`", // single level field that contains `dot` in name
+    "a.cstr", // two level nested field
+    "a.`c.str`", // two level nested field, and nested level contains `dot`
+    "`a.b`.cstr", // two level nested field, and top level contains `dot`
+    "`a.b`.`c.str`", // two level nested field, and both levels contain `dot`
+    "a.b.cstr" // three level nested field
+  ))
+
+  test("translate simple expression") { attrInts.zip(attrStrs)
+    .foreach { case ((attrInt, intColName), (attrStr, strColName)) =>
+
+    testTranslateFilter(EqualTo(attrInt, 1), Some(sources.EqualTo(intColName, 1)))
+    testTranslateFilter(EqualTo(1, attrInt), Some(sources.EqualTo(intColName, 1)))
 
     testTranslateFilter(EqualNullSafe(attrStr, Literal(null)),
-      Some(sources.EqualNullSafe("cstr", null)))
+      Some(sources.EqualNullSafe(strColName, null)))
     testTranslateFilter(EqualNullSafe(Literal(null), attrStr),
-      Some(sources.EqualNullSafe("cstr", null)))
+      Some(sources.EqualNullSafe(strColName, null)))
 
-    testTranslateFilter(GreaterThan(attrInt, 1), Some(sources.GreaterThan("cint", 1)))
-    testTranslateFilter(GreaterThan(1, attrInt), Some(sources.LessThan("cint", 1)))
+    testTranslateFilter(GreaterThan(attrInt, 1), Some(sources.GreaterThan(intColName, 1)))
+    testTranslateFilter(GreaterThan(1, attrInt), Some(sources.LessThan(intColName, 1)))
 
-    testTranslateFilter(LessThan(attrInt, 1), Some(sources.LessThan("cint", 1)))
-    testTranslateFilter(LessThan(1, attrInt), Some(sources.GreaterThan("cint", 1)))
+    testTranslateFilter(LessThan(attrInt, 1), Some(sources.LessThan(intColName, 1)))
+    testTranslateFilter(LessThan(1, attrInt), Some(sources.GreaterThan(intColName, 1)))
 
-    testTranslateFilter(GreaterThanOrEqual(attrInt, 1), Some(sources.GreaterThanOrEqual("cint", 1)))
-    testTranslateFilter(GreaterThanOrEqual(1, attrInt), Some(sources.LessThanOrEqual("cint", 1)))
+    testTranslateFilter(GreaterThanOrEqual(attrInt, 1),
+      Some(sources.GreaterThanOrEqual(intColName, 1)))
+    testTranslateFilter(GreaterThanOrEqual(1, attrInt),
+      Some(sources.LessThanOrEqual(intColName, 1)))
 
-    testTranslateFilter(LessThanOrEqual(attrInt, 1), Some(sources.LessThanOrEqual("cint", 1)))
-    testTranslateFilter(LessThanOrEqual(1, attrInt), Some(sources.GreaterThanOrEqual("cint", 1)))
+    testTranslateFilter(LessThanOrEqual(attrInt, 1),
+      Some(sources.LessThanOrEqual(intColName, 1)))
+    testTranslateFilter(LessThanOrEqual(1, attrInt),
+      Some(sources.GreaterThanOrEqual(intColName, 1)))
 
-    testTranslateFilter(InSet(attrInt, Set(1, 2, 3)), Some(sources.In("cint", Array(1, 2, 3))))
+    testTranslateFilter(InSet(attrInt, Set(1, 2, 3)), Some(sources.In(intColName, Array(1, 2, 3))))
 
-    testTranslateFilter(In(attrInt, Seq(1, 2, 3)), Some(sources.In("cint", Array(1, 2, 3))))
+    testTranslateFilter(In(attrInt, Seq(1, 2, 3)), Some(sources.In(intColName, Array(1, 2, 3))))
 
-    testTranslateFilter(IsNull(attrInt), Some(sources.IsNull("cint")))
-    testTranslateFilter(IsNotNull(attrInt), Some(sources.IsNotNull("cint")))
+    testTranslateFilter(IsNull(attrInt), Some(sources.IsNull(intColName)))
+    testTranslateFilter(IsNotNull(attrInt), Some(sources.IsNotNull(intColName)))
 
     // cint > 1 AND cint < 10
     testTranslateFilter(And(
       GreaterThan(attrInt, 1),
       LessThan(attrInt, 10)),
       Some(sources.And(
-        sources.GreaterThan("cint", 1),
-        sources.LessThan("cint", 10))))
+        sources.GreaterThan(intColName, 1),
+        sources.LessThan(intColName, 10))))
 
     // cint >= 8 OR cint <= 2
     testTranslateFilter(Or(
       GreaterThanOrEqual(attrInt, 8),
       LessThanOrEqual(attrInt, 2)),
       Some(sources.Or(
-        sources.GreaterThanOrEqual("cint", 8),
-        sources.LessThanOrEqual("cint", 2))))
+        sources.GreaterThanOrEqual(intColName, 8),
+        sources.LessThanOrEqual(intColName, 2))))
 
     testTranslateFilter(Not(GreaterThanOrEqual(attrInt, 8)),
-      Some(sources.Not(sources.GreaterThanOrEqual("cint", 8))))
+      Some(sources.Not(sources.GreaterThanOrEqual(intColName, 8))))
 
-    testTranslateFilter(StartsWith(attrStr, "a"), Some(sources.StringStartsWith("cstr", "a")))
+    testTranslateFilter(StartsWith(attrStr, "a"), Some(sources.StringStartsWith(strColName, "a")))
 
-    testTranslateFilter(EndsWith(attrStr, "a"), Some(sources.StringEndsWith("cstr", "a")))
+    testTranslateFilter(EndsWith(attrStr, "a"), Some(sources.StringEndsWith(strColName, "a")))
 
-    testTranslateFilter(Contains(attrStr, "a"), Some(sources.StringContains("cstr", "a")))
-  }
+    testTranslateFilter(Contains(attrStr, "a"), Some(sources.StringContains(strColName, "a")))
+  }}
 
-  test("translate complex expression") {
-    val attrInt = 'cint.int
+  test("translate complex expression") { attrInts.foreach { case (attrInt, intColName) =>
 
     // ABS(cint) - 2 <= 1
     testTranslateFilter(LessThanOrEqual(
@@ -102,11 +162,11 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
         LessThan(attrInt, 100))),
       Some(sources.Or(
         sources.And(
-          sources.GreaterThan("cint", 1),
-          sources.LessThan("cint", 10)),
+          sources.GreaterThan(intColName, 1),
+          sources.LessThan(intColName, 10)),
         sources.And(
-          sources.GreaterThan("cint", 50),
-          sources.LessThan("cint", 100)))))
+          sources.GreaterThan(intColName, 50),
+          sources.LessThan(intColName, 100)))))
 
     // SPARK-22548 Incorrect nested AND expression pushed down to JDBC data source
     // (cint > 1 AND ABS(cint) < 10) OR (cint < 50 AND cint > 100)
@@ -142,11 +202,11 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
         LessThan(attrInt, -10))),
       Some(sources.Or(
         sources.Or(
-          sources.EqualTo("cint", 1),
-          sources.EqualTo("cint", 10)),
+          sources.EqualTo(intColName, 1),
+          sources.EqualTo(intColName, 10)),
         sources.Or(
-          sources.GreaterThan("cint", 0),
-          sources.LessThan("cint", -10)))))
+          sources.GreaterThan(intColName, 0),
+          sources.LessThan(intColName, -10)))))
 
     // (cint = 1 OR ABS(cint) = 10) OR (cint > 0 OR cint < -10)
     testTranslateFilter(Or(
@@ -173,11 +233,11 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
         IsNotNull(attrInt))),
       Some(sources.And(
         sources.And(
-          sources.GreaterThan("cint", 1),
-          sources.LessThan("cint", 10)),
+          sources.GreaterThan(intColName, 1),
+          sources.LessThan(intColName, 10)),
         sources.And(
-          sources.EqualTo("cint", 6),
-          sources.IsNotNull("cint")))))
+          sources.EqualTo(intColName, 6),
+          sources.IsNotNull(intColName)))))
 
     // (cint > 1 AND cint < 10) AND (ABS(cint) = 6 AND cint IS NOT NULL)
     testTranslateFilter(And(
@@ -201,11 +261,11 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
         IsNotNull(attrInt))),
       Some(sources.And(
         sources.Or(
-          sources.GreaterThan("cint", 1),
-          sources.LessThan("cint", 10)),
+          sources.GreaterThan(intColName, 1),
+          sources.LessThan(intColName, 10)),
         sources.Or(
-          sources.EqualTo("cint", 6),
-          sources.IsNotNull("cint")))))
+          sources.EqualTo(intColName, 6),
+          sources.IsNotNull(intColName)))))
 
     // (cint > 1 OR cint < 10) AND (cint = 6 OR cint IS NOT NULL)
     testTranslateFilter(And(
@@ -217,7 +277,7 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
         // Functions such as 'Abs' are not supported
         EqualTo(Abs(attrInt), 6),
         IsNotNull(attrInt))), None)
-  }
+  }}
 
   test("SPARK-26865 DataSourceV2Strategy should push normalized filters") {
     val attrInt = 'cint.int
@@ -226,13 +286,38 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-31027 test `PushableColumn.unapply` that finds the column name of " +
+    "an expression that can be pushed down") {
+    attrInts.foreach { case (attrInt, colName) =>
+      assert(PushableColumnAndNestedColumn.unapply(attrInt) === Some(colName))
+
+      if (colName.contains(".")) {
+        assert(PushableColumnWithoutNestedColumn.unapply(attrInt) === None)
+      } else {
+        assert(PushableColumnWithoutNestedColumn.unapply(attrInt) === Some(colName))
+      }
+    }
+    attrStrs.foreach { case (attrStr, colName) =>
+      assert(PushableColumnAndNestedColumn.unapply(attrStr) === Some(colName))
+
+      if (colName.contains(".")) {
+        assert(PushableColumnWithoutNestedColumn.unapply(attrStr) === None)
+      } else {
+        assert(PushableColumnWithoutNestedColumn.unapply(attrStr) === Some(colName))
+      }
+    }
+
+    // `Abs(col)` can not be pushed down, so it returns `None`
+    assert(PushableColumnAndNestedColumn.unapply(Abs('col.int)) === None)
+  }
+
   /**
    * Translate the given Catalyst [[Expression]] into data source [[sources.Filter]]
    * then verify against the given [[sources.Filter]].
    */
   def testTranslateFilter(catalystFilter: Expression, result: Option[sources.Filter]): Unit = {
     assertResult(result) {
-      DataSourceStrategy.translateFilter(catalystFilter)
+      DataSourceStrategy.translateFilter(catalystFilter, true)
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
index 553773e2555cf..c412d953d7037 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
@@ -362,13 +362,15 @@ class FileIndexSuite extends SharedSparkSession {
       val wrongBasePath = new File(dir, "unknown")
       // basePath must be a directory
       wrongBasePath.mkdir()
-      val parameters = Map("basePath" -> wrongBasePath.getCanonicalPath)
-      val fileIndex = new InMemoryFileIndex(spark, Seq(path), parameters, None)
-      val msg = intercept[IllegalArgumentException] {
-        // trigger inferPartitioning()
-        fileIndex.partitionSpec()
-      }.getMessage
-      assert(msg === s"Wrong basePath ${wrongBasePath.getCanonicalPath} for the root path: $path")
+      withClue("SPARK-32368: 'basePath' can be case insensitive") {
+        val parameters = Map("bAsepAtH" -> wrongBasePath.getCanonicalPath)
+        val fileIndex = new InMemoryFileIndex(spark, Seq(path), parameters, None)
+        val msg = intercept[IllegalArgumentException] {
+          // trigger inferPartitioning()
+          fileIndex.partitionSpec()
+        }.getMessage
+        assert(msg === s"Wrong basePath ${wrongBasePath.getCanonicalPath} for the root path: $path")
+      }
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
index 5977e867f788a..37ec6e29213e7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
@@ -301,35 +301,15 @@ abstract class SchemaPruningSuite
     checkAnswer(query, Row("Y.", 1) :: Row("X.", 1) :: Row(null, 2) :: Row(null, 2) :: Nil)
   }
 
-  testSchemaPruning("select explode of nested field of array of struct") {
-    // Config combinations
-    val configs = Seq((true, true), (true, false), (false, true), (false, false))
-
-    configs.foreach { case (nestedPruning, nestedPruningOnExpr) =>
-      withSQLConf(
-          SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key -> nestedPruning.toString,
-          SQLConf.NESTED_PRUNING_ON_EXPRESSIONS.key -> nestedPruningOnExpr.toString) {
-        val query1 = spark.table("contacts")
-          .select(explode(col("friends.first")))
-        if (nestedPruning) {
-          // If `NESTED_SCHEMA_PRUNING_ENABLED` is enabled,
-          // even disabling `NESTED_PRUNING_ON_EXPRESSIONS`,
-          // nested schema is still pruned at scan node.
-          checkScan(query1, "struct<friends:array<struct<first:string>>>")
-        } else {
-          checkScan(query1, "struct<friends:array<struct<first:string,middle:string,last:string>>>")
-        }
-        checkAnswer(query1, Row("Susan") :: Nil)
-
-        val query2 = spark.table("contacts")
-          .select(explode(col("friends.first")), col("friends.middle"))
-        if (nestedPruning) {
-          checkScan(query2, "struct<friends:array<struct<first:string,middle:string>>>")
-        } else {
-          checkScan(query2, "struct<friends:array<struct<first:string,middle:string,last:string>>>")
-        }
-        checkAnswer(query2, Row("Susan", Array("Z.")) :: Nil)
-      }
+  testSchemaPruning("SPARK-32163: nested pruning should work even with cosmetic variations") {
+    withTempView("contact_alias") {
+      sql("select * from contacts")
+        .select(explode(col("friends.first")), col("friends"))
+        .createOrReplaceTempView("contact_alias")
+
+      val query = sql("select friends.middle, col from contact_alias")
+      checkScan(query, "struct<friends:array<struct<first:string,middle:string>>>")
+      checkAnswer(query, Row(Array("Z."), "Susan") :: Nil)
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
index 2cd142f913072..8462916daaab8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
@@ -304,7 +304,7 @@ class BinaryFileFormatSuite extends QueryTest with SharedSparkSession {
     val partitionedFile = mock(classOf[PartitionedFile])
     when(partitionedFile.filePath).thenReturn(file.getPath)
     val encoder = RowEncoder(requiredSchema).resolveAndBind()
-    encoder.fromRow(reader(partitionedFile).next())
+    encoder.createDeserializer().apply(reader(partitionedFile).next())
   }
 
   test("column pruning") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala
index e2abb39c986a7..53d287b32f8db 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala
@@ -238,7 +238,9 @@ object CSVBenchmark extends SqlBasedBenchmark {
 
       def timestampStr: Dataset[String] = {
         spark.range(0, rowsNum, 1, 1).mapPartitions { iter =>
-          iter.map(i => s"1970-01-01T01:02:03.${100 + i % 100}Z")
+          iter.map {
+            i => s"1970-01-01T01:02:03.${i % 200}Z".stripSuffix(".0Z")
+          }
         }.select($"value".as("timestamp")).as[String]
       }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index 97dfbbdb7fd2f..4e93ea3e8e42f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -1080,7 +1080,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
         .format("csv")
         .option("header", "true")
         .option("timestampFormat", "yyyy/MM/dd HH:mm")
-        .option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+        .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .save(timestampsWithFormatPath)
 
       // This will load back the timestamps as string.
@@ -1102,7 +1102,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
         .option("header", "true")
         .option("inferSchema", "true")
         .option("timestampFormat", "yyyy/MM/dd HH:mm")
-        .option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+        .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .load(timestampsWithFormatPath)
 
       checkAnswer(readBack, timestampsWithFormat)
@@ -1182,7 +1182,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
         .schema(schemaWithCorrField1)
         .csv(testFile(valueMalformedFile))
       checkAnswer(df2,
-        Row(0, null, "0,2013-111-11 12:13:14") ::
+        Row(0, null, "0,2013-111_11 12:13:14") ::
         Row(1, java.sql.Date.valueOf("1983-08-04"), null) ::
         Nil)
 
@@ -1199,7 +1199,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
         .schema(schemaWithCorrField2)
         .csv(testFile(valueMalformedFile))
       checkAnswer(df3,
-        Row(0, "0,2013-111-11 12:13:14", null) ::
+        Row(0, "0,2013-111_11 12:13:14", null) ::
         Row(1, null, java.sql.Date.valueOf("1983-08-04")) ::
         Nil)
 
@@ -1435,7 +1435,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
     assert(df.filter($"_corrupt_record".isNull).count() == 1)
     checkAnswer(
       df.select(columnNameOfCorruptRecord),
-      Row("0,2013-111-11 12:13:14") :: Row(null) :: Nil
+      Row("0,2013-111_11 12:13:14") :: Row(null) :: Nil
     )
   }
 
@@ -1450,10 +1450,22 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
       val ds = sampledTestData.coalesce(1)
       ds.write.text(path.getAbsolutePath)
 
-      val readback = spark.read
+      val readback1 = spark.read
         .option("inferSchema", true).option("samplingRatio", 0.1)
         .csv(path.getCanonicalPath)
-      assert(readback.schema == new StructType().add("_c0", IntegerType))
+      assert(readback1.schema == new StructType().add("_c0", IntegerType))
+
+      withClue("SPARK-32621: 'path' option can cause issues while inferring schema") {
+        // During infer, "path" option gets added again to the paths that have already been listed.
+        // This results in reading more data than necessary and causes different schema to be
+        // inferred when sampling ratio is involved.
+        val readback2 = spark.read
+          .option("inferSchema", true).option("samplingRatio", 0.1)
+          .option("path", path.getCanonicalPath)
+          .format("csv")
+          .load
+        assert(readback2.schema == new StructType().add("_c0", IntegerType))
+      }
     })
   }
 
@@ -1890,11 +1902,26 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
 
   test("SPARK-25387: bad input should not cause NPE") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
-    val input = spark.createDataset(Seq("\u0000\u0000\u0001234"))
+    val input = spark.createDataset(Seq("\u0001\u0000\u0001234"))
 
     checkAnswer(spark.read.schema(schema).csv(input), Row(null))
     checkAnswer(spark.read.option("multiLine", true).schema(schema).csv(input), Row(null))
-    assert(spark.read.csv(input).collect().toSet == Set(Row()))
+    assert(spark.read.schema(schema).csv(input).collect().toSet == Set(Row(null)))
+  }
+
+  test("SPARK-31261: bad csv input with `columnNameCorruptRecord` should not cause NPE") {
+    val schema = StructType(
+      StructField("a", IntegerType) :: StructField("_corrupt_record", StringType) :: Nil)
+    val input = spark.createDataset(Seq("\u0001\u0000\u0001234"))
+
+    checkAnswer(
+      spark.read
+        .option("columnNameOfCorruptRecord", "_corrupt_record")
+        .schema(schema)
+        .csv(input),
+      Row(null, "\u0001\u0000\u0001234"))
+    assert(spark.read.schema(schema).csv(input).collect().toSet ==
+      Set(Row(null, "\u0001\u0000\u0001234")))
   }
 
   test("field names of inferred schema shouldn't compare to the first row") {
@@ -2093,7 +2120,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
     Seq("csv", "").foreach { reader =>
       withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> reader) {
         withTempPath { path =>
-          val df = Seq(("0", "2013-111-11")).toDF("a", "b")
+          val df = Seq(("0", "2013-111_11")).toDF("a", "b")
           df.write
             .option("header", "true")
             .csv(path.getAbsolutePath)
@@ -2109,7 +2136,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
             .option("columnNameOfCorruptRecord", columnNameOfCorruptRecord)
             .schema(schemaWithCorrField)
             .csv(path.getAbsoluteFile.toString)
-          checkAnswer(readDF, Row(0, null, "0,2013-111-11") :: Nil)
+          checkAnswer(readDF, Row(0, null, "0,2013-111_11") :: Nil)
         }
       }
     }
@@ -2216,7 +2243,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
                 val readback = spark.read
                   .option("mode", mode)
                   .option("header", true)
-                  .option("timestampFormat", "uuuu-MM-dd HH:mm:ss")
+                  .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
                   .option("multiLine", multiLine)
                   .schema("c0 string, c1 integer, c2 timestamp")
                   .csv(path.getAbsolutePath)
@@ -2235,7 +2262,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
   }
 
   test("filters push down - malformed input in PERMISSIVE mode") {
-    val invalidTs = "2019-123-14 20:35:30"
+    val invalidTs = "2019-123_14 20:35:30"
     val invalidRow = s"0,$invalidTs,999"
     val validTs = "2019-12-14 20:35:30"
     Seq(true, false).foreach { filterPushdown =>
@@ -2252,7 +2279,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
               .option("mode", "PERMISSIVE")
               .option("columnNameOfCorruptRecord", "c3")
               .option("header", true)
-              .option("timestampFormat", "uuuu-MM-dd HH:mm:ss")
+              .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
               .schema("c0 integer, c1 timestamp, c2 integer, c3 string")
               .csv(path.getAbsolutePath)
               .where(condition)
@@ -2294,6 +2321,63 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
       }
     }
   }
+
+  test("SPARK-30810: parses and convert a CSV Dataset having different column from 'value'") {
+    val ds = spark.range(2).selectExpr("concat('a,b,', id) AS `a.text`").as[String]
+    val csv = spark.read.option("header", true).option("inferSchema", true).csv(ds)
+    assert(csv.schema.fieldNames === Seq("a", "b", "0"))
+    checkAnswer(csv, Row("a", "b", 1))
+  }
+
+  test("SPARK-30960: parse date/timestamp string with legacy format") {
+    val ds = Seq("2020-1-12 3:23:34.12, 2020-1-12 T").toDS()
+    val csv = spark.read.option("header", false).schema("t timestamp, d date").csv(ds)
+    checkAnswer(csv, Row(Timestamp.valueOf("2020-1-12 3:23:34.12"), Date.valueOf("2020-1-12")))
+  }
+
+  test("exception mode for parsing date/timestamp string") {
+    val ds = Seq("2020-01-27T20:06:11.847-0800").toDS()
+    val csv = spark.read
+      .option("header", false)
+      .option("timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSz")
+      .schema("t timestamp").csv(ds)
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "exception") {
+      val msg = intercept[SparkException] {
+        csv.collect()
+      }.getCause.getMessage
+      assert(msg.contains("Fail to parse"))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") {
+      checkAnswer(csv, Row(Timestamp.valueOf("2020-01-27 20:06:11.847")))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "corrected") {
+      checkAnswer(csv, Row(null))
+    }
+  }
+
+  test("SPARK-32614: don't treat rows starting with null char as comment") {
+    withTempPath { path =>
+      Seq("\u0000foo", "bar", "baz").toDS.write.text(path.getCanonicalPath)
+      val df = spark.read.format("csv")
+        .option("header", "false")
+        .option("inferSchema", "true")
+        .load(path.getCanonicalPath)
+      assert(df.count() == 3)
+    }
+  }
+
+  test("SPARK-32810: CSV data source should be able to read files with " +
+    "escaped glob metacharacter in the paths") {
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath
+      // test CSV writer / reader without specifying schema
+      val csvTableName = "[abc]"
+      spark.range(3).coalesce(1).write.csv(s"$basePath/$csvTableName")
+      val readback = spark.read
+        .csv(s"$basePath/${"""(\[|\]|\{|\})""".r.replaceAllIn(csvTableName, """\\$1""")}")
+      assert(readback.collect sameElements Array(Row("0"), Row("1"), Row("2")))
+    }
+  }
 }
 
 class CSVv1Suite extends CSVSuite {
@@ -2309,3 +2393,10 @@ class CSVv2Suite extends CSVSuite {
       .sparkConf
       .set(SQLConf.USE_V1_SOURCE_LIST, "")
 }
+
+class CSVLegacyTimeParserSuite extends CSVSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.LEGACY_TIME_PARSER_POLICY, "legacy")
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
index bcecaccc8cc89..0dbd6b5754afb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
@@ -430,7 +430,7 @@ object JsonBenchmark extends SqlBasedBenchmark {
       }
 
       readBench.addCase("infer timestamps from files", numIters) { _ =>
-        spark.read.json(timestampDir).noop()
+        spark.read.option("inferTimestamp", true).json(timestampDir).noop()
       }
 
       val dateSchema = new StructType().add("date", DateType)
@@ -445,7 +445,9 @@ object JsonBenchmark extends SqlBasedBenchmark {
 
       def timestampStr: Dataset[String] = {
         spark.range(0, rowsNum, 1, 1).mapPartitions { iter =>
-          iter.map(i => s"""{"timestamp":"1970-01-01T01:02:03.${100 + i % 100}Z"}""")
+          iter.map { i =>
+            s"""{"timestamp":"1970-01-01T01:02:03.${i % 200}Z"}""".stripSuffix(".0Z")
+          }
         }.select($"value".as("timestamp")).as[String]
       }
 
@@ -458,7 +460,7 @@ object JsonBenchmark extends SqlBasedBenchmark {
       }
 
       readBench.addCase("infer timestamps from Dataset[String]", numIters) { _ =>
-        spark.read.json(timestampStr).noop()
+        spark.read.option("inferTimestamp", true).json(timestampStr).noop()
       }
 
       def dateStr: Dataset[String] = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index d0e2e001034fb..8eb5432f8b61a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -21,6 +21,7 @@ import java.io._
 import java.nio.charset.{Charset, StandardCharsets, UnsupportedCharsetException}
 import java.nio.file.Files
 import java.sql.{Date, Timestamp}
+import java.time.LocalDate
 import java.util.Locale
 
 import com.fasterxml.jackson.core.JsonFactory
@@ -39,6 +40,7 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.types.StructType.fromDDL
+import org.apache.spark.sql.types.TestUDT.{MyDenseVector, MyDenseVectorUDT}
 import org.apache.spark.util.Utils
 
 class TestFileFilter extends PathFilter {
@@ -246,548 +248,523 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("Complex field and type inferring with null in sampling") {
-    val jsonDF = spark.read.json(jsonNullStruct)
-    val expectedSchema = StructType(
-      StructField("headers", StructType(
-        StructField("Charset", StringType, true) ::
-          StructField("Host", StringType, true) :: Nil)
-        , true) ::
-        StructField("ip", StringType, true) ::
-        StructField("nullstr", StringType, true):: Nil)
-
-    assert(expectedSchema === jsonDF.schema)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(jsonNullStruct)
+      val expectedSchema = StructType(
+        StructField("headers", StructType(
+          StructField("Charset", StringType, true) ::
+            StructField("Host", StringType, true) :: Nil)
+          , true) ::
+          StructField("ip", StringType, true) ::
+          StructField("nullstr", StringType, true):: Nil)
+
+      assert(expectedSchema === jsonDF.schema)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select nullstr, headers.Host from jsonTable"),
-      Seq(Row("", "1.abc.com"), Row("", null), Row("", null), Row(null, null))
-    )
+      checkAnswer(
+        sql("select nullstr, headers.Host from jsonTable"),
+        Seq(Row("", "1.abc.com"), Row("", null), Row("", null), Row(null, null))
+      )
+    }
   }
 
   test("Primitive field and type inferring") {
-    val jsonDF = spark.read.json(primitiveFieldAndType)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(primitiveFieldAndType)
 
-    val expectedSchema = StructType(
-      StructField("bigInteger", DecimalType(20, 0), true) ::
-      StructField("boolean", BooleanType, true) ::
-      StructField("double", DoubleType, true) ::
-      StructField("integer", LongType, true) ::
-      StructField("long", LongType, true) ::
-      StructField("null", StringType, true) ::
-      StructField("string", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("bigInteger", DecimalType(20, 0), true) ::
+        StructField("boolean", BooleanType, true) ::
+        StructField("double", DoubleType, true) ::
+        StructField("integer", LongType, true) ::
+        StructField("long", LongType, true) ::
+        StructField("null", StringType, true) ::
+        StructField("string", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row(new java.math.BigDecimal("92233720368547758070"),
-        true,
-        1.7976931348623157,
-        10,
-        21474836470L,
-        null,
-        "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row(new java.math.BigDecimal("92233720368547758070"),
+          true,
+          1.7976931348623157,
+          10,
+          21474836470L,
+          null,
+          "this is a simple string.")
+      )
+    }
   }
 
   test("Complex field and type inferring") {
-    val jsonDF = spark.read.json(complexFieldAndType1)
-
-    val expectedSchema = StructType(
-      StructField("arrayOfArray1", ArrayType(ArrayType(StringType, true), true), true) ::
-      StructField("arrayOfArray2", ArrayType(ArrayType(DoubleType, true), true), true) ::
-      StructField("arrayOfBigInteger", ArrayType(DecimalType(21, 0), true), true) ::
-      StructField("arrayOfBoolean", ArrayType(BooleanType, true), true) ::
-      StructField("arrayOfDouble", ArrayType(DoubleType, true), true) ::
-      StructField("arrayOfInteger", ArrayType(LongType, true), true) ::
-      StructField("arrayOfLong", ArrayType(LongType, true), true) ::
-      StructField("arrayOfNull", ArrayType(StringType, true), true) ::
-      StructField("arrayOfString", ArrayType(StringType, true), true) ::
-      StructField("arrayOfStruct", ArrayType(
-        StructType(
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(complexFieldAndType1)
+
+      val expectedSchema = StructType(
+        StructField("arrayOfArray1", ArrayType(ArrayType(StringType, true), true), true) ::
+        StructField("arrayOfArray2", ArrayType(ArrayType(DoubleType, true), true), true) ::
+        StructField("arrayOfBigInteger", ArrayType(DecimalType(21, 0), true), true) ::
+        StructField("arrayOfBoolean", ArrayType(BooleanType, true), true) ::
+        StructField("arrayOfDouble", ArrayType(DoubleType, true), true) ::
+        StructField("arrayOfInteger", ArrayType(LongType, true), true) ::
+        StructField("arrayOfLong", ArrayType(LongType, true), true) ::
+        StructField("arrayOfNull", ArrayType(StringType, true), true) ::
+        StructField("arrayOfString", ArrayType(StringType, true), true) ::
+        StructField("arrayOfStruct", ArrayType(
+          StructType(
+            StructField("field1", BooleanType, true) ::
+            StructField("field2", StringType, true) ::
+            StructField("field3", StringType, true) :: Nil), true), true) ::
+        StructField("struct", StructType(
           StructField("field1", BooleanType, true) ::
-          StructField("field2", StringType, true) ::
-          StructField("field3", StringType, true) :: Nil), true), true) ::
-      StructField("struct", StructType(
-        StructField("field1", BooleanType, true) ::
-        StructField("field2", DecimalType(20, 0), true) :: Nil), true) ::
-      StructField("structWithArrayFields", StructType(
-        StructField("field1", ArrayType(LongType, true), true) ::
-        StructField("field2", ArrayType(StringType, true), true) :: Nil), true) :: Nil)
+          StructField("field2", DecimalType(20, 0), true) :: Nil), true) ::
+        StructField("structWithArrayFields", StructType(
+          StructField("field1", ArrayType(LongType, true), true) ::
+          StructField("field2", ArrayType(StringType, true), true) :: Nil), true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    // Access elements of a primitive array.
-    checkAnswer(
-      sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from jsonTable"),
-      Row("str1", "str2", null)
-    )
+      // Access elements of a primitive array.
+      checkAnswer(
+        sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from jsonTable"),
+        Row("str1", "str2", null)
+      )
 
-    // Access an array of null values.
-    checkAnswer(
-      sql("select arrayOfNull from jsonTable"),
-      Row(Seq(null, null, null, null))
-    )
+      // Access an array of null values.
+      checkAnswer(
+        sql("select arrayOfNull from jsonTable"),
+        Row(Seq(null, null, null, null))
+      )
 
-    // Access elements of a BigInteger array (we use DecimalType internally).
-    checkAnswer(
-      sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from jsonTable"),
-      Row(new java.math.BigDecimal("922337203685477580700"),
-        new java.math.BigDecimal("-922337203685477580800"), null)
-    )
+      // Access elements of a BigInteger array (we use DecimalType internally).
+      checkAnswer(
+        sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from " +
+          "jsonTable"),
+        Row(new java.math.BigDecimal("922337203685477580700"),
+          new java.math.BigDecimal("-922337203685477580800"), null)
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray1[0], arrayOfArray1[1] from jsonTable"),
-      Row(Seq("1", "2", "3"), Seq("str1", "str2"))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray1[0], arrayOfArray1[1] from jsonTable"),
+        Row(Seq("1", "2", "3"), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray2[0], arrayOfArray2[1] from jsonTable"),
-      Row(Seq(1.0, 2.0, 3.0), Seq(1.1, 2.1, 3.1))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray2[0], arrayOfArray2[1] from jsonTable"),
+        Row(Seq(1.0, 2.0, 3.0), Seq(1.1, 2.1, 3.1))
+      )
 
-    // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
-    checkAnswer(
-      sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from jsonTable"),
-      Row("str2", 2.1)
-    )
+      // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
+      checkAnswer(
+        sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from jsonTable"),
+        Row("str2", 2.1)
+      )
 
-    // Access elements of an array of structs.
-    checkAnswer(
-      sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2], arrayOfStruct[3] " +
-        "from jsonTable"),
-      Row(
-        Row(true, "str1", null),
-        Row(false, null, null),
-        Row(null, null, null),
-        null)
-    )
+      // Access elements of an array of structs.
+      checkAnswer(
+        sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2], arrayOfStruct[3] " +
+          "from jsonTable"),
+        Row(
+          Row(true, "str1", null),
+          Row(false, null, null),
+          Row(null, null, null),
+          null)
+      )
 
-    // Access a struct and fields inside of it.
-    checkAnswer(
-      sql("select struct, struct.field1, struct.field2 from jsonTable"),
-      Row(
-        Row(true, new java.math.BigDecimal("92233720368547758070")),
-        true,
-        new java.math.BigDecimal("92233720368547758070")) :: Nil
-    )
+      // Access a struct and fields inside of it.
+      checkAnswer(
+        sql("select struct, struct.field1, struct.field2 from jsonTable"),
+        Row(
+          Row(true, new java.math.BigDecimal("92233720368547758070")),
+          true,
+          new java.math.BigDecimal("92233720368547758070")) :: Nil
+      )
 
-    // Access an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1, structWithArrayFields.field2 from jsonTable"),
-      Row(Seq(4, 5, 6), Seq("str1", "str2"))
-    )
+      // Access an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1, structWithArrayFields.field2 from jsonTable"),
+        Row(Seq(4, 5, 6), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from jsonTable"),
-      Row(5, null)
-    )
+      // Access elements of an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from " +
+          "jsonTable"),
+        Row(5, null)
+      )
+    }
   }
 
   test("GetField operation on complex data type") {
-    val jsonDF = spark.read.json(complexFieldAndType1)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(complexFieldAndType1)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select arrayOfStruct[0].field1, arrayOfStruct[0].field2 from jsonTable"),
-      Row(true, "str1")
-    )
+      checkAnswer(
+        sql("select arrayOfStruct[0].field1, arrayOfStruct[0].field2 from jsonTable"),
+        Row(true, "str1")
+      )
 
-    // Getting all values of a specific field from an array of structs.
-    checkAnswer(
-      sql("select arrayOfStruct.field1, arrayOfStruct.field2 from jsonTable"),
-      Row(Seq(true, false, null), Seq("str1", null, null))
-    )
+      // Getting all values of a specific field from an array of structs.
+      checkAnswer(
+        sql("select arrayOfStruct.field1, arrayOfStruct.field2 from jsonTable"),
+        Row(Seq(true, false, null), Seq("str1", null, null))
+      )
+    }
   }
 
   test("Type conflict in primitive field values") {
-    val jsonDF = spark.read.json(primitiveFieldValueTypeConflict)
-
-    val expectedSchema = StructType(
-      StructField("num_bool", StringType, true) ::
-      StructField("num_num_1", LongType, true) ::
-      StructField("num_num_2", DoubleType, true) ::
-      StructField("num_num_3", DoubleType, true) ::
-      StructField("num_str", StringType, true) ::
-      StructField("str_bool", StringType, true) :: Nil)
-
-    assert(expectedSchema === jsonDF.schema)
-
-    jsonDF.createOrReplaceTempView("jsonTable")
-
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row("true", 11L, null, 1.1, "13.1", "str1") ::
-        Row("12", null, 21474836470.9, null, null, "true") ::
-        Row("false", 21474836470L, 92233720368547758070d, 100, "str1", "false") ::
-        Row(null, 21474836570L, 1.1, 21474836470L, "92233720368547758070", null) :: Nil
-    )
-
-    // Number and Boolean conflict: resolve the type as number in this query.
-    checkAnswer(
-      sql("select num_bool - 10 from jsonTable where num_bool > 11"),
-      Row(2)
-    )
-
-    // Widening to LongType
-    checkAnswer(
-      sql("select num_num_1 - 100 from jsonTable where num_num_1 > 11"),
-      Row(21474836370L) :: Row(21474836470L) :: Nil
-    )
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(primitiveFieldValueTypeConflict)
 
-    checkAnswer(
-      sql("select num_num_1 - 100 from jsonTable where num_num_1 > 10"),
-      Row(-89) :: Row(21474836370L) :: Row(21474836470L) :: Nil
-    )
+      val expectedSchema = StructType(
+        StructField("num_bool", StringType, true) ::
+        StructField("num_num_1", LongType, true) ::
+        StructField("num_num_2", DoubleType, true) ::
+        StructField("num_num_3", DoubleType, true) ::
+        StructField("num_str", StringType, true) ::
+        StructField("str_bool", StringType, true) :: Nil)
 
-    // Widening to DecimalType
-    checkAnswer(
-      sql("select num_num_2 + 1.3 from jsonTable where num_num_2 > 1.1"),
-      Row(21474836472.2) ::
-        Row(92233720368547758071.3) :: Nil
-    )
+      assert(expectedSchema === jsonDF.schema)
 
-    // Widening to Double
-    checkAnswer(
-      sql("select num_num_3 + 1.2 from jsonTable where num_num_3 > 1.1"),
-      Row(101.2) :: Row(21474836471.2) :: Nil
-    )
-
-    // Number and String conflict: resolve the type as number in this query.
-    checkAnswer(
-      sql("select num_str + 1.2 from jsonTable where num_str > 14d"),
-      Row(92233720368547758071.2)
-    )
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    // Number and String conflict: resolve the type as number in this query.
-    checkAnswer(
-      sql("select num_str + 1.2 from jsonTable where num_str >= 92233720368547758060"),
-      Row(new java.math.BigDecimal("92233720368547758071.2").doubleValue)
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row("true", 11L, null, 1.1, "13.1", "str1") ::
+          Row("12", null, 21474836470.9, null, null, "true") ::
+          Row("false", 21474836470L, 92233720368547758070d, 100, "str1", "false") ::
+          Row(null, 21474836570L, 1.1, 21474836470L, "92233720368547758070", null) :: Nil
+      )
 
-    // String and Boolean conflict: resolve the type as string.
-    checkAnswer(
-      sql("select * from jsonTable where str_bool = 'str1'"),
-      Row("true", 11L, null, 1.1, "13.1", "str1")
-    )
-  }
+      // Number and Boolean conflict: resolve the type as number in this query.
+      checkAnswer(
+        sql("select num_bool - 10 from jsonTable where num_bool > 11"),
+        Row(2)
+      )
 
-  ignore("Type conflict in primitive field values (Ignored)") {
-    val jsonDF = spark.read.json(primitiveFieldValueTypeConflict)
-    jsonDF.createOrReplaceTempView("jsonTable")
+      // Widening to LongType
+      checkAnswer(
+        sql("select num_num_1 - 100 from jsonTable where num_num_1 > 11"),
+        Row(21474836370L) :: Row(21474836470L) :: Nil
+      )
 
-    // Right now, the analyzer does not promote strings in a boolean expression.
-    // Number and Boolean conflict: resolve the type as boolean in this query.
-    checkAnswer(
-      sql("select num_bool from jsonTable where NOT num_bool"),
-      Row(false)
-    )
+      checkAnswer(
+        sql("select num_num_1 - 100 from jsonTable where num_num_1 > 10"),
+        Row(-89) :: Row(21474836370L) :: Row(21474836470L) :: Nil
+      )
 
-    checkAnswer(
-      sql("select str_bool from jsonTable where NOT str_bool"),
-      Row(false)
-    )
+      // Widening to DecimalType
+      checkAnswer(
+        sql("select num_num_2 + 1.3 from jsonTable where num_num_2 > 1.1"),
+        Row(21474836472.2) ::
+          Row(92233720368547758071.3) :: Nil
+      )
 
-    // Right now, the analyzer does not know that num_bool should be treated as a boolean.
-    // Number and Boolean conflict: resolve the type as boolean in this query.
-    checkAnswer(
-      sql("select num_bool from jsonTable where num_bool"),
-      Row(true)
-    )
+      // Widening to Double
+      checkAnswer(
+        sql("select num_num_3 + 1.2 from jsonTable where num_num_3 > 1.1"),
+        Row(101.2) :: Row(21474836471.2) :: Nil
+      )
 
-    checkAnswer(
-      sql("select str_bool from jsonTable where str_bool"),
-      Row(false)
-    )
+      // Number and String conflict: resolve the type as number in this query.
+      checkAnswer(
+        sql("select num_str + 1.2 from jsonTable where num_str > 14d"),
+        Row(92233720368547758071.2)
+      )
 
-    // The plan of the following DSL is
-    // Project [(CAST(num_str#65:4, DoubleType) + 1.2) AS num#78]
-    //  Filter (CAST(CAST(num_str#65:4, DoubleType), DecimalType) > 92233720368547758060)
-    //    ExistingRdd [num_bool#61,num_num_1#62L,num_num_2#63,num_num_3#64,num_str#65,str_bool#66]
-    // We should directly cast num_str to DecimalType and also need to do the right type promotion
-    // in the Project.
-    checkAnswer(
-      jsonDF.
-        where('num_str >= BigDecimal("92233720368547758060")).
-        select(('num_str + 1.2).as("num")),
-      Row(new java.math.BigDecimal("92233720368547758071.2").doubleValue())
-    )
+      // Number and String conflict: resolve the type as number in this query.
+      checkAnswer(
+        sql("select num_str + 1.2 from jsonTable where num_str >= 92233720368547758060"),
+        Row(new java.math.BigDecimal("92233720368547758071.2").doubleValue)
+      )
 
-    // The following test will fail. The type of num_str is StringType.
-    // So, to evaluate num_str + 1.2, we first need to use Cast to convert the type.
-    // In our test data, one value of num_str is 13.1.
-    // The result of (CAST(num_str#65:4, DoubleType) + 1.2) for this value is 14.299999999999999,
-    // which is not 14.3.
-    // Number and String conflict: resolve the type as number in this query.
-    checkAnswer(
-      sql("select num_str + 1.2 from jsonTable where num_str > 13"),
-      Row(BigDecimal("14.3")) :: Row(BigDecimal("92233720368547758071.2")) :: Nil
-    )
+      // String and Boolean conflict: resolve the type as string.
+      checkAnswer(
+        sql("select * from jsonTable where str_bool = 'str1'"),
+        Row("true", 11L, null, 1.1, "13.1", "str1")
+      )
+    }
   }
 
   test("Type conflict in complex field values") {
-    val jsonDF = spark.read.json(complexFieldValueTypeConflict)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(complexFieldValueTypeConflict)
 
-    val expectedSchema = StructType(
-      StructField("array", ArrayType(LongType, true), true) ::
-      StructField("num_struct", StringType, true) ::
-      StructField("str_array", StringType, true) ::
-      StructField("struct", StructType(
-        StructField("field", StringType, true) :: Nil), true) ::
-      StructField("struct_array", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("array", ArrayType(LongType, true), true) ::
+        StructField("num_struct", StringType, true) ::
+        StructField("str_array", StringType, true) ::
+        StructField("struct", StructType(
+          StructField("field", StringType, true) :: Nil), true) ::
+        StructField("struct_array", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row(Seq(), "11", "[1,2,3]", Row(null), "[]") ::
-        Row(null, """{"field":false}""", null, null, "{}") ::
-        Row(Seq(4, 5, 6), null, "str", Row(null), "[7,8,9]") ::
-        Row(Seq(7), "{}", """["str1","str2",33]""", Row("str"), """{"field":true}""") :: Nil
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row(Seq(), "11", "[1,2,3]", Row(null), "[]") ::
+          Row(null, """{"field":false}""", null, null, "{}") ::
+          Row(Seq(4, 5, 6), null, "str", Row(null), "[7,8,9]") ::
+          Row(Seq(7), "{}", """["str1","str2",33]""", Row("str"), """{"field":true}""") :: Nil
+      )
+    }
   }
 
   test("Type conflict in array elements") {
-    val jsonDF = spark.read.json(arrayElementTypeConflict)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(arrayElementTypeConflict)
 
-    val expectedSchema = StructType(
-      StructField("array1", ArrayType(StringType, true), true) ::
-      StructField("array2", ArrayType(StructType(
-        StructField("field", LongType, true) :: Nil), true), true) ::
-      StructField("array3", ArrayType(StringType, true), true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("array1", ArrayType(StringType, true), true) ::
+        StructField("array2", ArrayType(StructType(
+          StructField("field", LongType, true) :: Nil), true), true) ::
+        StructField("array3", ArrayType(StringType, true), true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row(Seq("1", "1.1", "true", null, "[]", "{}", "[2,3,4]",
-        """{"field":"str"}"""), Seq(Row(214748364700L), Row(1)), null) ::
-      Row(null, null, Seq("""{"field":"str"}""", """{"field":1}""")) ::
-      Row(null, null, Seq("1", "2", "3")) :: Nil
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row(Seq("1", "1.1", "true", null, "[]", "{}", "[2,3,4]",
+          """{"field":"str"}"""), Seq(Row(214748364700L), Row(1)), null) ::
+        Row(null, null, Seq("""{"field":"str"}""", """{"field":1}""")) ::
+        Row(null, null, Seq("1", "2", "3")) :: Nil
+      )
 
-    // Treat an element as a number.
-    checkAnswer(
-      sql("select array1[0] + 1 from jsonTable where array1 is not null"),
-      Row(2)
-    )
+      // Treat an element as a number.
+      checkAnswer(
+        sql("select array1[0] + 1 from jsonTable where array1 is not null"),
+        Row(2)
+      )
+    }
   }
 
   test("Handling missing fields") {
-    val jsonDF = spark.read.json(missingFields)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(missingFields)
 
-    val expectedSchema = StructType(
-      StructField("a", BooleanType, true) ::
-      StructField("b", LongType, true) ::
-      StructField("c", ArrayType(LongType, true), true) ::
-      StructField("d", StructType(
-        StructField("field", BooleanType, true) :: Nil), true) ::
-      StructField("e", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("a", BooleanType, true) ::
+        StructField("b", LongType, true) ::
+        StructField("c", ArrayType(LongType, true), true) ::
+        StructField("d", StructType(
+          StructField("field", BooleanType, true) :: Nil), true) ::
+        StructField("e", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
+    }
   }
 
   test("Loading a JSON dataset from a text file") {
-    val dir = Utils.createTempDir()
-    dir.delete()
-    val path = dir.getCanonicalPath
-    primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
-    val jsonDF = spark.read.json(path)
+    withTempView("jsonTable") {
+      val dir = Utils.createTempDir()
+      dir.delete()
+      val path = dir.getCanonicalPath
+      primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
+      val jsonDF = spark.read.json(path)
 
-    val expectedSchema = StructType(
-      StructField("bigInteger", DecimalType(20, 0), true) ::
-      StructField("boolean", BooleanType, true) ::
-      StructField("double", DoubleType, true) ::
-      StructField("integer", LongType, true) ::
-      StructField("long", LongType, true) ::
-      StructField("null", StringType, true) ::
-      StructField("string", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("bigInteger", DecimalType(20, 0), true) ::
+        StructField("boolean", BooleanType, true) ::
+        StructField("double", DoubleType, true) ::
+        StructField("integer", LongType, true) ::
+        StructField("long", LongType, true) ::
+        StructField("null", StringType, true) ::
+        StructField("string", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row(new java.math.BigDecimal("92233720368547758070"),
-      true,
-      1.7976931348623157,
-      10,
-      21474836470L,
-      null,
-      "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row(new java.math.BigDecimal("92233720368547758070"),
+        true,
+        1.7976931348623157,
+        10,
+        21474836470L,
+        null,
+        "this is a simple string.")
+      )
+    }
   }
 
   test("Loading a JSON dataset primitivesAsString returns schema with primitive types as strings") {
-    val dir = Utils.createTempDir()
-    dir.delete()
-    val path = dir.getCanonicalPath
-    primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
-    val jsonDF = spark.read.option("primitivesAsString", "true").json(path)
-
-    val expectedSchema = StructType(
-      StructField("bigInteger", StringType, true) ::
-      StructField("boolean", StringType, true) ::
-      StructField("double", StringType, true) ::
-      StructField("integer", StringType, true) ::
-      StructField("long", StringType, true) ::
-      StructField("null", StringType, true) ::
-      StructField("string", StringType, true) :: Nil)
+    withTempView("jsonTable") {
+      val dir = Utils.createTempDir()
+      dir.delete()
+      val path = dir.getCanonicalPath
+      primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
+      val jsonDF = spark.read.option("primitivesAsString", "true").json(path)
+
+      val expectedSchema = StructType(
+        StructField("bigInteger", StringType, true) ::
+        StructField("boolean", StringType, true) ::
+        StructField("double", StringType, true) ::
+        StructField("integer", StringType, true) ::
+        StructField("long", StringType, true) ::
+        StructField("null", StringType, true) ::
+        StructField("string", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row("92233720368547758070",
-      "true",
-      "1.7976931348623157",
-      "10",
-      "21474836470",
-      null,
-      "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row("92233720368547758070",
+        "true",
+        "1.7976931348623157",
+        "10",
+        "21474836470",
+        null,
+        "this is a simple string.")
+      )
+    }
   }
 
   test("Loading a JSON dataset primitivesAsString returns complex fields as strings") {
-    val jsonDF = spark.read.option("primitivesAsString", "true").json(complexFieldAndType1)
-
-    val expectedSchema = StructType(
-      StructField("arrayOfArray1", ArrayType(ArrayType(StringType, true), true), true) ::
-      StructField("arrayOfArray2", ArrayType(ArrayType(StringType, true), true), true) ::
-      StructField("arrayOfBigInteger", ArrayType(StringType, true), true) ::
-      StructField("arrayOfBoolean", ArrayType(StringType, true), true) ::
-      StructField("arrayOfDouble", ArrayType(StringType, true), true) ::
-      StructField("arrayOfInteger", ArrayType(StringType, true), true) ::
-      StructField("arrayOfLong", ArrayType(StringType, true), true) ::
-      StructField("arrayOfNull", ArrayType(StringType, true), true) ::
-      StructField("arrayOfString", ArrayType(StringType, true), true) ::
-      StructField("arrayOfStruct", ArrayType(
-        StructType(
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.option("primitivesAsString", "true").json(complexFieldAndType1)
+
+      val expectedSchema = StructType(
+        StructField("arrayOfArray1", ArrayType(ArrayType(StringType, true), true), true) ::
+        StructField("arrayOfArray2", ArrayType(ArrayType(StringType, true), true), true) ::
+        StructField("arrayOfBigInteger", ArrayType(StringType, true), true) ::
+        StructField("arrayOfBoolean", ArrayType(StringType, true), true) ::
+        StructField("arrayOfDouble", ArrayType(StringType, true), true) ::
+        StructField("arrayOfInteger", ArrayType(StringType, true), true) ::
+        StructField("arrayOfLong", ArrayType(StringType, true), true) ::
+        StructField("arrayOfNull", ArrayType(StringType, true), true) ::
+        StructField("arrayOfString", ArrayType(StringType, true), true) ::
+        StructField("arrayOfStruct", ArrayType(
+          StructType(
+            StructField("field1", StringType, true) ::
+            StructField("field2", StringType, true) ::
+            StructField("field3", StringType, true) :: Nil), true), true) ::
+        StructField("struct", StructType(
           StructField("field1", StringType, true) ::
-          StructField("field2", StringType, true) ::
-          StructField("field3", StringType, true) :: Nil), true), true) ::
-      StructField("struct", StructType(
-        StructField("field1", StringType, true) ::
-        StructField("field2", StringType, true) :: Nil), true) ::
-      StructField("structWithArrayFields", StructType(
-        StructField("field1", ArrayType(StringType, true), true) ::
-        StructField("field2", ArrayType(StringType, true), true) :: Nil), true) :: Nil)
+          StructField("field2", StringType, true) :: Nil), true) ::
+        StructField("structWithArrayFields", StructType(
+          StructField("field1", ArrayType(StringType, true), true) ::
+          StructField("field2", ArrayType(StringType, true), true) :: Nil), true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    // Access elements of a primitive array.
-    checkAnswer(
-      sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from jsonTable"),
-      Row("str1", "str2", null)
-    )
+      // Access elements of a primitive array.
+      checkAnswer(
+        sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from jsonTable"),
+        Row("str1", "str2", null)
+      )
 
-    // Access an array of null values.
-    checkAnswer(
-      sql("select arrayOfNull from jsonTable"),
-      Row(Seq(null, null, null, null))
-    )
+      // Access an array of null values.
+      checkAnswer(
+        sql("select arrayOfNull from jsonTable"),
+        Row(Seq(null, null, null, null))
+      )
 
-    // Access elements of a BigInteger array (we use DecimalType internally).
-    checkAnswer(
-      sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from jsonTable"),
-      Row("922337203685477580700", "-922337203685477580800", null)
-    )
+      // Access elements of a BigInteger array (we use DecimalType internally).
+      checkAnswer(
+        sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from " +
+          "jsonTable"),
+        Row("922337203685477580700", "-922337203685477580800", null)
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray1[0], arrayOfArray1[1] from jsonTable"),
-      Row(Seq("1", "2", "3"), Seq("str1", "str2"))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray1[0], arrayOfArray1[1] from jsonTable"),
+        Row(Seq("1", "2", "3"), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray2[0], arrayOfArray2[1] from jsonTable"),
-      Row(Seq("1", "2", "3"), Seq("1.1", "2.1", "3.1"))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray2[0], arrayOfArray2[1] from jsonTable"),
+        Row(Seq("1", "2", "3"), Seq("1.1", "2.1", "3.1"))
+      )
 
-    // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
-    checkAnswer(
-      sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from jsonTable"),
-      Row("str2", "2.1")
-    )
+      // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
+      checkAnswer(
+        sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from jsonTable"),
+        Row("str2", "2.1")
+      )
 
-    // Access elements of an array of structs.
-    checkAnswer(
-      sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2], arrayOfStruct[3] " +
-        "from jsonTable"),
-      Row(
-        Row("true", "str1", null),
-        Row("false", null, null),
-        Row(null, null, null),
-        null)
-    )
+      // Access elements of an array of structs.
+      checkAnswer(
+        sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2], arrayOfStruct[3] " +
+          "from jsonTable"),
+        Row(
+          Row("true", "str1", null),
+          Row("false", null, null),
+          Row(null, null, null),
+          null)
+      )
 
-    // Access a struct and fields inside of it.
-    checkAnswer(
-      sql("select struct, struct.field1, struct.field2 from jsonTable"),
-      Row(
-        Row("true", "92233720368547758070"),
-        "true",
-        "92233720368547758070") :: Nil
-    )
+      // Access a struct and fields inside of it.
+      checkAnswer(
+        sql("select struct, struct.field1, struct.field2 from jsonTable"),
+        Row(
+          Row("true", "92233720368547758070"),
+          "true",
+          "92233720368547758070") :: Nil
+      )
 
-    // Access an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1, structWithArrayFields.field2 from jsonTable"),
-      Row(Seq("4", "5", "6"), Seq("str1", "str2"))
-    )
+      // Access an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1, structWithArrayFields.field2 from jsonTable"),
+        Row(Seq("4", "5", "6"), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from jsonTable"),
-      Row("5", null)
-    )
+      // Access elements of an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from " +
+          "jsonTable"),
+        Row("5", null)
+      )
+    }
   }
 
   test("Loading a JSON dataset prefersDecimal returns schema with float types as BigDecimal") {
-    val jsonDF = spark.read.option("prefersDecimal", "true").json(primitiveFieldAndType)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.option("prefersDecimal", "true").json(primitiveFieldAndType)
 
-    val expectedSchema = StructType(
-      StructField("bigInteger", DecimalType(20, 0), true) ::
-        StructField("boolean", BooleanType, true) ::
-        StructField("double", DecimalType(17, 16), true) ::
-        StructField("integer", LongType, true) ::
-        StructField("long", LongType, true) ::
-        StructField("null", StringType, true) ::
-        StructField("string", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("bigInteger", DecimalType(20, 0), true) ::
+          StructField("boolean", BooleanType, true) ::
+          StructField("double", DecimalType(17, 16), true) ::
+          StructField("integer", LongType, true) ::
+          StructField("long", LongType, true) ::
+          StructField("null", StringType, true) ::
+          StructField("string", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row(BigDecimal("92233720368547758070"),
-        true,
-        BigDecimal("1.7976931348623157"),
-        10,
-        21474836470L,
-        null,
-        "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row(BigDecimal("92233720368547758070"),
+          true,
+          BigDecimal("1.7976931348623157"),
+          10,
+          21474836470L,
+          null,
+          "this is a simple string.")
+      )
+    }
   }
 
   test("Find compatible types even if inferred DecimalType is not capable of other IntegralType") {
@@ -884,171 +861,182 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("Applying schemas") {
-    val dir = Utils.createTempDir()
-    dir.delete()
-    val path = dir.getCanonicalPath
-    primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
+    withTempView("jsonTable1", "jsonTable2") {
+      val dir = Utils.createTempDir()
+      dir.delete()
+      val path = dir.getCanonicalPath
+      primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
 
-    val schema = StructType(
-      StructField("bigInteger", DecimalType.SYSTEM_DEFAULT, true) ::
-      StructField("boolean", BooleanType, true) ::
-      StructField("double", DoubleType, true) ::
-      StructField("integer", IntegerType, true) ::
-      StructField("long", LongType, true) ::
-      StructField("null", StringType, true) ::
-      StructField("string", StringType, true) :: Nil)
+      val schema = StructType(
+        StructField("bigInteger", DecimalType.SYSTEM_DEFAULT, true) ::
+        StructField("boolean", BooleanType, true) ::
+        StructField("double", DoubleType, true) ::
+        StructField("integer", IntegerType, true) ::
+        StructField("long", LongType, true) ::
+        StructField("null", StringType, true) ::
+        StructField("string", StringType, true) :: Nil)
 
-    val jsonDF1 = spark.read.schema(schema).json(path)
+      val jsonDF1 = spark.read.schema(schema).json(path)
 
-    assert(schema === jsonDF1.schema)
+      assert(schema === jsonDF1.schema)
 
-    jsonDF1.createOrReplaceTempView("jsonTable1")
+      jsonDF1.createOrReplaceTempView("jsonTable1")
 
-    checkAnswer(
-      sql("select * from jsonTable1"),
-      Row(new java.math.BigDecimal("92233720368547758070"),
-      true,
-      1.7976931348623157,
-      10,
-      21474836470L,
-      null,
-      "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable1"),
+        Row(new java.math.BigDecimal("92233720368547758070"),
+        true,
+        1.7976931348623157,
+        10,
+        21474836470L,
+        null,
+        "this is a simple string.")
+      )
 
-    val jsonDF2 = spark.read.schema(schema).json(primitiveFieldAndType)
+      val jsonDF2 = spark.read.schema(schema).json(primitiveFieldAndType)
 
-    assert(schema === jsonDF2.schema)
+      assert(schema === jsonDF2.schema)
 
-    jsonDF2.createOrReplaceTempView("jsonTable2")
+      jsonDF2.createOrReplaceTempView("jsonTable2")
 
-    checkAnswer(
-      sql("select * from jsonTable2"),
-      Row(new java.math.BigDecimal("92233720368547758070"),
-      true,
-      1.7976931348623157,
-      10,
-      21474836470L,
-      null,
-      "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable2"),
+        Row(new java.math.BigDecimal("92233720368547758070"),
+        true,
+        1.7976931348623157,
+        10,
+        21474836470L,
+        null,
+        "this is a simple string.")
+      )
+    }
   }
 
   test("Applying schemas with MapType") {
-    val schemaWithSimpleMap = StructType(
-      StructField("map", MapType(StringType, IntegerType, true), false) :: Nil)
-    val jsonWithSimpleMap = spark.read.schema(schemaWithSimpleMap).json(mapType1)
-
-    jsonWithSimpleMap.createOrReplaceTempView("jsonWithSimpleMap")
+    withTempView("jsonWithSimpleMap", "jsonWithComplexMap") {
+      val schemaWithSimpleMap = StructType(
+        StructField("map", MapType(StringType, IntegerType, true), false) :: Nil)
+      val jsonWithSimpleMap = spark.read.schema(schemaWithSimpleMap).json(mapType1)
 
-    checkAnswer(
-      sql("select `map` from jsonWithSimpleMap"),
-      Row(Map("a" -> 1)) ::
-      Row(Map("b" -> 2)) ::
-      Row(Map("c" -> 3)) ::
-      Row(Map("c" -> 1, "d" -> 4)) ::
-      Row(Map("e" -> null)) :: Nil
-    )
+      jsonWithSimpleMap.createOrReplaceTempView("jsonWithSimpleMap")
 
-    withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
       checkAnswer(
-        sql("select `map`['c'] from jsonWithSimpleMap"),
-        Row(null) ::
-        Row(null) ::
-        Row(3) ::
-        Row(1) ::
-        Row(null) :: Nil
+        sql("select `map` from jsonWithSimpleMap"),
+        Row(Map("a" -> 1)) ::
+        Row(Map("b" -> 2)) ::
+        Row(Map("c" -> 3)) ::
+        Row(Map("c" -> 1, "d" -> 4)) ::
+        Row(Map("e" -> null)) :: Nil
       )
-    }
 
-    val innerStruct = StructType(
-      StructField("field1", ArrayType(IntegerType, true), true) ::
-      StructField("field2", IntegerType, true) :: Nil)
-    val schemaWithComplexMap = StructType(
-      StructField("map", MapType(StringType, innerStruct, true), false) :: Nil)
+      withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
+        checkAnswer(
+          sql("select `map`['c'] from jsonWithSimpleMap"),
+          Row(null) ::
+          Row(null) ::
+          Row(3) ::
+          Row(1) ::
+          Row(null) :: Nil
+        )
+      }
 
-    val jsonWithComplexMap = spark.read.schema(schemaWithComplexMap).json(mapType2)
+      val innerStruct = StructType(
+        StructField("field1", ArrayType(IntegerType, true), true) ::
+        StructField("field2", IntegerType, true) :: Nil)
+      val schemaWithComplexMap = StructType(
+        StructField("map", MapType(StringType, innerStruct, true), false) :: Nil)
 
-    jsonWithComplexMap.createOrReplaceTempView("jsonWithComplexMap")
+      val jsonWithComplexMap = spark.read.schema(schemaWithComplexMap).json(mapType2)
 
-    checkAnswer(
-      sql("select `map` from jsonWithComplexMap"),
-      Row(Map("a" -> Row(Seq(1, 2, 3, null), null))) ::
-      Row(Map("b" -> Row(null, 2))) ::
-      Row(Map("c" -> Row(Seq(), 4))) ::
-      Row(Map("c" -> Row(null, 3), "d" -> Row(Seq(null), null))) ::
-      Row(Map("e" -> null)) ::
-      Row(Map("f" -> Row(null, null))) :: Nil
-    )
+      jsonWithComplexMap.createOrReplaceTempView("jsonWithComplexMap")
 
-    withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
       checkAnswer(
-        sql("select `map`['a'].field1, `map`['c'].field2 from jsonWithComplexMap"),
-        Row(Seq(1, 2, 3, null), null) ::
-        Row(null, null) ::
-        Row(null, 4) ::
-        Row(null, 3) ::
-        Row(null, null) ::
-        Row(null, null) :: Nil
+        sql("select `map` from jsonWithComplexMap"),
+        Row(Map("a" -> Row(Seq(1, 2, 3, null), null))) ::
+        Row(Map("b" -> Row(null, 2))) ::
+        Row(Map("c" -> Row(Seq(), 4))) ::
+        Row(Map("c" -> Row(null, 3), "d" -> Row(Seq(null), null))) ::
+        Row(Map("e" -> null)) ::
+        Row(Map("f" -> Row(null, null))) :: Nil
       )
+
+      withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
+        checkAnswer(
+          sql("select `map`['a'].field1, `map`['c'].field2 from jsonWithComplexMap"),
+          Row(Seq(1, 2, 3, null), null) ::
+          Row(null, null) ::
+          Row(null, 4) ::
+          Row(null, 3) ::
+          Row(null, null) ::
+          Row(null, null) :: Nil
+        )
+      }
     }
   }
 
   test("SPARK-2096 Correctly parse dot notations") {
-    val jsonDF = spark.read.json(complexFieldAndType2)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(complexFieldAndType2)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select arrayOfStruct[0].field1, arrayOfStruct[0].field2 from jsonTable"),
-      Row(true, "str1")
-    )
-    checkAnswer(
-      sql(
-        """
-          |select complexArrayOfStruct[0].field1[1].inner2[0], complexArrayOfStruct[1].field2[0][1]
-          |from jsonTable
-        """.stripMargin),
-      Row("str2", 6)
-    )
+      checkAnswer(
+        sql("select arrayOfStruct[0].field1, arrayOfStruct[0].field2 from jsonTable"),
+        Row(true, "str1")
+      )
+      checkAnswer(
+        sql(
+          """
+            |select complexArrayOfStruct[0].field1[1].inner2[0],
+            |complexArrayOfStruct[1].field2[0][1]
+            |from jsonTable
+          """.stripMargin),
+        Row("str2", 6)
+      )
+    }
   }
 
   test("SPARK-3390 Complex arrays") {
-    val jsonDF = spark.read.json(complexFieldAndType2)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(complexFieldAndType2)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql(
-        """
-          |select arrayOfArray1[0][0][0], arrayOfArray1[1][0][1], arrayOfArray1[1][1][0]
-          |from jsonTable
-        """.stripMargin),
-      Row(5, 7, 8)
-    )
-    checkAnswer(
-      sql(
-        """
-          |select arrayOfArray2[0][0][0].inner1, arrayOfArray2[1][0],
-          |arrayOfArray2[1][1][1].inner2[0], arrayOfArray2[2][0][0].inner3[0][0].inner4
-          |from jsonTable
-        """.stripMargin),
-      Row("str1", Nil, "str4", 2)
-    )
+      checkAnswer(
+        sql(
+          """
+            |select arrayOfArray1[0][0][0], arrayOfArray1[1][0][1], arrayOfArray1[1][1][0]
+            |from jsonTable
+          """.stripMargin),
+        Row(5, 7, 8)
+      )
+      checkAnswer(
+        sql(
+          """
+            |select arrayOfArray2[0][0][0].inner1, arrayOfArray2[1][0],
+            |arrayOfArray2[1][1][1].inner2[0], arrayOfArray2[2][0][0].inner3[0][0].inner4
+            |from jsonTable
+          """.stripMargin),
+        Row("str1", Nil, "str4", 2)
+      )
+    }
   }
 
   test("SPARK-3308 Read top level JSON arrays") {
-    val jsonDF = spark.read.json(jsonArray)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(jsonArray)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql(
-        """
-          |select a, b, c
-          |from jsonTable
-        """.stripMargin),
-      Row("str_a_1", null, null) ::
-        Row("str_a_2", null, null) ::
-        Row(null, "str_b_3", null) ::
-        Row("str_a_4", "str_b_4", "str_c_4") :: Nil
-    )
+      checkAnswer(
+        sql(
+          """
+            |select a, b, c
+            |from jsonTable
+          """.stripMargin),
+        Row("str_a_1", null, null) ::
+          Row("str_a_2", null, null) ::
+          Row(null, "str_b_3", null) ::
+          Row("str_a_4", "str_b_4", "str_c_4") :: Nil
+      )
+    }
   }
 
   test("Corrupt records: FAILFAST mode") {
@@ -1190,158 +1178,162 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("SPARK-4068: nulls in arrays") {
-    val jsonDF = spark.read.json(nullsInArrays)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(nullsInArrays)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    val schema = StructType(
-      StructField("field1",
-        ArrayType(ArrayType(ArrayType(ArrayType(StringType, true), true), true), true), true) ::
-      StructField("field2",
-        ArrayType(ArrayType(
-          StructType(StructField("Test", LongType, true) :: Nil), true), true), true) ::
-      StructField("field3",
-        ArrayType(ArrayType(
-          StructType(StructField("Test", StringType, true) :: Nil), true), true), true) ::
-      StructField("field4",
-        ArrayType(ArrayType(ArrayType(LongType, true), true), true), true) :: Nil)
+      val schema = StructType(
+        StructField("field1",
+          ArrayType(ArrayType(ArrayType(ArrayType(StringType, true), true), true), true), true) ::
+        StructField("field2",
+          ArrayType(ArrayType(
+            StructType(StructField("Test", LongType, true) :: Nil), true), true), true) ::
+        StructField("field3",
+          ArrayType(ArrayType(
+            StructType(StructField("Test", StringType, true) :: Nil), true), true), true) ::
+        StructField("field4",
+          ArrayType(ArrayType(ArrayType(LongType, true), true), true), true) :: Nil)
 
-    assert(schema === jsonDF.schema)
+      assert(schema === jsonDF.schema)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT field1, field2, field3, field4
-          |FROM jsonTable
-        """.stripMargin),
-      Row(Seq(Seq(null), Seq(Seq(Seq("Test")))), null, null, null) ::
-        Row(null, Seq(null, Seq(Row(1))), null, null) ::
-        Row(null, null, Seq(Seq(null), Seq(Row("2"))), null) ::
-        Row(null, null, null, Seq(Seq(null, Seq(1, 2, 3)))) :: Nil
-    )
+      checkAnswer(
+        sql(
+          """
+            |SELECT field1, field2, field3, field4
+            |FROM jsonTable
+          """.stripMargin),
+        Row(Seq(Seq(null), Seq(Seq(Seq("Test")))), null, null, null) ::
+          Row(null, Seq(null, Seq(Row(1))), null, null) ::
+          Row(null, null, Seq(Seq(null), Seq(Row("2"))), null) ::
+          Row(null, null, null, Seq(Seq(null, Seq(1, 2, 3)))) :: Nil
+      )
+    }
   }
 
   test("SPARK-4228 DataFrame to JSON") {
-    val schema1 = StructType(
-      StructField("f1", IntegerType, false) ::
-      StructField("f2", StringType, false) ::
-      StructField("f3", BooleanType, false) ::
-      StructField("f4", ArrayType(StringType), nullable = true) ::
-      StructField("f5", IntegerType, true) :: Nil)
-
-    val rowRDD1 = unparsedStrings.map { r =>
-      val values = r.split(",").map(_.trim)
-      val v5 = try values(3).toInt catch {
-        case _: NumberFormatException => null
+    withTempView("applySchema1", "applySchema2", "primitiveTable", "complexTable") {
+      val schema1 = StructType(
+        StructField("f1", IntegerType, false) ::
+        StructField("f2", StringType, false) ::
+        StructField("f3", BooleanType, false) ::
+        StructField("f4", ArrayType(StringType), nullable = true) ::
+        StructField("f5", IntegerType, true) :: Nil)
+
+      val rowRDD1 = unparsedStrings.map { r =>
+        val values = r.split(",").map(_.trim)
+        val v5 = try values(3).toInt catch {
+          case _: NumberFormatException => null
+        }
+        Row(values(0).toInt, values(1), values(2).toBoolean, r.split(",").toList, v5)
       }
-      Row(values(0).toInt, values(1), values(2).toBoolean, r.split(",").toList, v5)
-    }
-
-    val df1 = spark.createDataFrame(rowRDD1, schema1)
-    df1.createOrReplaceTempView("applySchema1")
-    val df2 = df1.toDF
-    val result = df2.toJSON.collect()
-    // scalastyle:off
-    assert(result(0) === "{\"f1\":1,\"f2\":\"A1\",\"f3\":true,\"f4\":[\"1\",\" A1\",\" true\",\" null\"]}")
-    assert(result(3) === "{\"f1\":4,\"f2\":\"D4\",\"f3\":true,\"f4\":[\"4\",\" D4\",\" true\",\" 2147483644\"],\"f5\":2147483644}")
-    // scalastyle:on
-
-    val schema2 = StructType(
-      StructField("f1", StructType(
-        StructField("f11", IntegerType, false) ::
-        StructField("f12", BooleanType, false) :: Nil), false) ::
-      StructField("f2", MapType(StringType, IntegerType, true), false) :: Nil)
 
-    val rowRDD2 = unparsedStrings.map { r =>
-      val values = r.split(",").map(_.trim)
-      val v4 = try values(3).toInt catch {
-        case _: NumberFormatException => null
+      val df1 = spark.createDataFrame(rowRDD1, schema1)
+      df1.createOrReplaceTempView("applySchema1")
+      val df2 = df1.toDF
+      val result = df2.toJSON.collect()
+      // scalastyle:off
+      assert(result(0) === "{\"f1\":1,\"f2\":\"A1\",\"f3\":true,\"f4\":[\"1\",\" A1\",\" true\",\" null\"]}")
+      assert(result(3) === "{\"f1\":4,\"f2\":\"D4\",\"f3\":true,\"f4\":[\"4\",\" D4\",\" true\",\" 2147483644\"],\"f5\":2147483644}")
+      // scalastyle:on
+
+      val schema2 = StructType(
+        StructField("f1", StructType(
+          StructField("f11", IntegerType, false) ::
+          StructField("f12", BooleanType, false) :: Nil), false) ::
+        StructField("f2", MapType(StringType, IntegerType, true), false) :: Nil)
+
+      val rowRDD2 = unparsedStrings.map { r =>
+        val values = r.split(",").map(_.trim)
+        val v4 = try values(3).toInt catch {
+          case _: NumberFormatException => null
+        }
+        Row(Row(values(0).toInt, values(2).toBoolean), Map(values(1) -> v4))
       }
-      Row(Row(values(0).toInt, values(2).toBoolean), Map(values(1) -> v4))
-    }
 
-    val df3 = spark.createDataFrame(rowRDD2, schema2)
-    df3.createOrReplaceTempView("applySchema2")
-    val df4 = df3.toDF
-    val result2 = df4.toJSON.collect()
+      val df3 = spark.createDataFrame(rowRDD2, schema2)
+      df3.createOrReplaceTempView("applySchema2")
+      val df4 = df3.toDF
+      val result2 = df4.toJSON.collect()
 
-    assert(result2(1) === "{\"f1\":{\"f11\":2,\"f12\":false},\"f2\":{\"B2\":null}}")
-    assert(result2(3) === "{\"f1\":{\"f11\":4,\"f12\":true},\"f2\":{\"D4\":2147483644}}")
+      assert(result2(1) === "{\"f1\":{\"f11\":2,\"f12\":false},\"f2\":{\"B2\":null}}")
+      assert(result2(3) === "{\"f1\":{\"f11\":4,\"f12\":true},\"f2\":{\"D4\":2147483644}}")
 
-    val jsonDF = spark.read.json(primitiveFieldAndType)
-    val primTable = spark.read.json(jsonDF.toJSON)
-    primTable.createOrReplaceTempView("primitiveTable")
-    checkAnswer(
-        sql("select * from primitiveTable"),
-      Row(new java.math.BigDecimal("92233720368547758070"),
-        true,
-        1.7976931348623157,
-        10,
-        21474836470L,
-        "this is a simple string.")
+      val jsonDF = spark.read.json(primitiveFieldAndType)
+      val primTable = spark.read.json(jsonDF.toJSON)
+      primTable.createOrReplaceTempView("primitiveTable")
+      checkAnswer(
+          sql("select * from primitiveTable"),
+        Row(new java.math.BigDecimal("92233720368547758070"),
+          true,
+          1.7976931348623157,
+          10,
+          21474836470L,
+          "this is a simple string.")
       )
 
-    val complexJsonDF = spark.read.json(complexFieldAndType1)
-    val compTable = spark.read.json(complexJsonDF.toJSON)
-    compTable.createOrReplaceTempView("complexTable")
-    // Access elements of a primitive array.
-    checkAnswer(
-      sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from complexTable"),
-      Row("str1", "str2", null)
-    )
+      val complexJsonDF = spark.read.json(complexFieldAndType1)
+      val compTable = spark.read.json(complexJsonDF.toJSON)
+      compTable.createOrReplaceTempView("complexTable")
+      // Access elements of a primitive array.
+      checkAnswer(
+        sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from complexTable"),
+        Row("str1", "str2", null)
+      )
 
-    // Access an array of null values.
-    checkAnswer(
-      sql("select arrayOfNull from complexTable"),
-      Row(Seq(null, null, null, null))
-    )
+      // Access an array of null values.
+      checkAnswer(
+        sql("select arrayOfNull from complexTable"),
+        Row(Seq(null, null, null, null))
+      )
 
-    // Access elements of a BigInteger array (we use DecimalType internally).
-    checkAnswer(
-      sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] " +
-        " from complexTable"),
-      Row(new java.math.BigDecimal("922337203685477580700"),
-        new java.math.BigDecimal("-922337203685477580800"), null)
-    )
+      // Access elements of a BigInteger array (we use DecimalType internally).
+      checkAnswer(
+        sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] " +
+          " from complexTable"),
+        Row(new java.math.BigDecimal("922337203685477580700"),
+          new java.math.BigDecimal("-922337203685477580800"), null)
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray1[0], arrayOfArray1[1] from complexTable"),
-      Row(Seq("1", "2", "3"), Seq("str1", "str2"))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray1[0], arrayOfArray1[1] from complexTable"),
+        Row(Seq("1", "2", "3"), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray2[0], arrayOfArray2[1] from complexTable"),
-      Row(Seq(1.0, 2.0, 3.0), Seq(1.1, 2.1, 3.1))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray2[0], arrayOfArray2[1] from complexTable"),
+        Row(Seq(1.0, 2.0, 3.0), Seq(1.1, 2.1, 3.1))
+      )
 
-    // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
-    checkAnswer(
-      sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from complexTable"),
-      Row("str2", 2.1)
-    )
+      // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
+      checkAnswer(
+        sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from complexTable"),
+        Row("str2", 2.1)
+      )
 
-    // Access a struct and fields inside of it.
-    checkAnswer(
-      sql("select struct, struct.field1, struct.field2 from complexTable"),
-      Row(
-        Row(true, new java.math.BigDecimal("92233720368547758070")),
-        true,
-        new java.math.BigDecimal("92233720368547758070")) :: Nil
-    )
+      // Access a struct and fields inside of it.
+      checkAnswer(
+        sql("select struct, struct.field1, struct.field2 from complexTable"),
+        Row(
+          Row(true, new java.math.BigDecimal("92233720368547758070")),
+          true,
+          new java.math.BigDecimal("92233720368547758070")) :: Nil
+      )
 
-    // Access an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1, structWithArrayFields.field2 from complexTable"),
-      Row(Seq(4, 5, 6), Seq("str1", "str2"))
-    )
+      // Access an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1, structWithArrayFields.field2 from complexTable"),
+        Row(Seq(4, 5, 6), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] " +
-        "from complexTable"),
-      Row(5, null)
-    )
+      // Access elements of an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] " +
+          "from complexTable"),
+        Row(5, null)
+      )
+    }
   }
 
   test("Dataset toJSON doesn't construct rdd") {
@@ -1380,7 +1372,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
 
   test("SPARK-6245 JsonInferSchema.infer on empty RDD") {
     // This is really a test that it doesn't throw an exception
-    val options = new JSONOptions(Map.empty[String, String], "GMT")
+    val options = new JSONOptions(Map.empty[String, String], "UTC")
     val emptySchema = new JsonInferSchema(options).infer(
       empty.rdd,
       CreateJacksonParser.string)
@@ -1407,7 +1399,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("SPARK-8093 Erase empty structs") {
-    val options = new JSONOptions(Map.empty[String, String], "GMT")
+    val options = new JSONOptions(Map.empty[String, String], "UTC")
     val emptySchema = new JsonInferSchema(options).infer(
       emptyRecords.rdd,
       CreateJacksonParser.string)
@@ -1422,20 +1414,21 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     }
 
     withTempPath(root => {
-      val d1 = new File(root, "d1=1")
-      // root/dt=1/col1=abc
-      val p1_col1 = makePartition(
-        sparkContext.parallelize(2 to 5).map(i => s"""{"a": 1, "b": "str$i"}"""),
-        d1,
-        "col1",
-        "abc")
-
-      // root/dt=1/col1=abd
-      val p2 = makePartition(
-        sparkContext.parallelize(6 to 10).map(i => s"""{"a": 1, "b": "str$i"}"""),
-        d1,
-        "col1",
-        "abd")
+      withTempView("test_myjson_with_part") {
+        val d1 = new File(root, "d1=1")
+        // root/dt=1/col1=abc
+        val p1_col1 = makePartition(
+          sparkContext.parallelize(2 to 5).map(i => s"""{"a": 1, "b": "str$i"}"""),
+          d1,
+          "col1",
+          "abc")
+
+        // root/dt=1/col1=abd
+        val p2 = makePartition(
+          sparkContext.parallelize(6 to 10).map(i => s"""{"a": 1, "b": "str$i"}"""),
+          d1,
+          "col1",
+          "abd")
 
         spark.read.json(root.getAbsolutePath).createOrReplaceTempView("test_myjson_with_part")
         checkAnswer(sql(
@@ -1444,9 +1437,111 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
           "SELECT count(a) FROM test_myjson_with_part where d1 = 1 and col1='abd'"), Row(5))
         checkAnswer(sql(
           "SELECT count(a) FROM test_myjson_with_part where d1 = 1"), Row(9))
+      }
     })
   }
 
+  test("backward compatibility") {
+    // This test we make sure our JSON support can read JSON data generated by previous version
+    // of Spark generated through toJSON method and JSON data source.
+    // The data is generated by the following program.
+    // Here are a few notes:
+    //  - Spark 1.5.0 cannot save timestamp data. So, we manually added timestamp field (col13)
+    //      in the JSON object.
+    //  - For Spark before 1.5.1, we do not generate UDTs. So, we manually added the UDT value to
+    //      JSON objects generated by those Spark versions (col17).
+    //  - If the type is NullType, we do not write data out.
+
+    // Create the schema.
+    val struct =
+      StructType(
+        StructField("f1", FloatType, true) ::
+          StructField("f2", ArrayType(BooleanType), true) :: Nil)
+
+    val dataTypes =
+      Seq(
+        StringType, BinaryType, NullType, BooleanType,
+        ByteType, ShortType, IntegerType, LongType,
+        FloatType, DoubleType, DecimalType(25, 5), DecimalType(6, 5),
+        DateType, TimestampType,
+        ArrayType(IntegerType), MapType(StringType, LongType), struct,
+        new MyDenseVectorUDT())
+    val fields = dataTypes.zipWithIndex.map { case (dataType, index) =>
+      StructField(s"col$index", dataType, nullable = true)
+    }
+    val schema = StructType(fields)
+
+    val constantValues =
+      Seq(
+        "a string in binary".getBytes(StandardCharsets.UTF_8),
+        null,
+        true,
+        1.toByte,
+        2.toShort,
+        3,
+        Long.MaxValue,
+        0.25.toFloat,
+        0.75,
+        new java.math.BigDecimal(s"1234.23456"),
+        new java.math.BigDecimal(s"1.23456"),
+        java.sql.Date.valueOf("2015-01-01"),
+        java.sql.Timestamp.valueOf("2015-01-01 23:50:59.123"),
+        Seq(2, 3, 4),
+        Map("a string" -> 2000L),
+        Row(4.75.toFloat, Seq(false, true)),
+        new MyDenseVector(Array(0.25, 2.25, 4.25)))
+    val data =
+      Row.fromSeq(Seq("Spark " + spark.sparkContext.version) ++ constantValues) :: Nil
+
+    // Data generated by previous versions.
+    // scalastyle:off
+    val existingJSONData =
+      """{"col0":"Spark 1.2.2","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.3.1","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.3.1","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.4.1","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.4.1","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.5.0","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.5.0","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"16436","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" :: Nil
+    // scalastyle:on
+
+    // Generate data for the current version.
+    val df = spark.createDataFrame(spark.sparkContext.parallelize(data, 1), schema)
+    withTempPath { path =>
+      df.write.format("json").mode("overwrite").save(path.getCanonicalPath)
+
+      // df.toJSON will convert internal rows to external rows first and then generate
+      // JSON objects. While, df.write.format("json") will write internal rows directly.
+      val allJSON =
+        existingJSONData ++
+          df.toJSON.collect() ++
+          sparkContext.textFile(path.getCanonicalPath).collect()
+
+      Utils.deleteRecursively(path)
+      sparkContext.parallelize(allJSON, 1).saveAsTextFile(path.getCanonicalPath)
+
+      // Read data back with the schema specified.
+      val col0Values =
+        Seq(
+          "Spark 1.2.2",
+          "Spark 1.3.1",
+          "Spark 1.3.1",
+          "Spark 1.4.1",
+          "Spark 1.4.1",
+          "Spark 1.5.0",
+          "Spark 1.5.0",
+          "Spark " + spark.sparkContext.version,
+          "Spark " + spark.sparkContext.version)
+      val expectedResult = col0Values.map { v =>
+        Row.fromSeq(Seq(v) ++ constantValues)
+      }
+      checkAnswer(
+        spark.read.format("json").schema(schema).load(path.getCanonicalPath),
+        expectedResult
+      )
+    }
+  }
+
   test("SPARK-11544 test pathfilter") {
     withTempPath { dir =>
       val path = dir.getCanonicalPath
@@ -1461,6 +1556,15 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
         "mapreduce.input.pathFilter.class" -> classOf[TestFileFilter].getName
       )
       assert(spark.read.options(extraOptions).json(path).count() === 2)
+
+      withClue("SPARK-32621: 'path' option can cause issues while inferring schema") {
+        // During infer, "path" option is used again on top of the paths that have already been
+        // listed. When a partition is removed by TestFileFilter, this will cause a conflict while
+        // inferring partitions because the original path in the "path" option will list the
+        // partition directory that has been removed.
+        assert(
+          spark.read.options(extraOptions).format("json").option("path", path).load.count() === 2)
+      }
     }
   }
 
@@ -1680,7 +1784,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       timestampsWithFormat.write
         .format("json")
         .option("timestampFormat", "yyyy/MM/dd HH:mm")
-        .option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+        .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .save(timestampsWithFormatPath)
 
       // This will load back the timestamps as string.
@@ -1698,7 +1802,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       val readBack = spark.read
         .schema(customSchema)
         .option("timestampFormat", "yyyy/MM/dd HH:mm")
-        .option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+        .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .json(timestampsWithFormatPath)
 
       checkAnswer(readBack, timestampsWithFormat)
@@ -2046,9 +2150,18 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     )(withTempPath { path =>
       val ds = sampledTestData.coalesce(1)
       ds.write.text(path.getAbsolutePath)
-      val readback = spark.read.option("samplingRatio", 0.1).json(path.getCanonicalPath)
-
-      assert(readback.schema == new StructType().add("f1", LongType))
+      val readback1 = spark.read.option("samplingRatio", 0.1).json(path.getCanonicalPath)
+      assert(readback1.schema == new StructType().add("f1", LongType))
+
+      withClue("SPARK-32621: 'path' option can cause issues while inferring schema") {
+        // During infer, "path" option gets added again to the paths that have already been listed.
+        // This results in reading more data than necessary and causes different schema to be
+        // inferred when sampling ratio is involved.
+        val readback2 = spark.read
+          .option("samplingRatio", 0.1).option("path", path.getCanonicalPath)
+          .format("json").load
+        assert(readback2.schema == new StructType().add("f1", LongType))
+      }
     })
   }
 
@@ -2142,9 +2255,8 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
         .json(testFile(fileName))
         .count()
     }
-    val errMsg = exception.getMessage
 
-    assert(errMsg.contains("Malformed records are detected in record parsing"))
+    assert(exception.getMessage.contains("Malformed records are detected in record parsing"))
   }
 
   def checkEncoding(expectedEncoding: String, pathToJsonFiles: String,
@@ -2436,23 +2548,24 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     }
   }
 
-  test("SPARK-25040: empty strings should be disallowed") {
-    def failedOnEmptyString(dataType: DataType): Unit = {
-       val df = spark.read.schema(s"a ${dataType.catalogString}")
-        .option("mode", "FAILFAST").json(Seq("""{"a":""}""").toDS)
-      val errMessage = intercept[SparkException] {
-        df.collect()
-      }.getMessage
-      assert(errMessage.contains(
-        s"Failed to parse an empty string for data type ${dataType.catalogString}"))
-    }
 
-    def emptyString(dataType: DataType, expected: Any): Unit = {
-      val df = spark.read.schema(s"a ${dataType.catalogString}")
-        .option("mode", "FAILFAST").json(Seq("""{"a":""}""").toDS)
-      checkAnswer(df, Row(expected) :: Nil)
-    }
+  private def failedOnEmptyString(dataType: DataType): Unit = {
+    val df = spark.read.schema(s"a ${dataType.catalogString}")
+      .option("mode", "FAILFAST").json(Seq("""{"a":""}""").toDS)
+    val errMessage = intercept[SparkException] {
+      df.collect()
+    }.getMessage
+    assert(errMessage.contains(
+      s"Failed to parse an empty string for data type ${dataType.catalogString}"))
+  }
+
+  private def emptyString(dataType: DataType, expected: Any): Unit = {
+    val df = spark.read.schema(s"a ${dataType.catalogString}")
+      .option("mode", "FAILFAST").json(Seq("""{"a":""}""").toDS)
+    checkAnswer(df, Row(expected) :: Nil)
+  }
 
+  test("SPARK-25040: empty strings should be disallowed") {
     failedOnEmptyString(BooleanType)
     failedOnEmptyString(ByteType)
     failedOnEmptyString(ShortType)
@@ -2471,6 +2584,36 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     emptyString(BinaryType, "".getBytes(StandardCharsets.UTF_8))
   }
 
+  test("SPARK-25040: allowing empty strings when legacy config is enabled") {
+    def emptyStringAsNull(dataType: DataType): Unit = {
+      val df = spark.read.schema(s"a ${dataType.catalogString}")
+        .option("mode", "FAILFAST").json(Seq("""{"a":""}""").toDS)
+      checkAnswer(df, Row(null) :: Nil)
+    }
+
+    // Legacy mode prior to Spark 3.0.0
+    withSQLConf(SQLConf.LEGACY_ALLOW_EMPTY_STRING_IN_JSON.key -> "true") {
+      emptyStringAsNull(BooleanType)
+      emptyStringAsNull(ByteType)
+      emptyStringAsNull(ShortType)
+      emptyStringAsNull(IntegerType)
+      emptyStringAsNull(LongType)
+
+      failedOnEmptyString(FloatType)
+      failedOnEmptyString(DoubleType)
+      failedOnEmptyString(TimestampType)
+      failedOnEmptyString(DateType)
+
+      emptyStringAsNull(DecimalType.SYSTEM_DEFAULT)
+      emptyStringAsNull(ArrayType(IntegerType))
+      emptyStringAsNull(MapType(StringType, IntegerType, true))
+      emptyStringAsNull(StructType(StructField("f1", IntegerType, true) :: Nil))
+
+      emptyString(StringType, "")
+      emptyString(BinaryType, "".getBytes(StandardCharsets.UTF_8))
+    }
+  }
+
   test("return partial result for bad records") {
     val schema = "a double, b array<int>, c string, _corrupt_record string"
     val badRecords = Seq(
@@ -2485,7 +2628,9 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("inferring timestamp type") {
-    def schemaOf(jsons: String*): StructType = spark.read.json(jsons.toDS).schema
+    def schemaOf(jsons: String*): StructType = {
+      spark.read.option("inferTimestamp", true).json(jsons.toDS).schema
+    }
 
     assert(schemaOf(
       """{"a":"2018-12-17T10:11:12.123-01:00"}""",
@@ -2508,6 +2653,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       val timestampsWithFormatPath = s"${dir.getCanonicalPath}/timestampsWithFormat.json"
       val timestampsWithFormat = spark.read
         .option("timestampFormat", "dd/MM/yyyy HH:mm")
+        .option("inferTimestamp", true)
         .json(datesRecords)
       assert(timestampsWithFormat.schema === customSchema)
 
@@ -2520,12 +2666,59 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       val readBack = spark.read
         .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
         .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
+        .option("inferTimestamp", true)
         .json(timestampsWithFormatPath)
 
       assert(readBack.schema === customSchema)
       checkAnswer(readBack, timestampsWithFormat)
     }
   }
+
+  test("SPARK-30960, SPARK-31641: parse date/timestamp string with legacy format") {
+    val julianDay = -141704 // 1582-01-01 in Julian calendar
+    val ds = Seq(
+      s"{'t': '2020-1-12 3:23:34.12', 'd': '2020-1-12 T', 'd2': '12345', 'd3': '$julianDay'}"
+    ).toDS()
+    val json = spark.read.schema("t timestamp, d date, d2 date, d3 date").json(ds)
+    checkAnswer(json, Row(
+      Timestamp.valueOf("2020-1-12 3:23:34.12"),
+      Date.valueOf("2020-1-12"),
+      Date.valueOf(LocalDate.ofEpochDay(12345)),
+      Date.valueOf("1582-01-01")))
+  }
+
+  test("exception mode for parsing date/timestamp string") {
+    val ds = Seq("{'t': '2020-01-27T20:06:11.847-0800'}").toDS()
+    val json = spark.read
+      .schema("t timestamp")
+      .option("timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSz")
+      .json(ds)
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "exception") {
+      val msg = intercept[SparkException] {
+        json.collect()
+      }.getCause.getMessage
+      assert(msg.contains("Fail to parse"))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") {
+      checkAnswer(json, Row(Timestamp.valueOf("2020-01-27 20:06:11.847")))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "corrected") {
+      checkAnswer(json, Row(null))
+    }
+  }
+
+  test("SPARK-32810: JSON data source should be able to read files with " +
+    "escaped glob metacharacter in the paths") {
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath
+      // test JSON writer / reader without specifying schema
+      val jsonTableName = "{def}"
+      spark.range(3).coalesce(1).write.json(s"$basePath/$jsonTableName")
+      val readback = spark.read
+        .json(s"$basePath/${"""(\[|\]|\{|\})""".r.replaceAllIn(jsonTableName, """\\$1""")}")
+      assert(readback.collect sameElements Array(Row(0), Row(1), Row(2)))
+    }
+  }
 }
 
 class JsonV1Suite extends JsonSuite {
@@ -2541,3 +2734,10 @@ class JsonV2Suite extends JsonSuite {
       .sparkConf
       .set(SQLConf.USE_V1_SOURCE_LIST, "")
 }
+
+class JsonLegacyTimeParserSuite extends JsonSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.LEGACY_TIME_PARSER_POLICY, "legacy")
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala
index 719bf91e1786b..bfcef46339908 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala
@@ -17,16 +17,29 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
+import java.io.File
+
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.mapreduce.{JobID, TaskAttemptID, TaskID, TaskType}
+import org.apache.hadoop.mapreduce.lib.input.FileSplit
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 import org.apache.orc.TypeDescription
 
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase
 import org.apache.spark.sql.execution.vectorized.{OnHeapColumnVector, WritableColumnVector}
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{StructField, StructType}
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.unsafe.types.UTF8String.fromString
 
 class OrcColumnarBatchReaderSuite extends QueryTest with SharedSparkSession {
+
+  import testImplicits._
+
   private val dataSchema = StructType.fromDDL("col1 int, col2 int")
   private val partitionSchema = StructType.fromDDL("p1 string, p2 string")
   private val partitionValues = InternalRow(fromString("partValue1"), fromString("partValue2"))
@@ -77,4 +90,66 @@ class OrcColumnarBatchReaderSuite extends QueryTest with SharedSparkSession {
       assert(p1.getUTF8String(0) === partitionValues.getUTF8String(0))
     }
   }
+
+  test("SPARK-33593: partition column types") {
+    withTempPath { dir =>
+      Seq(1).toDF().repartition(1).write.orc(dir.getCanonicalPath)
+
+      val dataTypes =
+        Seq(StringType, BooleanType, ByteType, BinaryType, ShortType, IntegerType, LongType,
+          FloatType, DoubleType, DecimalType(25, 5), DateType, TimestampType)
+
+      val constantValues =
+        Seq(
+          UTF8String.fromString("a string"),
+          true,
+          1.toByte,
+          "Spark SQL".getBytes,
+          2.toShort,
+          3,
+          Long.MaxValue,
+          0.25.toFloat,
+          0.75D,
+          Decimal("1234.23456"),
+          DateTimeUtils.fromJavaDate(java.sql.Date.valueOf("2015-01-01")),
+          DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf("2015-01-01 23:50:59.123")))
+
+      dataTypes.zip(constantValues).foreach { case (dt, v) =>
+        val schema = StructType(StructField("col1", IntegerType) :: StructField("pcol", dt) :: Nil)
+        val partitionValues = new GenericInternalRow(Array(v))
+        val file = new File(SpecificParquetRecordReaderBase.listDirectory(dir).get(0))
+        val fileSplit = new FileSplit(new Path(file.getCanonicalPath), 0L, file.length, Array.empty)
+        val taskConf = sqlContext.sessionState.newHadoopConf()
+        val orcFileSchema = TypeDescription.fromString(schema.simpleString)
+        val vectorizedReader = new OrcColumnarBatchReader(4096)
+        val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
+        val taskAttemptContext = new TaskAttemptContextImpl(taskConf, attemptId)
+
+        try {
+          vectorizedReader.initialize(fileSplit, taskAttemptContext)
+          vectorizedReader.initBatch(
+            orcFileSchema,
+            schema.toArray,
+            Array(0, -1),
+            Array(-1, 0),
+            partitionValues)
+          vectorizedReader.nextKeyValue()
+          val row = vectorizedReader.getCurrentValue.getRow(0)
+
+          // Use `GenericMutableRow` by explicitly copying rather than `ColumnarBatch`
+          // in order to use get(...) method which is not implemented in `ColumnarBatch`.
+          val actual = row.copy().get(1, dt)
+          val expected = v
+          if (dt.isInstanceOf[BinaryType]) {
+            assert(actual.asInstanceOf[Array[Byte]]
+              sameElements expected.asInstanceOf[Array[Byte]])
+          } else {
+            assert(actual == expected)
+          }
+        } finally {
+          vectorizedReader.close()
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
index b8bf4b16fe53c..9caf0c836f711 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
@@ -595,22 +595,22 @@ abstract class OrcQueryTest extends OrcTest {
     }
 
     withSQLConf(SQLConf.IGNORE_CORRUPT_FILES.key -> "false") {
-      val m1 = intercept[SparkException] {
+      val e1 = intercept[SparkException] {
         testIgnoreCorruptFiles()
-      }.getMessage
-      assert(m1.contains("Malformed ORC file"))
-      val m2 = intercept[SparkException] {
+      }
+      assert(e1.getMessage.contains("Malformed ORC file"))
+      val e2 = intercept[SparkException] {
         testIgnoreCorruptFilesWithoutSchemaInfer()
-      }.getMessage
-      assert(m2.contains("Malformed ORC file"))
-      val m3 = intercept[SparkException] {
+      }
+      assert(e2.getMessage.contains("Malformed ORC file"))
+      val e3 = intercept[SparkException] {
         testAllCorruptFiles()
-      }.getMessage
-      assert(m3.contains("Could not read footer for file"))
-      val m4 = intercept[SparkException] {
+      }
+      assert(e3.getMessage.contains("Could not read footer for file"))
+      val e4 = intercept[SparkException] {
         testAllCorruptFilesWithoutSchemaInfer()
-      }.getMessage
-      assert(m4.contains("Malformed ORC file"))
+      }
+      assert(e4.getMessage.contains("Malformed ORC file"))
     }
   }
 
@@ -631,7 +631,7 @@ abstract class OrcQueryTest extends OrcTest {
   }
 }
 
-class OrcQuerySuite extends OrcQueryTest with SharedSparkSession {
+abstract class OrcQuerySuite extends OrcQueryTest with SharedSparkSession {
   import testImplicits._
 
   test("LZO compression options for writing to an ORC file") {
@@ -722,3 +722,10 @@ class OrcV1QuerySuite extends OrcQuerySuite {
       .sparkConf
       .set(SQLConf.USE_V1_SOURCE_LIST, "orc")
 }
+
+class OrcV2QuerySuite extends OrcQuerySuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
index 1e27593584786..5fa23a40e677d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.orc
 
 import java.io.File
 import java.nio.charset.StandardCharsets.UTF_8
-import java.sql.Timestamp
+import java.sql.{Date, Timestamp}
 import java.util.Locale
 
 import org.apache.hadoop.conf.Configuration
@@ -32,7 +32,7 @@ import org.apache.orc.impl.RecordReaderImpl
 import org.scalatest.BeforeAndAfterAll
 
 import org.apache.spark.{SPARK_VERSION_SHORT, SparkException}
-import org.apache.spark.sql.{Row, SPARK_VERSION_METADATA_KEY}
+import org.apache.spark.sql.{FakeFileSystemRequiringDSOption, Row, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.execution.datasources.SchemaMergeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -213,9 +213,7 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
           Seq(fs.listStatus(path1), fs.listStatus(path2), fs.listStatus(path3)).flatten
 
         val schema = SchemaMergeUtils.mergeSchemasInParallel(
-          spark,
-          fileStatuses,
-          schemaReader)
+          spark, Map.empty, fileStatuses, schemaReader)
 
         assert(schema.isDefined)
         assert(schema.get == StructType(Seq(
@@ -482,6 +480,79 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
       }
     }
   }
+
+  test("SPARK-31238: compatibility with Spark 2.4 in reading dates") {
+    Seq(false, true).foreach { vectorized =>
+      withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+        checkAnswer(
+          readResourceOrcFile("test-data/before_1582_date_v2_4.snappy.orc"),
+          Row(java.sql.Date.valueOf("1200-01-01")))
+      }
+    }
+  }
+
+  test("SPARK-31238, SPARK-31423: rebasing dates in write") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      Seq("1001-01-01", "1582-10-10").toDF("dateS")
+        .select($"dateS".cast("date").as("date"))
+        .write
+        .orc(path)
+
+      Seq(false, true).foreach { vectorized =>
+        withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+          checkAnswer(
+            spark.read.orc(path),
+            Seq(Row(Date.valueOf("1001-01-01")), Row(Date.valueOf("1582-10-15"))))
+        }
+      }
+    }
+  }
+
+  test("SPARK-31284: compatibility with Spark 2.4 in reading timestamps") {
+    Seq(false, true).foreach { vectorized =>
+      withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+        checkAnswer(
+          readResourceOrcFile("test-data/before_1582_ts_v2_4.snappy.orc"),
+          Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+      }
+    }
+  }
+
+  test("SPARK-31284, SPARK-31423: rebasing timestamps in write") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      Seq("1001-01-01 01:02:03.123456", "1582-10-10 11:12:13.654321").toDF("tsS")
+        .select($"tsS".cast("timestamp").as("ts"))
+        .write
+        .orc(path)
+
+      Seq(false, true).foreach { vectorized =>
+        withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+          checkAnswer(
+            spark.read.orc(path),
+            Seq(
+              Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")),
+              Row(java.sql.Timestamp.valueOf("1582-10-15 11:12:13.654321"))))
+        }
+      }
+    }
+  }
+
+  test("SPARK-33094: should propagate Hadoop config from DS options to underlying file system") {
+    withSQLConf(
+      "fs.file.impl" -> classOf[FakeFileSystemRequiringDSOption].getName,
+      "fs.file.impl.disable.cache" -> "true") {
+      Seq(false, true).foreach { mergeSchema =>
+        withTempPath { dir =>
+          val path = dir.getAbsolutePath
+          val conf = Map("ds_option" -> "value", "mergeSchema" -> mergeSchema.toString)
+          spark.range(1).write.options(conf).orc(path)
+          checkAnswer(spark.read.options(conf).orc(path), Row(0))
+        }
+      }
+    }
+  }
 }
 
 class OrcSourceSuite extends OrcSuite with SharedSparkSession {
@@ -531,4 +602,10 @@ class OrcSourceSuite extends OrcSuite with SharedSparkSession {
   test("SPARK-11412 read and merge orc schemas in parallel") {
     testMergeSchemasInParallel(OrcUtils.readOrcSchemasInParallel)
   }
+
+  test("SPARK-31580: Read a file written before ORC-569") {
+    // Test ORC file came from ORC-621
+    val df = readResourceOrcFile("test-data/TestStringDictionary.testRowIndex.orc")
+    assert(df.where("str < 'row 001000'").count() === 1000)
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
index 388744bd0fd6e..e929f904c798d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
@@ -22,6 +22,7 @@ import java.io.File
 import scala.reflect.ClassTag
 import scala.reflect.runtime.universe.TypeTag
 
+import org.apache.commons.io.FileUtils
 import org.scalatest.BeforeAndAfterAll
 
 import org.apache.spark.sql._
@@ -133,4 +134,13 @@ abstract class OrcTest extends QueryTest with FileBasedDataSourceTest with Befor
         throw new AnalysisException("Can not match OrcTable in the query.")
     }
   }
+
+  protected def readResourceOrcFile(name: String): DataFrame = {
+    val url = Thread.currentThread().getContextClassLoader.getResource(name)
+    // Copy to avoid URISyntaxException when `sql/hive` accesses the resources in `sql/core`
+    val file = File.createTempFile("orc-test", ".orc")
+    file.deleteOnExit();
+    FileUtils.copyURLToFile(url, file)
+    spark.read.orc(file.getAbsolutePath)
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
index 6d681afd23b18..fbfedf02dc87e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
@@ -42,7 +42,7 @@ class ParquetEncodingSuite extends ParquetCompatibilityTest with SharedSparkSess
 
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         reader.initialize(file.asInstanceOf[String], null)
         val batch = reader.resultBatch()
         assert(reader.nextBatch())
@@ -69,7 +69,7 @@ class ParquetEncodingSuite extends ParquetCompatibilityTest with SharedSparkSess
 
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         reader.initialize(file.asInstanceOf[String], null)
         val batch = reader.resultBatch()
         assert(reader.nextBatch())
@@ -100,7 +100,7 @@ class ParquetEncodingSuite extends ParquetCompatibilityTest with SharedSparkSess
 
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         reader.initialize(file, null /* set columns to null to project all columns */)
         val column = reader.resultBatch().column(0)
         assert(reader.nextBatch())
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 286bb1e920266..307234dcb86e9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -20,6 +20,10 @@ package org.apache.spark.sql.execution.datasources.parquet
 import java.math.{BigDecimal => JBigDecimal}
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
+import java.time.{LocalDate, LocalDateTime, ZoneId}
+
+import scala.reflect.ClassTag
+import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.parquet.filter2.predicate.{FilterApi, FilterPredicate, Operators}
 import org.apache.parquet.filter2.predicate.FilterApi._
@@ -32,7 +36,8 @@ import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.optimizer.InferFiltersFromConstraints
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
-import org.apache.spark.sql.execution.datasources.{DataSourceStrategy, HadoopFsRelation, LogicalRelation}
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.parseColumnPath
+import org.apache.spark.sql.execution.datasources.{DataSourceStrategy, HadoopFsRelation, LogicalRelation, PushableColumnAndNestedColumn}
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
 import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
 import org.apache.spark.sql.functions._
@@ -40,6 +45,7 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.ExtendedSQLTest
 import org.apache.spark.util.{AccumulatorContext, AccumulatorV2}
 
 /**
@@ -103,61 +109,109 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     checkFilterPredicate(predicate, filterClass, Seq(Row(expected)))(df)
   }
 
-  private def checkBinaryFilterPredicate
-      (predicate: Predicate, filterClass: Class[_ <: FilterPredicate], expected: Seq[Row])
-      (implicit df: DataFrame): Unit = {
-    def checkBinaryAnswer(df: DataFrame, expected: Seq[Row]) = {
-      assertResult(expected.map(_.getAs[Array[Byte]](0).mkString(",")).sorted) {
-        df.rdd.map(_.getAs[Array[Byte]](0).mkString(",")).collect().toSeq.sorted
+  /**
+   * Takes a sequence of products `data` to generate multi-level nested
+   * dataframes as new test data. It tests both non-nested and nested dataframes
+   * which are written and read back with Parquet datasource.
+   *
+   * This is different from [[ParquetTest.withParquetDataFrame]] which does not
+   * test nested cases.
+   */
+  private def withNestedParquetDataFrame[T <: Product: ClassTag: TypeTag](data: Seq[T])
+      (runTest: (DataFrame, String, Any => Any) => Unit): Unit =
+    withNestedParquetDataFrame(spark.createDataFrame(data))(runTest)
+
+  private def withNestedParquetDataFrame(inputDF: DataFrame)
+      (runTest: (DataFrame, String, Any => Any) => Unit): Unit = {
+    assert(inputDF.schema.fields.length == 1)
+    assert(!inputDF.schema.fields.head.dataType.isInstanceOf[StructType])
+    val df = inputDF.toDF("temp")
+    Seq(
+      (
+        df.withColumnRenamed("temp", "a"),
+        "a", // zero nesting
+        (x: Any) => x),
+      (
+        df.withColumn("a", struct(df("temp") as "b")).drop("temp"),
+        "a.b", // one level nesting
+        (x: Any) => Row(x)),
+      (
+        df.withColumn("a", struct(struct(df("temp") as "c") as "b")).drop("temp"),
+        "a.b.c", // two level nesting
+        (x: Any) => Row(Row(x))
+      ),
+      (
+        df.withColumnRenamed("temp", "a.b"),
+        "`a.b`", // zero nesting with column name containing `dots`
+        (x: Any) => x
+      ),
+      (
+        df.withColumn("a.b", struct(df("temp") as "c.d") ).drop("temp"),
+        "`a.b`.`c.d`", // one level nesting with column names containing `dots`
+        (x: Any) => Row(x)
+      )
+    ).foreach { case (newDF, colName, resultFun) =>
+      withTempPath { file =>
+        newDF.write.format(dataSourceName).save(file.getCanonicalPath)
+        readParquetFile(file.getCanonicalPath) { df => runTest(df, colName, resultFun) }
       }
     }
-
-    checkFilterPredicate(df, predicate, filterClass, checkBinaryAnswer _, expected)
-  }
-
-  private def checkBinaryFilterPredicate
-      (predicate: Predicate, filterClass: Class[_ <: FilterPredicate], expected: Array[Byte])
-      (implicit df: DataFrame): Unit = {
-    checkBinaryFilterPredicate(predicate, filterClass, Seq(Row(expected)))(df)
   }
 
-  private def testTimestampPushdown(data: Seq[Timestamp]): Unit = {
+  private def testTimestampPushdown(data: Seq[String], java8Api: Boolean): Unit = {
+    implicit class StringToTs(s: String) {
+      def ts: Timestamp = Timestamp.valueOf(s)
+    }
     assert(data.size === 4)
     val ts1 = data.head
     val ts2 = data(1)
     val ts3 = data(2)
     val ts4 = data(3)
 
-    withParquetDataFrame(data.map(i => Tuple1(i))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], data.map(i => Row.apply(i)))
+    import testImplicits._
+    val df = data.map(i => Tuple1(Timestamp.valueOf(i))).toDF()
+    withNestedParquetDataFrame(df) { case (parquetDF, colName, fun) =>
+      implicit val df: DataFrame = parquetDF
+
+      def resultFun(tsStr: String): Any = {
+        val parsed = if (java8Api) {
+          LocalDateTime.parse(tsStr.replace(" ", "T"))
+            .atZone(ZoneId.systemDefault())
+            .toInstant
+        } else {
+          Timestamp.valueOf(tsStr)
+        }
+        fun(parsed)
+      }
 
-      checkFilterPredicate('_1 === ts1, classOf[Eq[_]], ts1)
-      checkFilterPredicate('_1 <=> ts1, classOf[Eq[_]], ts1)
-      checkFilterPredicate('_1 =!= ts1, classOf[NotEq[_]],
-        Seq(ts2, ts3, ts4).map(i => Row.apply(i)))
+      val tsAttr = df(colName).expr
+      assert(df(colName).expr.dataType === TimestampType)
 
-      checkFilterPredicate('_1 < ts2, classOf[Lt[_]], ts1)
-      checkFilterPredicate('_1 > ts1, classOf[Gt[_]], Seq(ts2, ts3, ts4).map(i => Row.apply(i)))
-      checkFilterPredicate('_1 <= ts1, classOf[LtEq[_]], ts1)
-      checkFilterPredicate('_1 >= ts4, classOf[GtEq[_]], ts4)
+      checkFilterPredicate(tsAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(tsAttr.isNotNull, classOf[NotEq[_]],
+        data.map(i => Row.apply(resultFun(i))))
 
-      checkFilterPredicate(Literal(ts1) === '_1, classOf[Eq[_]], ts1)
-      checkFilterPredicate(Literal(ts1) <=> '_1, classOf[Eq[_]], ts1)
-      checkFilterPredicate(Literal(ts2) > '_1, classOf[Lt[_]], ts1)
-      checkFilterPredicate(Literal(ts3) < '_1, classOf[Gt[_]], ts4)
-      checkFilterPredicate(Literal(ts1) >= '_1, classOf[LtEq[_]], ts1)
-      checkFilterPredicate(Literal(ts4) <= '_1, classOf[GtEq[_]], ts4)
+      checkFilterPredicate(tsAttr === ts1.ts, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr <=> ts1.ts, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr =!= ts1.ts, classOf[NotEq[_]],
+        Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
 
-      checkFilterPredicate(!('_1 < ts4), classOf[GtEq[_]], ts4)
-      checkFilterPredicate('_1 < ts2 || '_1 > ts3, classOf[Operators.Or], Seq(Row(ts1), Row(ts4)))
-    }
-  }
+      checkFilterPredicate(tsAttr < ts2.ts, classOf[Lt[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr > ts1.ts, classOf[Gt[_]],
+        Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
+      checkFilterPredicate(tsAttr <= ts1.ts, classOf[LtEq[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr >= ts4.ts, classOf[GtEq[_]], resultFun(ts4))
 
-  private def testDecimalPushDown(data: DataFrame)(f: DataFrame => Unit): Unit = {
-    withTempPath { file =>
-      data.write.parquet(file.getCanonicalPath)
-      readParquetFile(file.toString)(f)
+      checkFilterPredicate(Literal(ts1.ts) === tsAttr, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts1.ts) <=> tsAttr, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts2.ts) > tsAttr, classOf[Lt[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts3.ts) < tsAttr, classOf[Gt[_]], resultFun(ts4))
+      checkFilterPredicate(Literal(ts1.ts) >= tsAttr, classOf[LtEq[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts4.ts) <= tsAttr, classOf[GtEq[_]], resultFun(ts4))
+
+      checkFilterPredicate(!(tsAttr < ts4.ts), classOf[GtEq[_]], resultFun(ts4))
+      checkFilterPredicate(tsAttr < ts2.ts || tsAttr > ts3.ts, classOf[Operators.Or],
+        Seq(Row(resultFun(ts1)), Row(resultFun(ts4))))
     }
   }
 
@@ -187,201 +241,265 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("filter pushdown - boolean") {
-    withParquetDataFrame((true :: false :: Nil).map(b => Tuple1.apply(Option(b)))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], Seq(Row(true), Row(false)))
+    val data = (true :: false :: Nil).map(b => Tuple1.apply(Option(b)))
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val booleanAttr = df(colName).expr
+      assert(df(colName).expr.dataType === BooleanType)
 
-      checkFilterPredicate('_1 === true, classOf[Eq[_]], true)
-      checkFilterPredicate('_1 <=> true, classOf[Eq[_]], true)
-      checkFilterPredicate('_1 =!= true, classOf[NotEq[_]], false)
+      checkFilterPredicate(booleanAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(booleanAttr.isNotNull, classOf[NotEq[_]],
+        Seq(Row(resultFun(true)), Row(resultFun(false))))
+
+      checkFilterPredicate(booleanAttr === true, classOf[Eq[_]], resultFun(true))
+      checkFilterPredicate(booleanAttr <=> true, classOf[Eq[_]], resultFun(true))
+      checkFilterPredicate(booleanAttr =!= true, classOf[NotEq[_]], resultFun(false))
     }
   }
 
   test("filter pushdown - tinyint") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toByte)))) { implicit df =>
-      assert(df.schema.head.dataType === ByteType)
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1.toByte, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1.toByte, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1.toByte, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2.toByte, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3.toByte, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1.toByte, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4.toByte, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1.toByte) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1.toByte) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2.toByte) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3.toByte) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1.toByte) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4.toByte) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4.toByte), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2.toByte || '_1 > 3.toByte,
-        classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toByte)))
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val tinyIntAttr = df(colName).expr
+      assert(df(colName).expr.dataType === ByteType)
+
+      checkFilterPredicate(tinyIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(tinyIntAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(tinyIntAttr === 1.toByte, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr <=> 1.toByte, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr =!= 1.toByte, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(tinyIntAttr < 2.toByte, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr > 3.toByte, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(tinyIntAttr <= 1.toByte, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr >= 4.toByte, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1.toByte) === tinyIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1.toByte) <=> tinyIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2.toByte) > tinyIntAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3.toByte) < tinyIntAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1.toByte) >= tinyIntAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4.toByte) <= tinyIntAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(tinyIntAttr < 4.toByte), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(tinyIntAttr < 2.toByte || tinyIntAttr > 3.toByte,
+        classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - smallint") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toShort)))) { implicit df =>
-      assert(df.schema.head.dataType === ShortType)
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1.toShort, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1.toShort, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1.toShort, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2.toShort, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3.toShort, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1.toShort, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4.toShort, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1.toShort) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1.toShort) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2.toShort) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3.toShort) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1.toShort) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4.toShort) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4.toShort), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2.toShort || '_1 > 3.toShort,
-        classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toShort)))
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val smallIntAttr = df(colName).expr
+      assert(df(colName).expr.dataType === ShortType)
+
+      checkFilterPredicate(smallIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(smallIntAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(smallIntAttr === 1.toShort, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr <=> 1.toShort, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr =!= 1.toShort, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(smallIntAttr < 2.toShort, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr > 3.toShort, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(smallIntAttr <= 1.toShort, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr >= 4.toShort, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1.toShort) === smallIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1.toShort) <=> smallIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2.toShort) > smallIntAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3.toShort) < smallIntAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1.toShort) >= smallIntAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4.toShort) <= smallIntAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(smallIntAttr < 4.toShort), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(smallIntAttr < 2.toShort || smallIntAttr > 3.toShort,
+        classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - integer") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i)))
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val intAttr = df(colName).expr
+      assert(df(colName).expr.dataType === IntegerType)
+
+      checkFilterPredicate(intAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(intAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(intAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(intAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(intAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(intAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(intAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(intAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(intAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === intAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> intAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > intAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < intAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= intAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= intAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(intAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(intAttr < 2 || intAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - long") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toLong)))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toLong)))
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val longAttr = df(colName).expr
+      assert(df(colName).expr.dataType === LongType)
+
+      checkFilterPredicate(longAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(longAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(longAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(longAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(longAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(longAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(longAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(longAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(longAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === longAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> longAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > longAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < longAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= longAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= longAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(longAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(longAttr < 2 || longAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - float") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toFloat)))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toFloat)))
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val floatAttr = df(colName).expr
+      assert(df(colName).expr.dataType === FloatType)
+
+      checkFilterPredicate(floatAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(floatAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(floatAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(floatAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(floatAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(floatAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(floatAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(floatAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(floatAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === floatAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> floatAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > floatAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < floatAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= floatAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= floatAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(floatAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(floatAttr < 2 || floatAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - double") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toDouble)))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toDouble)))
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val doubleAttr = df(colName).expr
+      assert(df(colName).expr.dataType === DoubleType)
+
+      checkFilterPredicate(doubleAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(doubleAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(doubleAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(doubleAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(doubleAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === doubleAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> doubleAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > doubleAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < doubleAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= doubleAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= doubleAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(doubleAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(doubleAttr < 2 || doubleAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - string") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(i.toString))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate(
-        '_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(i => Row.apply(i.toString)))
-
-      checkFilterPredicate('_1 === "1", classOf[Eq[_]], "1")
-      checkFilterPredicate('_1 <=> "1", classOf[Eq[_]], "1")
-      checkFilterPredicate(
-        '_1 =!= "1", classOf[NotEq[_]], (2 to 4).map(i => Row.apply(i.toString)))
-
-      checkFilterPredicate('_1 < "2", classOf[Lt[_]], "1")
-      checkFilterPredicate('_1 > "3", classOf[Gt[_]], "4")
-      checkFilterPredicate('_1 <= "1", classOf[LtEq[_]], "1")
-      checkFilterPredicate('_1 >= "4", classOf[GtEq[_]], "4")
-
-      checkFilterPredicate(Literal("1") === '_1, classOf[Eq[_]], "1")
-      checkFilterPredicate(Literal("1") <=> '_1, classOf[Eq[_]], "1")
-      checkFilterPredicate(Literal("2") > '_1, classOf[Lt[_]], "1")
-      checkFilterPredicate(Literal("3") < '_1, classOf[Gt[_]], "4")
-      checkFilterPredicate(Literal("1") >= '_1, classOf[LtEq[_]], "1")
-      checkFilterPredicate(Literal("4") <= '_1, classOf[GtEq[_]], "4")
-
-      checkFilterPredicate(!('_1 < "4"), classOf[GtEq[_]], "4")
-      checkFilterPredicate('_1 < "2" || '_1 > "3", classOf[Operators.Or], Seq(Row("1"), Row("4")))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toString)))
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val stringAttr = df(colName).expr
+      assert(df(colName).expr.dataType === StringType)
+
+      checkFilterPredicate(stringAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(stringAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i.toString))))
+
+      checkFilterPredicate(stringAttr === "1", classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr <=> "1", classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr =!= "1", classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i.toString))))
+
+      checkFilterPredicate(stringAttr < "2", classOf[Lt[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr > "3", classOf[Gt[_]], resultFun("4"))
+      checkFilterPredicate(stringAttr <= "1", classOf[LtEq[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr >= "4", classOf[GtEq[_]], resultFun("4"))
+
+      checkFilterPredicate(Literal("1") === stringAttr, classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(Literal("1") <=> stringAttr, classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(Literal("2") > stringAttr, classOf[Lt[_]], resultFun("1"))
+      checkFilterPredicate(Literal("3") < stringAttr, classOf[Gt[_]], resultFun("4"))
+      checkFilterPredicate(Literal("1") >= stringAttr, classOf[LtEq[_]], resultFun("1"))
+      checkFilterPredicate(Literal("4") <= stringAttr, classOf[GtEq[_]], resultFun("4"))
+
+      checkFilterPredicate(!(stringAttr < "4"), classOf[GtEq[_]], resultFun("4"))
+      checkFilterPredicate(stringAttr < "2" || stringAttr > "3", classOf[Operators.Or],
+        Seq(Row(resultFun("1")), Row(resultFun("4"))))
     }
   }
 
@@ -390,32 +508,38 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
       def b: Array[Byte] = int.toString.getBytes(StandardCharsets.UTF_8)
     }
 
-    withParquetDataFrame((1 to 4).map(i => Tuple1(i.b))) { implicit df =>
-      checkBinaryFilterPredicate('_1 === 1.b, classOf[Eq[_]], 1.b)
-      checkBinaryFilterPredicate('_1 <=> 1.b, classOf[Eq[_]], 1.b)
+    val data = (1 to 4).map(i => Tuple1(Option(i.b)))
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
 
-      checkBinaryFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkBinaryFilterPredicate(
-        '_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(i => Row.apply(i.b)).toSeq)
+      val binaryAttr: Expression = df(colName).expr
+      assert(df(colName).expr.dataType === BinaryType)
 
-      checkBinaryFilterPredicate(
-        '_1 =!= 1.b, classOf[NotEq[_]], (2 to 4).map(i => Row.apply(i.b)).toSeq)
+      checkFilterPredicate(binaryAttr === 1.b, classOf[Eq[_]], resultFun(1.b))
+      checkFilterPredicate(binaryAttr <=> 1.b, classOf[Eq[_]], resultFun(1.b))
 
-      checkBinaryFilterPredicate('_1 < 2.b, classOf[Lt[_]], 1.b)
-      checkBinaryFilterPredicate('_1 > 3.b, classOf[Gt[_]], 4.b)
-      checkBinaryFilterPredicate('_1 <= 1.b, classOf[LtEq[_]], 1.b)
-      checkBinaryFilterPredicate('_1 >= 4.b, classOf[GtEq[_]], 4.b)
+      checkFilterPredicate(binaryAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(binaryAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i.b))))
 
-      checkBinaryFilterPredicate(Literal(1.b) === '_1, classOf[Eq[_]], 1.b)
-      checkBinaryFilterPredicate(Literal(1.b) <=> '_1, classOf[Eq[_]], 1.b)
-      checkBinaryFilterPredicate(Literal(2.b) > '_1, classOf[Lt[_]], 1.b)
-      checkBinaryFilterPredicate(Literal(3.b) < '_1, classOf[Gt[_]], 4.b)
-      checkBinaryFilterPredicate(Literal(1.b) >= '_1, classOf[LtEq[_]], 1.b)
-      checkBinaryFilterPredicate(Literal(4.b) <= '_1, classOf[GtEq[_]], 4.b)
+      checkFilterPredicate(binaryAttr =!= 1.b, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i.b))))
 
-      checkBinaryFilterPredicate(!('_1 < 4.b), classOf[GtEq[_]], 4.b)
-      checkBinaryFilterPredicate(
-        '_1 < 2.b || '_1 > 3.b, classOf[Operators.Or], Seq(Row(1.b), Row(4.b)))
+      checkFilterPredicate(binaryAttr < 2.b, classOf[Lt[_]], resultFun(1.b))
+      checkFilterPredicate(binaryAttr > 3.b, classOf[Gt[_]], resultFun(4.b))
+      checkFilterPredicate(binaryAttr <= 1.b, classOf[LtEq[_]], resultFun(1.b))
+      checkFilterPredicate(binaryAttr >= 4.b, classOf[GtEq[_]], resultFun(4.b))
+
+      checkFilterPredicate(Literal(1.b) === binaryAttr, classOf[Eq[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(1.b) <=> binaryAttr, classOf[Eq[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(2.b) > binaryAttr, classOf[Lt[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(3.b) < binaryAttr, classOf[Gt[_]], resultFun(4.b))
+      checkFilterPredicate(Literal(1.b) >= binaryAttr, classOf[LtEq[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(4.b) <= binaryAttr, classOf[GtEq[_]], resultFun(4.b))
+
+      checkFilterPredicate(!(binaryAttr < 4.b), classOf[GtEq[_]], resultFun(4.b))
+      checkFilterPredicate(binaryAttr < 2.b || binaryAttr > 3.b, classOf[Operators.Or],
+        Seq(Row(resultFun(1.b)), Row(resultFun(4.b))))
     }
   }
 
@@ -425,70 +549,107 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     }
 
     val data = Seq("2018-03-18", "2018-03-19", "2018-03-20", "2018-03-21")
+    import testImplicits._
 
-    withParquetDataFrame(data.map(i => Tuple1(i.date))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], data.map(i => Row.apply(i.date)))
-
-      checkFilterPredicate('_1 === "2018-03-18".date, classOf[Eq[_]], "2018-03-18".date)
-      checkFilterPredicate('_1 <=> "2018-03-18".date, classOf[Eq[_]], "2018-03-18".date)
-      checkFilterPredicate('_1 =!= "2018-03-18".date, classOf[NotEq[_]],
-        Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(i.date)))
-
-      checkFilterPredicate('_1 < "2018-03-19".date, classOf[Lt[_]], "2018-03-18".date)
-      checkFilterPredicate('_1 > "2018-03-20".date, classOf[Gt[_]], "2018-03-21".date)
-      checkFilterPredicate('_1 <= "2018-03-18".date, classOf[LtEq[_]], "2018-03-18".date)
-      checkFilterPredicate('_1 >= "2018-03-21".date, classOf[GtEq[_]], "2018-03-21".date)
+    Seq(false, true).foreach { java8Api =>
+      withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+        val dates = data.map(i => Tuple1(Date.valueOf(i))).toDF()
+        withNestedParquetDataFrame(dates) { case (inputDF, colName, fun) =>
+          implicit val df: DataFrame = inputDF
 
-      checkFilterPredicate(
-        Literal("2018-03-18".date) === '_1, classOf[Eq[_]], "2018-03-18".date)
-      checkFilterPredicate(
-        Literal("2018-03-18".date) <=> '_1, classOf[Eq[_]], "2018-03-18".date)
-      checkFilterPredicate(
-        Literal("2018-03-19".date) > '_1, classOf[Lt[_]], "2018-03-18".date)
-      checkFilterPredicate(
-        Literal("2018-03-20".date) < '_1, classOf[Gt[_]], "2018-03-21".date)
-      checkFilterPredicate(
-        Literal("2018-03-18".date) >= '_1, classOf[LtEq[_]], "2018-03-18".date)
-      checkFilterPredicate(
-        Literal("2018-03-21".date) <= '_1, classOf[GtEq[_]], "2018-03-21".date)
+          def resultFun(dateStr: String): Any = {
+            val parsed = if (java8Api) LocalDate.parse(dateStr) else Date.valueOf(dateStr)
+            fun(parsed)
+          }
 
-      checkFilterPredicate(!('_1 < "2018-03-21".date), classOf[GtEq[_]], "2018-03-21".date)
-      checkFilterPredicate(
-        '_1 < "2018-03-19".date || '_1 > "2018-03-20".date,
-        classOf[Operators.Or],
-        Seq(Row("2018-03-18".date), Row("2018-03-21".date)))
+          val dateAttr: Expression = df(colName).expr
+          assert(df(colName).expr.dataType === DateType)
+
+          checkFilterPredicate(dateAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+          checkFilterPredicate(dateAttr.isNotNull, classOf[NotEq[_]],
+            data.map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(dateAttr === "2018-03-18".date, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr <=> "2018-03-18".date, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr =!= "2018-03-18".date, classOf[NotEq[_]],
+            Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(dateAttr < "2018-03-19".date, classOf[Lt[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr > "2018-03-20".date, classOf[Gt[_]],
+            resultFun("2018-03-21"))
+          checkFilterPredicate(dateAttr <= "2018-03-18".date, classOf[LtEq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr >= "2018-03-21".date, classOf[GtEq[_]],
+            resultFun("2018-03-21"))
+
+          checkFilterPredicate(Literal("2018-03-18".date) === dateAttr, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-18".date) <=> dateAttr, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-19".date) > dateAttr, classOf[Lt[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-20".date) < dateAttr, classOf[Gt[_]],
+            resultFun("2018-03-21"))
+          checkFilterPredicate(Literal("2018-03-18".date) >= dateAttr, classOf[LtEq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-21".date) <= dateAttr, classOf[GtEq[_]],
+            resultFun("2018-03-21"))
+
+          checkFilterPredicate(!(dateAttr < "2018-03-21".date), classOf[GtEq[_]],
+            resultFun("2018-03-21"))
+          checkFilterPredicate(
+            dateAttr < "2018-03-19".date || dateAttr > "2018-03-20".date,
+            classOf[Operators.Or],
+            Seq(Row(resultFun("2018-03-18")), Row(resultFun("2018-03-21"))))
+        }
+      }
     }
   }
 
   test("filter pushdown - timestamp") {
-    // spark.sql.parquet.outputTimestampType = TIMESTAMP_MILLIS
-    val millisData = Seq(Timestamp.valueOf("2018-06-14 08:28:53.123"),
-      Timestamp.valueOf("2018-06-15 08:28:53.123"),
-      Timestamp.valueOf("2018-06-16 08:28:53.123"),
-      Timestamp.valueOf("2018-06-17 08:28:53.123"))
-    withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-      ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString) {
-      testTimestampPushdown(millisData)
-    }
-
-    // spark.sql.parquet.outputTimestampType = TIMESTAMP_MICROS
-    val microsData = Seq(Timestamp.valueOf("2018-06-14 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-15 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-16 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-17 08:28:53.123456"))
-    withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-      ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) {
-      testTimestampPushdown(microsData)
-    }
-
-    // spark.sql.parquet.outputTimestampType = INT96 doesn't support pushdown
-    withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-      ParquetOutputTimestampType.INT96.toString) {
-      withParquetDataFrame(millisData.map(i => Tuple1(i))) { implicit df =>
-        val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
-        assertResult(None) {
-          createParquetFilters(schema).createFilter(sources.IsNull("_1"))
+    Seq(true, false).foreach { java8Api =>
+      withSQLConf(
+        SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString,
+        SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> "CORRECTED") {
+        // spark.sql.parquet.outputTimestampType = TIMESTAMP_MILLIS
+        val millisData = Seq(
+          "1000-06-14 08:28:53.123",
+          "1582-06-15 08:28:53.001",
+          "1900-06-16 08:28:53.0",
+          "2018-06-17 08:28:53.999")
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
+          ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString) {
+          testTimestampPushdown(millisData, java8Api)
+        }
+
+        // spark.sql.parquet.outputTimestampType = TIMESTAMP_MICROS
+        val microsData = Seq(
+          "1000-06-14 08:28:53.123456",
+          "1582-06-15 08:28:53.123456",
+          "1900-06-16 08:28:53.123456",
+          "2018-06-17 08:28:53.123456")
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
+          ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) {
+          testTimestampPushdown(microsData, java8Api)
+        }
+
+        // spark.sql.parquet.outputTimestampType = INT96 doesn't support pushdown
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
+          ParquetOutputTimestampType.INT96.toString) {
+          import testImplicits._
+          withTempPath { file =>
+            millisData.map(i => Tuple1(Timestamp.valueOf(i))).toDF
+              .write.format(dataSourceName).save(file.getCanonicalPath)
+            readParquetFile(file.getCanonicalPath) { df =>
+              val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
+              assertResult(None) {
+                createParquetFilters(schema).createFilter(sources.IsNull("_1"))
+              }
+            }
+          }
         }
       }
     }
@@ -502,33 +663,39 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
       (false, DecimalType.MAX_PRECISION) // binaryWriterUsingUnscaledBytes
     ).foreach { case (legacyFormat, precision) =>
       withSQLConf(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key -> legacyFormat.toString) {
-        val schema = StructType.fromDDL(s"a decimal($precision, 2)")
         val rdd =
           spark.sparkContext.parallelize((1 to 4).map(i => Row(new java.math.BigDecimal(i))))
-        val dataFrame = spark.createDataFrame(rdd, schema)
-        testDecimalPushDown(dataFrame) { implicit df =>
-          assert(df.schema === schema)
-          checkFilterPredicate('a.isNull, classOf[Eq[_]], Seq.empty[Row])
-          checkFilterPredicate('a.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-          checkFilterPredicate('a === 1, classOf[Eq[_]], 1)
-          checkFilterPredicate('a <=> 1, classOf[Eq[_]], 1)
-          checkFilterPredicate('a =!= 1, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-          checkFilterPredicate('a < 2, classOf[Lt[_]], 1)
-          checkFilterPredicate('a > 3, classOf[Gt[_]], 4)
-          checkFilterPredicate('a <= 1, classOf[LtEq[_]], 1)
-          checkFilterPredicate('a >= 4, classOf[GtEq[_]], 4)
-
-          checkFilterPredicate(Literal(1) === 'a, classOf[Eq[_]], 1)
-          checkFilterPredicate(Literal(1) <=> 'a, classOf[Eq[_]], 1)
-          checkFilterPredicate(Literal(2) > 'a, classOf[Lt[_]], 1)
-          checkFilterPredicate(Literal(3) < 'a, classOf[Gt[_]], 4)
-          checkFilterPredicate(Literal(1) >= 'a, classOf[LtEq[_]], 1)
-          checkFilterPredicate(Literal(4) <= 'a, classOf[GtEq[_]], 4)
-
-          checkFilterPredicate(!('a < 4), classOf[GtEq[_]], 4)
-          checkFilterPredicate('a < 2 || 'a > 3, classOf[Operators.Or], Seq(Row(1), Row(4)))
+        val dataFrame = spark.createDataFrame(rdd, StructType.fromDDL(s"a decimal($precision, 2)"))
+        withNestedParquetDataFrame(dataFrame) { case (inputDF, colName, resultFun) =>
+          implicit val df: DataFrame = inputDF
+
+          val decimalAttr: Expression = df(colName).expr
+          assert(df(colName).expr.dataType === DecimalType(precision, 2))
+
+          checkFilterPredicate(decimalAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+          checkFilterPredicate(decimalAttr.isNotNull, classOf[NotEq[_]],
+            (1 to 4).map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(decimalAttr === 1, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr <=> 1, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr =!= 1, classOf[NotEq[_]],
+            (2 to 4).map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(decimalAttr < 2, classOf[Lt[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr > 3, classOf[Gt[_]], resultFun(4))
+          checkFilterPredicate(decimalAttr <= 1, classOf[LtEq[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+          checkFilterPredicate(Literal(1) === decimalAttr, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(Literal(1) <=> decimalAttr, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(Literal(2) > decimalAttr, classOf[Lt[_]], resultFun(1))
+          checkFilterPredicate(Literal(3) < decimalAttr, classOf[Gt[_]], resultFun(4))
+          checkFilterPredicate(Literal(1) >= decimalAttr, classOf[LtEq[_]], resultFun(1))
+          checkFilterPredicate(Literal(4) <= decimalAttr, classOf[GtEq[_]], resultFun(4))
+
+          checkFilterPredicate(!(decimalAttr < 4), classOf[GtEq[_]], resultFun(4))
+          checkFilterPredicate(decimalAttr < 2 || decimalAttr > 3, classOf[Operators.Or],
+            Seq(Row(resultFun(1)), Row(resultFun(4))))
         }
       }
     }
@@ -648,10 +815,9 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
   test("Filter applied on merged Parquet schema with new column should work") {
     import testImplicits._
-    Seq("true", "false").foreach { vectorized =>
+    withAllParquetReaders {
       withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true",
-        SQLConf.PARQUET_SCHEMA_MERGING_ENABLED.key -> "true",
-        SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
+        SQLConf.PARQUET_SCHEMA_MERGING_ENABLED.key -> "true") {
         withTempPath { dir =>
           val path1 = s"${dir.getCanonicalPath}/table1"
           (1 to 3).map(i => (i, i.toString)).toDF("a", "b").write.parquet(path1)
@@ -1085,33 +1251,31 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("SPARK-17213: Broken Parquet filter push-down for string columns") {
-    Seq(true, false).foreach { vectorizedEnabled =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorizedEnabled.toString) {
-        withTempPath { dir =>
-          import testImplicits._
+    withAllParquetReaders {
+      withTempPath { dir =>
+        import testImplicits._
 
-          val path = dir.getCanonicalPath
-          // scalastyle:off nonascii
-          Seq("a", "é").toDF("name").write.parquet(path)
-          // scalastyle:on nonascii
+        val path = dir.getCanonicalPath
+        // scalastyle:off nonascii
+        Seq("a", "é").toDF("name").write.parquet(path)
+        // scalastyle:on nonascii
 
-          assert(spark.read.parquet(path).where("name > 'a'").count() == 1)
-          assert(spark.read.parquet(path).where("name >= 'a'").count() == 2)
+        assert(spark.read.parquet(path).where("name > 'a'").count() == 1)
+        assert(spark.read.parquet(path).where("name >= 'a'").count() == 2)
 
-          // scalastyle:off nonascii
-          assert(spark.read.parquet(path).where("name < 'é'").count() == 1)
-          assert(spark.read.parquet(path).where("name <= 'é'").count() == 2)
-          // scalastyle:on nonascii
-        }
+        // scalastyle:off nonascii
+        assert(spark.read.parquet(path).where("name < 'é'").count() == 1)
+        assert(spark.read.parquet(path).where("name <= 'é'").count() == 2)
+        // scalastyle:on nonascii
       }
     }
   }
 
-  test("SPARK-20364: Disable Parquet predicate pushdown for fields having dots in the names") {
+  test("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") {
     import testImplicits._
 
-    Seq(true, false).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString,
+    withAllParquetReaders {
+      withSQLConf(
           SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> true.toString,
           SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
         withTempPath { path =>
@@ -1120,6 +1284,28 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
           assert(readBack.count() == 1)
         }
       }
+
+      withSQLConf(
+          // Makes sure disabling 'spark.sql.parquet.recordFilter' still enables
+          // row group level filtering.
+          SQLConf.PARQUET_RECORD_FILTER_ENABLED.key -> "false",
+          SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true") {
+
+        withTempPath { path =>
+          val data = (1 to 1024)
+          data.toDF("col.dots").coalesce(1)
+            .write.option("parquet.block.size", 512)
+            .parquet(path.getAbsolutePath)
+          val df = spark.read.parquet(path.getAbsolutePath).filter("`col.dots` == 500")
+          // Here, we strip the Spark side filter and check the actual results from Parquet.
+          val actual = stripSparkFilter(df).collect().length
+          // Since those are filtered at row group level, the result count should be less
+          // than the total length but should not be a single record.
+          // Note that, if record level filtering is enabled, it should be a single record.
+          // If no filter is pushed down to Parquet, it should be the total length of data.
+          assert(actual > 1 && actual < data.length)
+        }
+      }
     }
   }
 
@@ -1390,8 +1576,30 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
       }
     }
   }
+
+  test("SPARK-30826: case insensitivity of StringStartsWith attribute") {
+    import testImplicits._
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      withTable("t1") {
+        withTempPath { dir =>
+          val path = dir.toURI.toString
+          Seq("42").toDF("COL").write.parquet(path)
+          spark.sql(
+            s"""
+               |CREATE TABLE t1 (col STRING)
+               |USING parquet
+               |OPTIONS (path '$path')
+           """.stripMargin)
+          checkAnswer(
+            spark.sql("SELECT * FROM t1 WHERE col LIKE '4%'"),
+            Row("42"))
+        }
+      }
+    }
+  }
 }
 
+@ExtendedSQLTest
 class ParquetV1FilterSuite extends ParquetFilterSuite {
   override protected def sparkConf: SparkConf =
     super
@@ -1406,51 +1614,72 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
       expected: Seq[Row]): Unit = {
     val output = predicate.collect { case a: Attribute => a }.distinct
 
-    withSQLConf(
-      SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true",
-      SQLConf.PARQUET_FILTER_PUSHDOWN_DATE_ENABLED.key -> "true",
-      SQLConf.PARQUET_FILTER_PUSHDOWN_TIMESTAMP_ENABLED.key -> "true",
-      SQLConf.PARQUET_FILTER_PUSHDOWN_DECIMAL_ENABLED.key -> "true",
-      SQLConf.PARQUET_FILTER_PUSHDOWN_STRING_STARTSWITH_ENABLED.key -> "true",
-      // Disable adding filters from constraints because it adds, for instance,
-      // is-not-null to pushed filters, which makes it hard to test if the pushed
-      // filter is expected or not (this had to be fixed with SPARK-13495).
-      SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> InferFiltersFromConstraints.ruleName,
-      SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
-      val query = df
-        .select(output.map(e => Column(e)): _*)
-        .where(Column(predicate))
+    Seq(("parquet", true), ("", false)).foreach { case (pushdownDsList, nestedPredicatePushdown) =>
+      withSQLConf(
+        SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true",
+        SQLConf.PARQUET_FILTER_PUSHDOWN_DATE_ENABLED.key -> "true",
+        SQLConf.PARQUET_FILTER_PUSHDOWN_TIMESTAMP_ENABLED.key -> "true",
+        SQLConf.PARQUET_FILTER_PUSHDOWN_DECIMAL_ENABLED.key -> "true",
+        SQLConf.PARQUET_FILTER_PUSHDOWN_STRING_STARTSWITH_ENABLED.key -> "true",
+        // Disable adding filters from constraints because it adds, for instance,
+        // is-not-null to pushed filters, which makes it hard to test if the pushed
+        // filter is expected or not (this had to be fixed with SPARK-13495).
+        SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> InferFiltersFromConstraints.ruleName,
+        SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
+        SQLConf.NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST.key -> pushdownDsList) {
+        val query = df
+          .select(output.map(e => Column(e)): _*)
+          .where(Column(predicate))
+
+        val nestedOrAttributes = predicate.collectFirst {
+          case g: GetStructField => g
+          case a: Attribute => a
+        }
+        assert(nestedOrAttributes.isDefined, "No GetStructField nor Attribute is detected.")
+
+        val parsed = parseColumnPath(
+          PushableColumnAndNestedColumn.unapply(nestedOrAttributes.get).get)
+
+        val containsNestedColumnOrDot = parsed.length > 1 || parsed(0).contains(".")
+
+        var maybeRelation: Option[HadoopFsRelation] = None
+        val maybeAnalyzedPredicate = query.queryExecution.optimizedPlan.collect {
+          case PhysicalOperation(_, filters,
+          LogicalRelation(relation: HadoopFsRelation, _, _, _)) =>
+            maybeRelation = Some(relation)
+            filters
+        }.flatten.reduceLeftOption(_ && _)
+        assert(maybeAnalyzedPredicate.isDefined, "No filter is analyzed from the given query")
+
+        val (_, selectedFilters, _) =
+          DataSourceStrategy.selectFilters(maybeRelation.get, maybeAnalyzedPredicate.toSeq)
+        // If predicates contains nested column or dot, we push down the predicates only if
+        // "parquet" is in `NESTED_PREDICATE_PUSHDOWN_V1_SOURCE_LIST`.
+        if (nestedPredicatePushdown || !containsNestedColumnOrDot) {
+          assert(selectedFilters.nonEmpty, "No filter is pushed down")
+          val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
+          val parquetFilters = createParquetFilters(schema)
+          // In this test suite, all the simple predicates are convertible here.
+          assert(parquetFilters.convertibleFilters(selectedFilters) === selectedFilters)
+          val pushedParquetFilters = selectedFilters.map { pred =>
+            val maybeFilter = parquetFilters.createFilter(pred)
+            assert(maybeFilter.isDefined, s"Couldn't generate filter predicate for $pred")
+            maybeFilter.get
+          }
+          // Doesn't bother checking type parameters here (e.g. `Eq[Integer]`)
+          assert(pushedParquetFilters.exists(_.getClass === filterClass),
+            s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
 
-      var maybeRelation: Option[HadoopFsRelation] = None
-      val maybeAnalyzedPredicate = query.queryExecution.optimizedPlan.collect {
-        case PhysicalOperation(_, filters,
-        LogicalRelation(relation: HadoopFsRelation, _, _, _)) =>
-          maybeRelation = Some(relation)
-          filters
-      }.flatten.reduceLeftOption(_ && _)
-      assert(maybeAnalyzedPredicate.isDefined, "No filter is analyzed from the given query")
-
-      val (_, selectedFilters, _) =
-        DataSourceStrategy.selectFilters(maybeRelation.get, maybeAnalyzedPredicate.toSeq)
-      assert(selectedFilters.nonEmpty, "No filter is pushed down")
-      val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
-      val parquetFilters = createParquetFilters(schema)
-      // In this test suite, all the simple predicates are convertible here.
-      assert(parquetFilters.convertibleFilters(selectedFilters) === selectedFilters)
-      val pushedParquetFilters = selectedFilters.map { pred =>
-        val maybeFilter = parquetFilters.createFilter(pred)
-        assert(maybeFilter.isDefined, s"Couldn't generate filter predicate for $pred")
-        maybeFilter.get
+          checker(stripSparkFilter(query), expected)
+        } else {
+          assert(selectedFilters.isEmpty, "There is filter pushed down")
+        }
       }
-      // Doesn't bother checking type parameters here (e.g. `Eq[Integer]`)
-      assert(pushedParquetFilters.exists(_.getClass === filterClass),
-        s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
-
-      checker(stripSparkFilter(query), expected)
     }
   }
 }
 
+@ExtendedSQLTest
 class ParquetV2FilterSuite extends ParquetFilterSuite {
   // TODO: enable Parquet V2 write path after file source V2 writers are workable.
   override protected def sparkConf: SparkConf =
@@ -1485,7 +1714,7 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
         case PhysicalOperation(_, filters,
             DataSourceV2ScanRelation(_, scan: ParquetScan, _)) =>
           assert(filters.nonEmpty, "No filter is analyzed from the given query")
-          val sourceFilters = filters.flatMap(DataSourceStrategy.translateFilter).toArray
+          val sourceFilters = filters.flatMap(DataSourceStrategy.translateFilter(_, true)).toArray
           val pushedFilters = scan.pushedFilters
           assert(pushedFilters.nonEmpty, "No filter is pushed down")
           val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 1550b3bbb6242..0984711dfcdfd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -17,6 +17,9 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
+import java.nio.file.{Files, Paths, StandardCopyOption}
+import java.sql.{Date, Timestamp}
+import java.time._
 import java.util.Locale
 
 import scala.collection.JavaConverters._
@@ -39,14 +42,15 @@ import org.apache.parquet.hadoop.util.HadoopInputFile
 import org.apache.parquet.io.api.RecordConsumer
 import org.apache.parquet.schema.{MessageType, MessageTypeParser}
 
-import org.apache.spark.{SPARK_VERSION_SHORT, SparkException}
+import org.apache.spark.{SPARK_VERSION_SHORT, SparkException, SparkUpgradeException}
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.{InternalRow, ScalaReflection}
 import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeRow}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
 import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -644,47 +648,39 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
   }
 
   test("read dictionary encoded decimals written as INT32") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // Decimal column in this file is encoded using plain dictionary
-          readResourceParquetFile("test-data/dec-in-i32.parquet"),
-          spark.range(1 << 4).select('id % 10 cast DecimalType(5, 2) as 'i32_dec))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // Decimal column in this file is encoded using plain dictionary
+        readResourceParquetFile("test-data/dec-in-i32.parquet"),
+        spark.range(1 << 4).select('id % 10 cast DecimalType(5, 2) as 'i32_dec))
     }
   }
 
   test("read dictionary encoded decimals written as INT64") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // Decimal column in this file is encoded using plain dictionary
-          readResourceParquetFile("test-data/dec-in-i64.parquet"),
-          spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'i64_dec))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // Decimal column in this file is encoded using plain dictionary
+        readResourceParquetFile("test-data/dec-in-i64.parquet"),
+        spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'i64_dec))
     }
   }
 
   test("read dictionary encoded decimals written as FIXED_LEN_BYTE_ARRAY") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // Decimal column in this file is encoded using plain dictionary
-          readResourceParquetFile("test-data/dec-in-fixed-len.parquet"),
-          spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'fixed_len_dec))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // Decimal column in this file is encoded using plain dictionary
+        readResourceParquetFile("test-data/dec-in-fixed-len.parquet"),
+        spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'fixed_len_dec))
     }
   }
 
   test("read dictionary and plain encoded timestamp_millis written as INT64") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // timestamp column in this file is encoded using combination of plain
-          // and dictionary encodings.
-          readResourceParquetFile("test-data/timemillis-in-i64.parquet"),
-          (1 to 3).map(i => Row(new java.sql.Timestamp(10))))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // timestamp column in this file is encoded using combination of plain
+        // and dictionary encodings.
+        readResourceParquetFile("test-data/timemillis-in-i64.parquet"),
+        (1 to 3).map(i => Row(new java.sql.Timestamp(10))))
     }
   }
 
@@ -719,7 +715,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       {
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         try {
           reader.initialize(file, null)
           val result = mutable.ArrayBuffer.empty[(Int, String)]
@@ -738,7 +734,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       {
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         try {
           reader.initialize(file, ("_2" :: Nil).asJava)
           val result = mutable.ArrayBuffer.empty[(String)]
@@ -756,7 +752,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       {
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         try {
           reader.initialize(file, ("_2" :: "_1" :: Nil).asJava)
           val result = mutable.ArrayBuffer.empty[(String, Int)]
@@ -775,7 +771,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       {
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         try {
           reader.initialize(file, List[String]().asJava)
           var result = 0
@@ -795,7 +791,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       Seq(1).toDF().repartition(1).write.parquet(dir.getCanonicalPath)
 
       val dataTypes =
-        Seq(StringType, BooleanType, ByteType, ShortType, IntegerType, LongType,
+        Seq(StringType, BooleanType, ByteType, BinaryType, ShortType, IntegerType, LongType,
           FloatType, DoubleType, DecimalType(25, 5), DateType, TimestampType)
 
       val constantValues =
@@ -803,6 +799,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
           UTF8String.fromString("a string"),
           true,
           1.toByte,
+          "Spark SQL".getBytes,
           2.toShort,
           3,
           Long.MaxValue,
@@ -816,7 +813,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         val schema = StructType(StructField("pcol", dt) :: Nil)
         val conf = sqlContext.conf
         val vectorizedReader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         val partitionValues = new GenericInternalRow(Array(v))
         val file = SpecificParquetRecordReaderBase.listDirectory(dir).get(0)
 
@@ -830,7 +827,11 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
           // in order to use get(...) method which is not implemented in `ColumnarBatch`.
           val actual = row.copy().get(1, dt)
           val expected = v
-          assert(actual == expected)
+          if (dt.isInstanceOf[BinaryType]) {
+            assert(actual.asInstanceOf[Array[Byte]] sameElements expected.asInstanceOf[Array[Byte]])
+          } else {
+            assert(actual == expected)
+          }
         } finally {
           vectorizedReader.close()
         }
@@ -863,20 +864,281 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
     }
   }
 
+  private def getMetaData(dir: java.io.File): Map[String, String] = {
+    val file = SpecificParquetRecordReaderBase.listDirectory(dir).get(0)
+    val conf = new Configuration()
+    val hadoopInputFile = HadoopInputFile.fromPath(new Path(file), conf)
+    val parquetReadOptions = HadoopReadOptions.builder(conf).build()
+    val m = ParquetFileReader.open(hadoopInputFile, parquetReadOptions)
+    val metadata = try {
+      m.getFileMetaData.getKeyValueMetaData
+    } finally {
+      m.close()
+    }
+    metadata.asScala.toMap
+  }
+
   test("Write Spark version into Parquet metadata") {
     withTempPath { dir =>
-      val path = dir.getAbsolutePath
-      spark.range(1).repartition(1).write.parquet(path)
-      val file = SpecificParquetRecordReaderBase.listDirectory(dir).get(0)
-
-      val conf = new Configuration()
-      val hadoopInputFile = HadoopInputFile.fromPath(new Path(file), conf)
-      val parquetReadOptions = HadoopReadOptions.builder(conf).build()
-      val m = ParquetFileReader.open(hadoopInputFile, parquetReadOptions)
-      val metaData = m.getFileMetaData.getKeyValueMetaData
-      m.close()
+      spark.range(1).repartition(1).write.parquet(dir.getAbsolutePath)
+      assert(getMetaData(dir)(SPARK_VERSION_METADATA_KEY) === SPARK_VERSION_SHORT)
+    }
+  }
+
+  // It generates input files for the test below:
+  // "SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps"
+  ignore("SPARK-31806: generate test files for checking compatibility with Spark 2.4") {
+    val resourceDir = "sql/core/src/test/resources/test-data"
+    val version = "2_4_5"
+    val N = 8
+    def save(
+        in: Seq[(String, String)],
+        t: String,
+        dstFile: String,
+        options: Map[String, String] = Map.empty): Unit = {
+      withTempDir { dir =>
+        in.toDF("dict", "plain")
+          .select($"dict".cast(t), $"plain".cast(t))
+          .repartition(1)
+          .write
+          .mode("overwrite")
+          .options(options)
+          .parquet(dir.getCanonicalPath)
+        Files.copy(
+          dir.listFiles().filter(_.getName.endsWith(".snappy.parquet")).head.toPath,
+          Paths.get(resourceDir, dstFile),
+          StandardCopyOption.REPLACE_EXISTING)
+      }
+    }
+    DateTimeTestUtils.withDefaultTimeZone(DateTimeTestUtils.LA) {
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> DateTimeTestUtils.LA.getId) {
+        save(
+          (1 to N).map(i => ("1001-01-01", s"1001-01-0$i")),
+          "date",
+          s"before_1582_date_v$version.snappy.parquet")
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "TIMESTAMP_MILLIS") {
+          save(
+            (1 to N).map(i => ("1001-01-01 01:02:03.123", s"1001-01-0$i 01:02:03.123")),
+            "timestamp",
+            s"before_1582_timestamp_millis_v$version.snappy.parquet")
+        }
+        val usTs = (1 to N).map(i => ("1001-01-01 01:02:03.123456", s"1001-01-0$i 01:02:03.123456"))
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "TIMESTAMP_MICROS") {
+          save(usTs, "timestamp", s"before_1582_timestamp_micros_v$version.snappy.parquet")
+        }
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "INT96") {
+          // Comparing to other logical types, Parquet-MR chooses dictionary encoding for the
+          // INT96 logical type because it consumes less memory for small column cardinality.
+          // Huge parquet files doesn't make sense to place to the resource folder. That's why
+          // we explicitly set `parquet.enable.dictionary` and generate two files w/ and w/o
+          // dictionary encoding.
+          save(
+            usTs,
+            "timestamp",
+            s"before_1582_timestamp_int96_plain_v$version.snappy.parquet",
+            Map("parquet.enable.dictionary" -> "false"))
+          save(
+            usTs,
+            "timestamp",
+            s"before_1582_timestamp_int96_dict_v$version.snappy.parquet",
+            Map("parquet.enable.dictionary" -> "true"))
+        }
+      }
+    }
+  }
+
+  test("SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps") {
+    val N = 8
+    // test reading the existing 2.4 files and new 3.0 files (with rebase on/off) together.
+    def checkReadMixedFiles[T](
+        fileName: String,
+        catalystType: String,
+        rowFunc: Int => (String, String),
+        toJavaType: String => T,
+        checkDefaultLegacyRead: String => Unit,
+        tsOutputType: String = "TIMESTAMP_MICROS"): Unit = {
+      withTempPaths(2) { paths =>
+        paths.foreach(_.delete())
+        val path2_4 = getResourceParquetFilePath("test-data/" + fileName)
+        val path3_0 = paths(0).getCanonicalPath
+        val path3_0_rebase = paths(1).getCanonicalPath
+        val df = Seq.tabulate(N)(rowFunc).toDF("dict", "plain")
+          .select($"dict".cast(catalystType), $"plain".cast(catalystType))
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> tsOutputType) {
+          checkDefaultLegacyRead(path2_4)
+          // By default we should fail to write ancient datetime values.
+          val e = intercept[SparkException](df.write.parquet(path3_0))
+          assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
+          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+            df.write.mode("overwrite").parquet(path3_0)
+          }
+          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+            df.write.parquet(path3_0_rebase)
+          }
+        }
+        // For Parquet files written by Spark 3.0, we know the writer info and don't need the
+        // config to guide the rebase behavior.
+        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
+          checkAnswer(
+            spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
+            (0 until N).flatMap { i =>
+              val (dictS, plainS) = rowFunc(i)
+              Seq.tabulate(3) { _ =>
+                Row(toJavaType(dictS), toJavaType(plainS))
+              }
+            })
+        }
+      }
+    }
+    def failInRead(path: String): Unit = {
+      val e = intercept[SparkException](spark.read.parquet(path).collect())
+      assert(e.getCause.isInstanceOf[SparkUpgradeException])
+    }
+    def successInRead(path: String): Unit = spark.read.parquet(path).collect()
+    Seq(
+      // By default we should fail to read ancient datetime values when parquet files don't
+      // contain Spark version.
+      "2_4_5" -> failInRead _,
+      "2_4_6" -> successInRead _).foreach { case (version, checkDefaultRead) =>
+      withAllParquetReaders {
+        checkReadMixedFiles(
+          s"before_1582_date_v$version.snappy.parquet",
+          "date",
+          (i: Int) => ("1001-01-01", s"1001-01-0${i + 1}"),
+          java.sql.Date.valueOf,
+          checkDefaultRead)
+        checkReadMixedFiles(
+          s"before_1582_timestamp_micros_v$version.snappy.parquet",
+          "timestamp",
+          (i: Int) => ("1001-01-01 01:02:03.123456", s"1001-01-0${i + 1} 01:02:03.123456"),
+          java.sql.Timestamp.valueOf,
+          checkDefaultRead)
+        checkReadMixedFiles(
+          s"before_1582_timestamp_millis_v$version.snappy.parquet",
+          "timestamp",
+          (i: Int) => ("1001-01-01 01:02:03.123", s"1001-01-0${i + 1} 01:02:03.123"),
+          java.sql.Timestamp.valueOf,
+          checkDefaultRead,
+          tsOutputType = "TIMESTAMP_MILLIS")
+        // INT96 is a legacy timestamp format and we always rebase the seconds for it.
+        Seq("plain", "dict").foreach { enc =>
+          checkAnswer(readResourceParquetFile(
+            s"test-data/before_1582_timestamp_int96_${enc}_v$version.snappy.parquet"),
+            Seq.tabulate(N) { i =>
+              Row(
+                java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456"),
+                java.sql.Timestamp.valueOf(s"1001-01-0${i + 1} 01:02:03.123456"))
+            })
+        }
+      }
+    }
+  }
 
-      assert(metaData.get(SPARK_VERSION_METADATA_KEY) === SPARK_VERSION_SHORT)
+  test("SPARK-31159: rebasing timestamps in write") {
+    val N = 8
+    Seq(false, true).foreach { dictionaryEncoding =>
+      Seq(
+        ("TIMESTAMP_MILLIS", "1001-01-01 01:02:03.123", "1001-01-07 01:09:05.123"),
+        ("TIMESTAMP_MICROS", "1001-01-01 01:02:03.123456", "1001-01-07 01:09:05.123456"),
+        ("INT96", "1001-01-01 01:02:03.123456", "1001-01-01 01:02:03.123456")
+      ).foreach { case (outType, tsStr, nonRebased) =>
+        withClue(s"output type $outType") {
+          withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outType) {
+            withTempPath { dir =>
+              val path = dir.getAbsolutePath
+              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+                Seq.tabulate(N)(_ => tsStr).toDF("tsS")
+                  .select($"tsS".cast("timestamp").as("ts"))
+                  .repartition(1)
+                  .write
+                  .option("parquet.enable.dictionary", dictionaryEncoding)
+                  .parquet(path)
+              }
+
+              withAllParquetReaders {
+                // The file metadata indicates if it needs rebase or not, so we can always get the
+                // correct result regardless of the "rebase mode" config.
+                Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+                  withSQLConf(
+                    SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> mode.toString) {
+                    checkAnswer(
+                      spark.read.parquet(path),
+                      Seq.tabulate(N)(_ => Row(Timestamp.valueOf(tsStr))))
+                  }
+                }
+
+                // Force to not rebase to prove the written datetime values are rebased
+                // and we will get wrong result if we don't rebase while reading.
+                withSQLConf("spark.test.forceNoRebase" -> "true") {
+                  checkAnswer(
+                    spark.read.parquet(path),
+                    Seq.tabulate(N)(_ => Row(Timestamp.valueOf(nonRebased))))
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-31159: rebasing dates in write") {
+    val N = 8
+    Seq(false, true).foreach { dictionaryEncoding =>
+      withTempPath { dir =>
+        val path = dir.getAbsolutePath
+        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+          Seq.tabulate(N)(_ => "1001-01-01").toDF("dateS")
+            .select($"dateS".cast("date").as("date"))
+            .repartition(1)
+            .write
+            .option("parquet.enable.dictionary", dictionaryEncoding)
+            .parquet(path)
+        }
+
+        withAllParquetReaders {
+          // The file metadata indicates if it needs rebase or not, so we can always get the
+          // correct result regardless of the "rebase mode" config.
+          Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+            withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
+              checkAnswer(
+                spark.read.parquet(path),
+                Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-01"))))
+            }
+          }
+
+          // Force to not rebase to prove the written datetime values are rebased and we will get
+          // wrong result if we don't rebase while reading.
+          withSQLConf("spark.test.forceNoRebase" -> "true") {
+            checkAnswer(
+              spark.read.parquet(path),
+              Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-07"))))
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-33163: write the metadata key 'org.apache.spark.legacyDateTime'") {
+    def saveTs(dir: java.io.File): Unit = {
+      Seq(Timestamp.valueOf("2020-10-15 01:02:03")).toDF()
+        .repartition(1)
+        .write
+        .parquet(dir.getAbsolutePath)
+    }
+    withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+      withTempPath { dir =>
+        saveTs(dir)
+        assert(getMetaData(dir)(SPARK_LEGACY_DATETIME) === "")
+      }
+    }
+    Seq(CORRECTED, EXCEPTION).foreach { mode =>
+      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> mode.toString) {
+        withTempPath { dir =>
+          saveTs(dir)
+          assert(getMetaData(dir).get(SPARK_LEGACY_DATETIME).isEmpty)
+        }
+      }
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
index 1ded34f24e436..a14f6416199a1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.datasources.parquet
 
 import java.io.File
+import java.time.ZoneOffset
 
 import org.apache.commons.io.FileUtils
 import org.apache.hadoop.fs.{FileSystem, Path, PathFilter}
@@ -118,21 +119,16 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
       ).map { s => java.sql.Timestamp.valueOf(s) }
       import testImplicits._
       // match the column names of the file from impala
-      withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-        SQLConf.ParquetOutputTimestampType.INT96.toString) {
-        val df = spark.createDataset(ts).toDF().repartition(1)
-          .withColumnRenamed("value", "ts")
-        df.write.parquet(tableDir.getAbsolutePath)
-      }
+      val df = spark.createDataset(ts).toDF().repartition(1).withColumnRenamed("value", "ts")
+      df.write.parquet(tableDir.getAbsolutePath)
       FileUtils.copyFile(new File(impalaPath), new File(tableDir, "part-00001.parq"))
 
       Seq(false, true).foreach { int96TimestampConversion =>
-        Seq(false, true).foreach { vectorized =>
+        withAllParquetReaders {
           withSQLConf(
               (SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key,
                 SQLConf.ParquetOutputTimestampType.INT96.toString),
-              (SQLConf.PARQUET_INT96_TIMESTAMP_CONVERSION.key, int96TimestampConversion.toString()),
-              (SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key, vectorized.toString())
+              (SQLConf.PARQUET_INT96_TIMESTAMP_CONVERSION.key, int96TimestampConversion.toString())
           ) {
             val readBack = spark.read.parquet(tableDir.getAbsolutePath).collect()
             assert(readBack.size === 6)
@@ -145,14 +141,15 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
               impalaFileData.map { ts =>
                 DateTimeUtils.toJavaTimestamp(DateTimeUtils.convertTz(
                   DateTimeUtils.fromJavaTimestamp(ts),
-                  DateTimeUtils.TimeZoneUTC,
-                  DateTimeUtils.getTimeZone(conf.sessionLocalTimeZone)))
+                  ZoneOffset.UTC,
+                  DateTimeUtils.getZoneId(conf.sessionLocalTimeZone)))
               }
             }
             val fullExpectations = (ts ++ impalaExpectations).map(_.toString).sorted.toArray
             val actual = readBack.map(_.getTimestamp(0).toString).sorted
             withClue(
-              s"int96TimestampConversion = $int96TimestampConversion; vectorized = $vectorized") {
+              s"int96TimestampConversion = $int96TimestampConversion; " +
+              s"vectorized = ${SQLConf.get.parquetVectorizedReaderEnabled}") {
               assert(fullExpectations === actual)
 
               // Now test that the behavior is still correct even with a filter which could get
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
index e63929470ce5f..32a9558e91f10 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
@@ -59,7 +59,8 @@ abstract class ParquetPartitionDiscoverySuite
 
   val timeZoneId = ZoneId.systemDefault()
   val df = DateFormatter(timeZoneId)
-  val tf = TimestampFormatter(timestampPartitionPattern, timeZoneId)
+  val tf = TimestampFormatter(
+    timestampPartitionPattern, timeZoneId, isParsing = true)
 
   protected override def beforeAll(): Unit = {
     super.beforeAll()
@@ -87,7 +88,7 @@ abstract class ParquetPartitionDiscoverySuite
     check("1990-02-24 12:00:30",
       Literal.create(Timestamp.valueOf("1990-02-24 12:00:30"), TimestampType))
 
-    val c = Calendar.getInstance(TimeZone.getTimeZone("GMT"))
+    val c = Calendar.getInstance(TimeZone.getTimeZone("UTC"))
     c.set(1990, 1, 24, 12, 0, 30)
     c.set(Calendar.MILLISECOND, 0)
     check("1990-02-24 12:00:30",
@@ -691,10 +692,10 @@ abstract class ParquetPartitionDiscoverySuite
     }
 
     withTempPath { dir =>
-      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .format("parquet").partitionBy(partitionColumns.map(_.name): _*).save(dir.toString)
       val fields = schema.map(f => Column(f.name).cast(f.dataType))
-      checkAnswer(spark.read.option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+      checkAnswer(spark.read.option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .load(dir.toString).select(fields: _*), row)
     }
   }
@@ -733,10 +734,10 @@ abstract class ParquetPartitionDiscoverySuite
     }
 
     withTempPath { dir =>
-      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .format("parquet").partitionBy(partitionColumns.map(_.name): _*).save(dir.toString)
       val fields = schema.map(f => Column(f.name))
-      checkAnswer(spark.read.option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+      checkAnswer(spark.read.option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .load(dir.toString).select(fields: _*), row)
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
index 917aaba2669ce..05d305a9b52ba 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
@@ -168,11 +168,9 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
       withTempPath { file =>
         val df = spark.createDataFrame(sparkContext.parallelize(data), schema)
         df.write.parquet(file.getCanonicalPath)
-        ("true" :: "false" :: Nil).foreach { vectorized =>
-          withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-            val df2 = spark.read.parquet(file.getCanonicalPath)
-            checkAnswer(df2, df.collect().toSeq)
-          }
+        withAllParquetReaders {
+          val df2 = spark.read.parquet(file.getCanonicalPath)
+          checkAnswer(df2, df.collect().toSeq)
         }
       }
     }
@@ -791,15 +789,13 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
   }
 
   test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") {
-    (true :: false :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-        withTempPath { path =>
-          // Repeated values for dictionary encoding.
-          Seq(Some("A"), Some("A"), None).toDF.repartition(1)
-            .write.parquet(path.getAbsolutePath)
-          val df = spark.read.parquet(path.getAbsolutePath)
-          checkAnswer(stripSparkFilter(df.where("NOT (value <=> 'A')")), df)
-        }
+    withAllParquetReaders {
+      withTempPath { path =>
+        // Repeated values for dictionary encoding.
+        Seq(Some("A"), Some("A"), None).toDF.repartition(1)
+          .write.parquet(path.getAbsolutePath)
+        val df = spark.read.parquet(path.getAbsolutePath)
+        checkAnswer(stripSparkFilter(df.where("NOT (value <=> 'A')")), df)
       }
     }
   }
@@ -821,10 +817,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
         withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> toTsType) {
           write(df2.write.mode(SaveMode.Append))
         }
-        Seq("true", "false").foreach { vectorized =>
-          withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-            checkAnswer(readback, df1.unionAll(df2))
-          }
+        withAllParquetReaders {
+          checkAnswer(readback, df1.unionAll(df2))
         }
       }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
index c64e95078e916..cab93bd96fff4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.execution.datasources.SchemaPruningSuite
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.tags.ExtendedSQLTest
 
 abstract class ParquetSchemaPruningSuite extends SchemaPruningSuite with AdaptiveSparkPlanHelper {
   override protected val dataSourceName: String = "parquet"
@@ -33,6 +34,7 @@ abstract class ParquetSchemaPruningSuite extends SchemaPruningSuite with Adaptiv
 
 }
 
+@ExtendedSQLTest
 class ParquetV1SchemaPruningSuite extends ParquetSchemaPruningSuite {
   override protected def sparkConf: SparkConf =
     super
@@ -40,6 +42,7 @@ class ParquetV1SchemaPruningSuite extends ParquetSchemaPruningSuite {
       .set(SQLConf.USE_V1_SOURCE_LIST, "parquet")
 }
 
+@ExtendedSQLTest
 class ParquetV2SchemaPruningSuite extends ParquetSchemaPruningSuite {
   // TODO: enable Parquet V2 write path after file source V2 writers are workable.
   override protected def sparkConf: SparkConf =
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
index 828ba6aee026b..db8ee724c01c1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
@@ -152,7 +152,17 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
   }
 
   protected def readResourceParquetFile(name: String): DataFrame = {
-    val url = Thread.currentThread().getContextClassLoader.getResource(name)
-    spark.read.parquet(url.toString)
+    spark.read.parquet(getResourceParquetFilePath(name))
+  }
+
+  protected def getResourceParquetFilePath(name: String): String = {
+    Thread.currentThread().getContextClassLoader.getResource(name).toString
+  }
+
+  def withAllParquetReaders(code: => Unit): Unit = {
+    // test the row-based reader
+    withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false")(code)
+    // test the vectorized reader
+    withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true")(code)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
index 539ff0d0e905c..7e97994476694 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
@@ -24,14 +24,14 @@ import java.nio.file.Files
 import org.apache.hadoop.io.SequenceFile.CompressionType
 import org.apache.hadoop.io.compress.GzipCodec
 
-import org.apache.spark.TestUtils
+import org.apache.spark.{SparkConf, TestUtils}
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{StringType, StructType}
 import org.apache.spark.util.Utils
 
-class TextSuite extends QueryTest with SharedSparkSession {
+abstract class TextSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
 
   test("reading text file") {
@@ -234,3 +234,17 @@ class TextSuite extends QueryTest with SharedSparkSession {
     assert(data.length == 4)
   }
 }
+
+class TextV1Suite extends TextSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "text")
+}
+
+class TextV2Suite extends TextSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
index 5e3b3441aa74f..f4812844cbae3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
@@ -19,12 +19,13 @@ package org.apache.spark.sql.execution.datasources.text
 
 import java.io.File
 
+import org.apache.spark.SparkConf
 import org.apache.spark.sql.{QueryTest, Row}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{StringType, StructType}
 
-class WholeTextFileSuite extends QueryTest with SharedSparkSession {
+abstract class WholeTextFileSuite extends QueryTest with SharedSparkSession {
 
   // Hadoop's FileSystem caching does not use the Configuration as part of its cache key, which
   // can cause Filesystem.get(Configuration) to return a cached instance created with a different
@@ -103,3 +104,17 @@ class WholeTextFileSuite extends QueryTest with SharedSparkSession {
     }
   }
 }
+
+class WholeTextFileV1Suite extends WholeTextFileSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "text")
+}
+
+class WholeTextFileV2Suite extends WholeTextFileSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
index 4cb845b2487d6..e9ef7c1a0c540 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
@@ -24,27 +24,14 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.execution.{CodegenSupport, LeafExecNode, WholeStageCodegenExec}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.functions._
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.test.SQLTestData.TestData
 import org.apache.spark.sql.types.StructType
 
-class DebuggingSuite extends SharedSparkSession {
-
-
-  var originalValue: String = _
-  // With on AQE, the WholeStageCodegenExec is added when running QueryStageExec.
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
+// Disable AQE because the WholeStageCodegenExec is added when running QueryStageExec
+class DebuggingSuite extends SharedSparkSession with DisableAdaptiveExecutionSuite {
 
   test("DataFrame.debug()") {
     testData.debug()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
index 5ce758e1e4eb8..ef0a596f21104 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.{BitwiseAnd, BitwiseOr, Cast, Literal, ShiftLeft}
 import org.apache.spark.sql.catalyst.plans.logical.BROADCAST
 import org.apache.spark.sql.execution.{SparkPlan, WholeStageCodegenExec}
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.exchange.EnsureRequirements
 import org.apache.spark.sql.functions._
@@ -39,7 +39,8 @@ import org.apache.spark.sql.types.{LongType, ShortType}
  * unsafe map in [[org.apache.spark.sql.execution.joins.UnsafeHashedRelation]] is not triggered
  * without serializing the hashed relation, which does not happen in local mode.
  */
-class BroadcastJoinSuite extends QueryTest with SQLTestUtils with AdaptiveSparkPlanHelper {
+abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
+  with AdaptiveSparkPlanHelper {
   import testImplicits._
 
   protected var spark: SparkSession = null
@@ -206,24 +207,25 @@ class BroadcastJoinSuite extends QueryTest with SQLTestUtils with AdaptiveSparkP
 
   test("broadcast hint in SQL") {
     import org.apache.spark.sql.catalyst.plans.logical.Join
-
-    spark.range(10).createOrReplaceTempView("t")
-    spark.range(10).createOrReplaceTempView("u")
-
-    for (name <- Seq("BROADCAST", "BROADCASTJOIN", "MAPJOIN")) {
-      val plan1 = sql(s"SELECT /*+ $name(t) */ * FROM t JOIN u ON t.id = u.id").queryExecution
-        .optimizedPlan
-      val plan2 = sql(s"SELECT /*+ $name(u) */ * FROM t JOIN u ON t.id = u.id").queryExecution
-        .optimizedPlan
-      val plan3 = sql(s"SELECT /*+ $name(v) */ * FROM t JOIN u ON t.id = u.id").queryExecution
-        .optimizedPlan
-
-      assert(plan1.asInstanceOf[Join].hint.leftHint.get.strategy.contains(BROADCAST))
-      assert(plan1.asInstanceOf[Join].hint.rightHint.isEmpty)
-      assert(plan2.asInstanceOf[Join].hint.leftHint.isEmpty)
-      assert(plan2.asInstanceOf[Join].hint.rightHint.get.strategy.contains(BROADCAST))
-      assert(plan3.asInstanceOf[Join].hint.leftHint.isEmpty)
-      assert(plan3.asInstanceOf[Join].hint.rightHint.isEmpty)
+    withTempView("t", "u") {
+      spark.range(10).createOrReplaceTempView("t")
+      spark.range(10).createOrReplaceTempView("u")
+
+      for (name <- Seq("BROADCAST", "BROADCASTJOIN", "MAPJOIN")) {
+        val plan1 = sql(s"SELECT /*+ $name(t) */ * FROM t JOIN u ON t.id = u.id").queryExecution
+          .optimizedPlan
+        val plan2 = sql(s"SELECT /*+ $name(u) */ * FROM t JOIN u ON t.id = u.id").queryExecution
+          .optimizedPlan
+        val plan3 = sql(s"SELECT /*+ $name(v) */ * FROM t JOIN u ON t.id = u.id").queryExecution
+          .optimizedPlan
+
+        assert(plan1.asInstanceOf[Join].hint.leftHint.get.strategy.contains(BROADCAST))
+        assert(plan1.asInstanceOf[Join].hint.rightHint.isEmpty)
+        assert(plan2.asInstanceOf[Join].hint.leftHint.isEmpty)
+        assert(plan2.asInstanceOf[Join].hint.rightHint.get.strategy.contains(BROADCAST))
+        assert(plan3.asInstanceOf[Join].hint.leftHint.isEmpty)
+        assert(plan3.asInstanceOf[Join].hint.rightHint.isEmpty)
+      }
     }
   }
 
@@ -237,33 +239,40 @@ class BroadcastJoinSuite extends QueryTest with SQLTestUtils with AdaptiveSparkP
     assert(HashJoin.rewriteKeyExpr(l :: l :: Nil) === l :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(l :: i :: Nil) === l :: i :: Nil)
 
-    assert(HashJoin.rewriteKeyExpr(i :: Nil) === Cast(i, LongType) :: Nil)
+    assert(HashJoin.rewriteKeyExpr(i :: Nil) ===
+      Cast(i, LongType, Some(conf.sessionLocalTimeZone)) :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: l :: Nil) === i :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: i :: Nil) ===
-      BitwiseOr(ShiftLeft(Cast(i, LongType), Literal(32)),
-        BitwiseAnd(Cast(i, LongType), Literal((1L << 32) - 1))) :: Nil)
+      BitwiseOr(ShiftLeft(Cast(i, LongType, Some(conf.sessionLocalTimeZone)), Literal(32)),
+        BitwiseAnd(Cast(i, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 32) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(i :: i :: i :: Nil) === i :: i :: i :: Nil)
 
-    assert(HashJoin.rewriteKeyExpr(s :: Nil) === Cast(s, LongType) :: Nil)
+    assert(HashJoin.rewriteKeyExpr(s :: Nil) ===
+      Cast(s, LongType, Some(conf.sessionLocalTimeZone)) :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: l :: Nil) === s :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: Nil) ===
-      BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+      BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: Nil) ===
       BitwiseOr(ShiftLeft(
-        BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-          BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+        BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+          BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))),
         Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: s :: Nil) ===
       BitwiseOr(ShiftLeft(
         BitwiseOr(ShiftLeft(
-          BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-            BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+          BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+            BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)),
+              Literal((1L << 16) - 1))),
           Literal(16)),
-          BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+          BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))),
         Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: s :: s :: Nil) ===
       s :: s :: s :: s :: s :: Nil)
 
@@ -398,4 +407,22 @@ class BroadcastJoinSuite extends QueryTest with SQLTestUtils with AdaptiveSparkP
         }
     }
   }
+
+  test("Broadcast timeout") {
+    val timeout = 5
+    val slowUDF = udf({ x: Int => Thread.sleep(timeout * 10 * 1000); x })
+    val df1 = spark.range(10).select($"id" as 'a)
+    val df2 = spark.range(5).select(slowUDF($"id") as 'a)
+    val testDf = df1.join(broadcast(df2), "a")
+    withSQLConf(SQLConf.BROADCAST_TIMEOUT.key -> timeout.toString) {
+      val e = intercept[Exception] {
+        testDf.collect()
+      }
+      assert(e.getMessage.contains(s"Could not execute broadcast in $timeout secs."))
+    }
+  }
 }
+
+class BroadcastJoinSuite extends BroadcastJoinSuiteBase with DisableAdaptiveExecutionSuite
+
+class BroadcastJoinSuiteAE extends BroadcastJoinSuiteBase with EnableAdaptiveExecutionSuite
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala
index 3526aa254c280..5e7985e28a8e5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala
@@ -358,6 +358,45 @@ class HashedRelationSuite extends SharedSparkSession {
     assert(java.util.Arrays.equals(os.toByteArray, os2.toByteArray))
   }
 
+  test("SPARK-31511: Make BytesToBytesMap iterators thread-safe") {
+    val ser = sparkContext.env.serializer.newInstance()
+    val key = Seq(BoundReference(0, LongType, false))
+
+    val unsafeProj = UnsafeProjection.create(
+      Seq(BoundReference(0, LongType, false), BoundReference(1, IntegerType, true)))
+    val rows = (0 until 10000).map(i => unsafeProj(InternalRow(Int.int2long(i), i + 1)).copy())
+    val unsafeHashed = UnsafeHashedRelation(rows.iterator, key, 1, mm)
+
+    val os = new ByteArrayOutputStream()
+    val thread1 = new Thread {
+      override def run(): Unit = {
+        val out = new ObjectOutputStream(os)
+        unsafeHashed.asInstanceOf[UnsafeHashedRelation].writeExternal(out)
+        out.flush()
+      }
+    }
+
+    val thread2 = new Thread {
+      override def run(): Unit = {
+        val threadOut = new ObjectOutputStream(new ByteArrayOutputStream())
+        unsafeHashed.asInstanceOf[UnsafeHashedRelation].writeExternal(threadOut)
+        threadOut.flush()
+      }
+    }
+
+    thread1.start()
+    thread2.start()
+    thread1.join()
+    thread2.join()
+
+    val unsafeHashed2 = ser.deserialize[UnsafeHashedRelation](ser.serialize(unsafeHashed))
+    val os2 = new ByteArrayOutputStream()
+    val out2 = new ObjectOutputStream(os2)
+    unsafeHashed2.writeExternal(out2)
+    out2.flush()
+    assert(java.util.Arrays.equals(os.toByteArray, os2.toByteArray))
+  }
+
   // This test require 4G heap to run, should run it manually
   ignore("build HashedRelation that is larger than 1G") {
     val unsafeProj = UnsafeProjection.create(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
index 7d09577075d5d..00c0315ae3779 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial}
 import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.execution.{FilterExec, RangeExec, SparkPlan, WholeStageCodegenExec}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
 import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
 import org.apache.spark.sql.functions._
@@ -33,22 +34,11 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.util.{AccumulatorContext, JsonProtocol}
 
-class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
+// Disable AQE because metric info is different with AQE on/off
+class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
+  with DisableAdaptiveExecutionSuite {
   import testImplicits._
 
-  var originalValue: String = _
-  // With AQE on/off, the metric info is different.
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
-
   /**
    * Generates a `DataFrame` by filling randomly generated bytes for hash collision.
    */
@@ -98,7 +88,7 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val ds = spark.range(10).filter('id < 5)
     testSparkPlanMetricsWithPredicates(ds.toDF(), 1, Map(
       0L -> (("WholeStageCodegen (1)", Map(
-        "duration total (min, med, max (stageId (attemptId): taskId))" -> {
+        "duration" -> {
           _.toString.matches(timingMetricPattern)
         })))), true)
   }
@@ -110,10 +100,10 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val df = testData2.groupBy().count() // 2 partitions
     val expected1 = Seq(
       Map("number of output rows" -> 2L,
-        "avg hash probe bucket list iters (min, med, max (stageId (attemptId): taskId))" ->
+        "avg hash probe bucket list iters" ->
           aggregateMetricsPattern),
       Map("number of output rows" -> 1L,
-        "avg hash probe bucket list iters (min, med, max (stageId (attemptId): taskId))" ->
+        "avg hash probe bucket list iters" ->
           aggregateMetricsPattern))
     val shuffleExpected1 = Map(
       "records read" -> 2L,
@@ -130,10 +120,10 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val df2 = testData2.groupBy('a).count()
     val expected2 = Seq(
       Map("number of output rows" -> 4L,
-        "avg hash probe bucket list iters (min, med, max (stageId (attemptId): taskId))" ->
+        "avg hash probe bucket list iters" ->
           aggregateMetricsPattern),
       Map("number of output rows" -> 3L,
-        "avg hash probe bucket list iters (min, med, max (stageId (attemptId): taskId))" ->
+        "avg hash probe bucket list iters" ->
           aggregateMetricsPattern))
 
     val shuffleExpected2 = Map(
@@ -181,12 +171,17 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
       }
       val metrics = getSparkPlanMetrics(df, 1, nodeIds, enableWholeStage).get
       nodeIds.foreach { nodeId =>
-        val probes = metrics(nodeId)._2("avg hash probe bucket list iters (min, med, max (stageId" +
-          " (attemptId): taskId))")
-        // Extract min, med, max from the string and strip off everthing else.
-        val index = probes.toString.stripPrefix("\n(").stripSuffix(")").indexOf(" (", 0)
-        probes.toString.stripPrefix("\n(").stripSuffix(")").slice(0, index).split(", ").foreach {
-          probe => assert(probe.toDouble > 1.0)
+        val probes = metrics(nodeId)._2("avg hash probe bucket list iters").toString
+        if (!probes.contains("\n")) {
+          // It's a single metrics value
+          assert(probes.toDouble > 1.0)
+        } else {
+          val mainValue = probes.split("\n").apply(1).stripPrefix("(").stripSuffix(")")
+          // Extract min, med, max from the string and strip off everthing else.
+          val index = mainValue.indexOf(" (", 0)
+          mainValue.slice(0, index).split(", ").foreach {
+            probe => assert(probe.toDouble > 1.0)
+          }
         }
       }
     }
@@ -231,13 +226,13 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val df = Seq(1, 3, 2).toDF("id").sort('id)
     testSparkPlanMetricsWithPredicates(df, 2, Map(
       0L -> (("Sort", Map(
-        "sort time total (min, med, max (stageId (attemptId): taskId))" -> {
+        "sort time" -> {
           _.toString.matches(timingMetricPattern)
         },
-        "peak memory total (min, med, max (stageId (attemptId): taskId))" -> {
+        "peak memory" -> {
           _.toString.matches(sizeMetricPattern)
         },
-        "spill size total (min, med, max (stageId (attemptId): taskId))" -> {
+        "spill size" -> {
           _.toString.matches(sizeMetricPattern)
         })))
     ))
@@ -468,27 +463,30 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     // TODO: test file source V2 as well when its statistics is correctly computed.
     withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") {
       withTempDir { tempDir =>
-        val dir = new File(tempDir, "pqS").getCanonicalPath
-
-        spark.range(10).write.parquet(dir)
-        spark.read.parquet(dir).createOrReplaceTempView("pqS")
-
-        // The executed plan looks like:
-        // Exchange RoundRobinPartitioning(2)
-        // +- BroadcastNestedLoopJoin BuildLeft, Cross
-        //   :- BroadcastExchange IdentityBroadcastMode
-        //   :  +- Exchange RoundRobinPartitioning(3)
-        //   :     +- *Range (0, 30, step=1, splits=2)
-        //   +- *FileScan parquet [id#465L] Batched: true, Format: Parquet, Location: ...(ignored)
-        val res3 = InputOutputMetricsHelper.run(
-          spark.range(30).repartition(3).crossJoin(sql("select * from pqS")).repartition(2).toDF()
-        )
-        // The query above is executed in the following stages:
-        //   1. range(30)                   => (30, 0, 30)
-        //   2. sql("select * from pqS")    => (0, 30, 0)
-        //   3. crossJoin(...) of 1. and 2. => (10, 0, 300)
-        //   4. shuffle & return results    => (0, 300, 0)
-        assert(res3 === (30L, 0L, 30L) :: (0L, 30L, 0L) :: (10L, 0L, 300L) :: (0L, 300L, 0L) :: Nil)
+        withTempView("pqS") {
+          val dir = new File(tempDir, "pqS").getCanonicalPath
+
+          spark.range(10).write.parquet(dir)
+          spark.read.parquet(dir).createOrReplaceTempView("pqS")
+
+          // The executed plan looks like:
+          // Exchange RoundRobinPartitioning(2)
+          // +- BroadcastNestedLoopJoin BuildLeft, Cross
+          //   :- BroadcastExchange IdentityBroadcastMode
+          //   :  +- Exchange RoundRobinPartitioning(3)
+          //   :     +- *Range (0, 30, step=1, splits=2)
+          //   +- *FileScan parquet [id#465L] Batched: true, Format: Parquet, Location: ...(ignored)
+          val res3 = InputOutputMetricsHelper.run(
+            spark.range(30).repartition(3).crossJoin(sql("select * from pqS")).repartition(2).toDF()
+          )
+          // The query above is executed in the following stages:
+          //   1. range(30)                   => (30, 0, 30)
+          //   2. sql("select * from pqS")    => (0, 30, 0)
+          //   3. crossJoin(...) of 1. and 2. => (10, 0, 300)
+          //   4. shuffle & return results    => (0, 300, 0)
+          assert(res3 === (30L, 0L, 30L) :: (0L, 30L, 0L) :: (10L, 0L, 300L) :: (0L, 300L, 0L) ::
+            Nil)
+        }
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
index 0c1148f7b82e4..2977b5339ab51 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
@@ -41,28 +41,28 @@ trait SQLMetricsTestUtils extends SQLTestUtils {
 
   protected def statusStore: SQLAppStatusStore = spark.sharedState.statusStore
 
-  // Pattern of size SQLMetric value, e.g. "\n96.2 MiB (32.1 MiB, 32.1 MiB, 32.1 MiB (stage 0
-  // (attempt 0): task 4))" OR "\n96.2 MiB (32.1 MiB, 32.1 MiB, 32.1 MiB)"
+  // Pattern of size SQLMetric value, e.g. "\n96.2 MiB (32.1 MiB, 32.1 MiB, 32.1 MiB (stage 0.0:
+  // task 4))" OR "\n96.2 MiB (32.1 MiB, 32.1 MiB, 32.1 MiB)"
   protected val sizeMetricPattern = {
     val bytes = "([0-9]+(\\.[0-9]+)?) (EiB|PiB|TiB|GiB|MiB|KiB|B)"
-    val maxMetrics = "\\(stage ([0-9])+ \\(attempt ([0-9])+\\)\\: task ([0-9])+\\)"
-    s"\\n$bytes \\($bytes, $bytes, $bytes( $maxMetrics)?\\)"
+    val maxMetrics = "\\(stage ([0-9])+\\.([0-9])+\\: task ([0-9])+\\)"
+    s"(.*\\n$bytes \\($bytes, $bytes, $bytes( $maxMetrics)?\\))|($bytes)"
   }
 
-  // Pattern of timing SQLMetric value, e.g. "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms (stage 3 (attempt
-  // 0): task 217))" OR "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms)"
+  // Pattern of timing SQLMetric value, e.g. "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms (stage 3.0):
+  // task 217))" OR "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms)" OR "1.0 ms"
   protected val timingMetricPattern = {
     val duration = "([0-9]+(\\.[0-9]+)?) (ms|s|m|h)"
-    val maxMetrics = "\\(stage ([0-9])+ \\(attempt ([0-9])+\\)\\: task ([0-9])+\\)"
-    s"\\n$duration \\($duration, $duration, $duration( $maxMetrics)?\\)"
+    val maxMetrics = "\\(stage ([0-9])+\\.([0-9])+\\: task ([0-9])+\\)"
+    s"(.*\\n$duration \\($duration, $duration, $duration( $maxMetrics)?\\))|($duration)"
   }
 
   // Pattern of size SQLMetric value for Aggregate tests.
-  // e.g "\n(1, 1, 0.9 (stage 1 (attempt 0): task 8)) OR "\n(1, 1, 0.9 )"
+  // e.g "\n(1, 1, 0.9 (stage 1.0: task 8)) OR "\n(1, 1, 0.9 )" OR "1"
   protected val aggregateMetricsPattern = {
     val iters = "([0-9]+(\\.[0-9]+)?)"
-    val maxMetrics = "\\(stage ([0-9])+ \\(attempt ([0-9])+\\)\\: task ([0-9])+\\)"
-    s"\\n\\($iters, $iters, $iters( $maxMetrics)?\\)"
+    val maxMetrics = "\\(stage ([0-9])+\\.([0-9])+\\: task ([0-9])+\\)"
+    s"(.*\\n\\($iters, $iters, $iters( $maxMetrics)?\\))|($iters)"
   }
 
   /**
@@ -98,7 +98,7 @@ trait SQLMetricsTestUtils extends SQLTestUtils {
     }
 
     val totalNumBytesMetric = executedNode.metrics.find(
-      _.name == "written output total (min, med, max (stageId (attemptId): taskId))").get
+      _.name == "written output").get
     val totalNumBytes = metrics(totalNumBytesMetric.accumulatorId).replaceAll(",", "")
       .split(" ").head.trim.toDouble
     assert(totalNumBytes > 0)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala
index c228740df07c8..60b8c9627844b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala
@@ -19,8 +19,13 @@ package org.apache.spark.sql.execution.streaming
 
 import org.scalatest.BeforeAndAfter
 
+import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.catalyst.plans.logical.Range
+import org.apache.spark.sql.connector.read.streaming
+import org.apache.spark.sql.connector.read.streaming.SparkDataStream
 import org.apache.spark.sql.functions.{count, window}
 import org.apache.spark.sql.streaming.StreamTest
+import org.apache.spark.sql.types.{LongType, StructType}
 
 class MicroBatchExecutionSuite extends StreamTest with BeforeAndAfter {
 
@@ -68,4 +73,122 @@ class MicroBatchExecutionSuite extends StreamTest with BeforeAndAfter {
       CheckNewAnswer((25, 1), (30, 1))   // This should not throw the error reported in SPARK-24156
     )
   }
+
+  test("no-data-batch re-executed after restart should call V1 source.getBatch()") {
+    val testSource = ReExecutedBatchTestSource(spark)
+    val df = testSource.toDF()
+      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withWatermark("eventTime", "10 seconds")
+      .groupBy(window($"eventTime", "5 seconds") as 'window)
+      .agg(count("*") as 'count)
+      .select($"window".getField("start").cast("long").as[Long])
+
+    /** Reset this test source so that it appears to be a new source requiring initialization */
+    def resetSource(): StreamAction = Execute("reset source") { _ =>
+      testSource.reset()  // Make it look like a new source that needs to be re-initialized
+      require(testSource.currentOffset === 0)
+      require(testSource.getBatchCallCount === 0)
+    }
+
+    /** Add data to this test source by incrementing its available offset */
+    def addData(numNewRows: Int): StreamAction = new AddData {
+      override def addData(query: Option[StreamExecution]): (SparkDataStream, streaming.Offset) = {
+        testSource.incrementAvailableOffset(numNewRows)
+        (testSource, testSource.getOffset.get)
+      }
+    }
+
+    testStream(df)(
+      addData(numNewRows = 10),   // generate values 1...10, sets watermark to 0
+      CheckAnswer(),
+      addData(numNewRows = 10),   // generate values 11...20, sets watermark to 10
+      ProcessAllAvailable(),      // let no-data-batch be executed
+      CheckAnswer(0, 5),          // start time of windows closed and outputted
+      Execute("verify source internal state before stop") { q =>
+        // Last batch should be a no-data batch
+        require(q.lastProgress.numInputRows === 0)
+        // Source should have expected internal state
+        require(testSource.currentOffset === 20)
+        // getBatch should be called only for 2 batches with data, not for no-data-batches
+        assert(testSource.getBatchCallCount === 2)
+      },
+      StopStream,
+
+      /* Verify that if the last no-data-batch was incomplete, getBatch() is called only once */
+      Execute("mark last batch as incomplete") { q =>
+        // Delete the last committed batch from the commit log to signify that the last batch
+        // (a no-data batch) did not complete and has to be re-executed on restart.
+        val commit = q.commitLog.getLatest().map(_._1).getOrElse(-1L)
+        q.commitLog.purgeAfter(commit - 1)
+      },
+      resetSource(),
+      StartStream(),
+      ProcessAllAvailable(),  // allow initialization and re-execution
+      Execute("verify source.getBatch() called after re-executed no-data-batch") { q =>
+        // After restart, getBatch() should be called once even for no-data batch
+        assert(testSource.getBatchCallCount === 1)
+        assert(testSource.currentOffset === 20)
+      },
+      addData(numNewRows = 10),   // generate values 21...30, sets watermark to 20
+      ProcessAllAvailable(),      // let no-data-batch be executed
+      CheckAnswer(0, 5, 10, 15),
+      StopStream,
+
+      /* Verify that if the last no-data-batch was complete, getBatch() is still called only once */
+      Execute("verify last batch was complete") { q =>
+        // Verify that the commit log records the last batch as completed
+        require(q.commitLog.getLatest().map(_._1).get === q.offsetLog.getLatest().map(_._1).get)
+      },
+      resetSource(),
+      StartStream(),
+      ProcessAllAvailable(),      // allow initialization to completed
+      Execute("verify source.getBatch() called even if no-data-batch was not re-executed") { q =>
+        // After restart, getBatch() should be called even for no-data batch, but only once
+        assert(testSource.getBatchCallCount === 1)
+        assert(testSource.currentOffset === 30)
+      },
+      addData(numNewRows = 10),   // generate values 31...40, sets watermark to 30
+      ProcessAllAvailable(),      // let no-data-batch be executed
+      CheckAnswer(0, 5, 10, 15, 20, 25)
+    )
+  }
+
+
+  case class ReExecutedBatchTestSource(spark: SparkSession) extends Source {
+    @volatile var currentOffset = 0L
+    @volatile var getBatchCallCount = 0
+
+    override def getOffset: Option[Offset] = {
+      if (currentOffset <= 0) None else Some(LongOffset(currentOffset))
+    }
+
+    override def getBatch(start: Option[Offset], end: Offset): DataFrame = {
+      getBatchCallCount = getBatchCallCount + 1
+      if (currentOffset == 0) currentOffset = getOffsetValue(end)
+      val plan = Range(
+        start.map(getOffsetValue).getOrElse(0L) + 1L, getOffsetValue(end) + 1L, 1, None,
+        isStreaming = true)
+      Dataset.ofRows(spark, plan)
+    }
+
+    def incrementAvailableOffset(numNewRows: Int): Unit = {
+      currentOffset = currentOffset + numNewRows
+    }
+
+    def reset(): Unit = {
+      currentOffset = 0L
+      getBatchCallCount = 0
+    }
+    def toDF(): DataFrame = Dataset.ofRows(spark, StreamingExecutionRelation(this, spark))
+    override def schema: StructType = new StructType().add("value", LongType)
+    override def stop(): Unit = {}
+    private def getOffsetValue(offset: Offset): Long = {
+      offset match {
+        case s: SerializedOffset => LongOffset(s).offset
+        case l: LongOffset => l.offset
+        case _ => throw new IllegalArgumentException("incorrect offset type: " + offset)
+      }
+    }
+  }
 }
+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
index 298afa880c930..24b8a973ade38 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
@@ -57,7 +57,7 @@ class AllExecutionsPageSuite extends SharedSparkSession with BeforeAndAfter {
 
     val html = renderSQLPage(request, tab, statusStore).toString().toLowerCase(Locale.ROOT)
     assert(html.contains("failed queries"))
-    assert(!html.contains("1970"))
+    assert(!html.contains("1970/01/01"))
   }
 
   test("sorting should be successful") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
index 55b551d0af078..0746059365004 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
@@ -37,6 +37,7 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.catalyst.util.quietly
 import org.apache.spark.sql.execution.{LeafExecNode, QueryExecution, SparkPlanInfo, SQLExecution}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.sql.functions.count
 import org.apache.spark.sql.internal.SQLConf
@@ -152,11 +153,14 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       expected.foreach { case (id, value) =>
         // The values in actual can be SQL metrics meaning that they contain additional formatting
         // when converted to string. Verify that they start with the expected value.
-        // TODO: this is brittle. There is no requirement that the actual string needs to start
-        // with the accumulator value.
         assert(actual.contains(id))
         val v = actual(id).trim
-        assert(v.startsWith(value.toString), s"Wrong value for accumulator $id")
+        if (v.contains("\n")) {
+          // The actual value can be "total (max, ...)\n6 ms (5 ms, ...)".
+          assert(v.split("\n")(1).startsWith(value.toString), s"Wrong value for accumulator $id")
+        } else {
+          assert(v.startsWith(value.toString), s"Wrong value for accumulator $id")
+        }
       }
     }
 
@@ -506,7 +510,7 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       override lazy val executedPlan = physicalPlan
     }
 
-    SQLExecution.withNewExecutionId(spark, dummyQueryExecution) {
+    SQLExecution.withNewExecutionId(dummyQueryExecution) {
       physicalPlan.execute().collect()
     }
 
@@ -620,13 +624,12 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
     assert(statusStore.execution(2) === None)
   }
 
-  test("SPARK-29894 test Codegen Stage Id in SparkPlanInfo") {
-    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-      // with AQE on, the WholeStageCodegen rule is applied when running QueryStageExec.
-      val df = createTestDataFrame.select(count("*"))
-      val sparkPlanInfo = SparkPlanInfo.fromSparkPlan(df.queryExecution.executedPlan)
-      assert(sparkPlanInfo.nodeName === "WholeStageCodegen (2)")
-    }
+  test("SPARK-29894 test Codegen Stage Id in SparkPlanInfo",
+    DisableAdaptiveExecution("WSCG rule is applied later in AQE")) {
+    // with AQE on, the WholeStageCodegen rule is applied when running QueryStageExec.
+    val df = createTestDataFrame.select(count("*"))
+    val sparkPlanInfo = SparkPlanInfo.fromSparkPlan(df.queryExecution.executedPlan)
+    assert(sparkPlanInfo.nodeName === "WholeStageCodegen (2)")
   }
 }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
index 37d028d6a713f..a369b2d6900f9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
@@ -447,13 +447,6 @@ class ColumnarBatchSuite extends SparkFunSuite {
       Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET, 2.234f)
       Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET + 4, 1.123f)
 
-      if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
-        // Ensure array contains Little Endian floats
-        val bb = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN)
-        Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET, bb.getFloat(0))
-        Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET + 4, bb.getFloat(4))
-      }
-
       column.putFloats(idx, 1, buffer, 4)
       column.putFloats(idx + 1, 1, buffer, 0)
       reference += 1.123f
@@ -491,6 +484,57 @@ class ColumnarBatchSuite extends SparkFunSuite {
       }
   }
 
+  testVector("[SPARK-31703] Float API - Little Endian", 1024, FloatType) {
+    column =>
+      val seed = System.currentTimeMillis()
+      val random = new Random(seed)
+      val reference = mutable.ArrayBuffer.empty[Float]
+
+      var idx = 0
+
+      val littleEndian = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN)
+      littleEndian.putFloat(0, 1.357f)
+      littleEndian.putFloat(4, 2.468f)
+      val arr = new Array[Byte](littleEndian.remaining)
+      littleEndian.get(arr)
+
+      column.putFloatsLittleEndian(idx, 1, arr, 4)
+      column.putFloatsLittleEndian(idx + 1, 1, arr, 0)
+      reference += 2.468f
+      reference += 1.357f
+      idx += 2
+
+      column.putFloatsLittleEndian(idx, 2, arr, 0)
+      reference += 1.357f
+      reference += 2.468f
+      idx += 2
+
+      while (idx < column.capacity) {
+        val single = random.nextBoolean()
+        if (single) {
+          val v = random.nextFloat()
+          column.putFloat(idx, v)
+          reference += v
+          idx += 1
+        } else {
+          val n = math.min(random.nextInt(column.capacity / 20), column.capacity - idx)
+          val v = random.nextFloat()
+          column.putFloats(idx, n, v)
+          var i = 0
+          while (i < n) {
+            reference += v
+            i += 1
+          }
+          idx += n
+        }
+      }
+
+      reference.zipWithIndex.foreach { v =>
+        assert(v._1 == column.getFloat(v._2),
+          "Seed = " + seed + " VectorType=" + column.getClass.getSimpleName)
+      }
+  }
+
   testVector("Double APIs", 1024, DoubleType) {
     column =>
       val seed = System.currentTimeMillis()
@@ -531,13 +575,6 @@ class ColumnarBatchSuite extends SparkFunSuite {
       Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET, 2.234)
       Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET + 8, 1.123)
 
-      if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
-        // Ensure array contains Little Endian doubles
-        val bb = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN)
-        Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET, bb.getDouble(0))
-        Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET + 8, bb.getDouble(8))
-      }
-
       column.putDoubles(idx, 1, buffer, 8)
       column.putDoubles(idx + 1, 1, buffer, 0)
       reference += 1.123
@@ -575,6 +612,57 @@ class ColumnarBatchSuite extends SparkFunSuite {
       }
   }
 
+  testVector("[SPARK-31703] Double API - Little Endian", 1024, DoubleType) {
+    column =>
+      val seed = System.currentTimeMillis()
+      val random = new Random(seed)
+      val reference = mutable.ArrayBuffer.empty[Double]
+
+      var idx = 0
+
+      val littleEndian = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN)
+      littleEndian.putDouble(0, 1.357)
+      littleEndian.putDouble(8, 2.468)
+      val arr = new Array[Byte](littleEndian.remaining)
+      littleEndian.get(arr)
+
+      column.putDoublesLittleEndian(idx, 1, arr, 8)
+      column.putDoublesLittleEndian(idx + 1, 1, arr, 0)
+      reference += 2.468
+      reference += 1.357
+      idx += 2
+
+      column.putDoublesLittleEndian(idx, 2, arr, 0)
+      reference += 1.357
+      reference += 2.468
+      idx += 2
+
+      while (idx < column.capacity) {
+        val single = random.nextBoolean()
+        if (single) {
+          val v = random.nextDouble()
+          column.putDouble(idx, v)
+          reference += v
+          idx += 1
+        } else {
+          val n = math.min(random.nextInt(column.capacity / 20), column.capacity - idx)
+          val v = random.nextDouble()
+          column.putDoubles(idx, n, v)
+          var i = 0
+          while (i < n) {
+            reference += v
+            i += 1
+          }
+          idx += n
+        }
+      }
+
+      reference.zipWithIndex.foreach { v =>
+        assert(v._1 == column.getDouble(v._2),
+          "Seed = " + seed + " VectorType=" + column.getClass.getSimpleName)
+      }
+  }
+
   testVector("String APIs", 7, StringType) {
     column =>
       val reference = mutable.ArrayBuffer.empty[String]
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
new file mode 100644
index 0000000000000..e18514c6f93f9
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.expressions
+
+import scala.collection.parallel.immutable.ParVector
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
+import org.apache.spark.sql.execution.HiveResult.hiveResultString
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession {
+
+  test("Replace _FUNC_ in ExpressionInfo") {
+    val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("upper"))
+    assert(info.getName === "upper")
+    assert(info.getClassName === "org.apache.spark.sql.catalyst.expressions.Upper")
+    assert(info.getUsage === "upper(str) - Returns `str` with all characters changed to uppercase.")
+    assert(info.getExamples.contains("> SELECT upper('SparkSql');"))
+    assert(info.getSince === "1.0.1")
+    assert(info.getNote === "")
+    assert(info.getExtended.contains("> SELECT upper('SparkSql');"))
+  }
+
+  test("group info in ExpressionInfo") {
+    val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("sum"))
+    assert(info.getGroup === "agg_funcs")
+
+    Seq("agg_funcs", "array_funcs", "datetime_funcs", "json_funcs", "map_funcs", "window_funcs")
+        .foreach { groupName =>
+      val info = new ExpressionInfo(
+        "testClass", null, "testName", null, "", "", "", groupName, "", "")
+      assert(info.getGroup === groupName)
+    }
+
+    val errMsg = intercept[IllegalArgumentException] {
+      val invalidGroupName = "invalid_group_funcs"
+      new ExpressionInfo("testClass", null, "testName", null, "", "", "", invalidGroupName, "", "")
+    }.getMessage
+    assert(errMsg.contains("'group' is malformed in the expression [testName]."))
+  }
+
+  test("error handling in ExpressionInfo") {
+    val errMsg1 = intercept[IllegalArgumentException] {
+      val invalidNote = "  invalid note"
+      new ExpressionInfo("testClass", null, "testName", null, "", "", invalidNote, "", "", "")
+    }.getMessage
+    assert(errMsg1.contains("'note' is malformed in the expression [testName]."))
+
+    val errMsg2 = intercept[IllegalArgumentException] {
+      val invalidSince = "-3.0.0"
+      new ExpressionInfo("testClass", null, "testName", null, "", "", "", "", invalidSince, "")
+    }.getMessage
+    assert(errMsg2.contains("'since' is malformed in the expression [testName]."))
+
+    val errMsg3 = intercept[IllegalArgumentException] {
+      val invalidDeprecated = "  invalid deprecated"
+      new ExpressionInfo("testClass", null, "testName", null, "", "", "", "", "", invalidDeprecated)
+    }.getMessage
+    assert(errMsg3.contains("'deprecated' is malformed in the expression [testName]."))
+  }
+
+  test("using _FUNC_ instead of function names in examples") {
+    val exampleRe = "(>.*;)".r
+    val setStmtRe = "(?i)^(>\\s+set\\s+).+".r
+    val ignoreSet = Set(
+      // Examples for CaseWhen show simpler syntax:
+      // `CASE WHEN ... THEN ... WHEN ... THEN ... END`
+      "org.apache.spark.sql.catalyst.expressions.CaseWhen",
+      // _FUNC_ is replaced by `locate` but `locate(... IN ...)` is not supported
+      "org.apache.spark.sql.catalyst.expressions.StringLocate",
+      // _FUNC_ is replaced by `%` which causes a parsing error on `SELECT %(2, 1.8)`
+      "org.apache.spark.sql.catalyst.expressions.Remainder",
+      // Examples demonstrate alternative names, see SPARK-20749
+      "org.apache.spark.sql.catalyst.expressions.Length")
+    spark.sessionState.functionRegistry.listFunction().foreach { funcId =>
+      val info = spark.sessionState.catalog.lookupFunctionInfo(funcId)
+      val className = info.getClassName
+      withClue(s"Expression class '$className'") {
+        val exprExamples = info.getOriginalExamples
+        if (!exprExamples.isEmpty && !ignoreSet.contains(className)) {
+          assert(exampleRe.findAllIn(exprExamples).toIterable
+            .filter(setStmtRe.findFirstIn(_).isEmpty) // Ignore SET commands
+            .forall(_.contains("_FUNC_")))
+        }
+      }
+    }
+  }
+
+  test("check outputs of expression examples") {
+    def unindentAndTrim(s: String): String = {
+      s.replaceAll("\n\\s+", "\n").trim
+    }
+    val beginSqlStmtRe = "  > ".r
+    val endSqlStmtRe = ";\n".r
+    def checkExampleSyntax(example: String): Unit = {
+      val beginStmtNum = beginSqlStmtRe.findAllIn(example).length
+      val endStmtNum = endSqlStmtRe.findAllIn(example).length
+      assert(beginStmtNum === endStmtNum,
+        "The number of ` > ` does not match to the number of `;`")
+    }
+    val exampleRe = """^(.+);\n(?s)(.+)$""".r
+    val ignoreSet = Set(
+      // One of examples shows getting the current timestamp
+      "org.apache.spark.sql.catalyst.expressions.UnixTimestamp",
+      "org.apache.spark.sql.catalyst.expressions.CurrentDate",
+      "org.apache.spark.sql.catalyst.expressions.CurrentTimestamp",
+      "org.apache.spark.sql.catalyst.expressions.Now",
+      // Random output without a seed
+      "org.apache.spark.sql.catalyst.expressions.Rand",
+      "org.apache.spark.sql.catalyst.expressions.Randn",
+      "org.apache.spark.sql.catalyst.expressions.Shuffle",
+      "org.apache.spark.sql.catalyst.expressions.Uuid",
+      // The example calls methods that return unstable results.
+      "org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection")
+
+    val parFuncs = new ParVector(spark.sessionState.functionRegistry.listFunction().toVector)
+    parFuncs.foreach { funcId =>
+      // Examples can change settings. We clone the session to prevent tests clashing.
+      val clonedSpark = spark.cloneSession()
+      // Coalescing partitions can change result order, so disable it.
+      clonedSpark.sessionState.conf.setConf(SQLConf.COALESCE_PARTITIONS_ENABLED, false)
+      val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId)
+      val className = info.getClassName
+      if (!ignoreSet.contains(className)) {
+        withClue(s"Function '${info.getName}', Expression class '$className'") {
+          val example = info.getExamples
+          checkExampleSyntax(example)
+          example.split("  > ").toList.foreach {
+            case exampleRe(sql, output) =>
+              val df = clonedSpark.sql(sql)
+              val actual = unindentAndTrim(
+                hiveResultString(df.queryExecution.executedPlan).mkString("\n"))
+              val expected = unindentAndTrim(output)
+              assert(actual === expected)
+            case _ =>
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
index d6a1fde2147b1..4630a429830fb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
@@ -89,9 +89,9 @@ class CatalogSuite extends SharedSparkSession {
     val columns = dbName
       .map { db => spark.catalog.listColumns(db, tableName) }
       .getOrElse { spark.catalog.listColumns(tableName) }
-    assume(tableMetadata.schema.nonEmpty, "bad test")
-    assume(tableMetadata.partitionColumnNames.nonEmpty, "bad test")
-    assume(tableMetadata.bucketSpec.isDefined, "bad test")
+    assert(tableMetadata.schema.nonEmpty, "bad test")
+    assert(tableMetadata.partitionColumnNames.nonEmpty, "bad test")
+    assert(tableMetadata.bucketSpec.isDefined, "bad test")
     assert(columns.collect().map(_.name).toSet == tableMetadata.schema.map(_.name).toSet)
     val bucketColumnNames = tableMetadata.bucketSpec.map(_.bucketColumnNames).getOrElse(Nil).toSet
     columns.collect().foreach { col =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala
new file mode 100644
index 0000000000000..0b5cd3dae4761
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.internal
+
+import java.io.File
+
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.catalog.CatalogTableType
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.StructType
+
+class DeprecatedCreateExternalTableSuite extends SharedSparkSession {
+  test("createExternalTable with explicit path") {
+    withTable("t") {
+      withTempDir { dir =>
+        val path = new File(dir, "test")
+        spark.range(100).write.parquet(path.getAbsolutePath)
+        spark.catalog.createExternalTable(
+          tableName = "t",
+          path = path.getAbsolutePath
+        )
+        assert(spark.sessionState.catalog.tableExists(TableIdentifier("t")))
+        val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
+        assert(table.tableType === CatalogTableType.EXTERNAL)
+        assert(table.provider === Some("parquet"))
+        assert(table.schema === new StructType().add("id", "long"))
+        assert(table.storage.locationUri.get == makeQualifiedPath(path.getAbsolutePath))
+      }
+    }
+  }
+
+  test("createExternalTable with 'path' options") {
+    withTable("t") {
+      withTempDir { dir =>
+        val path = new File(dir, "test")
+        spark.range(100).write.parquet(path.getAbsolutePath)
+        spark.catalog.createExternalTable(
+          tableName = "t",
+          source = "parquet",
+          options = Map("path" -> path.getAbsolutePath))
+        assert(spark.sessionState.catalog.tableExists(TableIdentifier("t")))
+        val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
+        assert(table.tableType === CatalogTableType.EXTERNAL)
+        assert(table.provider === Some("parquet"))
+        assert(table.schema === new StructType().add("id", "long"))
+        assert(table.storage.locationUri.get == makeQualifiedPath(path.getAbsolutePath))
+      }
+    }
+  }
+
+  test("createExternalTable with explicit schema") {
+    withTable("t") {
+      withTempDir { dir =>
+        val path = new File(dir, "test")
+        spark.range(100).write.parquet(path.getAbsolutePath)
+        spark.catalog.createExternalTable(
+          tableName = "t",
+          source = "parquet",
+          schema = new StructType().add("i", "int"),
+          options = Map("path" -> path.getAbsolutePath))
+        assert(spark.sessionState.catalog.tableExists(TableIdentifier("t")))
+        val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
+        assert(table.tableType === CatalogTableType.EXTERNAL)
+        assert(table.provider === Some("parquet"))
+        assert(table.schema === new StructType().add("i", "int"))
+        assert(table.storage.locationUri.get == makeQualifiedPath(path.getAbsolutePath))
+      }
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
index 0cc658c499615..567524ac75c2e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.internal
 
+import java.util.UUID
+
 import org.scalatest.Assertions._
 
 import org.apache.spark.{SparkException, SparkFunSuite, TaskContext}
@@ -26,6 +28,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.execution.{LeafExecNode, QueryExecution, SparkPlan}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
 import org.apache.spark.sql.execution.debug.codegenStringSeq
 import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.test.SQLTestUtils
@@ -96,10 +99,10 @@ class ExecutorSideSQLConfSuite extends SparkFunSuite with SQLTestUtils {
     }
   }
 
-  test("SPARK-22219: refactor to control to generate comment") {
+  test("SPARK-22219: refactor to control to generate comment",
+    DisableAdaptiveExecution("WSCG rule is applied later in AQE")) {
     Seq(true, false).foreach { flag =>
-      withSQLConf(StaticSQLConf.CODEGEN_COMMENTS.key -> flag.toString,
-        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+      withSQLConf(StaticSQLConf.CODEGEN_COMMENTS.key -> flag.toString) {
         // with AQE on, the WholeStageCodegen rule is applied when running QueryStageExec.
         val res = codegenStringSeq(spark.range(10).groupBy(col("id") * 2).count()
           .queryExecution.executedPlan)
@@ -144,17 +147,45 @@ class ExecutorSideSQLConfSuite extends SparkFunSuite with SQLTestUtils {
         }
 
         // set local configuration and assert
-        val confValue1 = "e"
+        val confValue1 = UUID.randomUUID().toString()
         createDataframe(confKey, confValue1).createOrReplaceTempView("m")
         spark.sparkContext.setLocalProperty(confKey, confValue1)
-        assert(sql("SELECT * FROM l WHERE EXISTS (SELECT * FROM m)").collect.size == 1)
+        assert(sql("SELECT * FROM l WHERE EXISTS (SELECT * FROM m)").collect().length == 1)
 
         // change the conf value and assert again
-        val confValue2 = "f"
+        val confValue2 = UUID.randomUUID().toString()
         createDataframe(confKey, confValue2).createOrReplaceTempView("n")
         spark.sparkContext.setLocalProperty(confKey, confValue2)
-        assert(sql("SELECT * FROM l WHERE EXISTS (SELECT * FROM n)").collect().size == 1)
+        assert(sql("SELECT * FROM l WHERE EXISTS (SELECT * FROM n)").collect().length == 1)
+      }
+    }
+  }
+
+  test("SPARK-22590 propagate local properties to broadcast execution thread") {
+    withSQLConf(StaticSQLConf.BROADCAST_EXCHANGE_MAX_THREAD_THRESHOLD.key -> "1") {
+      val df1 = Seq(true).toDF()
+      val confKey = "spark.sql.y"
+      val confValue1 = UUID.randomUUID().toString()
+      val confValue2 = UUID.randomUUID().toString()
+
+      def generateBroadcastDataFrame(confKey: String, confValue: String): Dataset[Boolean] = {
+        val df = spark.range(1).mapPartitions { _ =>
+          Iterator(TaskContext.get.getLocalProperty(confKey) == confValue)
+        }
+        df.hint("broadcast")
       }
+
+      // set local propert and assert
+      val df2 = generateBroadcastDataFrame(confKey, confValue1)
+      spark.sparkContext.setLocalProperty(confKey, confValue1)
+      val checks = df1.join(df2).collect()
+      assert(checks.forall(_.toSeq == Seq(true, true)))
+
+      // change local property and re-assert
+      val df3 = generateBroadcastDataFrame(confKey, confValue2)
+      spark.sparkContext.setLocalProperty(confKey, confValue2)
+      val checks2 = df1.join(df3).collect()
+      assert(checks2.forall(_.toSeq == Seq(true, true)))
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index 61be3672f3ebe..f1cd37f91d78a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.log4j.Level
 
 import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.MIT
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.sql.test.{SharedSparkSession, TestSQLContext}
 import org.apache.spark.util.Utils
@@ -115,6 +116,21 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-31234: reset will not change static sql configs and spark core configs") {
+    val conf = spark.sparkContext.getConf.getAll.toMap
+    val appName = conf.get("spark.app.name")
+    val driverHost = conf.get("spark.driver.host")
+    val master = conf.get("spark.master")
+    val warehouseDir = conf.get("spark.sql.warehouse.dir")
+    // ensure the conf here is not default value, and will not be reset to default value later
+    assert(warehouseDir.get.contains(this.getClass.getCanonicalName))
+    sql("RESET")
+    assert(conf.get("spark.app.name") === appName)
+    assert(conf.get("spark.driver.host") === driverHost)
+    assert(conf.get("spark.master") === master)
+    assert(conf.get("spark.sql.warehouse.dir") === warehouseDir)
+  }
+
   test("reset - public conf") {
     spark.sessionState.conf.clear()
     val original = spark.conf.get(SQLConf.GROUP_BY_ORDINAL)
@@ -170,33 +186,33 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     assert(e.getMessage === s"${SQLConf.CASE_SENSITIVE.key} should be boolean, but was 10")
   }
 
-  test("Test SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE's method") {
+  test("Test ADVISORY_PARTITION_SIZE_IN_BYTES's method") {
     spark.sessionState.conf.clear()
 
-    spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "100")
-    assert(spark.conf.get(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) === 100)
+    spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "100")
+    assert(spark.conf.get(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES) === 100)
 
-    spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "1k")
-    assert(spark.conf.get(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) === 1024)
+    spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "1k")
+    assert(spark.conf.get(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES) === 1024)
 
-    spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "1M")
-    assert(spark.conf.get(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) === 1048576)
+    spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "1M")
+    assert(spark.conf.get(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES) === 1048576)
 
-    spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "1g")
-    assert(spark.conf.get(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) === 1073741824)
+    spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "1g")
+    assert(spark.conf.get(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES) === 1073741824)
 
-    spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "-1")
-    assert(spark.conf.get(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) === -1)
+    spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "-1")
+    assert(spark.conf.get(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES) === -1)
 
     // Test overflow exception
     intercept[IllegalArgumentException] {
       // This value exceeds Long.MaxValue
-      spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "90000000000g")
+      spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "90000000000g")
     }
 
     intercept[IllegalArgumentException] {
       // This value less than Long.MinValue
-      spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "-90000000000g")
+      spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "-90000000000g")
     }
 
     spark.sessionState.conf.clear()
@@ -259,7 +275,7 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
 
     // check default value
     assert(spark.sessionState.conf.parquetOutputTimestampType ==
-      SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS)
+      SQLConf.ParquetOutputTimestampType.INT96)
 
     spark.sessionState.conf.setConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE, "timestamp_micros")
     assert(spark.sessionState.conf.parquetOutputTimestampType ==
@@ -285,8 +301,8 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     assert(spark.sessionState.conf.getConfString(fallback.key, "lzo") === "lzo")
 
     val displayValue = spark.sessionState.conf.getAllDefinedConfs
-      .find { case (key, _, _) => key == fallback.key }
-      .map { case (_, v, _) => v }
+      .find { case (key, _, _, _) => key == fallback.key }
+      .map { case (_, v, _, _) => v }
       .get
     assert(displayValue === fallback.defaultValueString)
 
@@ -297,8 +313,8 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     assert(spark.sessionState.conf.getConfString(fallback.key) === "lzo")
 
     val newDisplayValue = spark.sessionState.conf.getAllDefinedConfs
-      .find { case (key, _, _) => key == fallback.key }
-      .map { case (_, v, _) => v }
+      .find { case (key, _, _, _) => key == fallback.key }
+      .map { case (_, v, _, _) => v }
       .get
     assert(newDisplayValue === "lzo")
 
@@ -348,4 +364,23 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     }
     check(config2)
   }
+
+  test("spark.sql.session.timeZone should only accept valid zone id") {
+    spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, MIT.getId)
+    assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) === MIT.getId)
+    spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "America/Chicago")
+    assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) ===
+      "America/Chicago")
+
+    intercept[IllegalArgumentException] {
+      spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "pst")
+    }
+    intercept[IllegalArgumentException] {
+      spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "GMT+8:00")
+    }
+    val e = intercept[IllegalArgumentException] {
+      spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "Asia/shanghai")
+    }
+    assert(e.getMessage === "Cannot resolve the given timezone with ZoneId.of(_, ZoneId.SHORT_IDS)")
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala
new file mode 100644
index 0000000000000..4d33fc1855ab2
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.internal
+
+import java.net.URL
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FsUrlStreamHandlerFactory
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+
+/**
+ * Tests for [[org.apache.spark.sql.internal.SharedState]].
+ */
+class SharedStateSuite extends SharedSparkSession {
+
+  override protected def sparkConf: SparkConf = {
+    super.sparkConf
+      .set("spark.hadoop.fs.defaultFS", "file:///")
+  }
+
+  test("SPARK-31692: Url handler factory should have the hadoop configs from Spark conf") {
+    // Accessing shared state to init the object since it is `lazy val`
+    spark.sharedState
+    val field = classOf[URL].getDeclaredField("factory")
+    field.setAccessible(true)
+    val value = field.get(null)
+    assert(value.isInstanceOf[FsUrlStreamHandlerFactory])
+    val streamFactory = value.asInstanceOf[FsUrlStreamHandlerFactory]
+
+    val confField = classOf[FsUrlStreamHandlerFactory].getDeclaredField("conf")
+    confField.setAccessible(true)
+    val conf = confField.get(streamFactory)
+
+    assert(conf.isInstanceOf[Configuration])
+    assert(conf.asInstanceOf[Configuration].get("fs.defaultFS") == "file:///")
+  }
+
+  test("SPARK-33740: hadoop configs in hive-site.xml can overrides pre-existing hadoop ones") {
+    val conf = new SparkConf()
+    val hadoopConf = new Configuration()
+    SharedState.loadHiveConfFile(conf, hadoopConf)
+    assert(hadoopConf.get("hadoop.tmp.dir") === "/tmp/hive_one")
+    hadoopConf.clear()
+    SharedState.loadHiveConfFile(
+      conf.set("spark.hadoop.hadoop.tmp.dir", "noop"), hadoopConf)
+    assert(hadoopConf.get("hadoop.tmp.dir") === null)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 9cba95f7d7df2..1f584a569c3f3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -21,6 +21,8 @@ import java.math.BigDecimal
 import java.sql.{Date, DriverManager, SQLException, Timestamp}
 import java.util.{Calendar, GregorianCalendar, Properties}
 
+import scala.collection.JavaConverters._
+
 import org.h2.jdbc.JdbcSQLException
 import org.scalatest.{BeforeAndAfter, PrivateMethodTester}
 
@@ -635,12 +637,14 @@ class JDBCSuite extends QueryTest
   }
 
   test("test DATE types in cache") {
-    val rows = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties()).collect()
-    spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties())
-      .cache().createOrReplaceTempView("mycached_date")
-    val cachedRows = sql("select * from mycached_date").collect()
-    assert(rows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
-    assert(cachedRows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
+    withTempView("mycached_date") {
+      val rows = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties()).collect()
+      spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties())
+        .cache().createOrReplaceTempView("mycached_date")
+      val cachedRows = sql("select * from mycached_date").collect()
+      assert(rows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
+      assert(cachedRows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
+    }
   }
 
   test("test types for null value") {
@@ -692,7 +696,7 @@ class JDBCSuite extends QueryTest
   test("Remap types via JdbcDialects") {
     JdbcDialects.registerDialect(testH2Dialect)
     val df = spark.read.jdbc(urlWithUserAndPass, "TEST.PEOPLE", new Properties())
-    assert(df.schema.filter(_.dataType != org.apache.spark.sql.types.StringType).isEmpty)
+    assert(!df.schema.exists(_.dataType != org.apache.spark.sql.types.StringType))
     val rows = df.collect()
     assert(rows(0).get(0).isInstanceOf[String])
     assert(rows(0).get(1).isInstanceOf[String])
@@ -847,6 +851,8 @@ class JDBCSuite extends QueryTest
     assert(Postgres.getCatalystType(java.sql.Types.OTHER, "jsonb", 1, null) === Some(StringType))
     assert(Postgres.getCatalystType(java.sql.Types.ARRAY, "_numeric", 0, md) ==
       Some(ArrayType(DecimalType.SYSTEM_DEFAULT)))
+    assert(Postgres.getCatalystType(java.sql.Types.ARRAY, "_bpchar", 64, md) ==
+      Some(ArrayType(StringType)))
     assert(Postgres.getJDBCType(FloatType).map(_.databaseTypeDefinition).get == "FLOAT4")
     assert(Postgres.getJDBCType(DoubleType).map(_.databaseTypeDefinition).get == "FLOAT8")
     assert(Postgres.getJDBCType(ByteType).map(_.databaseTypeDefinition).get == "SMALLINT")
@@ -1300,7 +1306,8 @@ class JDBCSuite extends QueryTest
     testJdbcOptions(new JDBCOptions(parameters))
     testJdbcOptions(new JDBCOptions(CaseInsensitiveMap(parameters)))
     // test add/remove key-value from the case-insensitive map
-    var modifiedParameters = CaseInsensitiveMap(Map.empty) ++ parameters
+    var modifiedParameters =
+      (CaseInsensitiveMap(Map.empty) ++ parameters).asInstanceOf[Map[String, String]]
     testJdbcOptions(new JDBCOptions(modifiedParameters))
     modifiedParameters -= "dbtable"
     assert(modifiedParameters.get("dbTAblE").isEmpty)
@@ -1699,4 +1706,37 @@ class JDBCSuite extends QueryTest
     assert(JdbcDialects.get("jdbc:teradata://localhost/db") === TeradataDialect)
     assert(JdbcDialects.get("jdbc:Teradata://localhost/db") === TeradataDialect)
   }
+
+  test("SQLContext.jdbc (deprecated)") {
+    val sqlContext = spark.sqlContext
+    var jdbcDF = sqlContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE")
+    checkAnswer(jdbcDF, Row("fred", 1) :: Row("mary", 2) :: Row ("joe 'foo' \"bar\"", 3) :: Nil)
+
+    jdbcDF = sqlContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE", "THEID", 0, 4, 3)
+    checkNumPartitions(jdbcDF, 3)
+    checkAnswer(jdbcDF, Row("fred", 1) :: Row("mary", 2) :: Row ("joe 'foo' \"bar\"", 3) :: Nil)
+
+    val parts = Array[String]("THEID = 2")
+    jdbcDF = sqlContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE", parts)
+    checkAnswer(jdbcDF, Row("mary", 2) :: Nil)
+  }
+
+  test("SPARK-32364: JDBCOption constructor") {
+    val extraOptions = CaseInsensitiveMap[String](Map("UrL" -> "url1", "dBTable" -> "table1"))
+    val connectionProperties = new Properties()
+    connectionProperties.put("url", "url2")
+    connectionProperties.put("dbtable", "table2")
+
+    // connection property should override the options in extraOptions
+    val params = extraOptions ++ connectionProperties.asScala
+    assert(params.size == 2)
+    assert(params.get("uRl").contains("url2"))
+    assert(params.get("DbtaBle").contains("table2"))
+
+    // JDBCOptions constructor parameter should overwrite the existing conf
+    val options = new JDBCOptions(url, "table3", params)
+    assert(options.asProperties.size == 2)
+    assert(options.asProperties.get("url") == url)
+    assert(options.asProperties.get("dbtable") == "table3")
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
index c7266c886128c..df8ca33b893cc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.catalog.BucketSpec
 import org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
-import org.apache.spark.sql.execution.{DataSourceScanExec, SortExec}
+import org.apache.spark.sql.execution.{DataSourceScanExec, FileSourceScanExec, SortExec, SparkPlan}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
 import org.apache.spark.sql.execution.datasources.BucketingUtils
 import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
@@ -42,7 +42,7 @@ import org.apache.spark.util.collection.BitSet
 class BucketedReadWithoutHiveSupportSuite extends BucketedReadSuite with SharedSparkSession {
   protected override def beforeAll(): Unit = {
     super.beforeAll()
-    assume(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
   }
 }
 
@@ -100,6 +100,12 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
     }
   }
 
+  private def getFileScan(plan: SparkPlan): FileSourceScanExec = {
+    val fileScan = plan.collect { case f: FileSourceScanExec => f }
+    assert(fileScan.nonEmpty, plan)
+    fileScan.head
+  }
+
   // To verify if the bucket pruning works, this function checks two conditions:
   //   1) Check if the pruned buckets (before filtering) are empty.
   //   2) Verify the final result is the same as the expected one
@@ -119,8 +125,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
 
       // Filter could hide the bug in bucket pruning. Thus, skipping all the filters
       val plan = bucketedDataFrame.filter(filterCondition).queryExecution.executedPlan
-      val rdd = plan.find(_.isInstanceOf[DataSourceScanExec])
-      assert(rdd.isDefined, plan)
+      val fileScan = getFileScan(plan)
 
       // if nothing should be pruned, skip the pruning test
       if (bucketValues.nonEmpty) {
@@ -128,7 +133,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
         bucketValues.foreach { value =>
           matchedBuckets.set(BucketingUtils.getBucketIdFromValue(bucketColumn, numBuckets, value))
         }
-        val invalidBuckets = rdd.get.execute().mapPartitionsWithIndex { case (index, iter) =>
+        val invalidBuckets = fileScan.execute().mapPartitionsWithIndex { case (index, iter) =>
           // return indexes of partitions that should have been pruned and are not empty
           if (!matchedBuckets.get(index % numBuckets) && iter.nonEmpty) {
             Iterator(index)
@@ -183,7 +188,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
 
       // Case 4: InSet
       val inSetExpr = expressions.InSet($"j".expr,
-        Set(bucketValue, bucketValue + 1, bucketValue + 2, bucketValue + 3).map(lit(_).expr))
+        Set(bucketValue, bucketValue + 1, bucketValue + 2, bucketValue + 3))
       checkPrunedAnswers(
         bucketSpec,
         bucketValues = Seq(bucketValue, bucketValue + 1, bucketValue + 2, bucketValue + 3),
@@ -297,10 +302,9 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
 
       val bucketedDataFrame = spark.table("bucketed_table").select("i", "j", "k")
       val plan = bucketedDataFrame.queryExecution.executedPlan
-      val rdd = plan.find(_.isInstanceOf[DataSourceScanExec])
-      assert(rdd.isDefined, plan)
+      val fileScan = getFileScan(plan)
 
-      val emptyBuckets = rdd.get.execute().mapPartitionsWithIndex { case (index, iter) =>
+      val emptyBuckets = fileScan.execute().mapPartitionsWithIndex { case (index, iter) =>
         // return indexes of empty partitions
         if (iter.isEmpty) {
           Iterator(index)
@@ -762,10 +766,13 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
     withTable("bucketed_table") {
       df1.write.format("parquet").bucketBy(8, "i").saveAsTable("bucketed_table")
 
-      checkAnswer(spark.table("bucketed_table").select("j"), df1.select("j"))
+      val scanDF = spark.table("bucketed_table").select("j")
+      assert(!getFileScan(scanDF.queryExecution.executedPlan).bucketedScan)
+      checkAnswer(scanDF, df1.select("j"))
 
-      checkAnswer(spark.table("bucketed_table").groupBy("j").agg(max("k")),
-        df1.groupBy("j").agg(max("k")))
+      val aggDF = spark.table("bucketed_table").groupBy("j").agg(max("k"))
+      assert(!getFileScan(aggDF.queryExecution.executedPlan).bucketedScan)
+      checkAnswer(aggDF, df1.groupBy("j").agg(max("k")))
     }
   }
 
@@ -821,7 +828,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
   test("SPARK-29655 Read bucketed tables obeys spark.sql.shuffle.partitions") {
     withSQLConf(
       SQLConf.SHUFFLE_PARTITIONS.key -> "5",
-      SQLConf.SHUFFLE_MAX_NUM_POSTSHUFFLE_PARTITIONS.key -> "7")  {
+      SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "7")  {
       val bucketSpec = Some(BucketSpec(6, Seq("i", "j"), Nil))
       Seq(false, true).foreach { enableAdaptive =>
         withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> s"$enableAdaptive") {
@@ -836,4 +843,32 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
       }
     }
   }
+
+  test("SPARK-32767 Bucket join should work if SHUFFLE_PARTITIONS larger than bucket number") {
+    withSQLConf(
+      SQLConf.SHUFFLE_PARTITIONS.key -> "9",
+      SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "10")  {
+
+      val testSpec1 = BucketedTableTestSpec(
+        Some(BucketSpec(8, Seq("i", "j"), Seq("i", "j"))),
+        numPartitions = 1,
+        expectedShuffle = false,
+        expectedSort = false)
+      val testSpec2 = BucketedTableTestSpec(
+        Some(BucketSpec(6, Seq("i", "j"), Seq("i", "j"))),
+        numPartitions = 1,
+        expectedShuffle = true,
+        expectedSort = true)
+      Seq(false, true).foreach { enableAdaptive =>
+        withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> s"$enableAdaptive") {
+          Seq((testSpec1, testSpec2), (testSpec2, testSpec1)).foreach { specs =>
+            testBucketing(
+              bucketedTableTestSpecLeft = specs._1,
+              bucketedTableTestSpecRight = specs._2,
+              joinCondition = joinCondition(Seq("i", "j")))
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala
index 9713de988e379..a410f32d4af7e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
 class BucketedWriteWithoutHiveSupportSuite extends BucketedWriteSuite with SharedSparkSession {
   protected override def beforeAll(): Unit = {
     super.beforeAll()
-    assume(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
   }
 
   override protected def fileFormatsToTest: Seq[String] = Seq("parquet", "json")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
index e1022e377132c..a6c50904d395b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
@@ -22,9 +22,10 @@ import org.scalatest.BeforeAndAfterAll
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Cast, Expression, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Alias, AnsiCast, Attribute, Cast, Expression, Literal}
 import org.apache.spark.sql.execution.datasources.DataSourceAnalysis
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
 import org.apache.spark.sql.types.{DataType, IntegerType, StructType}
 
 class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
@@ -52,7 +53,12 @@ class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
   Seq(true, false).foreach { caseSensitive =>
     val conf = new SQLConf().copy(SQLConf.CASE_SENSITIVE -> caseSensitive)
     def cast(e: Expression, dt: DataType): Expression = {
-      Cast(e, dt, Option(conf.sessionLocalTimeZone))
+      conf.storeAssignmentPolicy match {
+        case StoreAssignmentPolicy.ANSI | StoreAssignmentPolicy.STRICT =>
+          AnsiCast(e, dt, Option(conf.sessionLocalTimeZone))
+        case _ =>
+          Cast(e, dt, Option(conf.sessionLocalTimeZone))
+      }
     }
     val rule = DataSourceAnalysis(conf)
     test(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala
index 1cb7a2156c3d3..33b2db57d9f0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala
@@ -24,66 +24,143 @@ import org.apache.spark.SparkFunSuite
  */
 class FiltersSuite extends SparkFunSuite {
 
-  test("EqualTo references") {
-    assert(EqualTo("a", "1").references.toSeq == Seq("a"))
-    assert(EqualTo("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
+  private def withFieldNames(f: (String, Array[String]) => Unit): Unit = {
+    Seq(("a", Array("a")),
+      ("a.b", Array("a", "b")),
+      ("`a.b`.c", Array("a.b", "c")),
+      ("`a.b`.`c.d`.`e.f`", Array("a.b", "c.d", "e.f"))
+    ).foreach { case (name, fieldNames) =>
+      f(name, fieldNames)
+    }
   }
 
-  test("EqualNullSafe references") {
-    assert(EqualNullSafe("a", "1").references.toSeq == Seq("a"))
-    assert(EqualNullSafe("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
-  }
+  test("EqualTo references") { withFieldNames { (name, fieldNames) =>
+    assert(EqualTo(name, "1").references.toSeq == Seq(name))
+    assert(EqualTo(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
 
-  test("GreaterThan references") {
-    assert(GreaterThan("a", "1").references.toSeq == Seq("a"))
-    assert(GreaterThan("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
-  }
+    assert(EqualTo(name, EqualTo("b", "2")).references.toSeq == Seq(name, "b"))
+    assert(EqualTo("b", EqualTo(name, "2")).references.toSeq == Seq("b", name))
 
-  test("GreaterThanOrEqual references") {
-    assert(GreaterThanOrEqual("a", "1").references.toSeq == Seq("a"))
-    assert(GreaterThanOrEqual("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
-  }
+    assert(EqualTo(name, EqualTo("b", "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(EqualTo("b", EqualTo(name, "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
 
-  test("LessThan references") {
-    assert(LessThan("a", "1").references.toSeq == Seq("a"))
-    assert(LessThan("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
-  }
+  test("EqualNullSafe references") { withFieldNames { (name, fieldNames) =>
+    assert(EqualNullSafe(name, "1").references.toSeq == Seq(name))
+    assert(EqualNullSafe(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
 
-  test("LessThanOrEqual references") {
-    assert(LessThanOrEqual("a", "1").references.toSeq == Seq("a"))
-    assert(LessThanOrEqual("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
-  }
+    assert(EqualNullSafe(name, EqualTo("b", "2")).references.toSeq == Seq(name, "b"))
+    assert(EqualNullSafe("b", EqualTo(name, "2")).references.toSeq == Seq("b", name))
 
-  test("In references") {
-    assert(In("a", Array("1")).references.toSeq == Seq("a"))
-    assert(In("a", Array("1", EqualTo("b", "2"))).references.toSeq == Seq("a", "b"))
-  }
+    assert(EqualNullSafe(name, EqualTo("b", "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(EqualNullSafe("b", EqualTo(name, "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
 
-  test("IsNull references") {
-    assert(IsNull("a").references.toSeq == Seq("a"))
-  }
+  test("GreaterThan references") { withFieldNames { (name, fieldNames) =>
+    assert(GreaterThan(name, "1").references.toSeq == Seq(name))
+    assert(GreaterThan(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
 
-  test("IsNotNull references") {
-    assert(IsNotNull("a").references.toSeq == Seq("a"))
-  }
+    assert(GreaterThan(name, EqualTo("b", "2")).references.toSeq == Seq(name, "b"))
+    assert(GreaterThan("b", EqualTo(name, "2")).references.toSeq == Seq("b", name))
 
-  test("And references") {
-    assert(And(EqualTo("a", "1"), EqualTo("b", "1")).references.toSeq == Seq("a", "b"))
-  }
+    assert(GreaterThan(name, EqualTo("b", "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(GreaterThan("b", EqualTo(name, "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
 
-  test("Or references") {
-    assert(Or(EqualTo("a", "1"), EqualTo("b", "1")).references.toSeq == Seq("a", "b"))
-  }
+  test("GreaterThanOrEqual references") { withFieldNames { (name, fieldNames) =>
+    assert(GreaterThanOrEqual(name, "1").references.toSeq == Seq(name))
+    assert(GreaterThanOrEqual(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
 
-  test("StringStartsWith references") {
-    assert(StringStartsWith("a", "str").references.toSeq == Seq("a"))
-  }
+    assert(GreaterThanOrEqual(name, EqualTo("b", "2")).references.toSeq == Seq(name, "b"))
+    assert(GreaterThanOrEqual("b", EqualTo(name, "2")).references.toSeq == Seq("b", name))
 
-  test("StringEndsWith references") {
-    assert(StringEndsWith("a", "str").references.toSeq == Seq("a"))
-  }
+    assert(GreaterThanOrEqual(name, EqualTo("b", "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(GreaterThanOrEqual("b", EqualTo(name, "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
 
-  test("StringContains references") {
-    assert(StringContains("a", "str").references.toSeq == Seq("a"))
-  }
+  test("LessThan references") { withFieldNames { (name, fieldNames) =>
+    assert(LessThan(name, "1").references.toSeq == Seq(name))
+    assert(LessThan(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+
+    assert(LessThan("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
+  }}
+
+  test("LessThanOrEqual references") { withFieldNames { (name, fieldNames) =>
+    assert(LessThanOrEqual(name, "1").references.toSeq == Seq(name))
+    assert(LessThanOrEqual(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+
+    assert(LessThanOrEqual(name, EqualTo("b", "2")).references.toSeq == Seq(name, "b"))
+    assert(LessThanOrEqual("b", EqualTo(name, "2")).references.toSeq == Seq("b", name))
+
+    assert(LessThanOrEqual(name, EqualTo("b", "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(LessThanOrEqual("b", EqualTo(name, "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
+
+  test("In references") { withFieldNames { (name, fieldNames) =>
+    assert(In(name, Array("1")).references.toSeq == Seq(name))
+    assert(In(name, Array("1")).v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+
+    assert(In(name, Array("1", EqualTo("b", "2"))).references.toSeq == Seq(name, "b"))
+    assert(In("b", Array("1", EqualTo(name, "2"))).references.toSeq == Seq("b", name))
+
+    assert(In(name, Array("1", EqualTo("b", "2"))).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(In("b", Array("1", EqualTo(name, "2"))).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
+
+  test("IsNull references") { withFieldNames { (name, fieldNames) =>
+    assert(IsNull(name).references.toSeq == Seq(name))
+    assert(IsNull(name).v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+  }}
+
+  test("IsNotNull references") { withFieldNames { (name, fieldNames) =>
+    assert(IsNotNull(name).references.toSeq == Seq(name))
+    assert(IsNull(name).v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+  }}
+
+  test("And references") { withFieldNames { (name, fieldNames) =>
+    assert(And(EqualTo(name, "1"), EqualTo("b", "1")).references.toSeq == Seq(name, "b"))
+    assert(And(EqualTo("b", "1"), EqualTo(name, "1")).references.toSeq == Seq("b", name))
+
+    assert(And(EqualTo(name, "1"), EqualTo("b", "1")).v2references.toSeq.map(_.toSeq) ==
+      Seq(fieldNames.toSeq, Seq("b")))
+    assert(And(EqualTo("b", "1"), EqualTo(name, "1")).v2references.toSeq.map(_.toSeq) ==
+      Seq(Seq("b"), fieldNames.toSeq))
+  }}
+
+  test("Or references") { withFieldNames { (name, fieldNames) =>
+    assert(Or(EqualTo(name, "1"), EqualTo("b", "1")).references.toSeq == Seq(name, "b"))
+    assert(Or(EqualTo("b", "1"), EqualTo(name, "1")).references.toSeq == Seq("b", name))
+
+    assert(Or(EqualTo(name, "1"), EqualTo("b", "1")).v2references.toSeq.map(_.toSeq) ==
+      Seq(fieldNames.toSeq, Seq("b")))
+    assert(Or(EqualTo("b", "1"), EqualTo(name, "1")).v2references.toSeq.map(_.toSeq) ==
+      Seq(Seq("b"), fieldNames.toSeq))
+  }}
+
+  test("StringStartsWith references") { withFieldNames { (name, fieldNames) =>
+    assert(StringStartsWith(name, "str").references.toSeq == Seq(name))
+    assert(StringStartsWith(name, "str").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+  }}
+
+  test("StringEndsWith references") { withFieldNames { (name, fieldNames) =>
+    assert(StringEndsWith(name, "str").references.toSeq == Seq(name))
+    assert(StringEndsWith(name, "str").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+  }}
+
+  test("StringContains references") { withFieldNames { (name, fieldNames) =>
+    assert(StringContains(name, "str").references.toSeq == Seq(name))
+    assert(StringContains(name, "str").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+  }}
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index bcff30a51c3f5..995d4b0e7230e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -523,15 +523,17 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
 
   test("new partitions should be added to catalog after writing to catalog table") {
     val table = "partitioned_catalog_table"
+    val tempTable = "partitioned_catalog_temp_table"
     val numParts = 210
     withTable(table) {
-      val df = (1 to numParts).map(i => (i, i)).toDF("part", "col1")
-      val tempTable = "partitioned_catalog_temp_table"
-      df.createOrReplaceTempView(tempTable)
-      sql(s"CREATE TABLE $table (part Int, col1 Int) USING parquet PARTITIONED BY (part)")
-      sql(s"INSERT INTO TABLE $table SELECT * from $tempTable")
-      val partitions = spark.sessionState.catalog.listPartitionNames(TableIdentifier(table))
-      assert(partitions.size == numParts)
+      withTempView(tempTable) {
+        val df = (1 to numParts).map(i => (i, i)).toDF("part", "col1")
+        df.createOrReplaceTempView(tempTable)
+        sql(s"CREATE TABLE $table (part Int, col1 Int) USING parquet PARTITIONED BY (part)")
+        sql(s"INSERT INTO TABLE $table SELECT * from $tempTable")
+        val partitions = spark.sessionState.catalog.listPartitionNames(TableIdentifier(table))
+        assert(partitions.size == numParts)
+      }
     }
   }
 
@@ -620,12 +622,12 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
         var msg = intercept[AnalysisException] {
           sql("insert into t select 1L, 2")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': LongType to IntegerType"))
+        assert(msg.contains("Cannot safely cast 'i': bigint to int"))
 
         msg = intercept[AnalysisException] {
           sql("insert into t select 1, 2.0")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'd': DecimalType(2,1) to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'd': decimal(2,1) to double"))
 
         msg = intercept[AnalysisException] {
           sql("insert into t select 1, 2.0D, 3")
@@ -657,18 +659,18 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
         var msg = intercept[AnalysisException] {
           sql("insert into t values('a', 'b')")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': StringType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': StringType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': string to int") &&
+          msg.contains("Cannot safely cast 'd': string to double"))
         msg = intercept[AnalysisException] {
           sql("insert into t values(now(), now())")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': TimestampType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': TimestampType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': timestamp to int") &&
+          msg.contains("Cannot safely cast 'd': timestamp to double"))
         msg = intercept[AnalysisException] {
           sql("insert into t values(true, false)")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': BooleanType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': BooleanType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': boolean to int") &&
+          msg.contains("Cannot safely cast 'd': boolean to double"))
       }
     }
   }
@@ -753,6 +755,27 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-30844: static partition should also follow StoreAssignmentPolicy") {
+    SQLConf.StoreAssignmentPolicy.values.foreach { policy =>
+      withSQLConf(
+        SQLConf.STORE_ASSIGNMENT_POLICY.key -> policy.toString) {
+        withTable("t") {
+          sql("create table t(a int, b string) using parquet partitioned by (a)")
+          policy match {
+            case SQLConf.StoreAssignmentPolicy.ANSI | SQLConf.StoreAssignmentPolicy.STRICT =>
+              val errorMsg = intercept[NumberFormatException] {
+                sql("insert into t partition(a='ansi') values('ansi')")
+              }.getMessage
+              assert(errorMsg.contains("invalid input syntax for type numeric: ansi"))
+            case SQLConf.StoreAssignmentPolicy.LEGACY =>
+              sql("insert into t partition(a='ansi') values('ansi')")
+              checkAnswer(sql("select * from t"), Row("ansi", null) :: Nil)
+          }
+        }
+      }
+    }
+  }
+
   test("SPARK-24860: dynamic partition overwrite specified per source without catalog table") {
     withTempPath { path =>
       Seq((1, 1), (2, 2)).toDF("i", "part")
@@ -802,21 +825,29 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
   }
 
   test("Stop task set if FileAlreadyExistsException was thrown") {
-    withSQLConf("fs.file.impl" -> classOf[FileExistingTestFileSystem].getName,
-        "fs.file.impl.disable.cache" -> "true") {
-      withTable("t") {
-        sql(
-          """
-            |CREATE TABLE t(i INT, part1 INT) USING PARQUET
-            |PARTITIONED BY (part1)
+    Seq(true, false).foreach { fastFail =>
+      withSQLConf("fs.file.impl" -> classOf[FileExistingTestFileSystem].getName,
+        "fs.file.impl.disable.cache" -> "true",
+        SQLConf.FASTFAIL_ON_FILEFORMAT_OUTPUT.key -> fastFail.toString) {
+        withTable("t") {
+          sql(
+            """
+              |CREATE TABLE t(i INT, part1 INT) USING PARQUET
+              |PARTITIONED BY (part1)
           """.stripMargin)
 
-        val df = Seq((1, 1)).toDF("i", "part1")
-        val err = intercept[SparkException] {
-          df.write.mode("overwrite").format("parquet").insertInto("t")
+          val df = Seq((1, 1)).toDF("i", "part1")
+          val err = intercept[SparkException] {
+            df.write.mode("overwrite").format("parquet").insertInto("t")
+          }
+
+          if (fastFail) {
+            assert(err.getCause.getMessage.contains("can not write to output file: " +
+              "org.apache.hadoop.fs.FileAlreadyExistsException"))
+          } else {
+            assert(err.getCause.getMessage.contains("Task failed while writing rows"))
+          }
         }
-        assert(err.getCause.getMessage.contains("can not write to output file: " +
-          "org.apache.hadoop.fs.FileAlreadyExistsException"))
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
index ab1d1f80e7397..6df1c5db14c26 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
@@ -24,7 +24,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext
 
 import org.apache.spark.TestUtils
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol
@@ -139,15 +139,15 @@ class PartitionedWriteSuite extends QueryTest with SharedSparkSession {
       checkPartitionValues(files.head, "2016-12-01 00:00:00")
     }
     withTempPath { f =>
-      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .partitionBy("ts").parquet(f.getAbsolutePath)
       val files = TestUtils.recursiveList(f).filter(_.getAbsolutePath.endsWith("parquet"))
       assert(files.length == 1)
-      // use timeZone option "GMT" to format partition value.
+      // use timeZone option utcTz.getId to format partition value.
       checkPartitionValues(files.head, "2016-12-01 08:00:00")
     }
     withTempPath { f =>
-      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
         df.write.partitionBy("ts").parquet(f.getAbsolutePath)
         val files = TestUtils.recursiveList(f).filter(_.getAbsolutePath.endsWith("parquet"))
         assert(files.length == 1)
@@ -156,4 +156,12 @@ class PartitionedWriteSuite extends QueryTest with SharedSparkSession {
       }
     }
   }
+
+  test("SPARK-31968: duplicate partition columns check") {
+    withTempPath { f =>
+      val e = intercept[AnalysisException](
+        Seq((3, 2)).toDF("a", "b").write.partitionBy("b", "b").csv(f.getAbsolutePath))
+      assert(e.getMessage.contains("Found duplicate column(s) b, b: `b`;"))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
index ce3ec7f97a537..1e0d9bd9990e2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
@@ -103,30 +103,32 @@ class SaveLoadSuite extends DataSourceTest with SharedSparkSession with BeforeAn
   }
 
   test("save and save again") {
-    df.write.json(path.toString)
-
-    val message = intercept[AnalysisException] {
+    withTempView("jsonTable2") {
       df.write.json(path.toString)
-    }.getMessage
 
-    assert(
-      message.contains("already exists"),
-      "We should complain that the path already exists.")
+      val message = intercept[AnalysisException] {
+        df.write.json(path.toString)
+      }.getMessage
 
-    if (path.exists()) Utils.deleteRecursively(path)
+      assert(
+        message.contains("already exists"),
+        "We should complain that the path already exists.")
 
-    df.write.json(path.toString)
-    checkLoad()
+      if (path.exists()) Utils.deleteRecursively(path)
 
-    df.write.mode(SaveMode.Overwrite).json(path.toString)
-    checkLoad()
+      df.write.json(path.toString)
+      checkLoad()
 
-    // verify the append mode
-    df.write.mode(SaveMode.Append).json(path.toString)
-    val df2 = df.union(df)
-    df2.createOrReplaceTempView("jsonTable2")
+      df.write.mode(SaveMode.Overwrite).json(path.toString)
+      checkLoad()
 
-    checkLoad(df2, "jsonTable2")
+      // verify the append mode
+      df.write.mode(SaveMode.Append).json(path.toString)
+      val df2 = df.union(df)
+      df2.createOrReplaceTempView("jsonTable2")
+
+      checkLoad(df2, "jsonTable2")
+    }
   }
 
   test("SPARK-23459: Improve error message when specified unknown column in partition columns") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
index 92ec2a0c172ef..6486e1aee8649 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
@@ -29,6 +29,7 @@ import org.scalatest.{BeforeAndAfter, Matchers}
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, Dataset}
 import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.execution.streaming.sources.MemorySink
 import org.apache.spark.sql.functions.{count, window}
@@ -592,6 +593,33 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     }
   }
 
+  test("SPARK-27340 Alias on TimeWindow expression cause watermark metadata lost") {
+    val inputData = MemoryStream[Int]
+    val aliasWindow = inputData.toDF()
+      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withWatermark("eventTime", "10 seconds")
+      .select(window($"eventTime", "5 seconds") as 'aliasWindow)
+    // Check the eventTime metadata is kept in the top level alias.
+    assert(aliasWindow.logicalPlan.output.exists(
+      _.metadata.contains(EventTimeWatermark.delayKey)))
+
+    val windowedAggregation = aliasWindow
+      .groupBy('aliasWindow)
+      .agg(count("*") as 'count)
+      .select($"aliasWindow".getField("start").cast("long").as[Long], $"count".as[Long])
+
+    testStream(windowedAggregation)(
+      AddData(inputData, 10, 11, 12, 13, 14, 15),
+      CheckNewAnswer(),
+      AddData(inputData, 25), // Advance watermark to 15 seconds
+      CheckNewAnswer((10, 5)),
+      assertNumStateRows(2),
+      AddData(inputData, 10), // Should not emit anything as data less than watermark
+      CheckNewAnswer(),
+      assertNumStateRows(2)
+    )
+  }
+
   test("test no-data flag") {
     val flagKey = SQLConf.STREAMING_NO_DATA_MICRO_BATCHES_ENABLED.key
 
@@ -773,7 +801,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
   }
 
   private val timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
-  timestampFormat.setTimeZone(ju.TimeZone.getTimeZone("UTC"))
+  timestampFormat.setTimeZone(ju.TimeZone.getTimeZone(UTC))
 
   private def formatTimestamp(sec: Long): String = {
     timestampFormat.format(new ju.Date(sec * 1000))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
index 877965100f018..aa2664c1e5393 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
@@ -555,10 +555,12 @@ abstract class FileStreamSinkSuite extends StreamTest {
             }
           }
 
-          val fs = new Path(outputDir.getCanonicalPath).getFileSystem(
-            spark.sessionState.newHadoopConf())
-          val sinkLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, spark,
-            outputDir.getCanonicalPath)
+          val outputDirPath = new Path(outputDir.getCanonicalPath)
+          val hadoopConf = spark.sessionState.newHadoopConf()
+          val fs = outputDirPath.getFileSystem(hadoopConf)
+          val logPath = FileStreamSink.getMetadataLogPath(fs, outputDirPath, conf)
+
+          val sinkLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, spark, logPath.toString)
 
           val allFiles = sinkLog.allFiles()
           // only files from non-empty partition should be logged
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
index fa320333143ec..7b16aebc531fb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
@@ -532,6 +532,18 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
     }
   }
 
+  test("SPARK-31935: Hadoop file system config should be effective in data source options") {
+    withTempDir { dir =>
+      val path = dir.getCanonicalPath
+      val defaultFs = "nonexistFS://nonexistFS"
+      val expectMessage = "No FileSystem for scheme nonexistFS"
+      val message = intercept[java.io.IOException] {
+        spark.readStream.option("fs.defaultFS", defaultFs).text(path)
+      }.getMessage
+      assert(message.filterNot(Set(':', '"').contains) == expectMessage)
+    }
+  }
+
   test("read from textfile") {
     withTempDirs { case (src, tmp) =>
       val textStream = spark.readStream.textFile(src.getCanonicalPath)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
index d36c64f61a726..b04f8b0d4d174 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
@@ -798,7 +798,7 @@ class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
         }
       },
       CheckNewAnswer(("c", "-1")),
-      assertNumStateRows(total = 0, updated = 0)
+      assertNumStateRows(total = 0, updated = 1)
     )
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
index b6618826487c6..d8d997e1768bd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
@@ -193,13 +193,15 @@ class StreamSuite extends StreamTest {
   }
 
   test("sql queries") {
-    val inputData = MemoryStream[Int]
-    inputData.toDF().createOrReplaceTempView("stream")
-    val evens = sql("SELECT * FROM stream WHERE value % 2 = 0")
-
-    testStream(evens)(
-      AddData(inputData, 1, 2, 3, 4),
-      CheckAnswer(2, 4))
+    withTempView("stream") {
+      val inputData = MemoryStream[Int]
+      inputData.toDF().createOrReplaceTempView("stream")
+      val evens = sql("SELECT * FROM stream WHERE value % 2 = 0")
+
+      testStream(evens)(
+        AddData(inputData, 1, 2, 3, 4),
+        CheckAnswer(2, 4))
+    }
   }
 
   test("DataFrame reuse") {
@@ -1244,9 +1246,10 @@ class StreamSuite extends StreamTest {
       failAfter(60.seconds) {
         val startTime = System.nanoTime()
         withSQLConf(SQLConf.STREAMING_STOP_TIMEOUT.key -> "2000") {
-          intercept[TimeoutException] {
+          val ex = intercept[TimeoutException] {
             sq.stop()
           }
+          assert(ex.getMessage.contains(sq.id.toString))
         }
         val duration = (System.nanoTime() - startTime) / 1e6
         assert(duration >= 2000,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
index 6d5ad873eedea..93bfd64455408 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
@@ -144,16 +144,22 @@ trait StreamTest extends QueryTest with SharedSparkSession with TimeLimits with
     }
   }
 
+  private def createToExternalRowConverter[A : Encoder](): A => Row = {
+    val encoder = encoderFor[A]
+    val toInternalRow = encoder.createSerializer()
+    val toExternalRow = RowEncoder(encoder.schema).resolveAndBind().createDeserializer()
+    toExternalRow.compose(toInternalRow)
+  }
+
   /**
    * Checks to make sure that the current data stored in the sink matches the `expectedAnswer`.
    * This operation automatically blocks until all added data has been processed.
    */
   object CheckAnswer {
     def apply[A : Encoder](data: A*): CheckAnswerRows = {
-      val encoder = encoderFor[A]
-      val toExternalRow = RowEncoder(encoder.schema).resolveAndBind()
+      val toExternalRow = createToExternalRowConverter[A]()
       CheckAnswerRows(
-        data.map(d => toExternalRow.fromRow(encoder.toRow(d))),
+        data.map(toExternalRow),
         lastOnly = false,
         isSorted = false)
     }
@@ -174,10 +180,9 @@ trait StreamTest extends QueryTest with SharedSparkSession with TimeLimits with
     }
 
     def apply[A: Encoder](isSorted: Boolean, data: A*): CheckAnswerRows = {
-      val encoder = encoderFor[A]
-      val toExternalRow = RowEncoder(encoder.schema).resolveAndBind()
+      val toExternalRow = createToExternalRowConverter[A]()
       CheckAnswerRows(
-        data.map(d => toExternalRow.fromRow(encoder.toRow(d))),
+        data.map(toExternalRow),
         lastOnly = true,
         isSorted = isSorted)
     }
@@ -215,9 +220,8 @@ trait StreamTest extends QueryTest with SharedSparkSession with TimeLimits with
     def apply(): CheckNewAnswerRows = CheckNewAnswerRows(Seq.empty)
 
     def apply[A: Encoder](data: A, moreData: A*): CheckNewAnswerRows = {
-      val encoder = encoderFor[A]
-      val toExternalRow = RowEncoder(encoder.schema).resolveAndBind()
-      CheckNewAnswerRows((data +: moreData).map(d => toExternalRow.fromRow(encoder.toRow(d))))
+      val toExternalRow = createToExternalRowConverter[A]()
+      CheckNewAnswerRows((data +: moreData).map(toExternalRow))
     }
 
     def apply(rows: Row*): CheckNewAnswerRows = CheckNewAnswerRows(rows)
@@ -301,6 +305,14 @@ trait StreamTest extends QueryTest with SharedSparkSession with TimeLimits with
     def apply(func: StreamExecution => Any): AssertOnQuery = apply("Execute")(func)
   }
 
+  /** Call `StreamingQuery.processAllAvailable()` to wait. */
+  object ProcessAllAvailable {
+    def apply(): AssertOnQuery = AssertOnQuery { query =>
+      query.processAllAvailable()
+      true
+    }
+  }
+
   object AwaitEpoch {
     def apply(epoch: Long): AssertOnQuery =
       Execute {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
index 741355381222d..85e1b85b84d26 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
@@ -202,47 +202,68 @@ class StreamingAggregationSuite extends StateStoreMetricsTest with Assertions {
         }
       }
 
-      def stateOperatorProgresses: Seq[StateOperatorProgress] = {
-        val operatorProgress = mutable.ArrayBuffer[StateOperatorProgress]()
-        var progress = query.recentProgress.last
-
-        operatorProgress ++= progress.stateOperators.map { op => op.copy(op.numRowsUpdated) }
-        if (progress.numInputRows == 0) {
-          // empty batch, merge metrics from previous batch as well
-          progress = query.recentProgress.takeRight(2).head
-          operatorProgress.zipWithIndex.foreach { case (sop, index) =>
-            // "numRowsUpdated" should be merged, as it could be updated in both batches.
-            // (for now it is only updated from previous batch, but things can be changed.)
-            // other metrics represent current status of state so picking up the latest values.
-            val newOperatorProgress = sop.copy(
-              sop.numRowsUpdated + progress.stateOperators(index).numRowsUpdated)
-            operatorProgress(index) = newOperatorProgress
-          }
-        }
+      // Pick the latest progress that actually ran a batch
+      def lastExecutedBatch: StreamingQueryProgress = {
+        query.recentProgress.filter(_.durationMs.containsKey("addBatch")).last
+      }
 
-        operatorProgress
+      def stateOperatorProgresses: Seq[StateOperatorProgress] = {
+        lastExecutedBatch.stateOperators
       }
     }
 
+    val clock = new StreamManualClock()
+
     testStream(aggWithWatermark)(
+      // batchId 0
       AddData(inputData, 15),
-      CheckAnswer(), // watermark = 5
+      StartStream(Trigger.ProcessingTime("interval 1 second"), clock),
+      CheckAnswer(), // watermark = 0
       AssertOnQuery { _.stateNodes.size === 1 },
       AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 0 },
       AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 1 },
       AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 1 },
+      AssertOnQuery { _.lastExecutedBatch.sink.numOutputRows == 0 },
+
+      // batchId 1 without data
+      AdvanceManualClock(1000L), // watermark = 5
+      Execute { q =>             // wait for the no data batch to complete
+        eventually(timeout(streamingTimeout)) { assert(q.lastProgress.batchId === 1) }
+      },
+      CheckAnswer(),
+      AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 0 },
+      AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 0 },
+      AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 1 },
+      AssertOnQuery { _.lastExecutedBatch.sink.numOutputRows == 0 },
+
+      // batchId 2 with data
       AddData(inputData, 10, 12, 14),
-      CheckAnswer(), // watermark = 5
-      AssertOnQuery { _.stateNodes.size === 1 },
+      AdvanceManualClock(1000L), // watermark = 5
+      CheckAnswer(),
       AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 0 },
       AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 1 },
       AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 2 },
+      AssertOnQuery { _.lastExecutedBatch.sink.numOutputRows == 0 },
+
+      // batchId 3 with data
       AddData(inputData, 25),
-      CheckAnswer((10, 3)), // watermark = 15
-      AssertOnQuery { _.stateNodes.size === 1 },
-      AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 1 },
+      AdvanceManualClock(1000L), // watermark = 5
+      CheckAnswer(),
+      AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 0 },
       AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 1 },
-      AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 2 }
+      AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 3 },
+      AssertOnQuery { _.lastExecutedBatch.sink.numOutputRows == 0 },
+
+      // batchId 4 without data
+      AdvanceManualClock(1000L), // watermark = 15
+      Execute { q =>             // wait for the no data batch to complete
+        eventually(timeout(streamingTimeout)) { assert(q.lastProgress.batchId === 4) }
+      },
+      CheckAnswer((10, 3)),
+      AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 1 },
+      AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 0 },
+      AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 2 },
+      AssertOnQuery { _.lastExecutedBatch.sink.numOutputRows == 1 }
     )
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
index cfd7204ea2931..51ddc7b49fcda 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
@@ -280,6 +280,12 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
         { // State should have been cleaned if flag is set, otherwise should not have been cleaned
           if (flag) assertNumStateRows(total = 1, updated = 1)
           else assertNumStateRows(total = 7, updated = 1)
+        },
+        AssertOnQuery { q =>
+          eventually(timeout(streamingTimeout)) {
+            q.lastProgress.sink.numOutputRows == 0L
+            true
+          }
         }
       )
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
index 3f218c9cb7fd9..7837b20cd6830 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.streaming
 
 import java.io.File
+import java.sql.Timestamp
 import java.util.{Locale, UUID}
 
 import scala.util.Random
@@ -991,4 +992,47 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
       )
     }
   }
+
+  test("SPARK-32148 stream-stream join regression on Spark 3.0.0") {
+    val input1 = MemoryStream[(Timestamp, String, String)]
+    val df1 = input1.toDF
+      .selectExpr("_1 as eventTime", "_2 as id", "_3 as comment")
+      .withWatermark(s"eventTime", "2 minutes")
+
+    val input2 = MemoryStream[(Timestamp, String, String)]
+    val df2 = input2.toDF
+      .selectExpr("_1 as eventTime", "_2 as id", "_3 as name")
+      .withWatermark(s"eventTime", "4 minutes")
+
+    val joined = df1.as("left")
+      .join(df2.as("right"),
+        expr("""
+               |left.id = right.id AND left.eventTime BETWEEN
+               |  right.eventTime - INTERVAL 30 seconds AND
+               |  right.eventTime + INTERVAL 30 seconds
+             """.stripMargin),
+        joinType = "leftOuter")
+
+    val inputDataForInput1 = Seq(
+      (Timestamp.valueOf("2020-01-01 00:00:00"), "abc", "has no join partner"),
+      (Timestamp.valueOf("2020-01-02 00:00:00"), "abc", "joined with A"),
+      (Timestamp.valueOf("2020-01-02 01:00:00"), "abc", "joined with B"))
+
+    val inputDataForInput2 = Seq(
+      (Timestamp.valueOf("2020-01-02 00:00:10"), "abc", "A"),
+      (Timestamp.valueOf("2020-01-02 00:59:59"), "abc", "B"),
+      (Timestamp.valueOf("2020-01-02 02:00:00"), "abc", "C"))
+
+    val expectedOutput = Seq(
+      (Timestamp.valueOf("2020-01-01 00:00:00"), "abc", "has no join partner", null, null, null),
+      (Timestamp.valueOf("2020-01-02 00:00:00"), "abc", "joined with A",
+        Timestamp.valueOf("2020-01-02 00:00:10"), "abc", "A"),
+      (Timestamp.valueOf("2020-01-02 01:00:00"), "abc", "joined with B",
+        Timestamp.valueOf("2020-01-02 00:59:59"), "abc", "B"))
+
+    testStream(joined)(
+      MultiAddData((input1, inputDataForInput1), (input2, inputDataForInput2)),
+      CheckNewAnswer(expectedOutput.head, expectedOutput.tail: _*)
+    )
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
index 9d0f829ac9684..a9aec6dbbdcd4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
@@ -49,7 +49,7 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     assert(spark.streams.active.isEmpty)
     // Skip check default `StreamingQueryStatusListener` which is for streaming UI.
     assert(spark.streams.listListeners()
-      .filterNot(_.isInstanceOf[StreamingQueryStatusListener]).isEmpty)
+      .forall(_.isInstanceOf[StreamingQueryStatusListener]))
     // Make sure we don't leak any events to the next test
     spark.sparkContext.listenerBus.waitUntilEmpty()
   }
@@ -254,8 +254,10 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
       assert(newEvent.name === event.name)
     }
 
-    testSerialization(new QueryStartedEvent(UUID.randomUUID, UUID.randomUUID, "name", 1L))
-    testSerialization(new QueryStartedEvent(UUID.randomUUID, UUID.randomUUID, null, 1L))
+    testSerialization(
+      new QueryStartedEvent(UUID.randomUUID, UUID.randomUUID, "name", "2016-12-05T20:54:20.827Z"))
+    testSerialization(
+      new QueryStartedEvent(UUID.randomUUID, UUID.randomUUID, null, "2016-12-05T20:54:20.827Z"))
   }
 
   test("QueryProgressEvent serialization") {
@@ -382,28 +384,27 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     }
   }
 
-  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2.0.0") {
+  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2_0_0") {
     // query-event-logs-version-2.0.0.txt has all types of events generated by
-    // Structured Streaming in Spark 2.0.0.
+    // Structured Streaming in Spark 2.0.0. Because we renamed the classes,
     // SparkListenerApplicationEnd is the only valid event and it's the last event. We use it
     // to verify that we can skip broken jsons generated by Structured Streaming.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.0.txt")
+    testReplayListenerBusWithBrokenEventJsons("query-event-logs-version-2.0.0.txt", 1)
   }
 
-  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2.0.1") {
+  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2_0_1") {
     // query-event-logs-version-2.0.1.txt has all types of events generated by
-    // Structured Streaming in Spark 2.0.1.
+    // Structured Streaming in Spark 2.0.1. Because we renamed the classes,
     // SparkListenerApplicationEnd is the only valid event and it's the last event. We use it
     // to verify that we can skip broken jsons generated by Structured Streaming.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.1.txt")
+    testReplayListenerBusWithBrokenEventJsons("query-event-logs-version-2.0.1.txt", 1)
   }
 
-  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2.0.2") {
+  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2_0_2") {
     // query-event-logs-version-2.0.2.txt has all types of events generated by
-    // Structured Streaming in Spark 2.0.2.
-    // SparkListenerApplicationEnd is the only valid event and it's the last event. We use it
-    // to verify that we can skip broken jsons generated by Structured Streaming.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.2.txt")
+    // Structured Streaming in Spark 2.0.2. SPARK-18516 refactored Structured Streaming query events
+    // in 2.1.0. This test is to verify we are able to load events generated by Spark 2.0.2.
+    testReplayListenerBusWithBrokenEventJsons("query-event-logs-version-2.0.2.txt", 5)
   }
 
   test("listener propagates observable metrics") {
@@ -432,9 +433,13 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     }
 
     try {
+      val noDataProgressIntervalKey = SQLConf.STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL.key
       spark.streams.addListener(listener)
       testStream(df, OutputMode.Append)(
-        StartStream(Trigger.ProcessingTime(100), triggerClock = clock),
+        StartStream(
+          Trigger.ProcessingTime(100),
+          triggerClock = clock,
+          Map(noDataProgressIntervalKey -> "100")),
         // Batch 1
         AddData(inputData, 1, 2),
         AdvanceManualClock(100),
@@ -463,7 +468,51 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     }
   }
 
-  private def testReplayListenerBusWithBorkenEventJsons(fileName: String): Unit = {
+  test("SPARK-31593: remove unnecessary streaming query progress update") {
+    withSQLConf(SQLConf.STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL.key -> "100") {
+      @volatile var numProgressEvent = 0
+      val listener = new StreamingQueryListener {
+        override def onQueryStarted(event: QueryStartedEvent): Unit = {}
+        override def onQueryProgress(event: QueryProgressEvent): Unit = {
+          numProgressEvent += 1
+        }
+        override def onQueryTerminated(event: QueryTerminatedEvent): Unit = {}
+      }
+      spark.streams.addListener(listener)
+
+      def checkProgressEvent(count: Int): StreamAction = {
+        AssertOnQuery { _ =>
+          eventually(Timeout(streamingTimeout)) {
+            assert(numProgressEvent == count)
+          }
+          true
+        }
+      }
+
+      try {
+        val input = new MemoryStream[Int](0, sqlContext)
+        val clock = new StreamManualClock()
+        val result = input.toDF().select("value")
+        testStream(result)(
+          StartStream(trigger = Trigger.ProcessingTime(10), triggerClock = clock),
+          AddData(input, 10),
+          checkProgressEvent(1),
+          AdvanceManualClock(10),
+          checkProgressEvent(2),
+          AdvanceManualClock(90),
+          checkProgressEvent(2),
+          AdvanceManualClock(10),
+          checkProgressEvent(3)
+        )
+      } finally {
+        spark.streams.removeListener(listener)
+      }
+    }
+  }
+
+  private def testReplayListenerBusWithBrokenEventJsons(
+      fileName: String,
+      expectedEventSize: Int): Unit = {
     val input = getClass.getResourceAsStream(s"/structured-streaming/$fileName")
     val events = mutable.ArrayBuffer[SparkListenerEvent]()
     try {
@@ -479,8 +528,8 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
       replayer.addListener(new SparkListener {})
       replayer.replay(input, fileName)
       // SparkListenerApplicationEnd is the only valid event
-      assert(events.size === 1)
-      assert(events(0).isInstanceOf[SparkListenerApplicationEnd])
+      assert(events.size === expectedEventSize)
+      assert(events.last.isInstanceOf[SparkListenerApplicationEnd])
     } finally {
       input.close()
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala
index 6f00b528cb8bd..08b3644745f9a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala
@@ -241,6 +241,7 @@ class StreamingQueryStatusAndProgressSuite extends StreamTest with Eventually {
           assert(nextProgress.numInputRows === 0)
           assert(nextProgress.stateOperators.head.numRowsTotal === 2)
           assert(nextProgress.stateOperators.head.numRowsUpdated === 0)
+          assert(nextProgress.sink.numOutputRows === 0)
         }
       } finally {
         query.stop()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
index 77f5c856ff0f4..1f408d55fd811 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
@@ -33,8 +33,11 @@ import org.scalatestplus.mockito.MockitoSugar
 
 import org.apache.spark.{SparkException, TestUtils}
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{Column, DataFrame, Dataset, Row}
+import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Dataset, Row}
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Literal, Rand, Randn, Shuffle, Uuid}
+import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
+import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Complete
 import org.apache.spark.sql.connector.read.InputPartition
 import org.apache.spark.sql.connector.read.streaming.{Offset => OffsetV2}
 import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
@@ -1106,6 +1109,90 @@ class StreamingQuerySuite extends StreamTest with BeforeAndAfter with Logging wi
     }
   }
 
+  test("SPARK-32456: SQL union in streaming query of append mode without watermark") {
+    val inputData1 = MemoryStream[Int]
+    val inputData2 = MemoryStream[Int]
+    withTempView("s1", "s2") {
+      inputData1.toDF().createOrReplaceTempView("s1")
+      inputData2.toDF().createOrReplaceTempView("s2")
+      val unioned = spark.sql(
+        "select s1.value from s1 union select s2.value from s2")
+      checkExceptionMessage(unioned)
+    }
+  }
+
+  test("SPARK-32456: distinct in streaming query of append mode without watermark") {
+    val inputData = MemoryStream[Int]
+    withTempView("deduptest") {
+      inputData.toDF().toDF("value").createOrReplaceTempView("deduptest")
+      val distinct = spark.sql("select distinct value from deduptest")
+      checkExceptionMessage(distinct)
+    }
+  }
+
+  test("SPARK-32456: distinct in streaming query of complete mode") {
+    val inputData = MemoryStream[Int]
+    withTempView("deduptest") {
+      inputData.toDF().toDF("value").createOrReplaceTempView("deduptest")
+      val distinct = spark.sql("select distinct value from deduptest")
+
+      testStream(distinct, Complete)(
+        AddData(inputData, 1, 2, 3, 3, 4),
+        CheckAnswer(Row(1), Row(2), Row(3), Row(4))
+      )
+    }
+  }
+
+  testQuietly("limit on empty batch should not cause state store error") {
+    // The source only produces two batches, the first batch is empty and the second batch has data.
+    val source = new Source {
+      var batchId = 0
+      override def stop(): Unit = {}
+      override def getOffset: Option[Offset] = {
+        Some(LongOffset(batchId + 1))
+      }
+      override def getBatch(start: Option[Offset], end: Offset): DataFrame = {
+        if (batchId == 0) {
+          batchId += 1
+          Dataset.ofRows(spark, LocalRelation(schema.toAttributes, Nil, isStreaming = true))
+        } else {
+          Dataset.ofRows(spark,
+            LocalRelation(schema.toAttributes, InternalRow(10) :: Nil, isStreaming = true))
+        }
+      }
+      override def schema: StructType = MockSourceProvider.fakeSchema
+    }
+
+    MockSourceProvider.withMockSources(source) {
+      val df = spark.readStream
+        .format("org.apache.spark.sql.streaming.util.MockSourceProvider")
+        .load()
+        .limit(1)
+
+      testStream(df)(
+        StartStream(),
+        AssertOnQuery { q =>
+          q.processAllAvailable()
+          true
+        },
+        CheckAnswer(10))
+    }
+  }
+
+  private def checkExceptionMessage(df: DataFrame): Unit = {
+    withTempDir { outputDir =>
+      withTempDir { checkpointDir =>
+        val exception = intercept[AnalysisException](
+          df.writeStream
+            .option("checkpointLocation", checkpointDir.getCanonicalPath)
+            .start(outputDir.getCanonicalPath))
+        assert(exception.getMessage.contains(
+          "Append output mode not supported when there are streaming aggregations on streaming " +
+            "DataFrames/DataSets without watermark"))
+      }
+    }
+  }
+
   /** Create a streaming DF that only execute one batch in which it returns the given static DF */
   private def createSingleTriggerStreamingDF(triggerDF: DataFrame): DataFrame = {
     require(!triggerDF.isStreaming)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala
index 8599ceb833ca4..0d17f2e0bc7fb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala
@@ -181,17 +181,19 @@ class ContinuousSuite extends ContinuousSuiteBase {
   }
 
   test("subquery alias") {
-    val input = ContinuousMemoryStream[Int]
-    input.toDF().createOrReplaceTempView("memory")
-    val test = spark.sql("select value from memory where value > 2")
+    withTempView("memory") {
+      val input = ContinuousMemoryStream[Int]
+      input.toDF().createOrReplaceTempView("memory")
+      val test = spark.sql("select value from memory where value > 2")
 
-    testStream(test)(
-      AddData(input, 0, 1),
-      CheckAnswer(),
-      StopStream,
-      AddData(input, 2, 3, 4),
-      StartStream(),
-      CheckAnswer(3, 4))
+      testStream(test)(
+        AddData(input, 0, 1),
+        CheckAnswer(),
+        StopStream,
+        AddData(input, 2, 3, 4),
+        StartStream(),
+        CheckAnswer(3, 4))
+    }
   }
 
   test("repeatedly restart") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
index f9fc540c2ab80..8bf7e276e12c2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
@@ -43,11 +43,13 @@ object LastOptions {
   var mockStreamSourceProvider = mock(classOf[StreamSourceProvider])
   var mockStreamSinkProvider = mock(classOf[StreamSinkProvider])
   var parameters: Map[String, String] = null
+  var sinkParameters: Map[String, String] = null
   var schema: Option[StructType] = null
   var partitionColumns: Seq[String] = Nil
 
   def clear(): Unit = {
     parameters = null
+    sinkParameters = null
     schema = null
     partitionColumns = null
     reset(mockStreamSourceProvider)
@@ -101,7 +103,7 @@ class DefaultSource extends StreamSourceProvider with StreamSinkProvider {
       parameters: Map[String, String],
       partitionColumns: Seq[String],
       outputMode: OutputMode): Sink = {
-    LastOptions.parameters = parameters
+    LastOptions.sinkParameters = parameters
     LastOptions.partitionColumns = partitionColumns
     LastOptions.mockStreamSinkProvider.createSink(spark, parameters, partitionColumns, outputMode)
     (_: Long, _: DataFrame) => {}
@@ -171,16 +173,48 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
 
     df.writeStream
       .format("org.apache.spark.sql.streaming.test")
-      .option("opt1", "1")
-      .options(Map("opt2" -> "2"))
+      .option("opt1", "5")
+      .options(Map("opt2" -> "4"))
       .options(map)
       .option("checkpointLocation", newMetadataDir)
       .start()
       .stop()
 
-    assert(LastOptions.parameters("opt1") == "1")
-    assert(LastOptions.parameters("opt2") == "2")
-    assert(LastOptions.parameters("opt3") == "3")
+    assert(LastOptions.sinkParameters("opt1") == "5")
+    assert(LastOptions.sinkParameters("opt2") == "4")
+    assert(LastOptions.sinkParameters("opt3") == "3")
+    assert(LastOptions.sinkParameters.contains("checkpointLocation"))
+  }
+
+  test("SPARK-32832: later option should override earlier options for load()") {
+    spark.readStream
+      .format("org.apache.spark.sql.streaming.test")
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .option("path", "5")
+      .load()
+    assert(LastOptions.parameters("path") == "5")
+  }
+
+  test("SPARK-32832: later option should override earlier options for start()") {
+    val ds = spark.readStream
+      .format("org.apache.spark.sql.streaming.test")
+      .load()
+    assert(LastOptions.parameters.isEmpty)
+
+    ds.writeStream
+      .format("org.apache.spark.sql.streaming.test")
+      .option("checkpointLocation", newMetadataDir)
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .option("path", "5")
+      .start()
+      .stop()
+    assert(LastOptions.sinkParameters("path") == "5")
   }
 
   test("partitioning") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala
index de43e470e8e13..2a1e18ab66bb7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala
@@ -97,7 +97,7 @@ class StreamingQueryPageSuite extends SharedSparkSession with BeforeAndAfter {
     when(streamQuery.name).thenReturn("query")
     when(streamQuery.id).thenReturn(id)
     when(streamQuery.runId).thenReturn(id)
-    when(streamQuery.submissionTime).thenReturn(1L)
+    when(streamQuery.startTimestamp).thenReturn(1L)
     when(streamQuery.lastProgress).thenReturn(progress)
     when(streamQuery.recentProgress).thenReturn(Array(progress))
     when(streamQuery.exception).thenReturn(None)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
index bd74ed340b408..6aa440e5609c5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
@@ -27,12 +27,13 @@ import org.apache.spark.sql.streaming
 class StreamingQueryStatusListenerSuite extends StreamTest {
 
   test("onQueryStarted, onQueryProgress, onQueryTerminated") {
-    val listener = new StreamingQueryStatusListener(spark.sqlContext.conf)
+    val listener = new StreamingQueryStatusListener(spark.sparkContext.conf)
 
     // hanlde query started event
     val id = UUID.randomUUID()
     val runId = UUID.randomUUID()
-    val startEvent = new StreamingQueryListener.QueryStartedEvent(id, runId, "test", 1L)
+    val startEvent = new StreamingQueryListener.QueryStartedEvent(
+      id, runId, "test", "2016-12-05T20:54:20.827Z")
     listener.onQueryStarted(startEvent)
 
     // result checking
@@ -73,12 +74,13 @@ class StreamingQueryStatusListenerSuite extends StreamTest {
   }
 
   test("same query start multiple times") {
-    val listener = new StreamingQueryStatusListener(spark.sqlContext.conf)
+    val listener = new StreamingQueryStatusListener(spark.sparkContext.conf)
 
     // handle first time start
     val id = UUID.randomUUID()
     val runId0 = UUID.randomUUID()
-    val startEvent0 = new StreamingQueryListener.QueryStartedEvent(id, runId0, "test", 1L)
+    val startEvent0 = new StreamingQueryListener.QueryStartedEvent(
+      id, runId0, "test", "2016-12-05T20:54:20.827Z")
     listener.onQueryStarted(startEvent0)
 
     // handle terminate event
@@ -87,7 +89,8 @@ class StreamingQueryStatusListenerSuite extends StreamTest {
 
     // handle second time start
     val runId1 = UUID.randomUUID()
-    val startEvent1 = new StreamingQueryListener.QueryStartedEvent(id, runId1, "test", 1L)
+    val startEvent1 = new StreamingQueryListener.QueryStartedEvent(
+      id, runId1, "test", "2016-12-05T20:54:20.827Z")
     listener.onQueryStarted(startEvent1)
 
     // result checking
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
new file mode 100644
index 0000000000000..fdf4c6634d79f
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming.ui
+
+import org.openqa.selenium.WebDriver
+import org.openqa.selenium.htmlunit.HtmlUnitDriver
+import org.scalatest._
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.time.SpanSugar._
+import org.scalatestplus.selenium.WebBrowser
+
+import org.apache.spark._
+import org.apache.spark.internal.config.UI.{UI_ENABLED, UI_PORT}
+import org.apache.spark.sql.LocalSparkSession.withSparkSession
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.util.quietly
+import org.apache.spark.sql.streaming.StreamingQueryException
+import org.apache.spark.ui.SparkUICssErrorHandler
+
+class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll {
+
+  implicit var webDriver: WebDriver = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    webDriver = new HtmlUnitDriver {
+      getWebClient.setCssErrorHandler(new SparkUICssErrorHandler)
+    }
+  }
+
+  private def newSparkSession(
+      master: String = "local",
+      additionalConfs: Map[String, String] = Map.empty): SparkSession = {
+    val conf = new SparkConf()
+      .setMaster(master)
+      .setAppName("ui-test")
+      .set(UI_ENABLED, true)
+      .set(UI_PORT, 0)
+    additionalConfs.foreach { case (k, v) => conf.set(k, v) }
+    val spark = SparkSession.builder().master(master).config(conf).getOrCreate()
+    assert(spark.sparkContext.ui.isDefined)
+    spark
+  }
+
+  def goToUi(spark: SparkSession, path: String): Unit = {
+    go to (spark.sparkContext.ui.get.webUrl.stripSuffix("/") + path)
+  }
+
+  test("SPARK-30984: Structured Streaming UI should be activated when running a streaming query") {
+    quietly {
+      withSparkSession(newSparkSession()) { spark =>
+        import spark.implicits._
+        try {
+          spark.range(1, 10).count()
+
+          goToUi(spark, "/StreamingQuery")
+
+          val h3Text = findAll(cssSelector("h3")).map(_.text).toSeq
+          h3Text should not contain ("Streaming Query")
+
+          val activeQuery =
+            spark.readStream.format("rate").load().writeStream.format("noop").start()
+          val completedQuery =
+            spark.readStream.format("rate").load().writeStream.format("noop").start()
+          completedQuery.stop()
+          val failedQuery = spark.readStream.format("rate").load().select("value").as[Long]
+            .map(_ / 0).writeStream.format("noop").start()
+          try {
+            failedQuery.awaitTermination()
+          } catch {
+            case _: StreamingQueryException =>
+          }
+
+          eventually(timeout(30.seconds), interval(100.milliseconds)) {
+            // Check the query list page
+            goToUi(spark, "/StreamingQuery")
+
+            findAll(cssSelector("h3")).map(_.text).toSeq should contain("Streaming Query")
+            findAll(cssSelector("""#activeQueries-table th""")).map(_.text).toSeq should be {
+              List("Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
+                "Avg Process /sec", "Lastest Batch")
+            }
+            val activeQueries =
+              findAll(cssSelector("""#activeQueries-table td""")).map(_.text).toSeq
+            activeQueries should contain(activeQuery.id.toString)
+            activeQueries should contain(activeQuery.runId.toString)
+            findAll(cssSelector("""#completedQueries-table th"""))
+              .map(_.text).toSeq should be {
+                List("Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
+                  "Avg Process /sec", "Lastest Batch", "Error")
+              }
+            val completedQueries =
+              findAll(cssSelector("""#completedQueries-table td""")).map(_.text).toSeq
+            completedQueries should contain(completedQuery.id.toString)
+            completedQueries should contain(completedQuery.runId.toString)
+            completedQueries should contain(failedQuery.id.toString)
+            completedQueries should contain(failedQuery.runId.toString)
+
+            // Check the query statistics page
+            val activeQueryLink =
+              findAll(cssSelector("""#activeQueries-table a""")).flatMap(_.attribute("href")).next
+            go to activeQueryLink
+
+            findAll(cssSelector("h3"))
+              .map(_.text).toSeq should contain("Streaming Query Statistics")
+            val summaryText = findAll(cssSelector("div strong")).map(_.text).toSeq
+            summaryText should contain ("Name:")
+            summaryText should contain ("Id:")
+            summaryText should contain ("RunId:")
+            findAll(cssSelector("""#stat-table th""")).map(_.text).toSeq should be {
+              List("", "Timelines", "Histograms")
+            }
+          }
+        } finally {
+          spark.streams.active.foreach(_.stop())
+        }
+      }
+    }
+  }
+
+  override def afterAll(): Unit = {
+    try {
+      if (webDriver != null) {
+        webDriver.quit()
+      }
+    } finally {
+      super.afterAll()
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
index fb939007697c2..c7ca0125c8a0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
@@ -224,6 +224,28 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
     assert(LastOptions.parameters("opt3") == "3")
   }
 
+  test("SPARK-32364: path argument of load function should override all existing options") {
+    spark.read
+      .format("org.apache.spark.sql.test")
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .load("5")
+    assert(LastOptions.parameters("path") == "5")
+  }
+
+  test("SPARK-32364: path argument of save function should override all existing options") {
+    Seq(1).toDF.write
+      .format("org.apache.spark.sql.test")
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .save("5")
+    assert(LastOptions.parameters("path") == "5")
+  }
+
   test("pass partitionBy as options") {
     Seq(1).toDF.write
       .format("org.apache.spark.sql.test")
@@ -282,7 +304,7 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
         plan = qe.analyzed
 
       }
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
     }
 
     spark.listenerManager.register(listener)
@@ -333,7 +355,7 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
         var msg = intercept[AnalysisException] {
           Seq((1L, 2.0)).toDF("i", "d").write.mode("append").saveAsTable("t")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': LongType to IntegerType"))
+        assert(msg.contains("Cannot safely cast 'i': bigint to int"))
 
         // Insert into table successfully.
         Seq((1, 2.0)).toDF("i", "d").write.mode("append").saveAsTable("t")
@@ -354,14 +376,14 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
         var msg = intercept[AnalysisException] {
           Seq(("a", "b")).toDF("i", "d").write.mode("append").saveAsTable("t")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': StringType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': StringType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': string to int") &&
+          msg.contains("Cannot safely cast 'd': string to double"))
 
         msg = intercept[AnalysisException] {
           Seq((true, false)).toDF("i", "d").write.mode("append").saveAsTable("t")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': BooleanType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': BooleanType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': boolean to int") &&
+          msg.contains("Cannot safely cast 'd': boolean to double"))
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
index 38893f846e5a4..7be15e9d87004 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
@@ -27,7 +27,8 @@ import scala.language.implicitConversions
 import scala.util.control.NonFatal
 
 import org.apache.hadoop.fs.Path
-import org.scalatest.{BeforeAndAfterAll, Suite}
+import org.scalactic.source.Position
+import org.scalatest.{BeforeAndAfterAll, Suite, Tag}
 import org.scalatest.concurrent.Eventually
 
 import org.apache.spark.SparkFunSuite
@@ -40,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTestBase
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.execution.FilterExec
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
 import org.apache.spark.sql.execution.datasources.DataSourceUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.util.UninterruptibleThread
@@ -114,6 +116,19 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
     }
   }
 
+  override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
+      (implicit pos: Position): Unit = {
+    if (testTags.exists(_.isInstanceOf[DisableAdaptiveExecution])) {
+      super.test(testName, testTags: _*) {
+        withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+          testFun
+        }
+      }
+    } else {
+      super.test(testName, testTags: _*)(testFun)
+    }
+  }
+
   /**
    * Run a test on a separate `UninterruptibleThread`.
    */
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala
index 6881812286b24..b17c93503804c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala
@@ -20,15 +20,17 @@ package org.apache.spark.sql.util
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark._
-import org.apache.spark.sql.{functions, AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.{functions, AnalysisException, Dataset, QueryTest, Row, SparkSession}
 import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, InsertIntoStatement, LogicalPlan, Project}
-import org.apache.spark.sql.execution.{QueryExecution, WholeStageCodegenExec}
+import org.apache.spark.sql.execution.{QueryExecution, QueryExecutionException, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.command.RunnableCommand
 import org.apache.spark.sql.execution.datasources.{CreateTable, InsertIntoHadoopFsRelationCommand}
 import org.apache.spark.sql.execution.datasources.json.JsonFileFormat
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.StringType
 
 class DataFrameCallbackSuite extends QueryTest
   with SharedSparkSession
@@ -40,7 +42,7 @@ class DataFrameCallbackSuite extends QueryTest
     val metrics = ArrayBuffer.empty[(String, QueryExecution, Long)]
     val listener = new QueryExecutionListener {
       // Only test successful case here, so no need to implement `onFailure`
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
 
       override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
         metrics += ((funcName, qe, duration))
@@ -67,10 +69,10 @@ class DataFrameCallbackSuite extends QueryTest
   }
 
   testQuietly("execute callback functions when a DataFrame action failed") {
-    val metrics = ArrayBuffer.empty[(String, QueryExecution, Throwable)]
+    val metrics = ArrayBuffer.empty[(String, QueryExecution, Exception)]
     val listener = new QueryExecutionListener {
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {
-        metrics += ((funcName, qe, error))
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
+        metrics += ((funcName, qe, exception))
       }
 
       // Only test failed case here, so no need to implement `onSuccess`
@@ -96,7 +98,7 @@ class DataFrameCallbackSuite extends QueryTest
     val metrics = ArrayBuffer.empty[Long]
     val listener = new QueryExecutionListener {
       // Only test successful case here, so no need to implement `onFailure`
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
 
       override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
         val metric = stripAQEPlan(qe.executedPlan) match {
@@ -136,7 +138,7 @@ class DataFrameCallbackSuite extends QueryTest
     val metrics = ArrayBuffer.empty[Long]
     val listener = new QueryExecutionListener {
       // Only test successful case here, so no need to implement `onFailure`
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
 
       override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
         metrics += qe.executedPlan.longMetric("dataSize").value
@@ -176,10 +178,10 @@ class DataFrameCallbackSuite extends QueryTest
 
   test("execute callback functions for DataFrameWriter") {
     val commands = ArrayBuffer.empty[(String, LogicalPlan)]
-    val errors = ArrayBuffer.empty[(String, Throwable)]
+    val exceptions = ArrayBuffer.empty[(String, Exception)]
     val listener = new QueryExecutionListener {
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {
-        errors += funcName -> error
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
+        exceptions += funcName -> exception
       }
 
       override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
@@ -225,9 +227,9 @@ class DataFrameCallbackSuite extends QueryTest
         spark.range(10).select($"id", $"id").write.insertInto("tab")
       }
       sparkContext.listenerBus.waitUntilEmpty()
-      assert(errors.length == 1)
-      assert(errors.head._1 == "insertInto")
-      assert(errors.head._2 == e)
+      assert(exceptions.length == 1)
+      assert(exceptions.head._1 == "insertInto")
+      assert(exceptions.head._2 == e)
     }
   }
 
@@ -238,7 +240,7 @@ class DataFrameCallbackSuite extends QueryTest
         metricMaps += qe.observedMetrics
       }
 
-      override def onFailure(funcName: String, qe: QueryExecution, exception: Throwable): Unit = {
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
         // No-op
       }
     }
@@ -278,4 +280,32 @@ class DataFrameCallbackSuite extends QueryTest
       spark.listenerManager.unregister(listener)
     }
   }
+
+  testQuietly("SPARK-31144: QueryExecutionListener should receive `java.lang.Error`") {
+    var e: Exception = null
+    val listener = new QueryExecutionListener {
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
+        e = exception
+      }
+      override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {}
+    }
+    spark.listenerManager.register(listener)
+
+    intercept[Error] {
+      Dataset.ofRows(spark, ErrorTestCommand("foo")).collect()
+    }
+    sparkContext.listenerBus.waitUntilEmpty()
+    assert(e != null && e.isInstanceOf[QueryExecutionException]
+      && e.getCause.isInstanceOf[Error] && e.getCause.getMessage == "foo")
+    spark.listenerManager.unregister(listener)
+  }
+}
+
+/** A test command that throws `java.lang.Error` during execution. */
+case class ErrorTestCommand(foo: String) extends RunnableCommand {
+
+  override val output: Seq[Attribute] = Seq(AttributeReference("foo", StringType)())
+
+  override def run(sparkSession: SparkSession): Seq[Row] =
+    throw new java.lang.Error(foo)
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala
index 2fd6cb220ea3f..ab854a0281c6c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala
@@ -57,7 +57,7 @@ private class CountingQueryExecutionListener extends QueryExecutionListener {
     CALLBACK_COUNT.incrementAndGet()
   }
 
-  override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {
+  override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
     CALLBACK_COUNT.incrementAndGet()
   }
 
diff --git a/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
index 9bfad1e83ee7b..6601bcb9018f4 100644
--- a/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
+++ b/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
@@ -21,7 +21,10 @@
 
 import org.apache.orc.storage.ql.exec.vector.*;
 
+import org.apache.spark.sql.catalyst.util.DateTimeUtils;
+import org.apache.spark.sql.catalyst.util.RebaseDateTime;
 import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DateType;
 import org.apache.spark.sql.types.Decimal;
 import org.apache.spark.sql.types.TimestampType;
 import org.apache.spark.sql.vectorized.ColumnarArray;
@@ -41,6 +44,7 @@ public class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVecto
   private DecimalColumnVector decimalData;
   private TimestampColumnVector timestampData;
   private final boolean isTimestamp;
+  private final boolean isDate;
 
   private int batchSize;
 
@@ -53,6 +57,12 @@ public class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVecto
       isTimestamp = false;
     }
 
+    if (type instanceof DateType) {
+      isDate = true;
+    } else {
+      isDate = false;
+    }
+
     baseData = vector;
     if (vector instanceof LongColumnVector) {
       longData = (LongColumnVector) vector;
@@ -129,14 +139,19 @@ public short getShort(int rowId) {
 
   @Override
   public int getInt(int rowId) {
-    return (int) longData.vector[getRowIndex(rowId)];
+    int value = (int) longData.vector[getRowIndex(rowId)];
+    if (isDate) {
+      return RebaseDateTime.rebaseJulianToGregorianDays(value);
+    } else {
+      return value;
+    }
   }
 
   @Override
   public long getLong(int rowId) {
     int index = getRowIndex(rowId);
     if (isTimestamp) {
-      return timestampData.time[index] * 1000 + timestampData.nanos[index] / 1000 % 1000;
+      return DateTimeUtils.fromJavaTimestamp(timestampData.asScratchTimestamp(index));
     } else {
       return longData.vector[index];
     }
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala
new file mode 100644
index 0000000000000..1dccf0ca1faef
--- /dev/null
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc
+
+import java.io.{DataInput, DataOutput, IOException}
+import java.sql.Date
+
+import org.apache.hadoop.io.WritableUtils
+import org.apache.orc.storage.serde2.io.DateWritable
+
+import org.apache.spark.sql.catalyst.util.RebaseDateTime.{rebaseGregorianToJulianDays, rebaseJulianToGregorianDays}
+
+/**
+ * The class accepts/returns days in Gregorian calendar and rebase them
+ * via conversion to local date in Julian calendar for dates before 1582-10-15
+ * in read/write for backward compatibility with Spark 2.4 and earlier versions.
+ *
+ * This is a clone of `org.apache.spark.sql.execution.datasources.DaysWritable`.
+ * The class is cloned because Hive ORC v1.2 uses different `DateWritable`:
+ *   - v1.2: `org.apache.orc.storage.serde2.io.DateWritable`
+ *   - v2.3 and `HiveInspectors`: `org.apache.hadoop.hive.serde2.io.DateWritable`
+ *
+ * @param gregorianDays The number of days since the epoch 1970-01-01 in
+ *                      Gregorian calendar.
+ * @param julianDays The number of days since the epoch 1970-01-01 in
+ *                   Julian calendar.
+ */
+class DaysWritable(
+    var gregorianDays: Int,
+    var julianDays: Int)
+  extends DateWritable {
+
+  def this() = this(0, 0)
+  def this(gregorianDays: Int) =
+    this(gregorianDays, rebaseGregorianToJulianDays(gregorianDays))
+  def this(dateWritable: DateWritable) = {
+    this(
+      gregorianDays = dateWritable match {
+        case daysWritable: DaysWritable => daysWritable.gregorianDays
+        case dateWritable: DateWritable =>
+        rebaseJulianToGregorianDays(dateWritable.getDays)
+      },
+      julianDays = dateWritable.getDays)
+  }
+
+  override def getDays: Int = julianDays
+  override def get(): Date = new Date(DateWritable.daysToMillis(julianDays))
+
+  override def set(d: Int): Unit = {
+    gregorianDays = d
+    julianDays = rebaseGregorianToJulianDays(d)
+  }
+
+  @throws[IOException]
+  override def write(out: DataOutput): Unit = {
+    WritableUtils.writeVInt(out, julianDays)
+  }
+
+  @throws[IOException]
+  override def readFields(in: DataInput): Unit = {
+    julianDays = WritableUtils.readVInt(in)
+    gregorianDays = rebaseJulianToGregorianDays(julianDays)
+  }
+}
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 995c5ed317de1..a068347634cda 100644
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
+import java.time.{Instant, LocalDate}
+
 import org.apache.orc.storage.common.`type`.HiveDecimal
 import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}
 import org.apache.orc.storage.ql.io.sarg.SearchArgument.Builder
@@ -24,6 +26,8 @@ import org.apache.orc.storage.ql.io.sarg.SearchArgumentFactory.newBuilder
 import org.apache.orc.storage.serde2.io.HiveDecimalWritable
 
 import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
 
@@ -64,9 +68,11 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * Create ORC filter as a SearchArgument instance.
    */
   def createFilter(schema: StructType, filters: Seq[Filter]): Option[SearchArgument] = {
-    val dataTypeMap = schema.map(f => f.name -> f.dataType).toMap
+    val dataTypeMap = getDataTypeMap(schema, SQLConf.get.caseSensitiveAnalysis)
     // Combines all convertible filters using `And` to produce a single conjunction
-    val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, filters))
+    // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
+    val newFilters = filters.filter(!_.containsNestedColumn)
+    val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, newFilters))
     conjunctionOptional.map { conjunction =>
       // Then tries to build a single ORC `SearchArgument` for the conjunction predicate.
       // The input predicate is fully convertible. There should not be any empty result in the
@@ -77,7 +83,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
 
   def convertibleFilters(
       schema: StructType,
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       filters: Seq[Filter]): Seq[Filter] = {
     import org.apache.spark.sql.sources._
 
@@ -135,7 +141,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
   /**
    * Get PredicateLeafType which is corresponding to the given DataType.
    */
-  private def getPredicateLeafType(dataType: DataType) = dataType match {
+  private[sql] def getPredicateLeafType(dataType: DataType) = dataType match {
     case BooleanType => PredicateLeaf.Type.BOOLEAN
     case ByteType | ShortType | IntegerType | LongType => PredicateLeaf.Type.LONG
     case FloatType | DoubleType => PredicateLeaf.Type.FLOAT
@@ -159,6 +165,10 @@ private[sql] object OrcFilters extends OrcFiltersBase {
       value.asInstanceOf[Number].doubleValue()
     case _: DecimalType =>
       new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
+    case _: DateType if value.isInstanceOf[LocalDate] =>
+      toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
+    case _: TimestampType if value.isInstanceOf[Instant] =>
+      toJavaTimestamp(instantToMicros(value.asInstanceOf[Instant]))
     case _ => value
   }
 
@@ -171,7 +181,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * @return the builder so far.
    */
   private def buildSearchArgument(
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       expression: Filter,
       builder: Builder): Builder = {
     import org.apache.spark.sql.sources._
@@ -207,60 +217,61 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * @return the builder so far.
    */
   private def buildLeafSearchArgument(
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       expression: Filter,
       builder: Builder): Option[Builder] = {
     def getType(attribute: String): PredicateLeaf.Type =
-      getPredicateLeafType(dataTypeMap(attribute))
+      getPredicateLeafType(dataTypeMap(attribute).fieldType)
 
     import org.apache.spark.sql.sources._
 
     // NOTE: For all case branches dealing with leaf predicates below, the additional `startAnd()`
     // call is mandatory. ORC `SearchArgument` builder requires that all leaf predicates must be
     // wrapped by a "parent" predicate (`And`, `Or`, or `Not`).
+    // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
+    // in order to distinguish predicate pushdown for nested columns.
     expression match {
-      case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().equals(quotedName, getType(attribute), castedValue).end())
+      case EqualTo(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .equals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().nullSafeEquals(quotedName, getType(attribute), castedValue).end())
+      case EqualNullSafe(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .nullSafeEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().lessThan(quotedName, getType(attribute), castedValue).end())
+      case LessThan(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .lessThan(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().lessThanEquals(quotedName, getType(attribute), castedValue).end())
+      case LessThanOrEqual(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .lessThanEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startNot().lessThanEquals(quotedName, getType(attribute), castedValue).end())
+      case GreaterThan(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startNot()
+          .lessThanEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startNot().lessThan(quotedName, getType(attribute), castedValue).end())
+      case GreaterThanOrEqual(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startNot()
+          .lessThan(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        Some(builder.startAnd().isNull(quotedName, getType(attribute)).end())
+      case IsNull(name) if dataTypeMap.contains(name) =>
+        Some(builder.startAnd()
+          .isNull(dataTypeMap(name).fieldName, getType(name)).end())
 
-      case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        Some(builder.startNot().isNull(quotedName, getType(attribute)).end())
+      case IsNotNull(name) if dataTypeMap.contains(name) =>
+        Some(builder.startNot()
+          .isNull(dataTypeMap(name).fieldName, getType(name)).end())
 
-      case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(attribute)))
-        Some(builder.startAnd().in(quotedName, getType(attribute),
+      case In(name, values) if dataTypeMap.contains(name) =>
+        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name).fieldType))
+        Some(builder.startAnd().in(dataTypeMap(name).fieldName, getType(name),
           castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
 
       case _ => None
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
index 68503aba22b40..7fbc1cd205b13 100644
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
-import java.sql.Date
-
 import org.apache.orc.storage.common.`type`.HiveDecimal
 import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch
 import org.apache.orc.storage.ql.io.sarg.{SearchArgument => OrcSearchArgument}
@@ -38,7 +36,9 @@ private[sql] object OrcShimUtils {
   private[sql] type Operator = OrcOperator
   private[sql] type SearchArgument = OrcSearchArgument
 
-  def getSqlDate(value: Any): Date = value.asInstanceOf[DateWritable].get
+  def getGregorianDays(value: Any): Int = {
+    new DaysWritable(value.asInstanceOf[DateWritable]).gregorianDays
+  }
 
   def getDecimal(value: Any): Decimal = {
     val decimal = value.asInstanceOf[HiveDecimalWritable].getHiveDecimal()
@@ -47,13 +47,13 @@ private[sql] object OrcShimUtils {
 
   def getDateWritable(reuseObj: Boolean): (SpecializedGetters, Int) => DateWritable = {
     if (reuseObj) {
-      val result = new DateWritable()
+      val result = new DaysWritable()
       (getter, ordinal) =>
         result.set(getter.getInt(ordinal))
         result
     } else {
       (getter: SpecializedGetters, ordinal: Int) =>
-        new DateWritable(getter.getInt(ordinal))
+        new DaysWritable(getter.getInt(ordinal))
     }
   }
 
diff --git a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
index ee5162bced8ac..a3c2343a73b97 100644
--- a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -24,9 +24,10 @@ import java.sql.{Date, Timestamp}
 import scala.collection.JavaConverters._
 
 import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}
+import org.apache.orc.storage.ql.io.sarg.SearchArgumentFactory.newBuilder
 
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{AnalysisException, Column, DataFrame}
+import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Row}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
@@ -245,29 +246,41 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
   }
 
   test("filter pushdown - timestamp") {
-    val timeString = "2015-08-20 14:57:00"
-    val timestamps = (1 to 4).map { i =>
-      val milliseconds = Timestamp.valueOf(timeString).getTime + i * 3600
-      new Timestamp(milliseconds)
-    }
-    withOrcDataFrame(timestamps.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) <=> $"_1",
-        PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(timestamps(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+    val input = Seq(
+      "1000-01-01 01:02:03",
+      "1582-10-01 00:11:22",
+      "1900-01-01 23:59:59",
+      "2020-05-25 10:11:12").map(Timestamp.valueOf)
+
+    withOrcFile(input.map(Tuple1(_))) { path =>
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          readFile(path) { implicit df =>
+            val timestamps = input.map(Literal(_))
+            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+
+            checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+            checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
+
+            checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate(
+              Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+            checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate(
+              Literal(timestamps(2)) < $"_1",
+              PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(
+              Literal(timestamps(0)) >= $"_1",
+              PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+          }
+        }
+      }
     }
   }
 
@@ -299,26 +312,33 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
   }
 
   test("filter pushdown - date") {
-    val dates = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
+    val input = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
       Date.valueOf(day)
     }
-    withOrcDataFrame(dates.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(dates(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(dates(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(dates(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(dates(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(dates(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(dates(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+    withOrcFile(input.map(Tuple1(_))) { path =>
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          readFile(path) { implicit df =>
+            val dates = input.map(Literal(_))
+            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+
+            checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+            checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
+
+            checkFilterPredicate(dates(0) === $"_1", PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate(dates(0) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+            checkFilterPredicate(dates(1) > $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate(dates(2) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(dates(0) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(dates(3) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+          }
+        }
+      }
     }
   }
 
@@ -450,5 +470,142 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
       ).get.toString
     }
   }
+
+  test("SPARK-32622: case sensitivity in predicate pushdown") {
+    withTempPath { dir =>
+      val count = 10
+      val tableName = "spark_32622"
+      val tableDir1 = dir.getAbsoluteFile + "/table1"
+
+      // Physical ORC files have both `A` and `a` fields.
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        spark.range(count).repartition(count).selectExpr("id - 1 as A", "id as a")
+          .write.mode("overwrite").orc(tableDir1)
+      }
+
+      // Metastore table has both `A` and `a` fields too.
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG, a LONG) USING ORC LOCATION '$tableDir1'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select a, A from $tableName"), (0 until count).map(c => Row(c, c - 1)))
+
+          val actual1 = stripSparkFilter(sql(s"select A from $tableName where A < 0"))
+          assert(actual1.count() == 1)
+
+          val actual2 = stripSparkFilter(sql(s"select A from $tableName where a < 0"))
+          assert(actual2.count() == 0)
+        }
+
+        // Exception thrown for ambiguous case.
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          val e = intercept[AnalysisException] {
+            sql(s"select a from $tableName where a < 0").collect()
+          }
+          assert(e.getMessage.contains(
+            "Reference 'a' is ambiguous"))
+        }
+      }
+
+      // Metastore table has only `A` field.
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG) USING ORC LOCATION '$tableDir1'
+             """.stripMargin)
+
+          val e = intercept[SparkException] {
+            sql(s"select A from $tableName where A < 0").collect()
+          }
+          assert(e.getCause.isInstanceOf[RuntimeException] && e.getCause.getMessage.contains(
+            """Found duplicate field(s) "A": [A, a] in case-insensitive mode"""))
+        }
+      }
+
+      // Physical ORC files have only `A` field.
+      val tableDir2 = dir.getAbsoluteFile + "/table2"
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        spark.range(count).repartition(count).selectExpr("id - 1 as A")
+          .write.mode("overwrite").orc(tableDir2)
+      }
+
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (a LONG) USING ORC LOCATION '$tableDir2'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select a from $tableName"), (0 until count).map(c => Row(c - 1)))
+
+          val actual = stripSparkFilter(sql(s"select a from $tableName where a < 0"))
+          assert(actual.count() == 1)
+        }
+      }
+
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG) USING ORC LOCATION '$tableDir2'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select A from $tableName"), (0 until count).map(c => Row(c - 1)))
+
+          val actual = stripSparkFilter(sql(s"select A from $tableName where A < 0"))
+          assert(actual.count() == 1)
+        }
+      }
+    }
+  }
+
+  test("SPARK-32646: Case-insensitive field resolution for pushdown when reading ORC") {
+    import org.apache.spark.sql.sources._
+
+    def getOrcFilter(
+        schema: StructType,
+        filters: Seq[Filter],
+        caseSensitive: String): Option[SearchArgument] = {
+      var orcFilter: Option[SearchArgument] = None
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive) {
+        orcFilter =
+          OrcFilters.createFilter(schema, filters)
+      }
+      orcFilter
+    }
+
+    def testFilter(
+        schema: StructType,
+        filters: Seq[Filter],
+        expected: SearchArgument): Unit = {
+      val caseSensitiveFilters = getOrcFilter(schema, filters, "true")
+      val caseInsensitiveFilters = getOrcFilter(schema, filters, "false")
+
+      assert(caseSensitiveFilters.isEmpty)
+      assert(caseInsensitiveFilters.isDefined)
+
+      assert(caseInsensitiveFilters.get.getLeaves().size() > 0)
+      assert(caseInsensitiveFilters.get.getLeaves().size() == expected.getLeaves().size())
+      (0 until expected.getLeaves().size()).foreach { index =>
+        assert(caseInsensitiveFilters.get.getLeaves().get(index) == expected.getLeaves().get(index))
+      }
+    }
+
+    val schema = StructType(Seq(StructField("cint", IntegerType)))
+    testFilter(schema, Seq(GreaterThan("CINT", 1)),
+      newBuilder.startNot()
+        .lessThanEquals("cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`().build())
+    testFilter(schema, Seq(
+      And(GreaterThan("CINT", 1), EqualTo("Cint", 2))),
+      newBuilder.startAnd()
+        .startNot()
+        .lessThanEquals("cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`()
+        .equals("cint", OrcFilters.getPredicateLeafType(IntegerType), 2L)
+        .`end`().build())
+  }
 }
 
diff --git a/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
index 2f1925e69a337..6e55fedfc4deb 100644
--- a/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
+++ b/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
@@ -21,7 +21,10 @@
 
 import org.apache.hadoop.hive.ql.exec.vector.*;
 
+import org.apache.spark.sql.catalyst.util.DateTimeUtils;
+import org.apache.spark.sql.catalyst.util.RebaseDateTime;
 import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DateType;
 import org.apache.spark.sql.types.Decimal;
 import org.apache.spark.sql.types.TimestampType;
 import org.apache.spark.sql.vectorized.ColumnarArray;
@@ -41,6 +44,7 @@ public class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVecto
   private DecimalColumnVector decimalData;
   private TimestampColumnVector timestampData;
   private final boolean isTimestamp;
+  private final boolean isDate;
 
   private int batchSize;
 
@@ -53,6 +57,12 @@ public class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVecto
       isTimestamp = false;
     }
 
+    if (type instanceof DateType) {
+      isDate = true;
+    } else {
+      isDate = false;
+    }
+
     baseData = vector;
     if (vector instanceof LongColumnVector) {
       longData = (LongColumnVector) vector;
@@ -129,14 +139,19 @@ public short getShort(int rowId) {
 
   @Override
   public int getInt(int rowId) {
-    return (int) longData.vector[getRowIndex(rowId)];
+    int value = (int) longData.vector[getRowIndex(rowId)];
+    if (isDate) {
+      return RebaseDateTime.rebaseJulianToGregorianDays(value);
+    } else {
+      return value;
+    }
   }
 
   @Override
   public long getLong(int rowId) {
     int index = getRowIndex(rowId);
     if (isTimestamp) {
-      return timestampData.time[index] * 1000 + timestampData.nanos[index] / 1000 % 1000;
+      return DateTimeUtils.fromJavaTimestamp(timestampData.asScratchTimestamp(index));
     } else {
       return longData.vector[index];
     }
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 948ab44a8c19c..9f1927eff3993 100644
--- a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
+import java.time.{Instant, LocalDate}
+
 import org.apache.hadoop.hive.common.`type`.HiveDecimal
 import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.Builder
@@ -24,6 +26,8 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable
 
 import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
 
@@ -64,9 +68,11 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * Create ORC filter as a SearchArgument instance.
    */
   def createFilter(schema: StructType, filters: Seq[Filter]): Option[SearchArgument] = {
-    val dataTypeMap = schema.map(f => f.name -> f.dataType).toMap
+    val dataTypeMap = getDataTypeMap(schema, SQLConf.get.caseSensitiveAnalysis)
     // Combines all convertible filters using `And` to produce a single conjunction
-    val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, filters))
+    // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
+    val newFilters = filters.filter(!_.containsNestedColumn)
+    val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, newFilters))
     conjunctionOptional.map { conjunction =>
       // Then tries to build a single ORC `SearchArgument` for the conjunction predicate.
       // The input predicate is fully convertible. There should not be any empty result in the
@@ -77,7 +83,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
 
   def convertibleFilters(
       schema: StructType,
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       filters: Seq[Filter]): Seq[Filter] = {
     import org.apache.spark.sql.sources._
 
@@ -135,7 +141,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
   /**
    * Get PredicateLeafType which is corresponding to the given DataType.
    */
-  private def getPredicateLeafType(dataType: DataType) = dataType match {
+  private[sql] def getPredicateLeafType(dataType: DataType) = dataType match {
     case BooleanType => PredicateLeaf.Type.BOOLEAN
     case ByteType | ShortType | IntegerType | LongType => PredicateLeaf.Type.LONG
     case FloatType | DoubleType => PredicateLeaf.Type.FLOAT
@@ -159,6 +165,10 @@ private[sql] object OrcFilters extends OrcFiltersBase {
       value.asInstanceOf[Number].doubleValue()
     case _: DecimalType =>
       new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
+    case _: DateType if value.isInstanceOf[LocalDate] =>
+      toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
+    case _: TimestampType if value.isInstanceOf[Instant] =>
+      toJavaTimestamp(instantToMicros(value.asInstanceOf[Instant]))
     case _ => value
   }
 
@@ -171,7 +181,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * @return the builder so far.
    */
   private def buildSearchArgument(
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       expression: Filter,
       builder: Builder): Builder = {
     import org.apache.spark.sql.sources._
@@ -207,60 +217,59 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * @return the builder so far.
    */
   private def buildLeafSearchArgument(
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       expression: Filter,
       builder: Builder): Option[Builder] = {
     def getType(attribute: String): PredicateLeaf.Type =
-      getPredicateLeafType(dataTypeMap(attribute))
+      getPredicateLeafType(dataTypeMap(attribute).fieldType)
 
     import org.apache.spark.sql.sources._
 
     // NOTE: For all case branches dealing with leaf predicates below, the additional `startAnd()`
     // call is mandatory. ORC `SearchArgument` builder requires that all leaf predicates must be
     // wrapped by a "parent" predicate (`And`, `Or`, or `Not`).
+    // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
+    // in order to distinguish predicate pushdown for nested columns.
     expression match {
-      case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().equals(quotedName, getType(attribute), castedValue).end())
+      case EqualTo(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .equals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().nullSafeEquals(quotedName, getType(attribute), castedValue).end())
+      case EqualNullSafe(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .nullSafeEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().lessThan(quotedName, getType(attribute), castedValue).end())
+      case LessThan(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .lessThan(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().lessThanEquals(quotedName, getType(attribute), castedValue).end())
+      case LessThanOrEqual(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .lessThanEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startNot().lessThanEquals(quotedName, getType(attribute), castedValue).end())
+      case GreaterThan(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startNot()
+          .lessThanEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startNot().lessThan(quotedName, getType(attribute), castedValue).end())
+      case GreaterThanOrEqual(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startNot()
+          .lessThan(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        Some(builder.startAnd().isNull(quotedName, getType(attribute)).end())
+      case IsNull(name) if dataTypeMap.contains(name) =>
+        Some(builder.startAnd().isNull(dataTypeMap(name).fieldName, getType(name)).end())
 
-      case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        Some(builder.startNot().isNull(quotedName, getType(attribute)).end())
+      case IsNotNull(name) if dataTypeMap.contains(name) =>
+        Some(builder.startNot().isNull(dataTypeMap(name).fieldName, getType(name)).end())
 
-      case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
-        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(attribute)))
-        Some(builder.startAnd().in(quotedName, getType(attribute),
+      case In(name, values) if dataTypeMap.contains(name) =>
+        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name).fieldType))
+        Some(builder.startAnd().in(dataTypeMap(name).fieldName, getType(name),
           castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
 
       case _ => None
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
index c32f024476e69..60c5b7a266c51 100644
--- a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
+++ b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
-import java.sql.Date
-
 import org.apache.hadoop.hive.common.`type`.HiveDecimal
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch
 import org.apache.hadoop.hive.ql.io.sarg.{SearchArgument => OrcSearchArgument}
@@ -26,6 +24,7 @@ import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf.{Operator => OrcOperator}
 import org.apache.hadoop.hive.serde2.io.{DateWritable, HiveDecimalWritable}
 
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
+import org.apache.spark.sql.execution.datasources.DaysWritable
 import org.apache.spark.sql.types.Decimal
 
 /**
@@ -38,7 +37,9 @@ private[sql] object OrcShimUtils {
   private[sql] type Operator = OrcOperator
   private[sql] type SearchArgument = OrcSearchArgument
 
-  def getSqlDate(value: Any): Date = value.asInstanceOf[DateWritable].get
+  def getGregorianDays(value: Any): Int = {
+    new DaysWritable(value.asInstanceOf[DateWritable]).gregorianDays
+  }
 
   def getDecimal(value: Any): Decimal = {
     val decimal = value.asInstanceOf[HiveDecimalWritable].getHiveDecimal()
@@ -47,13 +48,13 @@ private[sql] object OrcShimUtils {
 
   def getDateWritable(reuseObj: Boolean): (SpecializedGetters, Int) => DateWritable = {
     if (reuseObj) {
-      val result = new DateWritable()
+      val result = new DaysWritable()
       (getter, ordinal) =>
         result.set(getter.getInt(ordinal))
         result
     } else {
       (getter: SpecializedGetters, ordinal: Int) =>
-        new DateWritable(getter.getInt(ordinal))
+        new DaysWritable(getter.getInt(ordinal))
     }
   }
 
diff --git a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
index 1baa69e82bb18..cb69413277f63 100644
--- a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -24,9 +24,10 @@ import java.sql.{Date, Timestamp}
 import scala.collection.JavaConverters._
 
 import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
 
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{AnalysisException, Column, DataFrame}
+import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Row}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
@@ -246,29 +247,41 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
   }
 
   test("filter pushdown - timestamp") {
-    val timeString = "2015-08-20 14:57:00"
-    val timestamps = (1 to 4).map { i =>
-      val milliseconds = Timestamp.valueOf(timeString).getTime + i * 3600
-      new Timestamp(milliseconds)
-    }
-    withOrcDataFrame(timestamps.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(
-        Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(timestamps(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+    val input = Seq(
+      "1000-01-01 01:02:03",
+      "1582-10-01 00:11:22",
+      "1900-01-01 23:59:59",
+      "2020-05-25 10:11:12").map(Timestamp.valueOf)
+
+    withOrcFile(input.map(Tuple1(_))) { path =>
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          readFile(path) { implicit df =>
+            val timestamps = input.map(Literal(_))
+            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+
+            checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+            checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
+
+            checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate(
+              Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+            checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate(
+              Literal(timestamps(2)) < $"_1",
+              PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(
+              Literal(timestamps(0)) >= $"_1",
+              PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+          }
+        }
+      }
     }
   }
 
@@ -300,26 +313,33 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
   }
 
   test("filter pushdown - date") {
-    val dates = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
+    val input = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
       Date.valueOf(day)
     }
-    withOrcDataFrame(dates.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(dates(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(dates(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(dates(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(dates(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(dates(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(dates(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+    withOrcFile(input.map(Tuple1(_))) { path =>
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          readFile(path) { implicit df =>
+            val dates = input.map(Literal(_))
+            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+
+            checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+            checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
+
+            checkFilterPredicate(dates(0) === $"_1", PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate(dates(0) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+            checkFilterPredicate(dates(1) > $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate(dates(2) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(dates(0) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(dates(3) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+          }
+        }
+      }
     }
   }
 
@@ -451,5 +471,142 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
       ).get.toString
     }
   }
+
+  test("SPARK-32622: case sensitivity in predicate pushdown") {
+    withTempPath { dir =>
+      val count = 10
+      val tableName = "spark_32622"
+      val tableDir1 = dir.getAbsoluteFile + "/table1"
+
+      // Physical ORC files have both `A` and `a` fields.
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        spark.range(count).repartition(count).selectExpr("id - 1 as A", "id as a")
+          .write.mode("overwrite").orc(tableDir1)
+      }
+
+      // Metastore table has both `A` and `a` fields too.
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG, a LONG) USING ORC LOCATION '$tableDir1'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select a, A from $tableName"), (0 until count).map(c => Row(c, c - 1)))
+
+          val actual1 = stripSparkFilter(sql(s"select A from $tableName where A < 0"))
+          assert(actual1.count() == 1)
+
+          val actual2 = stripSparkFilter(sql(s"select A from $tableName where a < 0"))
+          assert(actual2.count() == 0)
+        }
+
+        // Exception thrown for ambiguous case.
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          val e = intercept[AnalysisException] {
+            sql(s"select a from $tableName where a < 0").collect()
+          }
+          assert(e.getMessage.contains(
+            "Reference 'a' is ambiguous"))
+        }
+      }
+
+      // Metastore table has only `A` field.
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG) USING ORC LOCATION '$tableDir1'
+             """.stripMargin)
+
+          val e = intercept[SparkException] {
+            sql(s"select A from $tableName where A < 0").collect()
+          }
+          assert(e.getCause.isInstanceOf[RuntimeException] && e.getCause.getMessage.contains(
+            """Found duplicate field(s) "A": [A, a] in case-insensitive mode"""))
+        }
+      }
+
+      // Physical ORC files have only `A` field.
+      val tableDir2 = dir.getAbsoluteFile + "/table2"
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        spark.range(count).repartition(count).selectExpr("id - 1 as A")
+          .write.mode("overwrite").orc(tableDir2)
+      }
+
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (a LONG) USING ORC LOCATION '$tableDir2'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select a from $tableName"), (0 until count).map(c => Row(c - 1)))
+
+          val actual = stripSparkFilter(sql(s"select a from $tableName where a < 0"))
+          assert(actual.count() == 1)
+        }
+      }
+
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG) USING ORC LOCATION '$tableDir2'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select A from $tableName"), (0 until count).map(c => Row(c - 1)))
+
+          val actual = stripSparkFilter(sql(s"select A from $tableName where A < 0"))
+          assert(actual.count() == 1)
+        }
+      }
+    }
+  }
+
+  test("SPARK-32646: Case-insensitive field resolution for pushdown when reading ORC") {
+    import org.apache.spark.sql.sources._
+
+    def getOrcFilter(
+        schema: StructType,
+        filters: Seq[Filter],
+        caseSensitive: String): Option[SearchArgument] = {
+      var orcFilter: Option[SearchArgument] = None
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive) {
+        orcFilter =
+          OrcFilters.createFilter(schema, filters)
+      }
+      orcFilter
+    }
+
+    def testFilter(
+        schema: StructType,
+        filters: Seq[Filter],
+        expected: SearchArgument): Unit = {
+      val caseSensitiveFilters = getOrcFilter(schema, filters, "true")
+      val caseInsensitiveFilters = getOrcFilter(schema, filters, "false")
+
+      assert(caseSensitiveFilters.isEmpty)
+      assert(caseInsensitiveFilters.isDefined)
+
+      assert(caseInsensitiveFilters.get.getLeaves().size() > 0)
+      assert(caseInsensitiveFilters.get.getLeaves().size() == expected.getLeaves().size())
+      (0 until expected.getLeaves().size()).foreach { index =>
+        assert(caseInsensitiveFilters.get.getLeaves().get(index) == expected.getLeaves().get(index))
+      }
+    }
+
+    val schema = StructType(Seq(StructField("cint", IntegerType)))
+    testFilter(schema, Seq(GreaterThan("CINT", 1)),
+      newBuilder.startNot()
+        .lessThanEquals("cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`().build())
+    testFilter(schema, Seq(
+      And(GreaterThan("CINT", 1), EqualTo("Cint", 2))),
+      newBuilder.startAnd()
+        .startNot()
+        .lessThanEquals("cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`()
+        .equals("cint", OrcFilters.getPredicateLeafType(IntegerType), 2L)
+        .`end`().build())
+  }
 }
 
diff --git a/sql/create-docs.sh b/sql/create-docs.sh
index 4353708d22f7b..8721df874ee73 100755
--- a/sql/create-docs.sh
+++ b/sql/create-docs.sh
@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-# Script to create SQL API docs. This requires `mkdocs` and to build
+# Script to create SQL API and config docs. This requires `mkdocs` and to build
 # Spark first. After running this script the html docs can be found in
 # $SPARK_HOME/sql/site
 
@@ -27,26 +27,31 @@ set -e
 FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
 SPARK_HOME="$(cd "`dirname "${BASH_SOURCE[0]}"`"/..; pwd)"
 
-if ! hash python 2>/dev/null; then
-  echo "Missing python in your path, skipping SQL documentation generation."
+if ! hash python3 2>/dev/null; then
+  echo "Missing python3 in your path, skipping SQL documentation generation."
   exit 0
 fi
 
 if ! hash mkdocs 2>/dev/null; then
   echo "Missing mkdocs in your path, trying to install mkdocs for SQL documentation generation."
-  pip install mkdocs
+  pip3 install mkdocs
 fi
 
 pushd "$FWDIR" > /dev/null
 
-# Now create the markdown file
 rm -fr docs
 mkdir docs
-echo "Generating markdown files for SQL documentation."
-"$SPARK_HOME/bin/spark-submit" gen-sql-markdown.py
 
-# Now create the HTML files
-echo "Generating HTML files for SQL documentation."
+echo "Generating SQL API Markdown files."
+"$SPARK_HOME/bin/spark-submit" gen-sql-api-docs.py
+
+echo "Generating SQL configuration table HTML file."
+"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py
+
+echo "Generating HTML files for SQL function table and examples."
+"$SPARK_HOME/bin/spark-submit" gen-sql-functions-docs.py
+
+echo "Generating HTML files for SQL API documentation."
 mkdocs build --clean
 rm -fr docs
 
diff --git a/sql/gen-sql-markdown.py b/sql/gen-sql-api-docs.py
similarity index 94%
rename from sql/gen-sql-markdown.py
rename to sql/gen-sql-api-docs.py
index e0529f8310613..61328997c1c58 100644
--- a/sql/gen-sql-markdown.py
+++ b/sql/gen-sql-api-docs.py
@@ -15,10 +15,12 @@
 # limitations under the License.
 #
 
-import sys
 import os
 from collections import namedtuple
 
+from pyspark.java_gateway import launch_gateway
+
+
 ExpressionInfo = namedtuple(
     "ExpressionInfo", "className name usage arguments examples note since deprecated")
 
@@ -41,7 +43,7 @@ def _list_function_infos(jvm):
             usage=usage,
             arguments=jinfo.getArguments().replace("_FUNC_", name),
             examples=jinfo.getExamples().replace("_FUNC_", name),
-            note=jinfo.getNote(),
+            note=jinfo.getNote().replace("_FUNC_", name),
             since=jinfo.getSince(),
             deprecated=jinfo.getDeprecated()))
     return sorted(infos, key=lambda i: i.name)
@@ -158,7 +160,7 @@ def _make_pretty_deprecated(deprecated):
         return "**Deprecated:**\n%s\n" % deprecated
 
 
-def generate_sql_markdown(jvm, path):
+def generate_sql_api_markdown(jvm, path):
     """
     Generates a markdown file after listing the function information. The output file
     is created in `path`.
@@ -219,8 +221,7 @@ def generate_sql_markdown(jvm, path):
 
 
 if __name__ == "__main__":
-    from pyspark.java_gateway import launch_gateway
-
     jvm = launch_gateway().jvm
-    markdown_file_path = "%s/docs/index.md" % os.path.dirname(sys.argv[0])
-    generate_sql_markdown(jvm, markdown_file_path)
+    spark_root_dir = os.path.dirname(os.path.dirname(__file__))
+    markdown_file_path = os.path.join(spark_root_dir, "sql/docs/index.md")
+    generate_sql_api_markdown(jvm, markdown_file_path)
diff --git a/sql/gen-sql-config-docs.py b/sql/gen-sql-config-docs.py
new file mode 100644
index 0000000000000..f1980bcc0d80d
--- /dev/null
+++ b/sql/gen-sql-config-docs.py
@@ -0,0 +1,135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+
+from collections import namedtuple
+from textwrap import dedent
+
+# To avoid adding a new direct dependency, we import markdown from within mkdocs.
+from mkdocs.structure.pages import markdown
+
+from pyspark.java_gateway import launch_gateway
+
+
+SQLConfEntry = namedtuple(
+    "SQLConfEntry", ["name", "default", "description", "version"])
+
+
+def get_sql_configs(jvm, group):
+    if group == "static":
+        config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listStaticSQLConfigs()
+    else:
+        config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listRuntimeSQLConfigs()
+    sql_configs = [
+        SQLConfEntry(
+            name=_sql_config._1(),
+            default=_sql_config._2(),
+            description=_sql_config._3(),
+            version=_sql_config._4()
+        )
+        for _sql_config in config_set
+    ]
+    return sql_configs
+
+
+def generate_sql_configs_table_html(sql_configs, path):
+    """
+    Generates an HTML table at `path` that lists all public SQL
+    configuration options.
+
+    The table will look something like this:
+
+    ```html
+    <table class="table">
+    <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
+
+    <tr>
+        <td><code>spark.sql.adaptive.enabled</code></td>
+        <td>false</td>
+        <td><p>When true, enable adaptive query execution.</p></td>
+        <td>2.1.0</td>
+    </tr>
+
+    ...
+
+    </table>
+    ```
+    """
+    value_reference_pattern = re.compile(r"^<value of (\S*)>$")
+
+    with open(path, 'w') as f:
+        f.write(dedent(
+            """
+            <table class="table">
+            <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
+            """
+        ))
+        for config in sorted(sql_configs, key=lambda x: x.name):
+            if config.name == "spark.sql.session.timeZone":
+                default = "(value of local timezone)"
+            elif config.name == "spark.sql.warehouse.dir":
+                default = "(value of <code>$PWD/spark-warehouse</code>)"
+            elif config.default == "<undefined>":
+                default = "(none)"
+            elif config.default.startswith("<value of "):
+                referenced_config_name = value_reference_pattern.match(config.default).group(1)
+                default = "(value of <code>{}</code>)".format(referenced_config_name)
+            else:
+                default = config.default
+
+            if default.startswith("<"):
+                raise Exception(
+                    "Unhandled reference in SQL config docs. Config '{name}' "
+                    "has default '{default}' that looks like an HTML tag."
+                    .format(
+                        name=config.name,
+                        default=config.default,
+                    )
+                )
+
+            f.write(dedent(
+                """
+                <tr>
+                    <td><code>{name}</code></td>
+                    <td>{default}</td>
+                    <td>{description}</td>
+                    <td>{version}</td>
+                </tr>
+                """
+                .format(
+                    name=config.name,
+                    default=default,
+                    description=markdown.markdown(config.description),
+                    version=config.version
+                )
+            ))
+        f.write("</table>\n")
+
+
+if __name__ == "__main__":
+    jvm = launch_gateway().jvm
+    docs_root_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs")
+
+    sql_configs = get_sql_configs(jvm, "runtime")
+    sql_configs_table_path = os.path.join(docs_root_dir, "generated-runtime-sql-config-table.html")
+    generate_sql_configs_table_html(sql_configs, path=sql_configs_table_path)
+
+    sql_configs = get_sql_configs(jvm, "static")
+    sql_configs_table_path = os.path.join(docs_root_dir, "generated-static-sql-config-table.html")
+    generate_sql_configs_table_html(sql_configs, path=sql_configs_table_path)
diff --git a/sql/gen-sql-functions-docs.py b/sql/gen-sql-functions-docs.py
new file mode 100644
index 0000000000000..c07734e273051
--- /dev/null
+++ b/sql/gen-sql-functions-docs.py
@@ -0,0 +1,230 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import itertools
+import os
+import re
+from collections import namedtuple
+
+# To avoid adding a new direct dependency, we import markdown from within mkdocs.
+from mkdocs.structure.pages import markdown
+
+from pyspark.java_gateway import launch_gateway
+
+
+ExpressionInfo = namedtuple("ExpressionInfo", "name usage examples group")
+
+groups = {
+    "agg_funcs", "array_funcs", "datetime_funcs",
+    "json_funcs", "map_funcs", "window_funcs",
+}
+
+
+def _list_grouped_function_infos(jvm):
+    """
+    Returns a list of function information grouped by each group value via JVM.
+    Sorts wrapped expression infos in each group by name and returns them.
+    """
+
+    jinfos = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listBuiltinFunctionInfos()
+    infos = []
+
+    for jinfo in filter(lambda x: x.getGroup() in groups, jinfos):
+        name = jinfo.getName()
+        usage = jinfo.getUsage()
+        usage = usage.replace("_FUNC_", name) if usage is not None else usage
+        infos.append(ExpressionInfo(
+            name=name,
+            usage=usage,
+            examples=jinfo.getExamples().replace("_FUNC_", name),
+            group=jinfo.getGroup()))
+
+    # Groups expression info by each group value
+    grouped_infos = itertools.groupby(sorted(infos, key=lambda x: x.group), key=lambda x: x.group)
+    # Then, sort expression infos in each group by name
+    return [(k, sorted(g, key=lambda x: x.name)) for k, g in grouped_infos]
+
+
+# TODO(SPARK-31499): Needs to add a column to describe arguments and their types
+def _make_pretty_usage(infos):
+    """
+    Makes the usage description pretty and returns a formatted string.
+
+    Expected input:
+
+        func(*) - ...
+
+        func(expr[, expr...]) - ...
+
+    Expected output:
+    <table class="table">
+      <thead>
+        <tr>
+          <th style="width:25%">Function</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>func(*)</td>
+          <td>...</td>
+        </tr>
+        <tr>
+          <td>func(expr[, expr...])</td>
+          <td>...</td>
+        </tr>
+      </tbody>
+      ...
+    </table>
+
+    """
+
+    result = []
+    result.append("<table class=\"table\">")
+    result.append("  <thead>")
+    result.append("    <tr>")
+    result.append("      <th style=\"width:25%\">Function</th>")
+    result.append("      <th>Description</th>")
+    result.append("    </tr>")
+    result.append("  </thead>")
+    result.append("  <tbody>")
+
+    for info in infos:
+        # Extracts (signature, description) pairs from `info.usage`.
+        # Expected formats are as follows;
+        #  - `_FUNC_(...) - description`, or
+        #  - `_FUNC_ - description`
+        usages = iter(re.split(r"(%s.*) - " % info.name, info.usage.strip())[1:])
+        for (sig, description) in zip(usages, usages):
+            result.append("    <tr>")
+            result.append("      <td>%s</td>" % sig)
+            result.append("      <td>%s</td>" % description.strip())
+            result.append("    </tr>")
+
+    result.append("  </tbody>")
+    result.append("</table>\n")
+    return "\n".join(result)
+
+
+def _make_pretty_examples(jspark, infos):
+    """
+    Makes the examples description pretty and returns a formatted string if `infos`
+    has any `examples` starting with the example prefix. Otherwise, returns None.
+
+    Expected input:
+
+        Examples:
+          > SELECT func(col)...;
+           ...
+          > SELECT func(col)...;
+           ...
+
+    Expected output:
+    <div class="codehilite"><pre><span></span>
+      <span class="c1">-- func</span>
+      <span class="k">SELECT</span>
+      ...
+    </pre></div>
+    ```
+
+    """
+
+    pretty_output = ""
+    for info in infos:
+        if info.examples.startswith("\n    Examples:"):
+            output = []
+            output.append("-- %s" % info.name)
+            query_examples = filter(lambda x: x.startswith("      > "), info.examples.split("\n"))
+            for query_example in query_examples:
+                query = query_example.lstrip("      > ")
+                print("    %s" % query)
+                query_output = jspark.sql(query).showString(20, 20, False)
+                output.append(query)
+                output.append(query_output)
+            pretty_output += "\n" + "\n".join(output)
+    if pretty_output != "":
+        return markdown.markdown(
+            "```sql%s```" % pretty_output, extensions=['codehilite', 'fenced_code'])
+
+
+def generate_functions_table_html(jvm, html_output_dir):
+    """
+    Generates a HTML file after listing the function information. The output file
+    is created under `html_output_dir`.
+
+    Expected output:
+
+    <table class="table">
+      <thead>
+        <tr>
+          <th style="width:25%">Function</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>func(*)</td>
+          <td>...</td>
+        </tr>
+        <tr>
+          <td>func(expr[, expr...])</td>
+          <td>...</td>
+        </tr>
+      </tbody>
+      ...
+    </table>
+
+    """
+    for key, infos in _list_grouped_function_infos(jvm):
+        function_table = _make_pretty_usage(infos)
+        key = key.replace("_", "-")
+        with open("%s/generated-%s-table.html" % (html_output_dir, key), 'w') as table_html:
+            table_html.write(function_table)
+
+
+def generate_functions_examples_html(jvm, jspark, html_output_dir):
+    """
+    Generates a HTML file after listing and executing the function information.
+    The output file is created under `html_output_dir`.
+
+    Expected output:
+
+    <div class="codehilite"><pre><span></span>
+      <span class="c1">-- func</span>
+      <span class="k">SELECT</span>
+      ...
+    </pre></div>
+
+    """
+    print("Running SQL examples to generate formatted output.")
+    for key, infos in _list_grouped_function_infos(jvm):
+        examples = _make_pretty_examples(jspark, infos)
+        key = key.replace("_", "-")
+        if examples is not None:
+            with open("%s/generated-%s-examples.html" % (
+                    html_output_dir, key), 'w') as examples_html:
+                examples_html.write(examples)
+
+
+if __name__ == "__main__":
+    jvm = launch_gateway().jvm
+    jspark = jvm.org.apache.spark.sql.SparkSession.builder().getOrCreate()
+    jspark.sparkContext().setLogLevel("ERROR")  # Make it less noisy.
+    spark_root_dir = os.path.dirname(os.path.dirname(__file__))
+    html_output_dir = os.path.join(spark_root_dir, "docs")
+    generate_functions_table_html(jvm, html_output_dir)
+    generate_functions_examples_html(jvm, jspark, html_output_dir)
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 75c7f77942396..f376afbd19599 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala
index f15193b0dc3cc..f9f2ceeed8a75 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hive.thriftserver
 import java.util.Locale
 import java.util.concurrent.atomic.AtomicBoolean
 
+import org.apache.hadoop.hive.common.ServerUtils
 import org.apache.hadoop.hive.conf.HiveConf
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
 import org.apache.hive.service.cli.thrift.{ThriftBinaryCLIService, ThriftHttpCLIService}
@@ -101,6 +102,8 @@ object HiveThriftServer2 extends Logging {
       SparkSQLEnv.sqlContext.sessionState.newHadoopConf())
 
     try {
+      // Cleanup the scratch dir before starting
+      ServerUtils.cleanUpScratchDir(executionHive.conf)
       val server = new HiveThriftServer2(SparkSQLEnv.sqlContext)
       server.init(executionHive.conf)
       server.start()
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index 76d07848f79a9..33f7f21276611 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -44,12 +44,13 @@ import org.apache.spark.unsafe.types.CalendarInterval
 import org.apache.spark.util.{Utils => SparkUtils}
 
 private[hive] class SparkExecuteStatementOperation(
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     statement: String,
     confOverlay: JMap[String, String],
     runInBackground: Boolean = true)
-    (sqlContext: SQLContext, sessionToActivePool: JMap[SessionHandle, String])
   extends ExecuteStatementOperation(parentSession, statement, confOverlay, runInBackground)
+  with SparkOperation
   with Logging {
 
   private var result: DataFrame = _
@@ -62,7 +63,6 @@ private[hive] class SparkExecuteStatementOperation(
   private var previousFetchStartOffset: Long = 0
   private var iter: Iterator[SparkRow] = _
   private var dataTypes: Array[DataType] = _
-  private var statementId: String = _
 
   private lazy val resultSchema: TableSchema = {
     if (result == null || result.schema.isEmpty) {
@@ -73,13 +73,6 @@ private[hive] class SparkExecuteStatementOperation(
     }
   }
 
-  override def close(): Unit = {
-    // RDDs will be cleaned automatically upon garbage collection.
-    logInfo(s"Close statement with $statementId")
-    cleanup(OperationState.CLOSED)
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
-
   def addNonNullColumnValue(from: SparkRow, to: ArrayBuffer[Any], ordinal: Int): Unit = {
     dataTypes(ordinal) match {
       case StringType =>
@@ -100,12 +93,15 @@ private[hive] class SparkExecuteStatementOperation(
         to += from.getByte(ordinal)
       case ShortType =>
         to += from.getShort(ordinal)
-      case DateType =>
-        to += from.getAs[Date](ordinal)
-      case TimestampType =>
-        to += from.getAs[Timestamp](ordinal)
       case BinaryType =>
         to += from.getAs[Array[Byte]](ordinal)
+      // SPARK-31859, SPARK-31861: Date and Timestamp need to be turned to String here to:
+      // - respect spark.sql.session.timeZone
+      // - work with spark.sql.datetime.java8API.enabled
+      // These types have always been sent over the wire as string, converted later.
+      case _: DateType | _: TimestampType =>
+        val hiveString = HiveResult.toHiveString((from.get(ordinal), dataTypes(ordinal)))
+        to += hiveString
       case CalendarIntervalType =>
         to += HiveResult.toHiveString((from.getAs[CalendarInterval](ordinal), CalendarIntervalType))
       case _: ArrayType | _: StructType | _: MapType | _: UserDefinedType[_] =>
@@ -114,7 +110,7 @@ private[hive] class SparkExecuteStatementOperation(
     }
   }
 
-  def getNextRowSet(order: FetchOrientation, maxRowsL: Long): RowSet = withSchedulerPool {
+  def getNextRowSet(order: FetchOrientation, maxRowsL: Long): RowSet = withLocalProperties {
     log.info(s"Received getNextRowSet request order=${order} and maxRowsL=${maxRowsL} " +
       s"with ${statementId}")
     validateDefaultFetchOrientation(order)
@@ -193,7 +189,6 @@ private[hive] class SparkExecuteStatementOperation(
 
   override def runInternal(): Unit = {
     setState(OperationState.PENDING)
-    statementId = UUID.randomUUID().toString
     logInfo(s"Submitting query '$statement' with $statementId")
     HiveThriftServer2.eventManager.onStatementStart(
       statementId,
@@ -217,7 +212,9 @@ private[hive] class SparkExecuteStatementOperation(
             override def run(): Unit = {
               registerCurrentOperationLog()
               try {
-                execute()
+                withLocalProperties {
+                  execute()
+                }
               } catch {
                 case e: HiveSQLException =>
                   setOperationException(e)
@@ -259,7 +256,7 @@ private[hive] class SparkExecuteStatementOperation(
     }
   }
 
-  private def execute(): Unit = withSchedulerPool {
+  private def execute(): Unit = {
     try {
       synchronized {
         if (getStatus.getState.isTerminal) {
@@ -274,16 +271,14 @@ private[hive] class SparkExecuteStatementOperation(
       val executionHiveClassLoader = sqlContext.sharedState.jarClassLoader
       Thread.currentThread().setContextClassLoader(executionHiveClassLoader)
 
+      // Always set the session state classloader to `executionHiveClassLoader` even for sync mode
+      if (!runInBackground) {
+        parentSession.getSessionState.getConf.setClassLoader(executionHiveClassLoader)
+      }
+
       sqlContext.sparkContext.setJobGroup(statementId, statement)
       result = sqlContext.sql(statement)
       logDebug(result.queryExecution.toString())
-      result.queryExecution.logical match {
-        case SetCommand(Some((SQLConf.THRIFTSERVER_POOL.key, Some(value)))) =>
-          sessionToActivePool.put(parentSession.getSessionHandle, value)
-          logInfo(s"Setting ${SparkContext.SPARK_SCHEDULER_POOL}=$value for future statements " +
-            "in this session.")
-        case _ =>
-      }
       HiveThriftServer2.eventManager.onStatementParsed(statementId,
         result.queryExecution.toString())
       iter = {
@@ -295,7 +290,7 @@ private[hive] class SparkExecuteStatementOperation(
           resultList.get.iterator
         }
       }
-      dataTypes = result.queryExecution.analyzed.output.map(_.dataType).toArray
+      dataTypes = result.schema.fields.map(_.dataType)
     } catch {
       // Actually do need to catch Throwable as some failures don't inherit from Exception and
       // HiveServer will silently swallow them.
@@ -341,38 +336,25 @@ private[hive] class SparkExecuteStatementOperation(
     synchronized {
       if (!getStatus.getState.isTerminal) {
         logInfo(s"Cancel query with $statementId")
-        cleanup(OperationState.CANCELED)
+        setState(OperationState.CANCELED)
+        cleanup()
         HiveThriftServer2.eventManager.onStatementCanceled(statementId)
       }
     }
   }
 
-  private def cleanup(state: OperationState): Unit = {
-    setState(state)
+  override protected def cleanup(): Unit = {
     if (runInBackground) {
       val backgroundHandle = getBackgroundHandle()
       if (backgroundHandle != null) {
         backgroundHandle.cancel(true)
       }
     }
+    // RDDs will be cleaned automatically upon garbage collection.
     if (statementId != null) {
       sqlContext.sparkContext.cancelJobGroup(statementId)
     }
   }
-
-  private def withSchedulerPool[T](body: => T): T = {
-    val pool = sessionToActivePool.get(parentSession.getSessionHandle)
-    if (pool != null) {
-      sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, pool)
-    }
-    try {
-      body
-    } finally {
-      if (pool != null) {
-        sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, null)
-      }
-    }
-  }
 }
 
 object SparkExecuteStatementOperation {
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala
index 2945cfd200e46..55070e035b944 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala
@@ -36,19 +36,13 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param parentSession a HiveSession from SessionManager
  */
 private[hive] class SparkGetCatalogsOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession)
-  extends GetCatalogsOperation(parentSession) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetCatalogsOperation(parentSession)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     val logMsg = "Listing catalogs"
     logInfo(s"$logMsg with $statementId")
     setState(OperationState.RUNNING)
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala
index ff7cbfeae13be..ca8ad5e6ad134 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala
@@ -48,26 +48,19 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param columnName column name
  */
 private[hive] class SparkGetColumnsOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String,
     tableName: String,
     columnName: String)
   extends GetColumnsOperation(parentSession, catalogName, schemaName, tableName, columnName)
-    with Logging {
+  with SparkOperation
+  with Logging {
 
   val catalog: SessionCatalog = sqlContext.sessionState.catalog
 
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
-
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName, tablePattern : $tableName"
     val logMsg = s"Listing columns '$cmdStr, columnName : $columnName'"
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala
index d9c12b6ca9e64..f5e647bfd4f38 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala
@@ -43,22 +43,16 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param functionName function name pattern
  */
 private[hive] class SparkGetFunctionsOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String,
     functionName: String)
-  extends GetFunctionsOperation(parentSession, catalogName, schemaName, functionName) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetFunctionsOperation(parentSession, catalogName, schemaName, functionName)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName"
     val logMsg = s"Listing functions '$cmdStr, functionName : $functionName'"
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala
index db19880d1b99f..74220986fcd34 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala
@@ -40,21 +40,15 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param schemaName database name, null or a concrete database name
  */
 private[hive] class SparkGetSchemasOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String)
-  extends GetSchemasOperation(parentSession, catalogName, schemaName) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetSchemasOperation(parentSession, catalogName, schemaName)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName"
     val logMsg = s"Listing databases '$cmdStr'"
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala
index b4093e58d3c07..1cf9c3a731af5 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala
@@ -37,16 +37,11 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param parentSession a HiveSession from SessionManager
  */
 private[hive] class SparkGetTableTypesOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession)
-  extends GetTableTypesOperation(parentSession) with SparkMetadataOperationUtils with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetTableTypesOperation(parentSession)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
     statementId = UUID.randomUUID().toString
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala
index 45c6d980aac47..a1d21e2d60c63 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala
@@ -46,24 +46,17 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param tableTypes list of allowed table types, e.g. "TABLE", "VIEW"
  */
 private[hive] class SparkGetTablesOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String,
     tableName: String,
     tableTypes: JList[String])
   extends GetTablesOperation(parentSession, catalogName, schemaName, tableName, tableTypes)
-    with SparkMetadataOperationUtils with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName"
     val tableTypesStr = if (tableTypes == null) "null" else tableTypes.asScala.mkString(",")
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala
index dd5668a93f82d..e38139d60df60 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala
@@ -36,16 +36,11 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param parentSession a HiveSession from SessionManager
  */
 private[hive] class SparkGetTypeInfoOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession)
-  extends GetTypeInfoOperation(parentSession) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetTypeInfoOperation(parentSession)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
     statementId = UUID.randomUUID().toString
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala
new file mode 100644
index 0000000000000..59516dc26bae6
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.thriftserver
+
+import org.apache.hive.service.cli.OperationState
+import org.apache.hive.service.cli.operation.Operation
+
+import org.apache.spark.SparkContext
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.{SparkSession, SQLContext}
+import org.apache.spark.sql.catalyst.catalog.CatalogTableType
+import org.apache.spark.sql.catalyst.catalog.CatalogTableType.{EXTERNAL, MANAGED, VIEW}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.util.Utils
+
+/**
+ * Utils for Spark operations.
+ */
+private[hive] trait SparkOperation extends Operation with Logging {
+
+  protected def sqlContext: SQLContext
+
+  protected var statementId = getHandle().getHandleIdentifier().getPublicId().toString()
+
+  protected def cleanup(): Unit = Unit // noop by default
+
+  abstract override def run(): Unit = {
+    withLocalProperties {
+      super.run()
+    }
+  }
+
+  abstract override def close(): Unit = {
+    super.close()
+    cleanup()
+    logInfo(s"Close statement with $statementId")
+    HiveThriftServer2.eventManager.onOperationClosed(statementId)
+  }
+
+  // Set thread local properties for the execution of the operation.
+  // This method should be applied during the execution of the operation, by all the child threads.
+  // The original spark context local properties will be restored after the operation.
+  //
+  // It is used to:
+  // - set appropriate SparkSession
+  // - set scheduler pool for the operation
+  def withLocalProperties[T](f: => T): T = {
+    val originalProps = Utils.cloneProperties(sqlContext.sparkContext.getLocalProperties)
+    val originalSession = SparkSession.getActiveSession
+
+    try {
+      // Set active SparkSession
+      SparkSession.setActiveSession(sqlContext.sparkSession)
+
+      // Set scheduler pool
+      sqlContext.sparkSession.conf.getOption(SQLConf.THRIFTSERVER_POOL.key) match {
+        case Some(pool) =>
+          sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, pool)
+        case None =>
+      }
+
+      // run the body
+      f
+    } finally {
+      // reset local properties, will also reset SPARK_SCHEDULER_POOL
+      sqlContext.sparkContext.setLocalProperties(originalProps)
+
+      originalSession match {
+        case Some(session) => SparkSession.setActiveSession(session)
+        case None => SparkSession.clearActiveSession()
+      }
+    }
+  }
+
+  def tableTypeString(tableType: CatalogTableType): String = tableType match {
+    case EXTERNAL | MANAGED => "TABLE"
+    case VIEW => "VIEW"
+    case t =>
+      throw new IllegalArgumentException(s"Unknown table type is found: $t")
+  }
+}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index b665d4a31b9b1..581aa68b6ba3f 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -44,7 +44,9 @@ import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.hive.HiveUtils
+import org.apache.spark.sql.hive.client.HiveClientImpl
 import org.apache.spark.sql.hive.security.HiveDelegationTokenProvider
+import org.apache.spark.sql.internal.SharedState
 import org.apache.spark.util.ShutdownHookManager
 
 /**
@@ -88,12 +90,7 @@ private[hive] object SparkSQLCLIDriver extends Logging {
     val hadoopConf = SparkHadoopUtil.get.newConfiguration(sparkConf)
     val extraConfigs = HiveUtils.formatTimeVarsForHiveClient(hadoopConf)
 
-    val cliConf = new HiveConf(classOf[SessionState])
-    (hadoopConf.iterator().asScala.map(kv => kv.getKey -> kv.getValue)
-      ++ sparkConf.getAll.toMap ++ extraConfigs).foreach {
-      case (k, v) =>
-        cliConf.set(k, v)
-    }
+    val cliConf = HiveClientImpl.newHiveConf(sparkConf, hadoopConf, extraConfigs)
 
     val sessionState = new CliSessionState(cliConf)
 
@@ -134,6 +131,7 @@ private[hive] object SparkSQLCLIDriver extends Logging {
       UserGroupInformation.getCurrentUser.addCredentials(credentials)
     }
 
+    SharedState.loadHiveConfFile(sparkConf, conf)
     SessionState.start(sessionState)
 
     // Clean up after we exit
@@ -192,8 +190,11 @@ private[hive] object SparkSQLCLIDriver extends Logging {
     // Execute -i init files (always in silent mode)
     cli.processInitFiles(sessionState)
 
-    newHiveConf.foreach { kv =>
-      SparkSQLEnv.sqlContext.setConf(kv._1, kv._2)
+    // We don't propagate hive.metastore.warehouse.dir, because it might has been adjusted in
+    // [[SharedState.loadHiveConfFile]] based on the user specified or default values of
+    // spark.sql.warehouse.dir and hive.metastore.warehouse.dir.
+    for ((k, v) <- newHiveConf if k != "hive.metastore.warehouse.dir") {
+      SparkSQLEnv.sqlContext.setConf(k, v)
     }
 
     if (sessionState.execString != null) {
@@ -379,10 +380,18 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
 
           ret = rc.getResponseCode
           if (ret != 0) {
-            // For analysis exception, only the error is printed out to the console.
-            rc.getException() match {
-              case e : AnalysisException =>
-                err.println(s"""Error in query: ${e.getMessage}""")
+            rc.getException match {
+              case e: AnalysisException => e.cause match {
+                case Some(_) if !sessionState.getIsSilent =>
+                  err.println(
+                    s"""Error in query: ${e.getMessage}
+                       |${org.apache.hadoop.util.StringUtils.stringifyException(e)}
+                     """.stripMargin)
+                // For analysis exceptions in silent mode or simple ones that only related to the
+                // query itself, such as `NoSuchDatabaseException`, only the error is printed out
+                // to the console.
+                case _ => err.println(s"""Error in query: ${e.getMessage}""")
+              }
               case _ => err.println(rc.getErrorMessage())
             }
             driver.close()
@@ -490,7 +499,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
             val ignoreErrors = HiveConf.getBoolVar(conf, HiveConf.ConfVars.CLIIGNOREERRORS)
             if (ret != 0 && !ignoreErrors) {
               CommandProcessorFactory.clean(conf.asInstanceOf[HiveConf])
-              ret
+              return ret
             }
           }
         }
@@ -506,35 +515,86 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
   }
 
   // Adapted splitSemiColon from Hive 2.3's CliDriver.splitSemiColon.
+  // Note: [SPARK-31595] if there is a `'` in a double quoted string, or a `"` in a single quoted
+  // string, the origin implementation from Hive will not drop the trailing semicolon as expected,
+  // hence we refined this function a little bit.
+  // Note: [SPARK-33100] Ignore a semicolon inside a bracketed comment in spark-sql.
   private def splitSemiColon(line: String): JList[String] = {
     var insideSingleQuote = false
     var insideDoubleQuote = false
+    var insideSimpleComment = false
+    var bracketedCommentLevel = 0
     var escape = false
     var beginIndex = 0
+    var leavingBracketedComment = false
+    var isStatement = false
     val ret = new JArrayList[String]
+
+    def insideBracketedComment: Boolean = bracketedCommentLevel > 0
+    def insideComment: Boolean = insideSimpleComment || insideBracketedComment
+    def statementInProgress(index: Int): Boolean = isStatement || (!insideComment &&
+      index > beginIndex && !s"${line.charAt(index)}".trim.isEmpty)
+
     for (index <- 0 until line.length) {
-      if (line.charAt(index) == '\'') {
+      // Checks if we need to decrement a bracketed comment level; the last character '/' of
+      // bracketed comments is still inside the comment, so `insideBracketedComment` must keep true
+      // in the previous loop and we decrement the level here if needed.
+      if (leavingBracketedComment) {
+        bracketedCommentLevel -= 1
+        leavingBracketedComment = false
+      }
+
+      if (line.charAt(index) == '\'' && !insideComment) {
         // take a look to see if it is escaped
-        if (!escape) {
+        // See the comment above about SPARK-31595
+        if (!escape && !insideDoubleQuote) {
           // flip the boolean variable
           insideSingleQuote = !insideSingleQuote
         }
-      } else if (line.charAt(index) == '\"') {
+      } else if (line.charAt(index) == '\"' && !insideComment) {
         // take a look to see if it is escaped
-        if (!escape) {
+        // See the comment above about SPARK-31595
+        if (!escape && !insideSingleQuote) {
           // flip the boolean variable
           insideDoubleQuote = !insideDoubleQuote
         }
+      } else if (line.charAt(index) == '-') {
+        val hasNext = index + 1 < line.length
+        if (insideDoubleQuote || insideSingleQuote || insideComment) {
+          // Ignores '-' in any case of quotes or comment.
+          // Avoids to start a comment(--) within a quoted segment or already in a comment.
+          // Sample query: select "quoted value --"
+          //                                    ^^ avoids starting a comment if it's inside quotes.
+        } else if (hasNext && line.charAt(index + 1) == '-') {
+          // ignore quotes and ; in simple comment
+          insideSimpleComment = true
+        }
       } else if (line.charAt(index) == ';') {
-        if (insideSingleQuote || insideDoubleQuote) {
+        if (insideSingleQuote || insideDoubleQuote || insideComment) {
           // do not split
         } else {
-          // split, do not include ; itself
-          ret.add(line.substring(beginIndex, index))
+          if (isStatement) {
+            // split, do not include ; itself
+            ret.add(line.substring(beginIndex, index))
+          }
           beginIndex = index + 1
+          isStatement = false
+        }
+      } else if (line.charAt(index) == '\n') {
+        // with a new line the inline simple comment should end.
+        if (!escape) {
+          insideSimpleComment = false
+        }
+      } else if (line.charAt(index) == '/' && !insideSimpleComment) {
+        val hasNext = index + 1 < line.length
+        if (insideSingleQuote || insideDoubleQuote) {
+          // Ignores '/' in any case of quotes
+        } else if (insideBracketedComment && line.charAt(index - 1) == '*' ) {
+          // Decrements `bracketedCommentLevel` at the beginning of the next loop
+          leavingBracketedComment = true
+        } else if (hasNext && !insideBracketedComment && line.charAt(index + 1) == '*') {
+          bracketedCommentLevel += 1
         }
-      } else {
-        // nothing to do
       }
       // set the escape
       if (escape) {
@@ -542,8 +602,12 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
       } else if (line.charAt(index) == '\\') {
         escape = true
       }
+
+      isStatement = statementInProgress(index)
+    }
+    if (isStatement) {
+      ret.add(line.substring(beginIndex))
     }
-    ret.add(line.substring(beginIndex))
     ret
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
index 362ac362e9718..12fba0eae6dce 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
@@ -61,7 +61,7 @@ private[hive] class SparkSQLDriver(val context: SQLContext = SparkSQLEnv.sqlCont
     try {
       context.sparkContext.setJobDescription(command)
       val execution = context.sessionState.executePlan(context.sql(command).logicalPlan)
-      hiveResponse = SQLExecution.withNewExecutionId(context.sparkSession, execution) {
+      hiveResponse = SQLExecution.withNewExecutionId(execution) {
         hiveResultString(execution.executedPlan)
       }
       tableSchema = getResultSetSchema(execution)
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
index b3171897141c2..e10e7ed1a2769 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
@@ -78,7 +78,6 @@ private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext:
     val ctx = sparkSqlOperationManager.sessionToContexts.getOrDefault(sessionHandle, sqlContext)
     ctx.sparkSession.sessionState.catalog.getTempViewNames().foreach(ctx.uncacheTable)
     super.closeSession(sessionHandle)
-    sparkSqlOperationManager.sessionToActivePool.remove(sessionHandle)
     sparkSqlOperationManager.sessionToContexts.remove(sessionHandle)
   }
 
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
index 3396560f43502..bc9c13eb0d4f8 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
@@ -38,7 +38,6 @@ private[thriftserver] class SparkSQLOperationManager()
   val handleToOperation = ReflectionUtils
     .getSuperField[JMap[OperationHandle, Operation]](this, "handleToOperation")
 
-  val sessionToActivePool = new ConcurrentHashMap[SessionHandle, String]()
   val sessionToContexts = new ConcurrentHashMap[SessionHandle, SQLContext]()
 
   override def newExecuteStatementOperation(
@@ -51,8 +50,8 @@ private[thriftserver] class SparkSQLOperationManager()
       s" initialized or had already closed.")
     val conf = sqlContext.sessionState.conf
     val runInBackground = async && conf.getConf(HiveUtils.HIVE_THRIFT_SERVER_ASYNC)
-    val operation = new SparkExecuteStatementOperation(parentSession, statement, confOverlay,
-      runInBackground)(sqlContext, sessionToActivePool)
+    val operation = new SparkExecuteStatementOperation(
+      sqlContext, parentSession, statement, confOverlay, runInBackground)
     handleToOperation.put(operation.getHandle, operation)
     logDebug(s"Created Operation for $statement with session=$parentSession, " +
       s"runInBackground=$runInBackground")
diff --git a/sql/hive-thriftserver/src/test/noclasspath/hive-site.xml b/sql/hive-thriftserver/src/test/noclasspath/hive-site.xml
new file mode 100644
index 0000000000000..d0bf04d36b00d
--- /dev/null
+++ b/sql/hive-thriftserver/src/test/noclasspath/hive-site.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+  <property>
+    <name>hive.in.test</name>
+      <value>true</value>
+      <description>Internal marker for test.</description>
+  </property>
+  <property>
+    <name>hive.metastore.warehouse.dir</name>
+    <value>/tmp/hive_one</value>
+  </property>
+</configuration>
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index 6609701be0ede..4a075d301b601 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -27,22 +27,23 @@ import scala.concurrent.Promise
 import scala.concurrent.duration._
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.hive.test.HiveTestJars
+import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer
 import org.apache.spark.util.{ThreadUtils, Utils}
 
 /**
- * A test suite for the `spark-sql` CLI tool.  Note that all test cases share the same temporary
- * Hive metastore and warehouse.
+ * A test suite for the `spark-sql` CLI tool.
  */
 class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
   val warehousePath = Utils.createTempDir()
   val metastorePath = Utils.createTempDir()
   val scratchDirPath = Utils.createTempDir()
+  val sparkWareHouseDir = Utils.createTempDir()
 
   override def beforeAll(): Unit = {
     super.beforeAll()
@@ -53,9 +54,9 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
 
   override def afterAll(): Unit = {
     try {
-      warehousePath.delete()
-      metastorePath.delete()
-      scratchDirPath.delete()
+      Utils.deleteRecursively(warehousePath)
+      Utils.deleteRecursively(metastorePath)
+      Utils.deleteRecursively(scratchDirPath)
     } finally {
       super.afterAll()
     }
@@ -70,30 +71,63 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
    *                       is taken as an immediate error condition. That is: if a line containing
    *                       with one of these strings is found, fail the test immediately.
    *                       The default value is `Seq("Error:")`
+   * @param maybeWarehouse an option for warehouse path, which will be set via
+   *                       `hive.metastore.warehouse.dir`.
+   * @param useExternalHiveFile whether to load the hive-site.xml from `src/test/noclasspath` or
+   *                            not, disabled by default
+   * @param metastore which path the embedded derby database for metastore locates. Use the the
+   *                  global `metastorePath` by default
    * @param queriesAndExpectedAnswers one or more tuples of query + answer
    */
   def runCliWithin(
       timeout: FiniteDuration,
       extraArgs: Seq[String] = Seq.empty,
-      errorResponses: Seq[String] = Seq("Error:"))(
+      errorResponses: Seq[String] = Seq("Error:"),
+      maybeWarehouse: Option[File] = Some(warehousePath),
+      useExternalHiveFile: Boolean = false,
+      metastore: File = metastorePath)(
       queriesAndExpectedAnswers: (String, String)*): Unit = {
 
-    val (queries, expectedAnswers) = queriesAndExpectedAnswers.unzip
     // Explicitly adds ENTER for each statement to make sure they are actually entered into the CLI.
-    val queriesString = queries.map(_ + "\n").mkString
+    val queriesString = queriesAndExpectedAnswers.map(_._1 + "\n").mkString
+    // spark-sql echoes the queries on STDOUT, expect first an echo of the query, then the answer.
+    val expectedAnswers = queriesAndExpectedAnswers.flatMap {
+      case (query, answer) =>
+        if (query == "") {
+          // empty query means a command launched with -e
+          Seq(answer)
+        } else {
+          // spark-sql echoes the submitted queries
+          val queryEcho = query.split("\n").toList match {
+            case firstLine :: tail =>
+              s"spark-sql> $firstLine" :: tail.map(l => s"         > $l")
+          }
+          // longer lines sometimes get split in the output,
+          // match the first 60 characters of each query line
+          queryEcho.map(_.take(60)) :+ answer
+        }
+    }
 
+    val extraHive = if (useExternalHiveFile) {
+      s"--driver-class-path ${System.getProperty("user.dir")}/src/test/noclasspath"
+    } else {
+      ""
+    }
+    val warehouseConf =
+      maybeWarehouse.map(dir => s"--hiveconf ${ConfVars.METASTOREWAREHOUSE}=$dir").getOrElse("")
     val command = {
       val cliScript = "../../bin/spark-sql".split("/").mkString(File.separator)
-      val jdbcUrl = s"jdbc:derby:;databaseName=$metastorePath;create=true"
+      val jdbcUrl = s"jdbc:derby:;databaseName=$metastore;create=true"
       s"""$cliScript
          |  --master local
          |  --driver-java-options -Dderby.system.durability=test
+         |  $extraHive
          |  --conf spark.ui.enabled=false
          |  --hiveconf ${ConfVars.METASTORECONNECTURLKEY}=$jdbcUrl
-         |  --hiveconf ${ConfVars.METASTOREWAREHOUSE}=$warehousePath
          |  --hiveconf ${ConfVars.SCRATCHDIR}=$scratchDirPath
          |  --hiveconf conf1=conftest
          |  --hiveconf conf2=1
+         |  $warehouseConf
        """.stripMargin.split("\\s+").toSeq ++ extraArgs
     }
 
@@ -105,10 +139,13 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     def captureOutput(source: String)(line: String): Unit = lock.synchronized {
       // This test suite sometimes gets extremely slow out of unknown reason on Jenkins.  Here we
       // add a timestamp to provide more diagnosis information.
-      buffer += s"${new Timestamp(new Date().getTime)} - $source> $line"
+      val newLine = s"${new Timestamp(new Date().getTime)} - $source> $line"
+      log.info(newLine)
+      buffer += newLine
 
       // If we haven't found all expected answers and another expected answer comes up...
       if (next < expectedAnswers.size && line.contains(expectedAnswers(next))) {
+        log.info(s"$source> found expected output line $next: '${expectedAnswers(next)}'")
         next += 1
         // If all expected answers have been found...
         if (next == expectedAnswers.size) {
@@ -136,6 +173,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
 
     try {
       ThreadUtils.awaitResult(foundAllExpectedAnswers.future, timeout)
+      log.info("Found all expected output.")
     } catch { case cause: Throwable =>
       val message =
         s"""
@@ -144,8 +182,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
            |=======================
            |Spark SQL CLI command line: ${command.mkString(" ")}
            |Exception: $cause
-           |Executed query $next "${queries(next)}",
-           |But failed to capture expected output "${expectedAnswers(next)}" within $timeout.
+           |Failed to capture next expected output "${expectedAnswers(next)}" within $timeout.
            |
            |${buffer.mkString("\n")}
            |===========================
@@ -155,7 +192,82 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
       logError(message, cause)
       fail(message, cause)
     } finally {
-      process.destroy()
+      if (!process.waitFor(1, MINUTES)) {
+        try {
+          fail("spark-sql did not exit gracefully.")
+        } finally {
+          process.destroy()
+        }
+      }
+    }
+  }
+
+  test("load warehouse dir from hive-site.xml") {
+    val metastore = Utils.createTempDir()
+    metastore.delete()
+    try {
+      runCliWithin(1.minute,
+        maybeWarehouse = None,
+        useExternalHiveFile = true,
+        metastore = metastore)(
+        "desc database default;" -> "hive_one",
+        "set spark.sql.warehouse.dir;" -> "hive_one")
+    } finally {
+      Utils.deleteRecursively(metastore)
+    }
+  }
+
+  test("load warehouse dir from --hiveconf") {
+    // --hiveconf will overrides hive-site.xml
+    runCliWithin(2.minute, useExternalHiveFile = true)(
+      "desc database default;" -> warehousePath.getAbsolutePath,
+      "create database cliTestDb;" -> "",
+      "desc database cliTestDb;" -> warehousePath.getAbsolutePath,
+      "set spark.sql.warehouse.dir;" -> warehousePath.getAbsolutePath)
+  }
+
+  test("load warehouse dir from --conf spark(.hadoop).hive.*") {
+    // override conf from hive-site.xml
+    val metastore = Utils.createTempDir()
+    metastore.delete()
+    try {
+      runCliWithin(2.minute,
+        extraArgs =
+          Seq("--conf", s"spark.hadoop.${ConfVars.METASTOREWAREHOUSE}=$sparkWareHouseDir"),
+        maybeWarehouse = None,
+        useExternalHiveFile = true,
+        metastore = metastore)(
+        "desc database default;" -> sparkWareHouseDir.getAbsolutePath,
+        "create database cliTestDb;" -> "",
+        "desc database cliTestDb;" -> sparkWareHouseDir.getAbsolutePath,
+        "set spark.sql.warehouse.dir;" -> sparkWareHouseDir.getAbsolutePath)
+
+      // override conf from --hiveconf too
+      runCliWithin(2.minute,
+        extraArgs = Seq("--conf", s"spark.${ConfVars.METASTOREWAREHOUSE}=$sparkWareHouseDir"),
+        metastore = metastore)(
+        "desc database default;" -> sparkWareHouseDir.getAbsolutePath,
+        "create database cliTestDb;" -> "",
+        "desc database cliTestDb;" -> sparkWareHouseDir.getAbsolutePath,
+        "set spark.sql.warehouse.dir;" -> sparkWareHouseDir.getAbsolutePath)
+    } finally {
+      Utils.deleteRecursively(metastore)
+    }
+  }
+
+  test("load warehouse dir from spark.sql.warehouse.dir") {
+    // spark.sql.warehouse.dir overrides all hive ones
+    val metastore = Utils.createTempDir()
+    metastore.delete()
+    try {
+      runCliWithin(2.minute,
+        extraArgs = Seq(
+            "--conf", s"${StaticSQLConf.WAREHOUSE_PATH.key}=${sparkWareHouseDir}1",
+            "--conf", s"spark.hadoop.${ConfVars.METASTOREWAREHOUSE}=${sparkWareHouseDir}2"),
+        metastore = metastore)(
+        "desc database default;" -> sparkWareHouseDir.getAbsolutePath.concat("1"))
+    } finally {
+      Utils.deleteRecursively(metastore)
     }
   }
 
@@ -168,7 +280,8 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
         -> "",
       "SHOW TABLES;"
         -> "hive_test",
-      s"LOAD DATA LOCAL INPATH '$dataFilePath' OVERWRITE INTO TABLE hive_test;"
+      s"""LOAD DATA LOCAL INPATH '$dataFilePath'
+         |OVERWRITE INTO TABLE hive_test;""".stripMargin
         -> "",
       "CACHE TABLE hive_test;"
         -> "",
@@ -185,18 +298,18 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
 
   test("Single command with --database") {
     runCliWithin(2.minute)(
-      "CREATE DATABASE hive_test_db;"
+      "CREATE DATABASE hive_db_test;"
         -> "",
-      "USE hive_test_db;"
+      "USE hive_db_test;"
         -> "",
-      "CREATE TABLE hive_test(key INT, val STRING);"
+      "CREATE TABLE hive_table_test(key INT, val STRING);"
         -> "",
       "SHOW TABLES;"
-        -> "hive_test"
+        -> "hive_table_test"
     )
 
-    runCliWithin(2.minute, Seq("--database", "hive_test_db", "-e", "SHOW TABLES;"))(
-      "" -> "hive_test"
+    runCliWithin(2.minute, Seq("--database", "hive_db_test", "-e", "SHOW TABLES;"))(
+      "" -> "hive_table_test"
     )
   }
 
@@ -208,12 +321,12 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
 
     runCliWithin(3.minute, Seq("--jars", s"$jarFile"))(
       """CREATE TABLE t1(key string, val string)
-        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
-      """.stripMargin
+        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';""".stripMargin
         -> "",
       "CREATE TABLE sourceTable (key INT, val STRING) USING hive;"
         -> "",
-      s"LOAD DATA LOCAL INPATH '$dataFilePath' OVERWRITE INTO TABLE sourceTable;"
+      s"""LOAD DATA LOCAL INPATH '$dataFilePath'
+         |OVERWRITE INTO TABLE sourceTable;""".stripMargin
         -> "",
       "INSERT INTO TABLE t1 SELECT key, val FROM sourceTable;"
         -> "",
@@ -234,12 +347,12 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
       3.minute,
       Seq("--conf", s"spark.hadoop.${ConfVars.HIVEAUXJARS}=$hiveContribJar"))(
       """CREATE TABLE addJarWithHiveAux(key string, val string)
-        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
-      """.stripMargin
+        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';""".stripMargin
         -> "",
       "CREATE TABLE sourceTableForWithHiveAux (key INT, val STRING) USING hive;"
         -> "",
-      s"LOAD DATA LOCAL INPATH '$dataFilePath' OVERWRITE INTO TABLE sourceTableForWithHiveAux;"
+      s"""LOAD DATA LOCAL INPATH '$dataFilePath'
+         |OVERWRITE INTO TABLE sourceTableForWithHiveAux;""".stripMargin
         -> "",
       "INSERT INTO TABLE addJarWithHiveAux SELECT key, val FROM sourceTableForWithHiveAux;"
         -> "",
@@ -308,19 +421,6 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     )
   }
 
-  test("SPARK-21451: spark.sql.warehouse.dir should respect options in --hiveconf") {
-    runCliWithin(1.minute)("set spark.sql.warehouse.dir;" -> warehousePath.getAbsolutePath)
-  }
-
-  test("SPARK-21451: Apply spark.hadoop.* configurations") {
-    val tmpDir = Utils.createTempDir(namePrefix = "SPARK-21451")
-    runCliWithin(
-      1.minute,
-      Seq("--conf", s"spark.hadoop.${ConfVars.METASTOREWAREHOUSE}=$tmpDir"))(
-      "set spark.sql.warehouse.dir;" -> tmpDir.getAbsolutePath)
-    tmpDir.delete()
-  }
-
   test("Support hive.aux.jars.path") {
     val hiveContribJar = HiveTestJars.getHiveContribJar().getCanonicalPath
     runCliWithin(
@@ -367,12 +467,12 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
       3.minute)(
       s"ADD JAR ${hiveContribJar};" -> "",
       """CREATE TABLE addJarWithSQL(key string, val string)
-        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
-      """.stripMargin
+        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';""".stripMargin
         -> "",
       "CREATE TABLE sourceTableForWithSQL(key INT, val STRING) USING hive;"
         -> "",
-      s"LOAD DATA LOCAL INPATH '$dataFilePath' OVERWRITE INTO TABLE sourceTableForWithSQL;"
+      s"""LOAD DATA LOCAL INPATH '$dataFilePath'
+         |OVERWRITE INTO TABLE sourceTableForWithSQL;""".stripMargin
         -> "",
       "INSERT INTO TABLE addJarWithSQL SELECT key, val FROM sourceTableForWithSQL;"
         -> "",
@@ -400,4 +500,78 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
         -> "1.000000000000000000"
     )
   }
+
+  test("SPARK-30049 Should not complain for quotes in commented lines") {
+    runCliWithin(1.minute)(
+      """SELECT concat('test', 'comment') -- someone's comment here
+        |;""".stripMargin -> "testcomment"
+    )
+  }
+
+  test("SPARK-31102 spark-sql fails to parse when contains comment") {
+    runCliWithin(1.minute)(
+      """SELECT concat('test', 'comment'),
+        |    -- someone's comment here
+        | 2;""".stripMargin -> "testcomment"
+    )
+  }
+
+  test("SPARK-30049 Should not complain for quotes in commented with multi-lines") {
+    runCliWithin(1.minute)(
+      """SELECT concat('test', 'comment') -- someone's comment here \
+        | comment continues here with single ' quote \
+        | extra ' \
+        |;""".stripMargin -> "testcomment"
+    )
+  }
+
+  test("SPARK-31595 Should allow unescaped quote mark in quoted string") {
+    runCliWithin(1.minute)(
+      "SELECT '\"legal string a';select 1 + 234;".stripMargin -> "235"
+    )
+    runCliWithin(1.minute)(
+      "SELECT \"legal 'string b\";select 22222 + 1;".stripMargin -> "22223"
+    )
+  }
+
+  test("AnalysisException with root cause will be printStacktrace") {
+    // If it is not in silent mode, will print the stacktrace
+    runCliWithin(
+      1.minute,
+      extraArgs = Seq("--hiveconf", "hive.session.silent=false",
+        "-e", "select date_sub(date'2011-11-11', '1.2');"),
+      errorResponses = Seq("NumberFormatException"))(
+      ("", "Error in query: The second argument of 'date_sub' function needs to be an integer."),
+      ("", "NumberFormatException: invalid input syntax for type numeric: 1.2"))
+    // If it is in silent mode, will print the error message only
+    runCliWithin(
+      1.minute,
+      extraArgs = Seq("--conf", "spark.hive.session.silent=true",
+        "-e", "select date_sub(date'2011-11-11', '1.2');"),
+      errorResponses = Seq("AnalysisException"))(
+      ("", "Error in query: The second argument of 'date_sub' function needs to be an integer."))
+  }
+
+  test("SPARK-33100: Ignore a semicolon inside a bracketed comment in spark-sql") {
+    runCliWithin(4.minute)(
+      "/* SELECT 'test';*/ SELECT 'test';" -> "test",
+      ";;/* SELECT 'test';*/ SELECT 'test';" -> "test",
+      "/* SELECT 'test';*/;; SELECT 'test';" -> "test",
+      "SELECT 'test'; -- SELECT 'test';" -> "test",
+      "SELECT 'test'; /* SELECT 'test';*/;" -> "test",
+      "/*$meta chars{^\\;}*/ SELECT 'test';" -> "test",
+      "/*\nmulti-line\n*/ SELECT 'test';" -> "test",
+      "/*/* multi-level bracketed*/ SELECT 'test';" -> "test"
+    )
+  }
+
+  test("SPARK-33100: test sql statements with hint in bracketed comment") {
+    runCliWithin(2.minute)(
+      "CREATE TEMPORARY VIEW t1 AS SELECT * FROM VALUES(1, 2) AS t1(k, v);" -> "",
+      "CREATE TEMPORARY VIEW t2 AS SELECT * FROM VALUES(2, 1) AS t2(k, v);" -> "",
+      "EXPLAIN SELECT /*+ MERGEJOIN(t1) */ t1.* FROM t1 JOIN t2 ON t1.k = t2.v;" -> "SortMergeJoin",
+      "EXPLAIN SELECT /* + MERGEJOIN(t1) */ t1.* FROM t1 JOIN t2 ON t1.k = t2.v;"
+        -> "BroadcastHashJoin"
+    )
+  }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala
index d056b3b2153cf..4564c2209a931 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.util.QueryExecutionListener
 
 class DummyQueryExecutionListener extends QueryExecutionListener {
   override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {}
-  override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+  override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
 }
 
 class DummyStreamingQueryListener extends StreamingQueryListener {
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 84eed7b2eda22..396e7f9d397db 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -545,7 +545,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
       }
 
       if (HiveUtils.isHive23) {
-        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.6"))
+        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.7"))
       } else {
         assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("1.2.1"))
       }
@@ -562,7 +562,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
       }
 
       if (HiveUtils.isHive23) {
-        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.6"))
+        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.7"))
       } else {
         assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("1.2.1"))
       }
@@ -771,6 +771,97 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
       client.closeSession(sessionHandle)
     }
   }
+
+  test("SPARK-29492: use add jar in sync mode") {
+    withCLIServiceClient { client =>
+      val user = System.getProperty("user.name")
+      val sessionHandle = client.openSession(user, "")
+      withJdbcStatement("smallKV", "addJar") { statement =>
+        val confOverlay = new java.util.HashMap[java.lang.String, java.lang.String]
+        val jarFile = HiveTestJars.getHiveHcatalogCoreJar().getCanonicalPath
+
+        Seq(s"ADD JAR $jarFile",
+          "CREATE TABLE smallKV(key INT, val STRING) USING hive",
+          s"LOAD DATA LOCAL INPATH '${TestData.smallKv}' OVERWRITE INTO TABLE smallKV")
+          .foreach(query => client.executeStatement(sessionHandle, query, confOverlay))
+
+        client.executeStatement(sessionHandle,
+          """CREATE TABLE addJar(key string)
+            |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
+          """.stripMargin, confOverlay)
+
+        client.executeStatement(sessionHandle,
+          "INSERT INTO TABLE addJar SELECT 'k1' as key FROM smallKV limit 1", confOverlay)
+
+        val operationHandle = client.executeStatement(
+          sessionHandle,
+          "SELECT key FROM addJar",
+          confOverlay)
+
+        // Fetch result first time
+        assertResult(1, "Fetching result first time from next row") {
+
+          val rows_next = client.fetchResults(
+            operationHandle,
+            FetchOrientation.FETCH_NEXT,
+            1000,
+            FetchType.QUERY_OUTPUT)
+          rows_next.numRows()
+        }
+      }
+    }
+  }
+
+  test("SPARK-31859 Thriftserver works with spark.sql.datetime.java8API.enabled=true") {
+    withJdbcStatement() { st =>
+      st.execute("set spark.sql.datetime.java8API.enabled=true")
+      val rs = st.executeQuery("select date '2020-05-28', timestamp '2020-05-28 00:00:00'")
+      rs.next()
+      assert(rs.getDate(1).toString() == "2020-05-28")
+      assert(rs.getTimestamp(2).toString() == "2020-05-28 00:00:00.0")
+    }
+  }
+
+  test("SPARK-31861 Thriftserver respects spark.sql.session.timeZone") {
+    withJdbcStatement() { st =>
+      st.execute("set spark.sql.session.timeZone=+03:15") // different than Thriftserver's JVM tz
+      val rs = st.executeQuery("select timestamp '2020-05-28 10:00:00'")
+      rs.next()
+      // The timestamp as string is the same as the literal
+      assert(rs.getString(1) == "2020-05-28 10:00:00.0")
+      // Parsing it to java.sql.Timestamp in the client will always result in a timestamp
+      // in client default JVM timezone. The string value of the Timestamp will match the literal,
+      // but if the JDBC application cares about the internal timezone and UTC offset of the
+      // Timestamp object, it should set spark.sql.session.timeZone to match its client JVM tz.
+      assert(rs.getTimestamp(1).toString() == "2020-05-28 10:00:00.0")
+    }
+  }
+
+  test("SPARK-31863 Session conf should persist between Thriftserver worker threads") {
+    val iter = 20
+    withJdbcStatement() { statement =>
+      // date 'now' is resolved during parsing, and relies on SQLConf.get to
+      // obtain the current set timezone. We exploit this to run this test.
+      // If the timezones are set correctly to 25 hours apart across threads,
+      // the dates should reflect this.
+
+      // iterate a few times for the odd chance the same thread is selected
+      for (_ <- 0 until iter) {
+        statement.execute("SET spark.sql.session.timeZone=GMT-12")
+        val firstResult = statement.executeQuery("SELECT date 'now'")
+        firstResult.next()
+        val beyondDateLineWest = firstResult.getDate(1)
+
+        statement.execute("SET spark.sql.session.timeZone=GMT+13")
+        val secondResult = statement.executeQuery("SELECT date 'now'")
+        secondResult.next()
+        val dateLineEast = secondResult.getDate(1)
+        assert(
+          dateLineEast after beyondDateLineWest,
+          "SQLConf changes should persist across execution threads")
+      }
+    }
+  }
 }
 
 class SingleSessionSuite extends HiveThriftJdbcTest {
@@ -780,7 +871,7 @@ class SingleSessionSuite extends HiveThriftJdbcTest {
     s"--conf ${HIVE_THRIFT_SERVER_SINGLESESSION.key}=true" :: Nil
 
   test("share the temporary functions across JDBC connections") {
-    withMultipleConnectionJdbcStatement()(
+    withMultipleConnectionJdbcStatement("test_udtf")(
       { statement =>
         val jarPath = "../hive/src/test/resources/TestUDTF.jar"
         val jarURL = s"file://${System.getProperty("user.dir")}/$jarPath"
@@ -884,6 +975,39 @@ class SingleSessionSuite extends HiveThriftJdbcTest {
   }
 }
 
+class HiveThriftCleanUpScratchDirSuite extends HiveThriftJdbcTest{
+  var tempScratchDir: File = _
+
+  override protected def beforeAll(): Unit = {
+    tempScratchDir = Utils.createTempDir()
+    tempScratchDir.setWritable(true, false)
+    assert(tempScratchDir.list().isEmpty)
+    new File(tempScratchDir.getAbsolutePath + File.separator + "SPARK-31626").createNewFile()
+    assert(tempScratchDir.list().nonEmpty)
+    super.beforeAll()
+  }
+
+  override def mode: ServerMode.Value = ServerMode.binary
+
+  override protected def extraConf: Seq[String] =
+    s" --hiveconf ${ConfVars.HIVE_START_CLEANUP_SCRATCHDIR}=true " ::
+       s"--hiveconf ${ConfVars.SCRATCHDIR}=${tempScratchDir.getAbsolutePath}" :: Nil
+
+  test("Cleanup the Hive scratchdir when starting the Hive Server") {
+    assert(!tempScratchDir.exists())
+    withJdbcStatement() { statement =>
+      val rs = statement.executeQuery("SELECT id FROM range(1)")
+      assert(rs.next())
+      assert(rs.getLong(1) === 0L)
+    }
+  }
+
+  override protected def afterAll(): Unit = {
+    Utils.deleteRecursively(tempScratchDir)
+    super.afterAll()
+  }
+}
+
 class HiveThriftHttpServerSuite extends HiveThriftJdbcTest {
   override def mode: ServerMode.Value = ServerMode.http
 
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
index ce610098156f3..e002bc0117c8b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
@@ -19,29 +19,25 @@ package org.apache.spark.sql.hive.thriftserver
 
 import java.sql.{DriverManager, Statement}
 
+import scala.collection.JavaConverters._
 import scala.concurrent.duration._
-import scala.util.{Random, Try}
+import scala.util.Try
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
+import org.apache.hive.service.cli.thrift.ThriftCLIService
 
-import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.test.SharedSparkSession
 
 trait SharedThriftServer extends SharedSparkSession {
 
   private var hiveServer2: HiveThriftServer2 = _
+  private var serverPort: Int = 0
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    // Chooses a random port between 10000 and 19999
-    var listeningPort = 10000 + Random.nextInt(10000)
-
     // Retries up to 3 times with different port numbers if the server fails to start
-    (1 to 3).foldLeft(Try(startThriftServer(listeningPort, 0))) { case (started, attempt) =>
-      started.orElse {
-        listeningPort += 1
-        Try(startThriftServer(listeningPort, attempt))
-      }
+    (1 to 3).foldLeft(Try(startThriftServer(0))) { case (started, attempt) =>
+      started.orElse(Try(startThriftServer(attempt)))
     }.recover {
       case cause: Throwable =>
         throw cause
@@ -59,8 +55,7 @@ trait SharedThriftServer extends SharedSparkSession {
 
   protected def withJdbcStatement(fs: (Statement => Unit)*): Unit = {
     val user = System.getProperty("user.name")
-
-    val serverPort = hiveServer2.getHiveConf.get(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname)
+    require(serverPort != 0, "Failed to bind an actual port for HiveThriftServer2")
     val connections =
       fs.map { _ => DriverManager.getConnection(s"jdbc:hive2://localhost:$serverPort", user, "") }
     val statements = connections.map(_.createStatement())
@@ -73,11 +68,19 @@ trait SharedThriftServer extends SharedSparkSession {
     }
   }
 
-  private def startThriftServer(port: Int, attempt: Int): Unit = {
-    logInfo(s"Trying to start HiveThriftServer2: port=$port, attempt=$attempt")
+  private def startThriftServer(attempt: Int): Unit = {
+    logInfo(s"Trying to start HiveThriftServer2:, attempt=$attempt")
     val sqlContext = spark.newSession().sqlContext
-    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, port.toString)
+    // Set the HIVE_SERVER2_THRIFT_PORT to 0, so it could randomly pick any free port to use.
+    // It's much more robust than set a random port generated by ourselves ahead
+    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, "0")
     hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
+    hiveServer2.getServices.asScala.foreach {
+      case t: ThriftCLIService if t.getPortNumber != 0 =>
+        serverPort = t.getPortNumber
+        logInfo(s"Started HiveThriftServer2: port=$serverPort, attempt=$attempt")
+      case _ =>
+    }
 
     // Wait for thrift server to be ready to serve the query, via executing simple query
     // till the query succeeds. See SPARK-30345 for more details.
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
index 13df3fabc4919..4c2f29e0bf394 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
@@ -17,10 +17,25 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
+import java.util
+import java.util.concurrent.Semaphore
+
+import scala.concurrent.duration._
+
+import org.apache.hadoop.hive.conf.HiveConf
+import org.apache.hive.service.cli.OperationState
+import org.apache.hive.service.cli.session.{HiveSession, HiveSessionImpl}
+import org.mockito.Mockito.{doReturn, mock, spy, when, RETURNS_DEEP_STUBS}
+import org.mockito.invocation.InvocationOnMock
+
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.{DataFrame, SQLContext}
+import org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2EventManager
+import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, NullType, StringType, StructField, StructType}
 
-class SparkExecuteStatementOperationSuite extends SparkFunSuite {
+class SparkExecuteStatementOperationSuite extends SparkFunSuite with SharedSparkSession {
+
   test("SPARK-17112 `select null` via JDBC triggers IllegalArgumentException in ThriftServer") {
     val field1 = StructField("NULL", NullType)
     val field2 = StructField("(IF(true, NULL, NULL))", NullType)
@@ -42,4 +57,68 @@ class SparkExecuteStatementOperationSuite extends SparkFunSuite {
     assert(columns.get(1).getType().getName == "INT")
     assert(columns.get(1).getComment() == "")
   }
+
+  Seq(
+    (OperationState.CANCELED, (_: SparkExecuteStatementOperation).cancel()),
+    (OperationState.CLOSED, (_: SparkExecuteStatementOperation).close())
+  ).foreach { case (finalState, transition) =>
+    test("SPARK-32057 SparkExecuteStatementOperation should not transiently become ERROR " +
+      s"before being set to $finalState") {
+      val hiveSession = new HiveSessionImpl(ThriftserverShimUtils.testedProtocolVersions.head,
+      "username", "password", new HiveConf, "ip address")
+      hiveSession.open(new util.HashMap)
+
+      HiveThriftServer2.eventManager = mock(classOf[HiveThriftServer2EventManager])
+
+      val spySqlContext = spy(sqlContext)
+
+      // When cancel() is called on the operation, cleanup causes an exception to be thrown inside
+      // of execute(). This should not cause the state to become ERROR. The exception here will be
+      // triggered in our custom cleanup().
+      val signal = new Semaphore(0)
+      val dataFrame = mock(classOf[DataFrame], RETURNS_DEEP_STUBS)
+      when(dataFrame.collect()).thenAnswer((_: InvocationOnMock) => {
+        signal.acquire()
+        throw new RuntimeException("Operation was cancelled by test cleanup.")
+      })
+      val statement = "stmt"
+      doReturn(dataFrame, Nil: _*).when(spySqlContext).sql(statement)
+
+      val executeStatementOperation = new MySparkExecuteStatementOperation(spySqlContext,
+        hiveSession, statement, signal, finalState)
+
+      val run = new Thread() {
+        override def run(): Unit = executeStatementOperation.runInternal()
+      }
+      assert(executeStatementOperation.getStatus.getState === OperationState.INITIALIZED)
+      run.start()
+      eventually(timeout(5.seconds)) {
+        assert(executeStatementOperation.getStatus.getState === OperationState.RUNNING)
+      }
+      transition(executeStatementOperation)
+      run.join()
+      assert(executeStatementOperation.getStatus.getState === finalState)
+    }
+  }
+
+  private class MySparkExecuteStatementOperation(
+      sqlContext: SQLContext,
+      hiveSession: HiveSession,
+      statement: String,
+      signal: Semaphore,
+      finalState: OperationState)
+    extends SparkExecuteStatementOperation(sqlContext, hiveSession, statement,
+      new util.HashMap, false) {
+
+    override def cleanup(): Unit = {
+      super.cleanup()
+      signal.release()
+      // At this point, operation should already be in finalState (set by either close() or
+      // cancel()). We want to check if it stays in finalState after the exception thrown by
+      // releasing the semaphore propagates. We hence need to sleep for a short while.
+      Thread.sleep(1000)
+      // State should not be ERROR
+      assert(getStatus.getState === finalState)
+    }
+  }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
index ffd1fc48f19fe..f28faea2be868 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
+import org.apache.commons.io.FileUtils
 import test.custom.listener.{DummyQueryExecutionListener, DummyStreamingQueryListener}
 
 import org.apache.spark.SparkFunSuite
@@ -25,10 +26,19 @@ import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.hive.HiveUtils.{HIVE_METASTORE_JARS, HIVE_METASTORE_VERSION}
 import org.apache.spark.sql.hive.test.TestHiveContext
 import org.apache.spark.sql.internal.StaticSQLConf.{QUERY_EXECUTION_LISTENERS, STREAMING_QUERY_LISTENERS, WAREHOUSE_PATH}
+import org.apache.spark.util.Utils
 
 class SparkSQLEnvSuite extends SparkFunSuite {
   test("SPARK-29604 external listeners should be initialized with Spark classloader") {
+    val metastorePath = Utils.createTempDir("spark_derby")
+    FileUtils.forceDelete(metastorePath)
+
+    val jdbcUrl = s"jdbc:derby:;databaseName=$metastorePath;create=true"
+
     withSystemProperties(
+      "javax.jdo.option.ConnectionURL" -> jdbcUrl,
+      "derby.system.durability" -> "test",
+      "spark.ui.enabled" -> "false",
       QUERY_EXECUTION_LISTENERS.key -> classOf[DummyQueryExecutionListener].getCanonicalName,
       STREAMING_QUERY_LISTENERS.key -> classOf[DummyStreamingQueryListener].getCanonicalName,
       WAREHOUSE_PATH.key -> TestHiveContext.makeWarehouseDir().toURI.getPath,
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
index a63b5dac0aac3..fd45e7a48c0eb 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
@@ -192,6 +192,11 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
         assert(rs.next())
         assert(rs.getBigDecimal(1) === new java.math.BigDecimal("1.00"))
       }
+      testExecuteStatementWithProtocolVersion(version,
+        "SELECT cast(null as decimal) ") { rs =>
+        assert(rs.next())
+        assert(rs.getBigDecimal(1) === null)
+      }
     }
 
     test(s"$version get string type") {
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index d9ac9ab441f0c..8b16674f05a84 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -23,6 +23,7 @@ import java.util.{Locale, MissingFormatArgumentException}
 
 import scala.util.control.NonFatal
 
+import org.apache.commons.io.FileUtils
 import org.apache.commons.lang3.exception.ExceptionUtils
 
 import org.apache.spark.SparkException
@@ -32,6 +33,7 @@ import org.apache.spark.sql.catalyst.util.fileToString
 import org.apache.spark.sql.execution.HiveResult
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.GitHubActionsUnstableTest
 
 /**
  * Re-run all the tests in SQLQueryTestSuite via Thrift Server.
@@ -51,8 +53,18 @@ import org.apache.spark.sql.types._
  *   2. Support DESC command.
  *   3. Support SHOW command.
  */
+@GitHubActionsUnstableTest
 class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServer {
 
+  override protected def testFile(fileName: String): String = {
+    val url = Thread.currentThread().getContextClassLoader.getResource(fileName)
+    // Copy to avoid URISyntaxException during accessing the resources in `sql/core`
+    val file = File.createTempFile("thriftserver-test", ".data")
+    file.deleteOnExit()
+    FileUtils.copyURLToFile(url, file)
+    file.getAbsolutePath
+  }
+
   /** List of test cases to ignore, in lower cases. */
   override def blackList: Set[String] = super.blackList ++ Set(
     // Missing UDF
@@ -80,8 +92,6 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ
     // We do not test with configSet.
     withJdbcStatement { statement =>
 
-      loadTestData(statement)
-
       configSet.foreach { case (k, v) =>
         statement.execute(s"SET $k = $v")
       }
@@ -262,61 +272,6 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ
     }
   }
 
-  /** Load built-in test tables. */
-  private def loadTestData(statement: Statement): Unit = {
-    // Prepare the data
-    statement.execute(
-      """
-        |CREATE OR REPLACE TEMPORARY VIEW testdata as
-        |SELECT id AS key, CAST(id AS string) AS value FROM range(1, 101)
-      """.stripMargin)
-    statement.execute(
-      """
-        |CREATE OR REPLACE TEMPORARY VIEW arraydata as
-        |SELECT * FROM VALUES
-        |(ARRAY(1, 2, 3), ARRAY(ARRAY(1, 2, 3))),
-        |(ARRAY(2, 3, 4), ARRAY(ARRAY(2, 3, 4))) AS v(arraycol, nestedarraycol)
-      """.stripMargin)
-    statement.execute(
-      """
-        |CREATE OR REPLACE TEMPORARY VIEW mapdata as
-        |SELECT * FROM VALUES
-        |MAP(1, 'a1', 2, 'b1', 3, 'c1', 4, 'd1', 5, 'e1'),
-        |MAP(1, 'a2', 2, 'b2', 3, 'c2', 4, 'd2'),
-        |MAP(1, 'a3', 2, 'b3', 3, 'c3'),
-        |MAP(1, 'a4', 2, 'b4'),
-        |MAP(1, 'a5') AS v(mapcol)
-      """.stripMargin)
-    statement.execute(
-      s"""
-         |CREATE TEMPORARY VIEW aggtest
-         |  (a int, b float)
-         |USING csv
-         |OPTIONS (path '${baseResourcePath.getParent}/test-data/postgresql/agg.data',
-         |  header 'false', delimiter '\t')
-      """.stripMargin)
-    statement.execute(
-      s"""
-         |CREATE OR REPLACE TEMPORARY VIEW onek
-         |  (unique1 int, unique2 int, two int, four int, ten int, twenty int, hundred int,
-         |    thousand int, twothousand int, fivethous int, tenthous int, odd int, even int,
-         |    stringu1 string, stringu2 string, string4 string)
-         |USING csv
-         |OPTIONS (path '${baseResourcePath.getParent}/test-data/postgresql/onek.data',
-         |  header 'false', delimiter '\t')
-      """.stripMargin)
-    statement.execute(
-      s"""
-         |CREATE OR REPLACE TEMPORARY VIEW tenk1
-         |  (unique1 int, unique2 int, two int, four int, ten int, twenty int, hundred int,
-         |    thousand int, twothousand int, fivethous int, tenthous int, odd int, even int,
-         |    stringu1 string, stringu2 string, string4 string)
-         |USING csv
-         |  OPTIONS (path '${baseResourcePath.getParent}/test-data/postgresql/tenk.data',
-         |  header 'false', delimiter '\t')
-      """.stripMargin)
-  }
-
   // Returns true if sql is retrieving data.
   private def isNeedSort(sql: String): Boolean = {
     val upperCase = sql.toUpperCase(Locale.ROOT)
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
index 3e1fce78ae71c..73547d752024b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
@@ -17,6 +17,9 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
+import org.apache.spark.tags.GitHubActionsUnstableTest
+
+@GitHubActionsUnstableTest
 class ThriftServerWithSparkContextSuite extends SharedThriftServer {
 
   test("SPARK-29911: Uncache cached tables when session closed") {
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
index 547c6056b4fe8..462b93a0f09fe 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
@@ -123,22 +123,6 @@ private static TColumnValue stringValue(HiveVarchar value) {
     return TColumnValue.stringVal(tStringValue);
   }
 
-  private static TColumnValue dateValue(Date value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return new TColumnValue(TColumnValue.stringVal(tStringValue));
-  }
-
-  private static TColumnValue timestampValue(Timestamp value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
   private static TColumnValue stringValue(HiveIntervalYearMonth value) {
     TStringValue tStrValue = new TStringValue();
     if (value != null) {
@@ -178,15 +162,16 @@ public static TColumnValue toTColumnValue(Type type, Object value) {
     case VARCHAR_TYPE:
       return stringValue((HiveVarchar)value);
     case DATE_TYPE:
-      return dateValue((Date)value);
     case TIMESTAMP_TYPE:
-      return timestampValue((Timestamp)value);
+      // SPARK-31859, SPARK-31861: converted to string already in SparkExecuteStatementOperation
+      return stringValue((String)value);
     case INTERVAL_YEAR_MONTH_TYPE:
       return stringValue((HiveIntervalYearMonth) value);
     case INTERVAL_DAY_TIME_TYPE:
       return stringValue((HiveIntervalDayTime) value);
     case DECIMAL_TYPE:
-      return stringValue(((BigDecimal)value).toPlainString());
+      String plainStr = value == null ? null : ((BigDecimal)value).toPlainString();
+      return stringValue(plainStr);
     case BINARY_TYPE:
       String strVal = value == null ? null : UTF8String.fromBytes((byte[])value).toString();
       return stringValue(strVal);
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java
index 51bb28748d9e2..4b331423948fa 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java
@@ -280,7 +280,10 @@ public void cancel() throws HiveSQLException {
     throw new UnsupportedOperationException("SQLOperation.cancel()");
   }
 
-  public abstract void close() throws HiveSQLException;
+  public void close() throws HiveSQLException {
+    setState(OperationState.CLOSED);
+    cleanupOperationLog();
+  }
 
   public abstract TableSchema getResultSetSchema() throws HiveSQLException;
 
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
index 859f9c8b449e5..ad6fb3ba37a0e 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
@@ -148,14 +148,20 @@ public synchronized void start() {
     }
   }
 
+  private final Object timeoutCheckerLock = new Object();
+
   private void startTimeoutChecker() {
     final long interval = Math.max(checkInterval, 3000L);  // minimum 3 seconds
-    Runnable timeoutChecker = new Runnable() {
+    final Runnable timeoutChecker = new Runnable() {
       @Override
       public void run() {
-        for (sleepInterval(interval); !shutdown; sleepInterval(interval)) {
+        sleepFor(interval);
+        while (!shutdown) {
           long current = System.currentTimeMillis();
           for (HiveSession session : new ArrayList<HiveSession>(handleToSession.values())) {
+            if (shutdown) {
+              break;
+            }
             if (sessionTimeout > 0 && session.getLastAccessTime() + sessionTimeout <= current
                 && (!checkOperation || session.getNoOperationTime() > sessionTimeout)) {
               SessionHandle handle = session.getSessionHandle();
@@ -170,24 +176,34 @@ public void run() {
               session.closeExpiredOperations();
             }
           }
+          sleepFor(interval);
         }
       }
 
-      private void sleepInterval(long interval) {
-        try {
-          Thread.sleep(interval);
-        } catch (InterruptedException e) {
-          // ignore
+      private void sleepFor(long interval) {
+        synchronized (timeoutCheckerLock) {
+          try {
+            timeoutCheckerLock.wait(interval);
+          } catch (InterruptedException e) {
+            // Ignore, and break.
+          }
         }
       }
     };
     backgroundOperationPool.execute(timeoutChecker);
   }
 
+  private void shutdownTimeoutChecker() {
+    shutdown = true;
+    synchronized (timeoutCheckerLock) {
+      timeoutCheckerLock.notify();
+    }
+  }
+
   @Override
   public synchronized void stop() {
     super.stop();
-    shutdown = true;
+    shutdownTimeoutChecker();
     if (backgroundOperationPool != null) {
       backgroundOperationPool.shutdown();
       long timeout = hiveConf.getTimeVar(
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index 21b8bf7de75ce..e1ee503b81209 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -76,6 +76,10 @@ public void run() {
             keyStorePassword, sslVersionBlacklist);
       }
 
+      // In case HIVE_SERVER2_THRIFT_PORT or hive.server2.thrift.port is configured with 0 which
+      // represents any free port, we should set it to the actual one
+      portNum = serverSocket.getServerSocket().getLocalPort();
+
       // Server args
       int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE);
       int requestTimeout = (int) hiveConf.getTimeVar(
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 504e63dbc5e5e..1099a00b67eb7 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -143,6 +143,9 @@ public void run() {
       // TODO: check defaults: maxTimeout, keepalive, maxBodySize, bodyRecieveDuration, etc.
       // Finally, start the server
       httpServer.start();
+      // In case HIVE_SERVER2_THRIFT_HTTP_PORT or hive.server2.thrift.http.port is configured with
+      // 0 which represents any free port, we should set it to the actual one
+      portNum = connector.getLocalPort();
       String msg = "Started " + ThriftHttpCLIService.class.getSimpleName() + " in " + schemeName
           + " mode on port " + connector.getLocalPort()+ " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
index f881ab159ea67..85adf55df15e0 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
@@ -124,22 +124,6 @@ private static TColumnValue stringValue(HiveVarchar value) {
     return TColumnValue.stringVal(tStringValue);
   }
 
-  private static TColumnValue dateValue(Date value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return new TColumnValue(TColumnValue.stringVal(tStringValue));
-  }
-
-  private static TColumnValue timestampValue(Timestamp value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
   private static TColumnValue stringValue(HiveIntervalYearMonth value) {
     TStringValue tStrValue = new TStringValue();
     if (value != null) {
@@ -181,15 +165,16 @@ public static TColumnValue toTColumnValue(TypeDescriptor typeDescriptor, Object
     case VARCHAR_TYPE:
       return stringValue((HiveVarchar)value);
     case DATE_TYPE:
-      return dateValue((Date)value);
     case TIMESTAMP_TYPE:
-      return timestampValue((Timestamp)value);
+      // SPARK-31859, SPARK-31861: converted to string already in SparkExecuteStatementOperation
+      return stringValue((String)value);
     case INTERVAL_YEAR_MONTH_TYPE:
       return stringValue((HiveIntervalYearMonth) value);
     case INTERVAL_DAY_TIME_TYPE:
       return stringValue((HiveIntervalDayTime) value);
     case DECIMAL_TYPE:
-      return stringValue(((BigDecimal)value).toPlainString());
+      String plainStr = value == null ? null : ((BigDecimal)value).toPlainString();
+      return stringValue(plainStr);
     case BINARY_TYPE:
       String strVal = value == null ? null : UTF8String.fromBytes((byte[])value).toString();
       return stringValue(strVal);
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java
index f26c715add987..558c68f85c16b 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java
@@ -298,7 +298,10 @@ public void cancel() throws HiveSQLException {
     throw new UnsupportedOperationException("SQLOperation.cancel()");
   }
 
-  public abstract void close() throws HiveSQLException;
+  public void close() throws HiveSQLException {
+    setState(OperationState.CLOSED);
+    cleanupOperationLog();
+  }
 
   public abstract TableSchema getResultSetSchema() throws HiveSQLException;
 
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
index 49221b13bb892..5a381d170b4f9 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
@@ -148,14 +148,20 @@ public synchronized void start() {
     }
   }
 
+  private final Object timeoutCheckerLock = new Object();
+
   private void startTimeoutChecker() {
     final long interval = Math.max(checkInterval, 3000L);  // minimum 3 seconds
-    Runnable timeoutChecker = new Runnable() {
+    final Runnable timeoutChecker = new Runnable() {
       @Override
       public void run() {
-        for (sleepInterval(interval); !shutdown; sleepInterval(interval)) {
+        sleepFor(interval);
+        while (!shutdown) {
           long current = System.currentTimeMillis();
           for (HiveSession session : new ArrayList<HiveSession>(handleToSession.values())) {
+            if (shutdown) {
+              break;
+            }
             if (sessionTimeout > 0 && session.getLastAccessTime() + sessionTimeout <= current
                 && (!checkOperation || session.getNoOperationTime() > sessionTimeout)) {
               SessionHandle handle = session.getSessionHandle();
@@ -170,24 +176,34 @@ public void run() {
               session.closeExpiredOperations();
             }
           }
+          sleepFor(interval);
         }
       }
 
-      private void sleepInterval(long interval) {
-        try {
-          Thread.sleep(interval);
-        } catch (InterruptedException e) {
-          // ignore
+      private void sleepFor(long interval) {
+        synchronized (timeoutCheckerLock) {
+          try {
+            timeoutCheckerLock.wait(interval);
+          } catch (InterruptedException e) {
+            // Ignore, and break.
+          }
         }
       }
     };
     backgroundOperationPool.execute(timeoutChecker);
   }
 
+  private void shutdownTimeoutChecker() {
+    shutdown = true;
+    synchronized (timeoutCheckerLock) {
+      timeoutCheckerLock.notify();
+    }
+  }
+
   @Override
   public synchronized void stop() {
     super.stop();
-    shutdown = true;
+    shutdownTimeoutChecker();
     if (backgroundOperationPool != null) {
       backgroundOperationPool.shutdown();
       long timeout = hiveConf.getTimeVar(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index fc19c65daaf54..a7de9c0f3d0d2 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -77,6 +77,10 @@ public void run() {
             keyStorePassword, sslVersionBlacklist);
       }
 
+      // In case HIVE_SERVER2_THRIFT_PORT or hive.server2.thrift.port is configured with 0 which
+      // represents any free port, we should set it to the actual one
+      portNum = serverSocket.getServerSocket().getLocalPort();
+
       // Server args
       int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE);
       int requestTimeout = (int) hiveConf.getTimeVar(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 08626e7eb146d..73d5f84476af0 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -144,6 +144,9 @@ public void run() {
       // TODO: check defaults: maxTimeout, keepalive, maxBodySize, bodyRecieveDuration, etc.
       // Finally, start the server
       httpServer.start();
+      // In case HIVE_SERVER2_THRIFT_HTTP_PORT or hive.server2.thrift.http.port is configured with
+      // 0 which represents any free port, we should set it to the actual one
+      portNum = connector.getLocalPort();
       String msg = "Started " + ThriftHttpCLIService.class.getSimpleName() + " in " + schemeName
           + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index 29825e5116ef9..4112512e56fee 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.hive.execution
 
 import java.io.File
-import java.util.{Locale, TimeZone}
 
 import org.scalatest.BeforeAndAfter
 
@@ -27,22 +26,21 @@ import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHive
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
+import org.apache.spark.tags.SlowHiveTest
 
 /**
  * Runs the test cases that are included in the hive distribution.
  */
+@SlowHiveTest
 class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   // TODO: bundle in jar files... get from classpath
   private lazy val hiveQueryDir = TestHive.getHiveFile(
     "ql/src/test/queries/clientpositive".split("/").mkString(File.separator))
 
-  private val originalTimeZone = TimeZone.getDefault
-  private val originalLocale = Locale.getDefault
   private val originalColumnBatchSize = TestHive.conf.columnBatchSize
   private val originalInMemoryPartitionPruning = TestHive.conf.inMemoryPartitionPruning
   private val originalCrossJoinEnabled = TestHive.conf.crossJoinEnabled
   private val originalSessionLocalTimeZone = TestHive.conf.sessionLocalTimeZone
-  private val originalCreateHiveTable = TestHive.conf.createHiveTableByDefaultEnabled
 
   def testCases: Seq[(String, File)] = {
     hiveQueryDir.listFiles.map(f => f.getName.stripSuffix(".q") -> f)
@@ -51,10 +49,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   override def beforeAll(): Unit = {
     super.beforeAll()
     TestHive.setCacheTables(true)
-    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    // Add Locale setting
-    Locale.setDefault(Locale.US)
     // Set a relatively small column batch size for testing purposes
     TestHive.setConf(SQLConf.COLUMN_BATCH_SIZE, 5)
     // Enable in-memory partition pruning for testing purposes
@@ -66,21 +60,16 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
     // Fix session local timezone to America/Los_Angeles for those timezone sensitive tests
     // (timestamp_*)
     TestHive.setConf(SQLConf.SESSION_LOCAL_TIMEZONE, "America/Los_Angeles")
-    TestHive.setConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED, true)
     RuleExecutor.resetMetrics()
   }
 
   override def afterAll(): Unit = {
     try {
       TestHive.setCacheTables(false)
-      TimeZone.setDefault(originalTimeZone)
-      Locale.setDefault(originalLocale)
       TestHive.setConf(SQLConf.COLUMN_BATCH_SIZE, originalColumnBatchSize)
       TestHive.setConf(SQLConf.IN_MEMORY_PARTITION_PRUNING, originalInMemoryPartitionPruning)
       TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, originalCrossJoinEnabled)
       TestHive.setConf(SQLConf.SESSION_LOCAL_TIMEZONE, originalSessionLocalTimeZone)
-      TestHive.setConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED,
-        originalCreateHiveTable)
 
       // For debugging dump some statistics about how much time was spent in various optimizer rules
       logWarning(RuleExecutor.dumpTimeSpent())
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
index ed23f65815917..2c0970c85449f 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.hive.execution
 
 import java.io.File
-import java.util.{Locale, TimeZone}
 
 import org.scalatest.BeforeAndAfter
 
@@ -33,17 +32,11 @@ import org.apache.spark.util.Utils
  * files, every `createQueryTest` calls should explicitly set `reset` to `false`.
  */
 class HiveWindowFunctionQuerySuite extends HiveComparisonTest with BeforeAndAfter {
-  private val originalTimeZone = TimeZone.getDefault
-  private val originalLocale = Locale.getDefault
   private val testTempDir = Utils.createTempDir()
 
   override def beforeAll(): Unit = {
     super.beforeAll()
     TestHive.setCacheTables(true)
-    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    // Add Locale setting
-    Locale.setDefault(Locale.US)
 
     // Create the table used in windowing.q
     sql("DROP TABLE IF EXISTS part")
@@ -103,8 +96,6 @@ class HiveWindowFunctionQuerySuite extends HiveComparisonTest with BeforeAndAfte
   override def afterAll(): Unit = {
     try {
       TestHive.setCacheTables(false)
-      TimeZone.setDefault(originalTimeZone)
-      Locale.setDefault(originalLocale)
       TestHive.reset()
     } finally {
       super.afterAll()
@@ -747,17 +738,11 @@ class HiveWindowFunctionQuerySuite extends HiveComparisonTest with BeforeAndAfte
 
 class HiveWindowFunctionQueryFileSuite
   extends HiveCompatibilitySuite with BeforeAndAfter {
-  private val originalTimeZone = TimeZone.getDefault
-  private val originalLocale = Locale.getDefault
   private val testTempDir = Utils.createTempDir()
 
   override def beforeAll(): Unit = {
     super.beforeAll()
     TestHive.setCacheTables(true)
-    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    // Add Locale setting
-    Locale.setDefault(Locale.US)
 
     // The following settings are used for generating golden files with Hive.
     // We have to use kryo to correctly let Hive serialize plans with window functions.
@@ -772,8 +757,6 @@ class HiveWindowFunctionQueryFileSuite
   override def afterAll(): Unit = {
     try {
       TestHive.setCacheTables(false)
-      TimeZone.setDefault(originalTimeZone)
-      Locale.setDefault(originalLocale)
       TestHive.reset()
     } finally {
       super.afterAll()
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index c37582386347b..1b4c6dc110e97 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
new file mode 100644
index 0000000000000..02a5117f005e8
--- /dev/null
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.apache.spark.SparkContext
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.{SparkSession, SQLContext}
+
+
+/**
+ * An instance of the Spark SQL execution engine that integrates with data stored in Hive.
+ * Configuration for Hive is read from hive-site.xml on the classpath.
+ */
+@deprecated("Use SparkSession.builder.enableHiveSupport instead", "2.0.0")
+class HiveContext private[hive](_sparkSession: SparkSession)
+  extends SQLContext(_sparkSession) with Logging {
+
+  self =>
+
+  def this(sc: SparkContext) = {
+    this(SparkSession.builder().sparkContext(HiveUtils.withHiveExternalCatalog(sc)).getOrCreate())
+  }
+
+  def this(sc: JavaSparkContext) = this(sc.sc)
+
+  /**
+   * Returns a new HiveContext as new session, which will have separated SQLConf, UDF/UDAF,
+   * temporary tables and SessionState, but sharing the same CacheManager, IsolatedClientLoader
+   * and Hive client (both of execution and metadata) with existing HiveContext.
+   */
+  override def newSession(): HiveContext = {
+    new HiveContext(sparkSession.newSession())
+  }
+
+  /**
+   * Invalidate and refresh all the cached the metadata of the given table. For performance reasons,
+   * Spark SQL or the external data source library it uses might cache certain metadata about a
+   * table, such as the location of blocks. When those change outside of Spark SQL, users should
+   * call this function to invalidate the cache.
+   *
+   * @since 1.3.0
+   */
+  def refreshTable(tableName: String): Unit = {
+    sparkSession.catalog.refreshTable(tableName)
+  }
+
+}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index ca292f65efeee..2d4e45aecdbb1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -61,6 +61,10 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
   import HiveExternalCatalog._
   import CatalogTableType._
 
+  // SPARK-32256: Make sure `VersionInfo` is initialized before touching the isolated classloader.
+  // This is to ensure Hive can get the Hadoop version when using the isolated classloader.
+  org.apache.hadoop.util.VersionInfo.getVersion()
+
   /**
    * A Hive client used to interact with the metastore.
    */
@@ -634,7 +638,15 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
         k.startsWith(DATASOURCE_PREFIX) || k.startsWith(STATISTICS_PREFIX) ||
           k.startsWith(CREATED_SPARK_VERSION)
       }
-      val newTableProps = propsFromOldTable ++ tableDefinition.properties + partitionProviderProp
+      val newFormatIfExists = tableDefinition.provider.flatMap { p =>
+        if (DDLUtils.isDatasourceTable(tableDefinition)) {
+          Some(DATASOURCE_PROVIDER -> p)
+        } else {
+          None
+        }
+      }
+      val newTableProps =
+        propsFromOldTable ++ tableDefinition.properties + partitionProviderProp ++ newFormatIfExists
 
       // // Add old table's owner if we need to restore
       val owner = Option(tableDefinition.owner).filter(_.nonEmpty).getOrElse(oldTableDef.owner)
@@ -853,6 +865,11 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
     client.listTables(db, pattern)
   }
 
+  override def listViews(db: String, pattern: String): Seq[String] = withClient {
+    requireDbExists(db)
+    client.listTablesByType(db, pattern, CatalogTableType.VIEW)
+  }
+
   override def loadTable(
       db: String,
       table: String,
@@ -934,9 +951,10 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
   // Hive metastore is not case preserving and the partition columns are always lower cased. We need
   // to lower case the column names in partition specification before calling partition related Hive
   // APIs, to match this behaviour.
-  private def lowerCasePartitionSpec(spec: TablePartitionSpec): TablePartitionSpec = {
+  private def toMetaStorePartitionSpec(spec: TablePartitionSpec): TablePartitionSpec = {
     // scalastyle:off caselocale
-    spec.map { case (k, v) => k.toLowerCase -> v }
+    val lowNames = spec.map { case (k, v) => k.toLowerCase -> v }
+    ExternalCatalogUtils.convertNullPartitionValues(lowNames)
     // scalastyle:on caselocale
   }
 
@@ -985,8 +1003,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       }
       p.copy(storage = p.storage.copy(locationUri = Some(partitionPath.toUri)))
     }
-    val lowerCasedParts = partsWithLocation.map(p => p.copy(spec = lowerCasePartitionSpec(p.spec)))
-    client.createPartitions(db, table, lowerCasedParts, ignoreIfExists)
+    val metaStoreParts = partsWithLocation
+      .map(p => p.copy(spec = toMetaStorePartitionSpec(p.spec)))
+    client.createPartitions(db, table, metaStoreParts, ignoreIfExists)
   }
 
   override def dropPartitions(
@@ -998,7 +1017,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       retainData: Boolean): Unit = withClient {
     requireTableExists(db, table)
     client.dropPartitions(
-      db, table, parts.map(lowerCasePartitionSpec), ignoreIfNotExists, purge, retainData)
+      db, table, parts.map(toMetaStorePartitionSpec), ignoreIfNotExists, purge, retainData)
   }
 
   override def renamePartitions(
@@ -1007,7 +1026,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       specs: Seq[TablePartitionSpec],
       newSpecs: Seq[TablePartitionSpec]): Unit = withClient {
     client.renamePartitions(
-      db, table, specs.map(lowerCasePartitionSpec), newSpecs.map(lowerCasePartitionSpec))
+      db, table, specs.map(toMetaStorePartitionSpec), newSpecs.map(toMetaStorePartitionSpec))
 
     val tableMeta = getTable(db, table)
     val partitionColumnNames = tableMeta.partitionColumnNames
@@ -1023,7 +1042,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       val fs = tablePath.getFileSystem(hadoopConf)
       val newParts = newSpecs.map { spec =>
         val rightPath = renamePartitionDirectory(fs, tablePath, partitionColumnNames, spec)
-        val partition = client.getPartition(db, table, lowerCasePartitionSpec(spec))
+        val partition = client.getPartition(db, table, toMetaStorePartitionSpec(spec))
         partition.copy(storage = partition.storage.copy(locationUri = Some(rightPath.toUri)))
       }
       alterPartitions(db, table, newParts)
@@ -1133,12 +1152,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       db: String,
       table: String,
       newParts: Seq[CatalogTablePartition]): Unit = withClient {
-    val lowerCasedParts = newParts.map(p => p.copy(spec = lowerCasePartitionSpec(p.spec)))
-
-    val rawTable = getRawTable(db, table)
-
+    val metaStoreParts = newParts.map(p => p.copy(spec = toMetaStorePartitionSpec(p.spec)))
     // convert partition statistics to properties so that we can persist them through hive api
-    val withStatsProps = lowerCasedParts.map { p =>
+    val withStatsProps = metaStoreParts.map { p =>
       if (p.stats.isDefined) {
         val statsProperties = statsToProperties(p.stats.get)
         p.copy(parameters = p.parameters ++ statsProperties)
@@ -1154,7 +1170,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       db: String,
       table: String,
       spec: TablePartitionSpec): CatalogTablePartition = withClient {
-    val part = client.getPartition(db, table, lowerCasePartitionSpec(spec))
+    val part = client.getPartition(db, table, toMetaStorePartitionSpec(spec))
     restorePartitionMetadata(part, getTable(db, table))
   }
 
@@ -1192,7 +1208,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       db: String,
       table: String,
       spec: TablePartitionSpec): Option[CatalogTablePartition] = withClient {
-    client.getPartitionOption(db, table, lowerCasePartitionSpec(spec)).map { part =>
+    client.getPartitionOption(db, table, toMetaStorePartitionSpec(spec)).map { part =>
       restorePartitionMetadata(part, getTable(db, table))
     }
   }
@@ -1207,7 +1223,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
     val catalogTable = getTable(db, table)
     val partColNameMap = buildLowerCasePartColNameMap(catalogTable).mapValues(escapePathName)
     val clientPartitionNames =
-      client.getPartitionNames(catalogTable, partialSpec.map(lowerCasePartitionSpec))
+      client.getPartitionNames(catalogTable, partialSpec.map(toMetaStorePartitionSpec))
     clientPartitionNames.map { partitionPath =>
       val partSpec = PartitioningUtils.parsePathFragmentAsSeq(partitionPath)
       partSpec.map { case (partName, partValue) =>
@@ -1226,11 +1242,12 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       table: String,
       partialSpec: Option[TablePartitionSpec] = None): Seq[CatalogTablePartition] = withClient {
     val partColNameMap = buildLowerCasePartColNameMap(getTable(db, table))
-    val res = client.getPartitions(db, table, partialSpec.map(lowerCasePartitionSpec)).map { part =>
-      part.copy(spec = restorePartitionSpec(part.spec, partColNameMap))
+    val metaStoreSpec = partialSpec.map(toMetaStorePartitionSpec)
+    val res = client.getPartitions(db, table, metaStoreSpec)
+      .map { part => part.copy(spec = restorePartitionSpec(part.spec, partColNameMap))
     }
 
-    partialSpec match {
+    metaStoreSpec match {
       // This might be a bug of Hive: When the partition value inside the partial partition spec
       // contains dot, and we ask Hive to list partitions w.r.t. the partial partition spec, Hive
       // treats dot as matching any single character and may return more partitions than we
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 0cd9b3641bd4a..16e9014340244 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.hive
 
 import java.lang.reflect.{ParameterizedType, Type, WildcardType}
-import java.util.concurrent.TimeUnit._
 
 import scala.collection.JavaConverters._
 
@@ -33,6 +32,7 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.execution.datasources.DaysWritable
 import org.apache.spark.sql.types
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -466,7 +466,7 @@ private[hive] trait HiveInspectors {
         _ => constant
       case poi: WritableConstantTimestampObjectInspector =>
         val t = poi.getWritableConstantValue
-        val constant = SECONDS.toMicros(t.getSeconds) + NANOSECONDS.toMicros(t.getNanos)
+        val constant = DateTimeUtils.fromJavaTimestamp(t.getTimestamp)
         _ => constant
       case poi: WritableConstantIntObjectInspector =>
         val constant = poi.getWritableConstantValue.get()
@@ -618,7 +618,7 @@ private[hive] trait HiveInspectors {
         case x: DateObjectInspector if x.preferWritable() =>
           data: Any => {
             if (data != null) {
-              DateTimeUtils.fromJavaDate(x.getPrimitiveWritableObject(data).get())
+              new DaysWritable(x.getPrimitiveWritableObject(data)).gregorianDays
             } else {
               null
             }
@@ -634,8 +634,7 @@ private[hive] trait HiveInspectors {
         case x: TimestampObjectInspector if x.preferWritable() =>
           data: Any => {
             if (data != null) {
-              val t = x.getPrimitiveWritableObject(data)
-              SECONDS.toMicros(t.getSeconds) + NANOSECONDS.toMicros(t.getNanos)
+              DateTimeUtils.fromJavaTimestamp(x.getPrimitiveWritableObject(data).getTimestamp)
             } else {
               null
             }
@@ -1011,8 +1010,12 @@ private[hive] trait HiveInspectors {
       new hadoopIo.BytesWritable(value.asInstanceOf[Array[Byte]])
     }
 
-  private def getDateWritable(value: Any): hiveIo.DateWritable =
-    if (value == null) null else new hiveIo.DateWritable(value.asInstanceOf[Int])
+  private def getDateWritable(value: Any): DaysWritable =
+    if (value == null) {
+      null
+    } else {
+      new DaysWritable(value.asInstanceOf[Int])
+    }
 
   private def getTimestampWritable(value: Any): hiveIo.TimestampWritable =
     if (value == null) {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
index b117c582a3e6e..e25610757a69b 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
@@ -25,7 +25,9 @@ import org.apache.spark.sql.catalyst.optimizer.Optimizer
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.{SparkOptimizer, SparkPlanner}
+import org.apache.spark.sql.execution.aggregate.ResolveEncodersInScalaAgg
 import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin
+import org.apache.spark.sql.execution.command.CommandCheck
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.execution.datasources.v2.TableCapabilityCheck
 import org.apache.spark.sql.hive.client.HiveClient
@@ -75,7 +77,9 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
         new FindDataSourceTable(session) +:
         new ResolveSQLOnFile(session) +:
         new FallBackFileSourceV2(session) +:
-        new ResolveSessionCatalog(catalogManager, conf, catalog.isView) +:
+        ResolveEncodersInScalaAgg +:
+        new ResolveSessionCatalog(
+          catalogManager, conf, catalog.isTempView, catalog.isTempFunction) +:
         customResolutionRules
 
     override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =
@@ -92,22 +96,12 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
       PreWriteCheck +:
         PreReadCheck +:
         TableCapabilityCheck +:
+        CommandCheck(conf) +:
         customCheckRules
   }
 
-  /**
-   * Logical query plan optimizer that takes into account Hive.
-   */
-  override protected def optimizer: Optimizer = {
-    new SparkOptimizer(catalogManager, catalog, experimentalMethods) {
-      override def postHocOptimizationBatches: Seq[Batch] = Seq(
-        Batch("Prune Hive Table Partitions", Once, new PruneHiveTablePartitions(session))
-      )
-
-      override def extendedOperatorOptimizationRules: Seq[Rule[LogicalPlan]] =
-        super.extendedOperatorOptimizationRules ++ customOperatorOptimizationRules
-    }
-  }
+  override def customEarlyScanPushDownRules: Seq[Rule[LogicalPlan]] =
+    Seq(new PruneHiveTablePartitions(session))
 
   /**
    * Planner that takes into account Hive-specific strategies.
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
index 3beef6b1df457..04a6a8f8aa9a5 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
@@ -118,9 +118,12 @@ private[hive] object HiveShim {
    *
    * @param functionClassName UDF class name
    * @param instance optional UDF instance which contains additional information (for macro)
+   * @param clazz optional class instance to create UDF instance
    */
-  private[hive] case class HiveFunctionWrapper(var functionClassName: String,
-    private var instance: AnyRef = null) extends java.io.Externalizable {
+  private[hive] case class HiveFunctionWrapper(
+      var functionClassName: String,
+      private var instance: AnyRef = null,
+      private var clazz: Class[_ <: AnyRef] = null) extends java.io.Externalizable {
 
     // for Serialization
     def this() = this(null)
@@ -232,8 +235,10 @@ private[hive] object HiveShim {
         in.readFully(functionInBytes)
 
         // deserialize the function object via Hive Utilities
+        clazz = Utils.getContextOrSparkClassLoader.loadClass(functionClassName)
+          .asInstanceOf[Class[_ <: AnyRef]]
         instance = deserializePlan[AnyRef](new java.io.ByteArrayInputStream(functionInBytes),
-          Utils.getContextOrSparkClassLoader.loadClass(functionClassName))
+          clazz)
       }
     }
 
@@ -241,8 +246,11 @@ private[hive] object HiveShim {
       if (instance != null) {
         instance.asInstanceOf[UDFType]
       } else {
-        val func = Utils.getContextOrSparkClassLoader
-          .loadClass(functionClassName).getConstructor().newInstance().asInstanceOf[UDFType]
+        if (clazz == null) {
+          clazz = Utils.getContextOrSparkClassLoader.loadClass(functionClassName)
+            .asInstanceOf[Class[_ <: AnyRef]]
+        }
+        val func = clazz.getConstructor().newInstance().asInstanceOf[UDFType]
         if (!func.isInstanceOf[UDF]) {
           // We cache the function if it's no the Simple UDF,
           // as we always have to create new instance for Simple UDF
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index 9c4b8a5819a33..04caf57efdc74 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -61,10 +61,11 @@ private[spark] object HiveUtils extends Logging {
   /** The version of hive used internally by Spark SQL. */
   val builtinHiveVersion: String = if (isHive23) hiveVersion else "1.2.1"
 
-  val HIVE_METASTORE_VERSION = buildConf("spark.sql.hive.metastore.version")
+  val HIVE_METASTORE_VERSION = buildStaticConf("spark.sql.hive.metastore.version")
     .doc("Version of the Hive metastore. Available options are " +
-        "<code>0.12.0</code> through <code>2.3.6</code> and " +
+        "<code>0.12.0</code> through <code>2.3.7</code> and " +
         "<code>3.0.0</code> through <code>3.1.2</code>.")
+    .version("1.4.0")
     .stringConf
     .createWithDefault(builtinHiveVersion)
 
@@ -73,10 +74,10 @@ private[spark] object HiveUtils extends Logging {
   // already rely on this config.
   val FAKE_HIVE_VERSION = buildConf("spark.sql.hive.version")
     .doc(s"deprecated, please use ${HIVE_METASTORE_VERSION.key} to get the Hive version in Spark.")
-    .stringConf
-    .createWithDefault(builtinHiveVersion)
+    .version("1.1.1")
+    .fallbackConf(HIVE_METASTORE_VERSION)
 
-  val HIVE_METASTORE_JARS = buildConf("spark.sql.hive.metastore.jars")
+  val HIVE_METASTORE_JARS = buildStaticConf("spark.sql.hive.metastore.jars")
     .doc(s"""
       | Location of the jars that should be used to instantiate the HiveMetastoreClient.
       | This property can be one of three options: "
@@ -89,12 +90,14 @@ private[spark] object HiveUtils extends Logging {
       |   Use Hive jars of specified version downloaded from Maven repositories.
       | 3. A classpath in the standard format for both Hive and Hadoop.
       """.stripMargin)
+    .version("1.4.0")
     .stringConf
     .createWithDefault("builtin")
 
   val CONVERT_METASTORE_PARQUET = buildConf("spark.sql.hive.convertMetastoreParquet")
     .doc("When set to true, the built-in Parquet reader and writer are used to process " +
       "parquet tables created by using the HiveQL syntax, instead of Hive serde.")
+    .version("1.1.1")
     .booleanConf
     .createWithDefault(true)
 
@@ -103,12 +106,14 @@ private[spark] object HiveUtils extends Logging {
       .doc("When true, also tries to merge possibly different but compatible Parquet schemas in " +
         "different Parquet data files. This configuration is only effective " +
         "when \"spark.sql.hive.convertMetastoreParquet\" is true.")
+      .version("1.3.1")
       .booleanConf
       .createWithDefault(false)
 
   val CONVERT_METASTORE_ORC = buildConf("spark.sql.hive.convertMetastoreOrc")
     .doc("When set to true, the built-in ORC reader and writer are used to process " +
       "ORC tables created by using the HiveQL syntax, instead of Hive serde.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -118,6 +123,7 @@ private[spark] object HiveUtils extends Logging {
         "`spark.sql.hive.convertMetastoreOrc` is true, the built-in ORC/Parquet writer is used" +
         "to process inserting into partitioned ORC/Parquet tables created by using the HiveSQL " +
         "syntax.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -126,15 +132,17 @@ private[spark] object HiveUtils extends Logging {
       "instead of Hive serde in CTAS. This flag is effective only if " +
       "`spark.sql.hive.convertMetastoreParquet` or `spark.sql.hive.convertMetastoreOrc` is " +
       "enabled respectively for Parquet and ORC formats")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
-  val HIVE_METASTORE_SHARED_PREFIXES = buildConf("spark.sql.hive.metastore.sharedPrefixes")
+  val HIVE_METASTORE_SHARED_PREFIXES = buildStaticConf("spark.sql.hive.metastore.sharedPrefixes")
     .doc("A comma separated list of class prefixes that should be loaded using the classloader " +
       "that is shared between Spark SQL and a specific version of Hive. An example of classes " +
       "that should be shared is JDBC drivers that are needed to talk to the metastore. Other " +
       "classes that need to be shared are those that interact with classes that are already " +
       "shared. For example, custom appenders that are used by log4j.")
+    .version("1.4.0")
     .stringConf
     .toSequence
     .createWithDefault(jdbcPrefixes)
@@ -142,16 +150,18 @@ private[spark] object HiveUtils extends Logging {
   private def jdbcPrefixes = Seq(
     "com.mysql.jdbc", "org.postgresql", "com.microsoft.sqlserver", "oracle.jdbc")
 
-  val HIVE_METASTORE_BARRIER_PREFIXES = buildConf("spark.sql.hive.metastore.barrierPrefixes")
+  val HIVE_METASTORE_BARRIER_PREFIXES = buildStaticConf("spark.sql.hive.metastore.barrierPrefixes")
     .doc("A comma separated list of class prefixes that should explicitly be reloaded for each " +
       "version of Hive that Spark SQL is communicating with. For example, Hive UDFs that are " +
       "declared in a prefix that typically would be shared (i.e. <code>org.apache.spark.*</code>).")
+    .version("1.4.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val HIVE_THRIFT_SERVER_ASYNC = buildConf("spark.sql.hive.thriftServer.async")
     .doc("When set to true, Hive Thrift server executes SQL queries in an asynchronous way.")
+    .version("1.5.0")
     .booleanConf
     .createWithDefault(true)
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
index e31dffa4795c5..3ea80eaf6f714 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
@@ -61,6 +61,15 @@ private[hive] trait HiveClient {
   /** Returns the names of tables in the given database that matches the given pattern. */
   def listTables(dbName: String, pattern: String): Seq[String]
 
+  /**
+   * Returns the names of tables with specific tableType in the given database that matches
+   * the given pattern.
+   */
+  def listTablesByType(
+      dbName: String,
+      pattern: String,
+      tableType: CatalogTableType): Seq[String]
+
   /** Sets the name of current database. */
   def setCurrentDatabase(databaseName: String): Unit
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index b5c5f0e9381bc..f3118368b9829 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.hive.client
 
 import java.io.{File, PrintStream}
 import java.lang.{Iterable => JIterable}
+import java.lang.reflect.InvocationTargetException
 import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Locale, Map => JMap}
 import java.util.concurrent.TimeUnit._
@@ -48,7 +49,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.metrics.source.HiveCatalogMetrics
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException, NoSuchPartitionsException, PartitionsAlreadyExistException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.expressions.Expression
@@ -58,7 +59,6 @@ import org.apache.spark.sql.execution.QueryExecutionException
 import org.apache.spark.sql.hive.HiveExternalCatalog
 import org.apache.spark.sql.hive.HiveExternalCatalog.{DATASOURCE_SCHEMA, DATASOURCE_SCHEMA_NUMPARTS, DATASOURCE_SCHEMA_PART_PREFIX}
 import org.apache.spark.sql.hive.HiveUtils
-import org.apache.spark.sql.hive.client.HiveClientImpl._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.util.{CircularBuffer, Utils}
@@ -99,6 +99,8 @@ private[hive] class HiveClientImpl(
   extends HiveClient
   with Logging {
 
+  import HiveClientImpl._
+
   // Circular buffer to hold what hive prints to STDOUT and ERR.  Only printed when failures occur.
   private val outputBuffer = new CircularBuffer()
 
@@ -159,36 +161,7 @@ private[hive] class HiveClientImpl(
       s"(version ${version.fullVersion}) is ${conf.getVar(ConfVars.METASTOREWAREHOUSE)}")
 
   private def newState(): SessionState = {
-    val hiveConf = new HiveConf(classOf[SessionState])
-    // HiveConf is a Hadoop Configuration, which has a field of classLoader and
-    // the initial value will be the current thread's context class loader
-    // (i.e. initClassLoader at here).
-    // We call hiveConf.setClassLoader(initClassLoader) at here to make
-    // this action explicit.
-    hiveConf.setClassLoader(initClassLoader)
-
-    // 1: Take all from the hadoopConf to this hiveConf.
-    // This hadoopConf contains user settings in Hadoop's core-site.xml file
-    // and Hive's hive-site.xml file. Note, we load hive-site.xml file manually in
-    // SharedState and put settings in this hadoopConf instead of relying on HiveConf
-    // to load user settings. Otherwise, HiveConf's initialize method will override
-    // settings in the hadoopConf. This issue only shows up when spark.sql.hive.metastore.jars
-    // is not set to builtin. When spark.sql.hive.metastore.jars is builtin, the classpath
-    // has hive-site.xml. So, HiveConf will use that to override its default values.
-    // 2: we set all spark confs to this hiveConf.
-    // 3: we set all entries in config to this hiveConf.
-    val confMap = (hadoopConf.iterator().asScala.map(kv => kv.getKey -> kv.getValue) ++
-      sparkConf.getAll.toMap ++ extraConfig).toMap
-    confMap.foreach { case (k, v) => hiveConf.set(k, v) }
-    SQLConf.get.redactOptions(confMap).foreach { case (k, v) =>
-      logDebug(
-        s"""
-           |Applying Hadoop/Hive/Spark and extra properties to Hive Conf:
-           |$k=$v
-         """.stripMargin)
-    }
-    // Disable CBO because we removed the Calcite dependency.
-    hiveConf.setBoolean("hive.cbo.enable", false)
+    val hiveConf = newHiveConf(sparkConf, hadoopConf, extraConfig, Some(initClassLoader))
     val state = new SessionState(hiveConf)
     if (clientLoader.cachedHive != null) {
       Hive.set(clientLoader.cachedHive.asInstanceOf[Hive])
@@ -318,7 +291,15 @@ private[hive] class HiveClientImpl(
     // with the HiveConf in `state` to override the context class loader of the current
     // thread.
     shim.setCurrentSessionState(state)
-    val ret = try f finally {
+    val ret = try {
+      f
+    } catch {
+      case e: NoClassDefFoundError
+        if HiveUtils.isHive23 && e.getMessage.contains("org/apache/hadoop/hive/serde2/SerDe") =>
+        throw new ClassNotFoundException("The SerDe interface removed since Hive 2.3(HIVE-15167)." +
+          " Please migrate your custom SerDes to Hive 2.3 or build your own Spark with" +
+          " hive-1.2 profile. See HIVE-15167 for more details.", e)
+    } finally {
       state.getConf.setClassLoader(originalConfLoader)
       Thread.currentThread().setContextClassLoader(original)
       HiveCatalogMetrics.incrementHiveClientCalls(1)
@@ -625,7 +606,17 @@ private[hive] class HiveClientImpl(
       table: String,
       parts: Seq[CatalogTablePartition],
       ignoreIfExists: Boolean): Unit = withHiveState {
-    shim.createPartitions(client, db, table, parts, ignoreIfExists)
+    def replaceExistException(e: Throwable): Unit = e match {
+      case _: HiveException if e.getCause.isInstanceOf[AlreadyExistsException] =>
+        throw new PartitionsAlreadyExistException(db, table, parts.map(_.spec))
+      case _ => throw e
+    }
+    try {
+      shim.createPartitions(client, db, table, parts, ignoreIfExists)
+    } catch {
+      case e: InvocationTargetException => replaceExistException(e.getCause)
+      case e: Throwable => replaceExistException(e)
+    }
   }
 
   override def dropPartitions(
@@ -646,9 +637,7 @@ private[hive] class HiveClientImpl(
         // (b='1', c='1') and (b='1', c='2'), a partial spec of (b='1') will match both.
         val parts = client.getPartitions(hiveTable, s.asJava).asScala
         if (parts.isEmpty && !ignoreIfNotExists) {
-          throw new AnalysisException(
-            s"No partition is dropped. One partition spec '$s' does not exist in table '$table' " +
-            s"database '$db'")
+          throw new NoSuchPartitionsException(db, table, Seq(s))
         }
         parts.map(_.getValues)
       }.distinct
@@ -775,6 +764,24 @@ private[hive] class HiveClientImpl(
     client.getTablesByPattern(dbName, pattern).asScala
   }
 
+  override def listTablesByType(
+      dbName: String,
+      pattern: String,
+      tableType: CatalogTableType): Seq[String] = withHiveState {
+    val hiveTableType = toHiveTableType(tableType)
+    try {
+      // Try with Hive API getTablesByType first, it's supported from Hive 2.3+.
+      shim.getTablesByType(client, dbName, pattern, hiveTableType)
+    } catch {
+      case _: UnsupportedOperationException =>
+        // Fallback to filter logic if getTablesByType not supported.
+        val tableNames = client.getTablesByPattern(dbName, pattern).asScala
+        getRawTablesByName(dbName, tableNames)
+          .filter(_.getTableType == hiveTableType)
+          .map(_.getTableName)
+    }
+  }
+
   /**
    * Runs the specified SQL query using Hive.
    */
@@ -831,7 +838,12 @@ private[hive] class HiveClientImpl(
             state.out.println(tokens(0) + " " + cmd_1)
             // scalastyle:on println
           }
-          Seq(proc.run(cmd_1).getResponseCode.toString)
+          val response: CommandProcessorResponse = proc.run(cmd_1)
+          // Throw an exception if there is an error in query processing.
+          if (response.getResponseCode != 0) {
+            throw new QueryExecutionException(response.getErrorMessage)
+          }
+          Seq(response.getResponseCode.toString)
       }
     } catch {
       case e: Exception =>
@@ -977,7 +989,7 @@ private[hive] class HiveClientImpl(
   }
 }
 
-private[hive] object HiveClientImpl {
+private[hive] object HiveClientImpl extends Logging {
   /** Converts the native StructField to Hive's FieldSchema. */
   def toHiveColumn(c: StructField): FieldSchema = {
     val typeString = if (c.metadata.contains(HIVE_TYPE_STRING)) {
@@ -1026,25 +1038,29 @@ private[hive] object HiveClientImpl {
   private def toOutputFormat(name: String) =
     Utils.classForName[org.apache.hadoop.hive.ql.io.HiveOutputFormat[_, _]](name)
 
+  def toHiveTableType(catalogTableType: CatalogTableType): HiveTableType = {
+    catalogTableType match {
+      case CatalogTableType.EXTERNAL => HiveTableType.EXTERNAL_TABLE
+      case CatalogTableType.MANAGED => HiveTableType.MANAGED_TABLE
+      case CatalogTableType.VIEW => HiveTableType.VIRTUAL_VIEW
+      case t =>
+        throw new IllegalArgumentException(
+          s"Unknown table type is found at toHiveTableType: $t")
+    }
+  }
+
   /**
    * Converts the native table metadata representation format CatalogTable to Hive's Table.
    */
   def toHiveTable(table: CatalogTable, userName: Option[String] = None): HiveTable = {
     val hiveTable = new HiveTable(table.database, table.identifier.table)
+    hiveTable.setTableType(toHiveTableType(table.tableType))
     // For EXTERNAL_TABLE, we also need to set EXTERNAL field in the table properties.
     // Otherwise, Hive metastore will change the table to a MANAGED_TABLE.
     // (metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java#L1095-L1105)
-    hiveTable.setTableType(table.tableType match {
-      case CatalogTableType.EXTERNAL =>
-        hiveTable.setProperty("EXTERNAL", "TRUE")
-        HiveTableType.EXTERNAL_TABLE
-      case CatalogTableType.MANAGED =>
-        HiveTableType.MANAGED_TABLE
-      case CatalogTableType.VIEW => HiveTableType.VIRTUAL_VIEW
-      case t =>
-        throw new IllegalArgumentException(
-          s"Unknown table type is found at toHiveTable: $t")
-    })
+    if (table.tableType == CatalogTableType.EXTERNAL) {
+      hiveTable.setProperty("EXTERNAL", "TRUE")
+    }
     // Note: In Hive the schema and partition columns must be disjoint sets
     val (partCols, schema) = table.schema.map(toHiveColumn).partition { c =>
       table.partitionColumnNames.contains(c.getName)
@@ -1219,4 +1235,50 @@ private[hive] object HiveClientImpl {
     StatsSetupConst.RAW_DATA_SIZE,
     StatsSetupConst.TOTAL_SIZE
   )
+
+  def newHiveConf(
+      sparkConf: SparkConf,
+      hadoopConf: JIterable[JMap.Entry[String, String]],
+      extraConfig: Map[String, String],
+      classLoader: Option[ClassLoader] = None): HiveConf = {
+    val hiveConf = new HiveConf(classOf[SessionState])
+    // HiveConf is a Hadoop Configuration, which has a field of classLoader and
+    // the initial value will be the current thread's context class loader.
+    // We call hiveConf.setClassLoader(initClassLoader) at here to ensure it use the classloader
+    // we want.
+    classLoader.foreach(hiveConf.setClassLoader)
+    // 1: Take all from the hadoopConf to this hiveConf.
+    // This hadoopConf contains user settings in Hadoop's core-site.xml file
+    // and Hive's hive-site.xml file. Note, we load hive-site.xml file manually in
+    // SharedState and put settings in this hadoopConf instead of relying on HiveConf
+    // to load user settings. Otherwise, HiveConf's initialize method will override
+    // settings in the hadoopConf. This issue only shows up when spark.sql.hive.metastore.jars
+    // is not set to builtin. When spark.sql.hive.metastore.jars is builtin, the classpath
+    // has hive-site.xml. So, HiveConf will use that to override its default values.
+    // 2: we set all spark confs to this hiveConf.
+    // 3: we set all entries in config to this hiveConf.
+    val confMap = (hadoopConf.iterator().asScala.map(kv => kv.getKey -> kv.getValue) ++
+      sparkConf.getAll.toMap ++ extraConfig).toMap
+    confMap.foreach { case (k, v) => hiveConf.set(k, v) }
+    SQLConf.get.redactOptions(confMap).foreach { case (k, v) =>
+      logDebug(s"Applying Hadoop/Hive/Spark and extra properties to Hive Conf:$k=$v")
+    }
+    // Disable CBO because we removed the Calcite dependency.
+    hiveConf.setBoolean("hive.cbo.enable", false)
+    // If this is true, SessionState.start will create a file to log hive job which will not be
+    // deleted on exit and is useless for spark
+    if (hiveConf.getBoolean("hive.session.history.enabled", false)) {
+      logWarning("Detected HiveConf hive.session.history.enabled is true and will be reset to" +
+        " false to disable useless hive logic")
+      hiveConf.setBoolean("hive.session.history.enabled", false)
+    }
+    // If this is tez engine, SessionState.start might bring extra logic to initialize tez stuff,
+    // which is useless for spark.
+    if (hiveConf.get("hive.execution.engine") == "tez") {
+      logWarning("Detected HiveConf hive.execution.engine is 'tez' and will be reset to 'mr'" +
+        " to disable useless hive logic")
+      hiveConf.set("hive.execution.engine", "mr")
+    }
+    hiveConf
+  }
 }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 50ce536a160c8..d11bf94a1fd86 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -29,6 +29,7 @@ import scala.util.control.NonFatal
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.hive.conf.HiveConf
 import org.apache.hadoop.hive.metastore.IMetaStoreClient
+import org.apache.hadoop.hive.metastore.TableType
 import org.apache.hadoop.hive.metastore.api.{Database, EnvironmentContext, Function => HiveFunction, FunctionType, MetaException, PrincipalType, ResourceType, ResourceUri}
 import org.apache.hadoop.hive.ql.Driver
 import org.apache.hadoop.hive.ql.io.AcidUtils
@@ -90,6 +91,12 @@ private[client] sealed abstract class Shim {
 
   def alterPartitions(hive: Hive, tableName: String, newParts: JList[Partition]): Unit
 
+  def getTablesByType(
+      hive: Hive,
+      dbName: String,
+      pattern: String,
+      tableType: TableType): Seq[String]
+
   def createPartitions(
       hive: Hive,
       db: String,
@@ -363,6 +370,15 @@ private[client] class Shim_v0_12 extends Shim with Logging {
     conf.getIntVar(HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY) * 1000L
   }
 
+  override def getTablesByType(
+      hive: Hive,
+      dbName: String,
+      pattern: String,
+      tableType: TableType): Seq[String] = {
+    throw new UnsupportedOperationException("Hive 2.2 and lower versions don't support " +
+      "getTablesByType. Please use Hive 2.3 or higher version.")
+  }
+
   override def loadPartition(
       hive: Hive,
       loadPath: Path,
@@ -692,7 +708,8 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
         .map(col => col.getName).toSet
 
       def unapply(attr: Attribute): Option[String] = {
-        if (varcharKeys.contains(attr.name)) {
+        val resolver = SQLConf.get.resolver
+        if (varcharKeys.exists(c => resolver(c, attr.name))) {
           None
         } else if (attr.dataType.isInstanceOf[IntegralType] || attr.dataType == StringType) {
           Some(attr.name)
@@ -712,7 +729,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
       def unapply(expr: Expression): Option[Attribute] = {
         expr match {
           case attr: Attribute => Some(attr)
-          case Cast(child @ AtomicType(), dt: AtomicType, _)
+          case Cast(child @ IntegralType(), dt: IntegralType, _)
               if Cast.canUpCast(child.dataType.asInstanceOf[AtomicType], dt) => unapply(child)
           case _ => None
         }
@@ -1220,7 +1237,24 @@ private[client] class Shim_v2_1 extends Shim_v2_0 {
 
 private[client] class Shim_v2_2 extends Shim_v2_1
 
-private[client] class Shim_v2_3 extends Shim_v2_1
+private[client] class Shim_v2_3 extends Shim_v2_1 {
+  private lazy val getTablesByTypeMethod =
+    findMethod(
+      classOf[Hive],
+      "getTablesByType",
+      classOf[String],
+      classOf[String],
+      classOf[TableType])
+
+  override def getTablesByType(
+      hive: Hive,
+      dbName: String,
+      pattern: String,
+      tableType: TableType): Seq[String] = {
+    getTablesByTypeMethod.invoke(hive, dbName, pattern, tableType)
+      .asInstanceOf[JList[String]].asScala
+  }
+}
 
 private[client] class Shim_v3_0 extends Shim_v2_3 {
   // Spark supports only non-ACID operations
@@ -1293,10 +1327,7 @@ private[client] class Shim_v3_0 extends Shim_v2_3 {
       inheritTableSpecs: Boolean,
       isSkewedStoreAsSubdir: Boolean,
       isSrcLocal: Boolean): Unit = {
-    val session = SparkSession.getActiveSession
-    assert(session.nonEmpty)
-    val database = session.get.sessionState.catalog.getCurrentDatabase
-    val table = hive.getTable(database, tableName)
+    val table = hive.getTable(tableName)
     val loadFileType = if (replace) {
       clazzLoadFileType.getEnumConstants.find(_.toString.equalsIgnoreCase("REPLACE_ALL"))
     } else {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index 5da7b70cfc7aa..42a0ec0253b85 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -104,7 +104,8 @@ private[hive] object IsolatedClientLoader extends Logging {
     case "2.0" | "2.0.0" | "2.0.1" => hive.v2_0
     case "2.1" | "2.1.0" | "2.1.1" => hive.v2_1
     case "2.2" | "2.2.0" => hive.v2_2
-    case "2.3" | "2.3.0" | "2.3.1" | "2.3.2" | "2.3.3" | "2.3.4" | "2.3.5" | "2.3.6" => hive.v2_3
+    case "2.3" | "2.3.0" | "2.3.1" | "2.3.2" | "2.3.3" | "2.3.4" | "2.3.5" | "2.3.6" | "2.3.7" =>
+      hive.v2_3
     case "3.0" | "3.0.0" => hive.v3_0
     case "3.1" | "3.1.0" | "3.1.1" | "3.1.2" => hive.v3_1
     case version =>
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala
index 26319119a596a..8526d86454604 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala
@@ -89,7 +89,7 @@ package object client {
 
     // Since HIVE-14496, Hive materialized view need calcite-core.
     // For spark, only VersionsSuite currently creates a hive materialized view for testing.
-    case object v2_3 extends HiveVersion("2.3.6",
+    case object v2_3 extends HiveVersion("2.3.7",
       exclusions = Seq("org.apache.calcite:calcite-druid",
         "org.apache.calcite.avatica:avatica",
         "org.apache.curator:*",
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala
index b66c302a7d7ea..7ef637ed553ad 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.hive.client.HiveClientImpl
 import org.apache.spark.sql.util.SchemaUtils
 
@@ -63,12 +64,16 @@ case class InsertIntoHiveDirCommand(
       s"when inserting into ${storage.locationUri.get}",
       sparkSession.sessionState.conf.caseSensitiveAnalysis)
 
-    val hiveTable = HiveClientImpl.toHiveTable(CatalogTable(
+    val table = CatalogTable(
       identifier = TableIdentifier(storage.locationUri.get.toString, Some("default")),
+      provider = Some(DDLUtils.HIVE_PROVIDER),
       tableType = org.apache.spark.sql.catalyst.catalog.CatalogTableType.VIEW,
       storage = storage,
       schema = outputColumns.toStructType
-    ))
+    )
+    DDLUtils.checkDataColNames(table)
+
+    val hiveTable = HiveClientImpl.toHiveTable(table)
     hiveTable.getMetadata.put(serdeConstants.SERIALIZATION_LIB,
       storage.serde.getOrElse(classOf[LazySimpleSerDe].getName))
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
index 801be64702519..39d5b711ab87e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
 import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, ExternalCatalog, ExternalCatalogUtils}
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.command.CommandUtils
 import org.apache.spark.sql.hive.HiveShim.{ShimFileSinkDesc => FileSinkDesc}
@@ -106,7 +107,7 @@ case class InsertIntoHiveTable(
     }
 
     // un-cache this table.
-    sparkSession.catalog.uncacheTable(table.identifier.quotedString)
+    CommandUtils.uncacheTableOrView(sparkSession, table.identifier.quotedString)
     sparkSession.sessionState.catalog.refreshTable(table.identifier)
 
     CommandUtils.updateTableStats(sparkSession, table)
@@ -130,6 +131,7 @@ case class InsertIntoHiveTable(
     val numDynamicPartitions = partition.values.count(_.isEmpty)
     val numStaticPartitions = partition.values.count(_.nonEmpty)
     val partitionSpec = partition.map {
+      case (key, Some(null)) => key -> ExternalCatalogUtils.DEFAULT_PARTITION_NAME
       case (key, Some(value)) => key -> value
       case (key, None) => key -> ""
     }
@@ -223,9 +225,13 @@ case class InsertIntoHiveTable(
                 ExternalCatalogUtils.unescapePathName(splitPart(1))
             }.toMap
 
+            val caseInsensitiveDpMap = CaseInsensitiveMap(dpMap)
+
             val updatedPartitionSpec = partition.map {
+              case (key, Some(null)) => key -> ExternalCatalogUtils.DEFAULT_PARTITION_NAME
               case (key, Some(value)) => key -> value
-              case (key, None) if dpMap.contains(key) => key -> dpMap(key)
+              case (key, None) if caseInsensitiveDpMap.contains(key) =>
+                key -> caseInsensitiveDpMap(key)
               case (key, _) =>
                 throw new SparkException(s"Dynamic partition key $key is not among " +
                   "written partition paths.")
@@ -239,7 +245,7 @@ case class InsertIntoHiveTable(
             if (fs.exists(partitionPath)) {
               if (!fs.delete(partitionPath, true)) {
                 throw new RuntimeException(
-                  "Cannot remove partition directory '" + partitionPath.toString)
+                  s"Cannot remove partition directory '$partitionPath'")
               }
             }
           }
@@ -291,7 +297,7 @@ case class InsertIntoHiveTable(
               if (fs.exists(path)) {
                 if (!fs.delete(path, true)) {
                   throw new RuntimeException(
-                    "Cannot remove partition directory '" + path.toString)
+                    s"Cannot remove partition directory '$path'")
                 }
                 // Don't let Hive do overwrite operation since it is slower.
                 doHiveOverwrite = false
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala
index a0349f627d107..da6e4c52cf3a7 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala
@@ -30,6 +30,14 @@ import org.apache.spark.sql.execution.datasources.DataSourceStrategy
 import org.apache.spark.sql.internal.SQLConf
 
 /**
+ * Prune hive table partitions using partition filters on [[HiveTableRelation]]. The pruned
+ * partitions will be kept in [[HiveTableRelation.prunedPartitions]], and the statistics of
+ * the hive table relation will be updated based on pruned partitions.
+ *
+ * This rule is executed in optimization phase, so the statistics can be updated before physical
+ * planning, which is useful for some spark strategy, eg.
+ * [[org.apache.spark.sql.execution.SparkStrategies.JoinSelection]].
+ *
  * TODO: merge this with PruneFileSourcePartitions after we completely make hive as a data source.
  */
 private[sql] class PruneHiveTablePartitions(session: SparkSession)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
index e12f663304e7a..c7183fd7385a6 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hive.execution
 import java.io._
 import java.nio.charset.StandardCharsets
 import java.util.Properties
+import java.util.concurrent.TimeUnit
 import javax.annotation.Nullable
 
 import scala.collection.JavaConverters._
@@ -42,6 +43,7 @@ import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.hive.HiveInspectors
 import org.apache.spark.sql.hive.HiveShim._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.DataType
 import org.apache.spark.util.{CircularBuffer, RedirectThread, SerializableConfiguration, Utils}
 
@@ -94,9 +96,8 @@ case class ScriptTransformationExec(
       // This new thread will consume the ScriptTransformation's input rows and write them to the
       // external process. That process's output will be read by this current thread.
       val writerThread = new ScriptTransformationWriterThread(
-        inputIterator,
+        inputIterator.map(outputProjection),
         input.map(_.dataType),
-        outputProjection,
         inputSerde,
         inputSoi,
         ioschema,
@@ -137,6 +138,15 @@ case class ScriptTransformationExec(
             throw writerThread.exception.get
           }
 
+          // There can be a lag between reader read EOF and the process termination.
+          // If the script fails to startup, this kind of error may be missed.
+          // So explicitly waiting for the process termination.
+          val timeout = conf.getConf(SQLConf.SCRIPT_TRANSFORMATION_EXIT_TIMEOUT)
+          val exitRes = proc.waitFor(timeout, TimeUnit.SECONDS)
+          if (!exitRes) {
+            log.warn(s"Transformation script process exits timeout in $timeout seconds")
+          }
+
           if (!proc.isAlive) {
             val exitCode = proc.exitValue()
             if (exitCode != 0) {
@@ -174,7 +184,6 @@ case class ScriptTransformationExec(
                     // Ideally the proc should *not* be alive at this point but
                     // there can be a lag between EOF being written out and the process
                     // being terminated. So explicitly waiting for the process to be done.
-                    proc.waitFor()
                     checkFailureAndPropagate()
                     return false
                 }
@@ -249,16 +258,15 @@ case class ScriptTransformationExec(
 private class ScriptTransformationWriterThread(
     iter: Iterator[InternalRow],
     inputSchema: Seq[DataType],
-    outputProjection: Projection,
     @Nullable inputSerde: AbstractSerDe,
-    @Nullable inputSoi: ObjectInspector,
+    @Nullable inputSoi: StructObjectInspector,
     ioschema: HiveScriptIOSchema,
     outputStream: OutputStream,
     proc: Process,
     stderrBuffer: CircularBuffer,
     taskContext: TaskContext,
     conf: Configuration
-  ) extends Thread("Thread-ScriptTransformation-Feed") with Logging {
+  ) extends Thread("Thread-ScriptTransformation-Feed") with HiveInspectors with Logging {
 
   setDaemon(true)
 
@@ -278,8 +286,8 @@ private class ScriptTransformationWriterThread(
     var threwException: Boolean = true
     val len = inputSchema.length
     try {
-      iter.map(outputProjection).foreach { row =>
-        if (inputSerde == null) {
+      if (inputSerde == null) {
+        iter.foreach { row =>
           val data = if (len == 0) {
             ioschema.inputRowFormatMap("TOK_TABLEROWFORMATLINES")
           } else {
@@ -295,10 +303,21 @@ private class ScriptTransformationWriterThread(
             sb.toString()
           }
           outputStream.write(data.getBytes(StandardCharsets.UTF_8))
-        } else {
-          val writable = inputSerde.serialize(
-            row.asInstanceOf[GenericInternalRow].values, inputSoi)
+        }
+      } else {
+        // Convert Spark InternalRows to hive data via `HiveInspectors.wrapperFor`.
+        val hiveData = new Array[Any](inputSchema.length)
+        val fieldOIs = inputSoi.getAllStructFieldRefs.asScala.map(_.getFieldObjectInspector).toArray
+        val wrappers = fieldOIs.zip(inputSchema).map { case (f, dt) => wrapperFor(f, dt) }
+
+        iter.foreach { row =>
+          var i = 0
+          while (i < fieldOIs.length) {
+            hiveData(i) = if (row.isNullAt(i)) null else wrappers(i)(row.get(i, inputSchema(i)))
+            i += 1
+          }
 
+          val writable = inputSerde.serialize(hiveData, inputSoi)
           if (scriptInputWriter != null) {
             scriptInputWriter.write(writable)
           } else {
@@ -374,14 +393,13 @@ case class HiveScriptIOSchema (
   val outputRowFormatMap = outputRowFormat.toMap.withDefault((k) => defaultFormat(k))
 
 
-  def initInputSerDe(input: Seq[Expression]): Option[(AbstractSerDe, ObjectInspector)] = {
+  def initInputSerDe(input: Seq[Expression]): Option[(AbstractSerDe, StructObjectInspector)] = {
     inputSerdeClass.map { serdeClass =>
       val (columns, columnTypes) = parseAttrs(input)
       val serde = initSerDe(serdeClass, columns, columnTypes, inputSerdeProps)
       val fieldObjectInspectors = columnTypes.map(toInspector)
       val objectInspector = ObjectInspectorFactory
         .getStandardStructObjectInspector(columns.asJava, fieldObjectInspectors.asJava)
-        .asInstanceOf[ObjectInspector]
       (serde, objectInspector)
     }
   }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 05d608a2016a5..3117781a84bbf 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -41,6 +41,11 @@ import org.apache.spark.sql.hive.HiveShim._
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
+/**
+ * Here we cannot extends `ImplicitTypeCasts` to compatible with UDF input data type, the reason is:
+ * we use children data type to reflect UDF method first and will get exception if it fails so that
+ * we can never go into `ImplicitTypeCasts`.
+ */
 private[hive] case class HiveSimpleUDF(
     name: String, funcWrapper: HiveFunctionWrapper, children: Seq[Expression])
   extends Expression
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
index 7f2eb14956dc1..d1ee1baadcbce 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
@@ -70,14 +70,12 @@ class OrcFileFormat extends FileFormat with DataSourceRegister with Serializable
     val orcOptions = new OrcOptions(options, sparkSession.sessionState.conf)
     if (orcOptions.mergeSchema) {
       SchemaMergeUtils.mergeSchemasInParallel(
-        sparkSession,
-        files,
-        OrcFileOperator.readOrcSchemasInParallel)
+        sparkSession, options, files, OrcFileOperator.readOrcSchemasInParallel)
     } else {
       val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
       OrcFileOperator.readSchema(
         files.map(_.getPath.toString),
-        Some(sparkSession.sessionState.newHadoopConf()),
+        Some(sparkSession.sessionState.newHadoopConfWithOptions(options)),
         ignoreCorruptFiles
       )
     }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala
index cd1bffb6b7ab7..f9c514567c639 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
 
 import org.apache.spark.SparkException
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.execution.datasources.orc.{OrcFilters => DatasourceOrcFilters}
 import org.apache.spark.sql.execution.datasources.orc.OrcFilters.buildTree
 import org.apache.spark.sql.hive.HiveUtils
@@ -73,9 +74,11 @@ private[orc] object OrcFilters extends Logging {
     if (HiveUtils.isHive23) {
       DatasourceOrcFilters.createFilter(schema, filters).asInstanceOf[Option[SearchArgument]]
     } else {
-      val dataTypeMap = schema.map(f => f.name -> f.dataType).toMap
+      val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
+      // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
+      val newFilters = filters.filter(!_.containsNestedColumn)
       // Combines all convertible filters using `And` to produce a single conjunction
-      val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, filters))
+      val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, newFilters))
       conjunctionOptional.map { conjunction =>
         // Then tries to build a single ORC `SearchArgument` for the conjunction predicate.
         // The input predicate is fully convertible. There should not be any empty result in the
diff --git a/sql/hive/src/test/noclasspath/TestUDTF-spark-26560.jar b/sql/hive/src/test/noclasspath/TestUDTF-spark-26560.jar
deleted file mode 100644
index b73b17d5c7880..0000000000000
Binary files a/sql/hive/src/test/noclasspath/TestUDTF-spark-26560.jar and /dev/null differ
diff --git a/sql/hive/src/test/noclasspath/hive-test-udfs.jar b/sql/hive/src/test/noclasspath/hive-test-udfs.jar
new file mode 100644
index 0000000000000..a5bfa456f6686
Binary files /dev/null and b/sql/hive/src/test/noclasspath/hive-test-udfs.jar differ
diff --git a/sql/hive/src/test/resources/data/scripts/test_transform.py b/sql/hive/src/test/resources/data/scripts/test_transform.py
index ac6d11d8b919c..dedb370f6c90e 100755
--- a/sql/hive/src/test/resources/data/scripts/test_transform.py
+++ b/sql/hive/src/test/resources/data/scripts/test_transform.py
@@ -1,3 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
 import sys
 
 delim = sys.argv[1]
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #2-0-732ed232ac592c5e7f7c913a88874fd2 b/sql/hive/src/test/resources/golden/timestamp cast #3-0-732ed232ac592c5e7f7c913a88874fd2
similarity index 100%
rename from sql/hive/src/test/resources/golden/timestamp cast #2-0-732ed232ac592c5e7f7c913a88874fd2
rename to sql/hive/src/test/resources/golden/timestamp cast #3-0-732ed232ac592c5e7f7c913a88874fd2
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #6-0-6d2da5cfada03605834e38bc4075bc79 b/sql/hive/src/test/resources/golden/timestamp cast #4-0-6d2da5cfada03605834e38bc4075bc79
similarity index 100%
rename from sql/hive/src/test/resources/golden/timestamp cast #6-0-6d2da5cfada03605834e38bc4075bc79
rename to sql/hive/src/test/resources/golden/timestamp cast #4-0-6d2da5cfada03605834e38bc4075bc79
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #4-0-732ed232ac592c5e7f7c913a88874fd2 b/sql/hive/src/test/resources/golden/timestamp cast #4-0-732ed232ac592c5e7f7c913a88874fd2
deleted file mode 100644
index 5625e59da8873..0000000000000
--- a/sql/hive/src/test/resources/golden/timestamp cast #4-0-732ed232ac592c5e7f7c913a88874fd2	
+++ /dev/null
@@ -1 +0,0 @@
-1.2
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #8-0-6d2da5cfada03605834e38bc4075bc79 b/sql/hive/src/test/resources/golden/timestamp cast #8-0-6d2da5cfada03605834e38bc4075bc79
deleted file mode 100644
index 1d94c8a014fb4..0000000000000
--- a/sql/hive/src/test/resources/golden/timestamp cast #8-0-6d2da5cfada03605834e38bc4075bc79	
+++ /dev/null
@@ -1 +0,0 @@
--1.2
diff --git a/sql/hive/src/test/resources/test_script.py b/sql/hive/src/test/resources/test_script.py
new file mode 100644
index 0000000000000..82ef7b38f0c1b
--- /dev/null
+++ b/sql/hive/src/test/resources/test_script.py
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+for line in sys.stdin:
+    (a, b, c, d, e) = line.split('\t')
+    sys.stdout.write('\t'.join([a, b, c, d, e]))
+    sys.stdout.flush()
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
index 7b3fb68174234..e3141f504ea90 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
@@ -19,12 +19,15 @@ package org.apache.spark.sql.hive
 
 import java.io.File
 
-import org.apache.spark.sql.{AnalysisException, Dataset, QueryTest, SaveMode}
+import org.apache.commons.io.FileUtils
+
+import org.apache.spark.sql.{AnalysisException, Dataset, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation}
 import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
 import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.storage.RDDBlockId
@@ -102,10 +105,10 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
   }
 
   test("uncache of nonexistent tables") {
-    val expectedErrorMsg = "Table or view not found: nonexistentTable"
+    val expectedErrorMsg = "Table or view not found:"
     // make sure table doesn't exist
     var e = intercept[AnalysisException](spark.table("nonexistentTable")).getMessage
-    assert(e.contains(expectedErrorMsg))
+    assert(e.contains(s"$expectedErrorMsg nonexistentTable"))
     e = intercept[AnalysisException] {
       uncacheTable("nonexistentTable")
     }.getMessage
@@ -113,7 +116,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     e = intercept[AnalysisException] {
       sql("UNCACHE TABLE nonexistentTable")
     }.getMessage
-    assert(e.contains(expectedErrorMsg))
+    assert(e.contains(s"$expectedErrorMsg default.nonexistentTable"))
     sql("UNCACHE TABLE IF EXISTS nonexistentTable")
   }
 
@@ -403,13 +406,13 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
           // Cache the table 'cachedTable' in default db without qualified table name , and then
           // check whether the table is cached with expected name.
           sql("CACHE TABLE cachedTable OPTIONS('storageLevel' 'DISK_ONLY')")
-          assertCached(sql("SELECT * FROM cachedTable"), "`cachedTable`", DISK_ONLY)
+          assertCached(sql("SELECT * FROM cachedTable"), "`default`.`cachedTable`", DISK_ONLY)
           assert(spark.catalog.isCached("cachedTable"), "Table 'cachedTable' should be cached.")
 
           // Refresh the table 'cachedTable' in default db with unqualified table name, and then
           // check whether the table is still cached with the same name.
           sql("REFRESH TABLE cachedTable")
-          assertCached(sql("SELECT * FROM cachedTable"), "`cachedTable`", DISK_ONLY)
+          assertCached(sql("SELECT * FROM cachedTable"), "`default`.`cachedTable`", DISK_ONLY)
           assert(spark.catalog.isCached("cachedTable"),
             "Table 'cachedTable' should be cached after refreshing with its unqualified name.")
 
@@ -420,7 +423,8 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
           // 'default.cachedTable', instead of 'cachedTable'
           activateDatabase(db) {
             sql("REFRESH TABLE default.cachedTable")
-            assertCached(sql("SELECT * FROM default.cachedTable"), "`cachedTable`", DISK_ONLY)
+            assertCached(
+              sql("SELECT * FROM default.cachedTable"), "`default`.`cachedTable`", DISK_ONLY)
             assert(spark.catalog.isCached("default.cachedTable"),
               "Table 'cachedTable' should be cached after refreshing with its qualified name.")
           }
@@ -428,4 +432,99 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       }
     }
   }
+
+  test("SPARK-33963: do not use table stats while looking in table cache") {
+    val t = "table_on_test"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (col int)")
+      assert(!spark.catalog.isCached(t))
+      sql(s"CACHE TABLE $t")
+      assert(spark.catalog.isCached(t))
+    }
+  }
+
+  test("SPARK-33950: refresh cache after partition dropping") {
+    withTable("t") {
+      sql(s"CREATE TABLE t (id int, part int) USING hive PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      sql("INSERT INTO t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(1, 1)))
+      sql("ALTER TABLE t DROP PARTITION (part=0)")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 1)))
+    }
+  }
+
+  test("SPARK-34011: refresh cache after partition renaming") {
+    withTable("t") {
+      sql("CREATE TABLE t (id int, part int) USING hive PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      sql("INSERT INTO t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      QueryTest.checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(1, 1)))
+      sql("ALTER TABLE t PARTITION (part=0) RENAME TO PARTITION (part=2)")
+      assert(spark.catalog.isCached("t"))
+      QueryTest.checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 2), Row(1, 1)))
+    }
+  }
+
+  test("SPARK-34055: refresh cache in partition adding") {
+    withTable("t") {
+      sql("CREATE TABLE t (id int, part int) USING hive PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0)))
+
+      // Create new partition (part = 1) in the filesystem
+      val information = sql("SHOW TABLE EXTENDED LIKE 't' PARTITION (part = 0)")
+        .select("information")
+        .first().getString(0)
+      val part0Loc = information
+        .split("\\r?\\n")
+        .filter(_.startsWith("Location:"))
+        .head
+        .replace("Location: file:", "")
+      val part1Loc = part0Loc.replace("part=0", "part=1")
+      FileUtils.copyDirectory(new File(part0Loc), new File(part1Loc))
+
+      sql(s"ALTER TABLE t ADD PARTITION (part=1) LOCATION '$part1Loc'")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(0, 1)))
+    }
+  }
+
+  test("SPARK-34060: update stats of cached table") {
+    withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+      def checkTableSize(expected: String): Unit = {
+        val stats =
+          sql("DESCRIBE TABLE EXTENDED t")
+            .select("data_type")
+            .where("col_name = 'Statistics'")
+            .first()
+            .getString(0)
+        assert(stats.contains(expected))
+      }
+
+      sql("CREATE TABLE t (id int, part int) USING hive PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      sql("INSERT INTO t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(1, 1)))
+      checkTableSize("4 bytes")
+
+      sql("ALTER TABLE t DROP PARTITION (part=0)")
+      assert(spark.catalog.isCached("t"))
+      checkTableSize("2 bytes")
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 1)))
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextCompatibilitySuite.scala
new file mode 100644
index 0000000000000..a80db765846e9
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextCompatibilitySuite.scala
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.scalatest.BeforeAndAfterEach
+
+import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
+
+
+class HiveContextCompatibilitySuite extends SparkFunSuite with BeforeAndAfterEach {
+
+  override protected val enableAutoThreadAudit = false
+  private var sc: SparkContext = null
+  private var hc: HiveContext = null
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    sc = SparkContext.getOrCreate(new SparkConf().setMaster("local").setAppName("test"))
+    HiveUtils.newTemporaryConfiguration(useInMemoryDerby = true).foreach { case (k, v) =>
+      sc.hadoopConfiguration.set(k, v)
+    }
+    hc = new HiveContext(sc)
+  }
+
+  override def afterEach(): Unit = {
+    try {
+      hc.sharedState.cacheManager.clearCache()
+      hc.sessionState.catalog.reset()
+    } finally {
+      super.afterEach()
+    }
+  }
+
+  override def afterAll(): Unit = {
+    try {
+      sc = null
+      hc = null
+    } finally {
+      super.afterAll()
+    }
+  }
+
+  test("basic operations") {
+    val _hc = hc
+    import _hc.implicits._
+    val df1 = (1 to 20).map { i => (i, i) }.toDF("a", "x")
+    val df2 = (1 to 100).map { i => (i, i % 10, i % 2 == 0) }.toDF("a", "b", "c")
+      .select($"a", $"b")
+      .filter($"a" > 10 && $"b" > 6 && $"c")
+    val df3 = df1.join(df2, "a")
+    val res = df3.collect()
+    val expected = Seq((18, 18, 8)).toDF("a", "x", "b").collect()
+    assert(res.toSeq == expected.toSeq)
+    df3.createOrReplaceTempView("mai_table")
+    val df4 = hc.table("mai_table")
+    val res2 = df4.collect()
+    assert(res2.toSeq == expected.toSeq)
+  }
+
+  test("basic DDLs") {
+    val _hc = hc
+    import _hc.implicits._
+    val databases = hc.sql("SHOW DATABASES").collect().map(_.getString(0))
+    assert(databases.toSeq == Seq("default"))
+    hc.sql("CREATE DATABASE mee_db")
+    hc.sql("USE mee_db")
+    val databases2 = hc.sql("SHOW DATABASES").collect().map(_.getString(0))
+    assert(databases2.toSet == Set("default", "mee_db"))
+    val df = (1 to 10).map { i => ("bob" + i.toString, i) }.toDF("name", "age")
+    df.createOrReplaceTempView("mee_table")
+    hc.sql("CREATE TABLE moo_table (name string, age int)")
+    hc.sql("INSERT INTO moo_table SELECT * FROM mee_table")
+    assert(
+      hc.sql("SELECT * FROM moo_table order by name").collect().toSeq ==
+      df.collect().toSeq.sortBy(_.getString(0)))
+    val tables = hc.sql("SHOW TABLES IN mee_db").select("tableName").collect().map(_.getString(0))
+    assert(tables.toSet == Set("moo_table", "mee_table"))
+    hc.sql("DROP TABLE moo_table")
+    hc.sql("DROP TABLE mee_table")
+    val tables2 = hc.sql("SHOW TABLES IN mee_db").select("tableName").collect().map(_.getString(0))
+    assert(tables2.isEmpty)
+    hc.sql("USE default")
+    hc.sql("DROP DATABASE mee_db CASCADE")
+    val databases3 = hc.sql("SHOW DATABASES").collect().map(_.getString(0))
+    assert(databases3.toSeq == Seq("default"))
+  }
+
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
index 79e569b51ca1d..473a93bf129df 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
@@ -17,11 +17,14 @@
 
 package org.apache.spark.sql.hive
 
+import java.net.URI
+
 import org.apache.hadoop.conf.Configuration
 
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog._
+import org.apache.spark.sql.execution.QueryExecutionException
 import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.types.StructType
 
@@ -154,4 +157,65 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite {
     catalog.createTable(hiveTable, ignoreIfExists = false)
     assert(catalog.getTable("db1", "spark_29498").owner === owner)
   }
+
+  test("SPARK-30868 throw an exception if HiveClient#runSqlHive fails") {
+    val client = externalCatalog.client
+    // test add jars which doesn't exists
+    val jarPath = "file:///tmp/not_exists.jar"
+    assertThrows[QueryExecutionException](client.runSqlHive(s"ADD JAR $jarPath"))
+
+    // test change to the database which doesn't exists
+    assertThrows[QueryExecutionException](client.runSqlHive(
+      s"use db_not_exists"))
+
+    // test create hive table failed with unsupported into type
+    assertThrows[QueryExecutionException](client.runSqlHive(
+      s"CREATE TABLE t(n into)"))
+
+    // test desc table failed with wrong `FORMATED` keyword
+    assertThrows[QueryExecutionException](client.runSqlHive(
+      s"DESC FORMATED t"))
+
+    // test wrong insert query
+    assertThrows[QueryExecutionException](client.runSqlHive(
+      "INSERT overwrite directory \"fs://localhost/tmp\" select 1 as a"))
+  }
+
+  test("SPARK-31061: alterTable should be able to change table provider") {
+    val catalog = newBasicCatalog()
+    val parquetTable = CatalogTable(
+      identifier = TableIdentifier("parq_tbl", Some("db1")),
+      tableType = CatalogTableType.MANAGED,
+      storage = storageFormat.copy(locationUri = Some(new URI("file:/some/path"))),
+      schema = new StructType().add("col1", "int").add("col2", "string"),
+      provider = Some("parquet"))
+    catalog.createTable(parquetTable, ignoreIfExists = false)
+
+    val rawTable = externalCatalog.getTable("db1", "parq_tbl")
+    assert(rawTable.provider === Some("parquet"))
+
+    val fooTable = parquetTable.copy(provider = Some("foo"))
+    catalog.alterTable(fooTable)
+    val alteredTable = externalCatalog.getTable("db1", "parq_tbl")
+    assert(alteredTable.provider === Some("foo"))
+  }
+
+  test("SPARK-31061: alterTable should be able to change table provider from hive") {
+    val catalog = newBasicCatalog()
+    val hiveTable = CatalogTable(
+      identifier = TableIdentifier("parq_tbl", Some("db1")),
+      tableType = CatalogTableType.MANAGED,
+      storage = storageFormat,
+      schema = new StructType().add("col1", "int").add("col2", "string"),
+      provider = Some("hive"))
+    catalog.createTable(hiveTable, ignoreIfExists = false)
+
+    val rawTable = externalCatalog.getTable("db1", "parq_tbl")
+    assert(rawTable.provider === Some("hive"))
+
+    val fooTable = rawTable.copy(provider = Some("foo"))
+    catalog.alterTable(fooTable)
+    val alteredTable = externalCatalog.getTable("db1", "parq_tbl")
+    assert(alteredTable.provider === Some("foo"))
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index 3b5a1247bc09c..b81b7e8ec0c0f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.CatalogTableType
 import org.apache.spark.sql.internal.StaticSQLConf.WAREHOUSE_PATH
 import org.apache.spark.sql.test.SQLTestUtils
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
 import org.apache.spark.util.Utils
 
 /**
@@ -46,6 +46,7 @@ import org.apache.spark.util.Utils
  * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the
  * downloading for this spark version.
  */
+@SlowHiveTest
 @ExtendedHiveTest
 class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
   private val isTestAtLeastJava9 = SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)
@@ -233,7 +234,7 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils {
   // Tests the latest version of every release line.
   val testingVersions: Seq[String] = {
     import scala.io.Source
-    try {
+    val versions: Seq[String] = try {
       Source.fromURL(s"${releaseMirror}/spark").mkString
         .split("\n")
         .filter(_.contains("""<li><a href="spark-"""))
@@ -242,8 +243,9 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils {
         .filter(_ < org.apache.spark.SPARK_VERSION)
     } catch {
       // do not throw exception during object initialization.
-      case NonFatal(_) => Nil
+      case NonFatal(_) => Seq("3.0.1", "2.4.7") // A temporary fallback to use a specific version
     }
+    versions.filter(v => v.startsWith("3") || !TestUtils.isPythonVersionAtLeast38())
   }
 
   protected var spark: SparkSession = _
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 5912992694e84..13c48f38e7f78 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.hive
 
 import java.util
-import java.util.{Locale, TimeZone}
 
 import org.apache.hadoop.hive.ql.udf.UDAFPercentile
 import org.apache.hadoop.hive.serde2.io.DoubleWritable
@@ -74,11 +73,6 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
       .get())
   }
 
-  // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-  TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-  // Add Locale setting
-  Locale.setDefault(Locale.US)
-
   val data =
     Literal(true) ::
     Literal(null) ::
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
index 94a55b911f092..db8ebcd45f3eb 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
@@ -114,7 +114,7 @@ class HiveMetadataCacheSuite extends QueryTest with SQLTestUtils with TestHiveSi
             val e2 = intercept[SparkException] {
               sql("select * from test").count()
             }
-            assert(e2.getMessage.contains("FileNotFoundException"))
+            assert(e.getMessage.contains("FileNotFoundException"))
             spark.catalog.refreshByPath(dir.getAbsolutePath)
             assert(sql("select * from test").count() == 3)
           }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index 20bafd832d0da..95e99c653d6f6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -62,7 +62,7 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton with SQLTestUtils {
       spark.sql("create view vw1 as select 1 as id")
       val plan = spark.sql("select id from vw1").queryExecution.analyzed
       val aliases = plan.collect {
-        case x @ SubqueryAlias(AliasIdentifier("vw1", Some("default")), _) => x
+        case x @ SubqueryAlias(AliasIdentifier("vw1", Seq("spark_catalog", "default")), _) => x
       }
       assert(aliases.size == 1)
     }
@@ -97,7 +97,7 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton with SQLTestUtils {
           |c22 map<int,char(10)>,
           |c23 struct<a:int,b:int>,
           |c24 struct<c:varchar(10),d:int>
-          |)
+          |) USING hive
         """.stripMargin)
 
       val schema = hiveClient.getTable("default", "t").schema
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala
index 590ef949ffbd7..6496d3154feea 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala
@@ -118,11 +118,6 @@ class HiveSchemaInferenceSuite
         properties = Map.empty),
       true)
 
-    // Add partition records (if specified)
-    if (!partitionCols.isEmpty) {
-      spark.catalog.recoverPartitions(TEST_TABLE_NAME)
-    }
-
     // Check that the table returned by HiveExternalCatalog has schemaPreservesCase set to false
     // and that the raw table returned by the Hive client doesn't have any Spark SQL properties
     // set (table needs to be obtained from client since HiveExternalCatalog filters these
@@ -130,6 +125,12 @@ class HiveSchemaInferenceSuite
     assert(!externalCatalog.getTable(DATABASE, TEST_TABLE_NAME).schemaPreservesCase)
     val rawTable = client.getTable(DATABASE, TEST_TABLE_NAME)
     assert(rawTable.properties.filterKeys(_.startsWith(DATASOURCE_SCHEMA_PREFIX)) == Map.empty)
+
+    // Add partition records (if specified)
+    if (!partitionCols.isEmpty) {
+      spark.catalog.recoverPartitions(TEST_TABLE_NAME)
+    }
+
     schema
   }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
index 6e2dcfc04d498..78535b094b83d 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
@@ -45,7 +45,6 @@ class HiveSharedStateSuite extends SparkFunSuite {
       GLOBAL_TEMP_DATABASE.key -> tmpDb)
 
     val state = new SharedState(sc, initialConfigs)
-    assert(state.warehousePath !== invalidPath, "warehouse path can't determine by session options")
     assert(sc.conf.get(WAREHOUSE_PATH.key) !== invalidPath,
       "warehouse conf in session options can't affect application wide spark conf")
     assert(sc.hadoopConfiguration.get(ConfVars.METASTOREWAREHOUSE.varname) !== invalidPath,
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
index e5d572c90af38..cfcf70c0e79f0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
@@ -19,38 +19,50 @@ package org.apache.spark.sql.hive
 
 import org.apache.spark.sql.{AnalysisException, ShowCreateTableSuite}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable}
+import org.apache.spark.sql.catalyst.catalog.CatalogTable
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
 
 class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSingleton {
 
-  private var origCreateHiveTableConfig = false
-
-  protected override def beforeAll(): Unit = {
-    super.beforeAll()
-    origCreateHiveTableConfig =
-      SQLConf.get.getConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED)
-    SQLConf.get.setConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED, true)
-  }
-
-  protected override def afterAll(): Unit = {
-    SQLConf.get.setConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED,
-      origCreateHiveTableConfig)
-    super.afterAll()
+  test("view") {
+    Seq(true, false).foreach { serde =>
+      withView("v1") {
+        sql("CREATE VIEW v1 AS SELECT 1 AS a")
+        checkCreateView("v1", serde)
+      }
+    }
   }
 
-  test("view") {
-    withView("v1") {
-      sql("CREATE VIEW v1 AS SELECT 1 AS a")
-      checkCreateHiveTableOrView("v1", "VIEW")
+  test("view with output columns") {
+    Seq(true, false).foreach { serde =>
+      withView("v1") {
+        sql("CREATE VIEW v1 (a, b COMMENT 'b column') AS SELECT 1 AS a, 2 AS b")
+        checkCreateView("v1", serde)
+      }
     }
   }
 
-  test("view  with output columns") {
-    withView("v1") {
-      sql("CREATE VIEW v1 (b) AS SELECT 1 AS a")
-      checkCreateHiveTableOrView("v1", "VIEW")
+  test("view with table comment and properties") {
+    Seq(true, false).foreach { serde =>
+      withView("v1") {
+        sql(
+          s"""
+             |CREATE VIEW v1 (
+             |  c1 COMMENT 'bla',
+             |  c2
+             |)
+             |COMMENT 'table comment'
+             |TBLPROPERTIES (
+             |  'prop1' = 'value1',
+             |  'prop2' = 'value2'
+             |)
+             |AS SELECT 1 AS c1, '2' AS c2
+         """.stripMargin
+        )
+
+        checkCreateView("v1", serde)
+      }
     }
   }
 
@@ -68,7 +80,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
          """.stripMargin
       )
 
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -88,7 +100,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
            """.stripMargin
         )
 
-        checkCreateHiveTableOrView("t1")
+        checkCreateTable("t1", serde = true)
       }
     }
   }
@@ -108,7 +120,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
          """.stripMargin
       )
 
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -126,7 +138,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
          """.stripMargin
       )
 
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -141,7 +153,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
          """.stripMargin
       )
 
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -163,7 +175,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
          """.stripMargin
       )
 
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -176,7 +188,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
            |INTO 2 BUCKETS
          """.stripMargin
       )
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -219,29 +231,9 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
       val createTable = "CREATE TABLE `t1` (`a` STRUCT<`b`: STRING>) USING hive"
       sql(createTable)
       val shownDDL = getShowDDL("SHOW CREATE TABLE t1")
-      assert(shownDDL == createTable.dropRight(" USING hive".length))
+      assert(shownDDL == "CREATE TABLE `default`.`t1` (`a` STRUCT<`b`: STRING>)")
 
-      checkCreateHiveTableOrView("t1")
-    }
-  }
-
-  /**
-   * This method compares the given table with the table created by the DDL generated by
-   * `SHOW CREATE TABLE AS SERDE`.
-   */
-  private def checkCreateHiveTableOrView(tableName: String, checkType: String = "TABLE"): Unit = {
-    val table = TableIdentifier(tableName, Some("default"))
-    val db = table.database.getOrElse("default")
-    val expected = spark.sharedState.externalCatalog.getTable(db, table.table)
-    val shownDDL = sql(s"SHOW CREATE TABLE ${table.quotedString} AS SERDE").head().getString(0)
-    sql(s"DROP $checkType ${table.quotedString}")
-
-    try {
-      sql(shownDDL)
-      val actual = spark.sharedState.externalCatalog.getTable(db, table.table)
-      checkCatalogTables(expected, actual)
-    } finally {
-      sql(s"DROP $checkType IF EXISTS ${table.table}")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -343,7 +335,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
       )
 
       val cause = intercept[AnalysisException] {
-        checkCreateHiveTableOrView("t1")
+        checkCreateTable("t1", serde = true)
       }
 
       assert(cause.getMessage.contains("Use `SHOW CREATE TABLE` without `AS SERDE` instead"))
@@ -445,27 +437,6 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
     }
   }
 
-  test("hive view is not supported by show create table without as serde") {
-    withTable("t1") {
-      withView("v1") {
-        sql("CREATE TABLE t1 (c1 STRING, c2 STRING)")
-
-        createRawHiveTable(
-          s"""
-             |CREATE VIEW v1
-             |AS SELECT * from t1
-           """.stripMargin
-        )
-
-        val cause = intercept[AnalysisException] {
-          sql("SHOW CREATE TABLE v1")
-        }
-
-        assert(cause.getMessage.contains("view isn't supported"))
-      }
-    }
-  }
-
   test("partitioned, bucketed hive table in Spark DDL") {
     withTable("t1") {
       sql(
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
index 31ff62ed0a530..3a7e92ee1c00b 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -38,12 +38,13 @@ import org.apache.spark.sql.hive.test.{HiveTestJars, TestHiveContext}
 import org.apache.spark.sql.internal.SQLConf.SHUFFLE_PARTITIONS
 import org.apache.spark.sql.internal.StaticSQLConf.WAREHOUSE_PATH
 import org.apache.spark.sql.types.{DecimalType, StructType}
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
 import org.apache.spark.util.{ResetSystemProperties, Utils}
 
 /**
  * This suite tests spark-submit with applications using HiveContext.
  */
+@SlowHiveTest
 @ExtendedHiveTest
 class HiveSparkSubmitSuite
   extends SparkSubmitTestUtils
@@ -787,7 +788,7 @@ object SPARK_18360 {
       .enableHiveSupport().getOrCreate()
 
     val defaultDbLocation = spark.catalog.getDatabase("default").locationUri
-    assert(new Path(defaultDbLocation) == new Path(spark.sharedState.warehousePath))
+    assert(new Path(defaultDbLocation) == new Path(spark.conf.get(WAREHOUSE_PATH)))
 
     val hiveClient =
       spark.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog].client
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala
new file mode 100644
index 0000000000000..ee8e6f4f78be5
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression}
+import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.test.SQLTestUtils
+import org.apache.spark.sql.types.{IntegerType, StringType}
+import org.apache.spark.util.Utils
+
+class HiveUDFDynamicLoadSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
+
+  case class UDFTestInformation(
+      identifier: String,
+      funcName: String,
+      className: String,
+      fnVerifyQuery: () => Unit,
+      fnCreateHiveUDFExpression: () => Expression)
+
+  private val udfTestInfos: Seq[UDFTestInformation] = Array(
+    // UDF
+    // UDFExampleAdd2 is slightly modified version of UDFExampleAdd in hive/contrib,
+    // which adds two integers or doubles.
+    UDFTestInformation(
+      "UDF",
+      "udf_add2",
+      "org.apache.hadoop.hive.contrib.udf.example.UDFExampleAdd2",
+      () => {
+        checkAnswer(sql("SELECT udf_add2(1, 2)"), Row(3) :: Nil)
+      },
+      () => {
+        HiveSimpleUDF(
+          "default.udf_add2",
+          HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udf.example.UDFExampleAdd2"),
+          Array(
+            AttributeReference("a", IntegerType, nullable = false)(),
+            AttributeReference("b", IntegerType, nullable = false)()))
+      }),
+
+    // GenericUDF
+    // GenericUDFTrim2 is cloned version of GenericUDFTrim in hive/contrib.
+    UDFTestInformation(
+      "GENERIC_UDF",
+      "generic_udf_trim2",
+      "org.apache.hadoop.hive.contrib.udf.example.GenericUDFTrim2",
+      () => {
+        checkAnswer(sql("SELECT generic_udf_trim2(' hello ')"), Row("hello") :: Nil)
+      },
+      () => {
+        HiveGenericUDF(
+          "default.generic_udf_trim2",
+          HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udf.example.GenericUDFTrim2"),
+          Array(AttributeReference("a", StringType, nullable = false)())
+        )
+      }
+    ),
+
+    // AbstractGenericUDAFResolver
+    // GenericUDAFSum2 is cloned version of GenericUDAFSum in hive/exec.
+    UDFTestInformation(
+      "GENERIC_UDAF",
+      "generic_udaf_sum2",
+      "org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum2",
+      () => {
+        import spark.implicits._
+        val df = Seq((0: Integer) -> 0, (1: Integer) -> 1, (2: Integer) -> 2, (3: Integer) -> 3)
+          .toDF("key", "value").createOrReplaceTempView("t")
+        checkAnswer(sql("SELECT generic_udaf_sum2(value) FROM t GROUP BY key % 2"),
+          Row(2) :: Row(4) :: Nil)
+      },
+      () => {
+        HiveUDAFFunction(
+          "default.generic_udaf_sum2",
+          HiveFunctionWrapper("org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum2"),
+          Array(AttributeReference("a", IntegerType, nullable = false)())
+        )
+      }
+    ),
+
+    // UDAF
+    // UDAFExampleMax2 is cloned version of UDAFExampleMax in hive/contrib.
+    UDFTestInformation(
+      "UDAF",
+      "udaf_max2",
+      "org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax2",
+      () => {
+        import spark.implicits._
+        val df = Seq((0: Integer) -> 0, (1: Integer) -> 1, (2: Integer) -> 2, (3: Integer) -> 3)
+          .toDF("key", "value").createOrReplaceTempView("t")
+        checkAnswer(sql("SELECT udaf_max2(value) FROM t GROUP BY key % 2"),
+          Row(2) :: Row(3) :: Nil)
+      },
+      () => {
+        HiveUDAFFunction(
+          "default.udaf_max2",
+          HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax2"),
+          Array(AttributeReference("a", IntegerType, nullable = false)()),
+          isUDAFBridgeRequired = true
+        )
+      }
+    ),
+
+    // GenericUDTF
+    // GenericUDTFCount3 is slightly modified version of GenericUDTFCount2 in hive/contrib,
+    // which emits the count for three times.
+    UDFTestInformation(
+      "GENERIC_UDTF",
+      "udtf_count3",
+      "org.apache.hadoop.hive.contrib.udtf.example.GenericUDTFCount3",
+      () => {
+        checkAnswer(
+          sql("SELECT udtf_count3(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"),
+          Row(3) :: Row(3) :: Row(3) :: Nil)
+      },
+      () => {
+        HiveGenericUDTF(
+          "default.udtf_count3",
+          HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udtf.example.GenericUDTFCount3"),
+          Array.empty[Expression]
+        )
+      }
+    )
+  )
+
+  udfTestInfos.foreach { udfInfo =>
+    // The test jars are built from below commit:
+    // https://github.com/HeartSaVioR/hive/commit/12f3f036b6efd0299cd1d457c0c0a65e0fd7e5f2
+    // which contain new UDF classes to be dynamically loaded and tested via Spark.
+
+    // This jar file should not be placed to the classpath.
+    val jarPath = "src/test/noclasspath/hive-test-udfs.jar"
+    val jarUrl = s"file://${System.getProperty("user.dir")}/$jarPath"
+
+    test("Spark should be able to run Hive UDF using jar regardless of " +
+      s"current thread context classloader (${udfInfo.identifier}") {
+      Utils.withContextClassLoader(Utils.getSparkClassLoader) {
+        withUserDefinedFunction(udfInfo.funcName -> false) {
+          val sparkClassLoader = Thread.currentThread().getContextClassLoader
+
+          sql(s"CREATE FUNCTION ${udfInfo.funcName} AS '${udfInfo.className}' USING JAR '$jarUrl'")
+
+          assert(Thread.currentThread().getContextClassLoader eq sparkClassLoader)
+
+          // JAR will be loaded at first usage, and it will change the current thread's
+          // context classloader to jar classloader in sharedState.
+          // See SessionState.addJar for details.
+          udfInfo.fnVerifyQuery()
+
+          assert(Thread.currentThread().getContextClassLoader ne sparkClassLoader)
+          assert(Thread.currentThread().getContextClassLoader eq
+            spark.sqlContext.sharedState.jarClassLoader)
+
+          val udfExpr = udfInfo.fnCreateHiveUDFExpression()
+          // force initializing - this is what we do in HiveSessionCatalog
+          udfExpr.dataType
+
+          // Roll back to the original classloader and run query again. Without this line, the test
+          // would pass, as thread's context classloader is changed to jar classloader. But thread
+          // context classloader can be changed from others as well which would fail the query; one
+          // example is spark-shell, which thread context classloader rolls back automatically. This
+          // mimics the behavior of spark-shell.
+          Thread.currentThread().setContextClassLoader(sparkClassLoader)
+
+          udfInfo.fnVerifyQuery()
+
+          val newExpr = udfExpr.makeCopy(udfExpr.productIterator.map(_.asInstanceOf[AnyRef])
+            .toArray)
+          newExpr.dataType
+        }
+      }
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
index 80afc9d8f44bc..e1b0637963b75 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
@@ -53,7 +53,8 @@ class PartitionProviderCompatibilitySuite
       s"ALTER TABLE $tableName PARTITION (partCol=1) SET LOCATION '/foo'",
       s"ALTER TABLE $tableName DROP PARTITION (partCol=1)",
       s"DESCRIBE $tableName PARTITION (partCol=1)",
-      s"SHOW PARTITIONS $tableName")
+      s"SHOW PARTITIONS $tableName",
+      s"SHOW TABLE EXTENDED LIKE '$tableName' PARTITION (partCol=1)")
 
     withSQLConf(SQLConf.HIVE_MANAGE_FILESOURCE_PARTITIONS.key -> "true") {
       for (cmd <- unsupportedCommands) {
@@ -124,10 +125,15 @@ class PartitionProviderCompatibilitySuite
         }
         // disabled
         withSQLConf(SQLConf.HIVE_MANAGE_FILESOURCE_PARTITIONS.key -> "false") {
-          val e = intercept[AnalysisException] {
-            spark.sql(s"show partitions test")
+          Seq(
+            "SHOW PARTITIONS test",
+            "SHOW TABLE EXTENDED LIKE 'test' PARTITION (partCol=1)"
+          ).foreach { showPartitions =>
+            val e = intercept[AnalysisException] {
+              spark.sql(showPartitions)
+            }
+            assert(e.getMessage.contains("filesource partition management is disabled"))
           }
-          assert(e.getMessage.contains("filesource partition management is disabled"))
           spark.sql("refresh table test")
           assert(spark.sql("select * from test").count() == 5)
         }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
index 3af163af0968c..f9ae1e00cfa36 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
@@ -405,8 +405,8 @@ class PartitionedTablePerfStatsSuite
           })
           executorPool.shutdown()
           executorPool.awaitTermination(30, TimeUnit.SECONDS)
-          assert(HiveCatalogMetrics.METRIC_FILES_DISCOVERED.getCount() == 50)
-          assert(HiveCatalogMetrics.METRIC_PARALLEL_LISTING_JOB_COUNT.getCount() == 1)
+          assert(HiveCatalogMetrics.METRIC_FILES_DISCOVERED.getCount() == 100)
+          assert(HiveCatalogMetrics.METRIC_PARALLEL_LISTING_JOB_COUNT.getCount() == 2)
         }
       }
     }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
index 488175a22bad7..046ab01640444 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
@@ -993,12 +993,16 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
           assert(fetched1.get.colStats.size == 2)
 
           withTempPaths(numPaths = 2) { case Seq(dir1, dir2) =>
-            val file1 = new File(dir1 + "/data")
+            val partDir1 = new File(new File(dir1, "ds=2008-04-09"), "hr=11")
+            val file1 = new File(partDir1, "data")
+            file1.getParentFile.mkdirs()
             Utils.tryWithResource(new PrintWriter(file1)) { writer =>
               writer.write("1,a")
             }
 
-            val file2 = new File(dir2 + "/data")
+            val partDir2 = new File(new File(dir2, "ds=2008-04-09"), "hr=12")
+            val file2 = new File(partDir2, "data")
+            file2.getParentFile.mkdirs()
             Utils.tryWithResource(new PrintWriter(file2)) { writer =>
               writer.write("1,a")
             }
@@ -1007,8 +1011,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
             sql(
               s"""
                  |ALTER TABLE $table ADD
-                 |PARTITION (ds='2008-04-09', hr='11') LOCATION '${dir1.toURI.toString}'
-                 |PARTITION (ds='2008-04-09', hr='12') LOCATION '${dir2.toURI.toString}'
+                 |PARTITION (ds='2008-04-09', hr='11') LOCATION '${partDir1.toURI.toString}'
+                 |PARTITION (ds='2008-04-09', hr='12') LOCATION '${partDir1.toURI.toString}'
             """.stripMargin)
             if (autoUpdate) {
               val fetched2 = checkTableStats(table, hasSizeInBytes = true, expectedRowCounts = None)
@@ -1520,10 +1524,12 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     val ext_tbl = "SPARK_30269_external"
     withTempDir { dir =>
       withTable(tbl, ext_tbl) {
-        sql(s"CREATE TABLE $tbl (key INT, value STRING, ds STRING) PARTITIONED BY (ds)")
+        sql(s"CREATE TABLE $tbl (key INT, value STRING, ds STRING)" +
+          "USING parquet PARTITIONED BY (ds)")
         sql(
           s"""
              | CREATE TABLE $ext_tbl (key INT, value STRING, ds STRING)
+             | USING PARQUET
              | PARTITIONED BY (ds)
              | LOCATION '${dir.toURI}'
            """.stripMargin)
@@ -1556,4 +1562,20 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
       }
     }
   }
+
+  test("SPARK-34084: auto update table stats") {
+    Seq("parquet", "hive").foreach { format =>
+      withTable("t") {
+        withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "false") {
+          sql(s"CREATE TABLE t (col0 int, part int) USING $format PARTITIONED BY (part)")
+          sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+          assert(getCatalogTable("t").stats.isEmpty)
+        }
+        withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+          sql("ALTER TABLE t ADD PARTITION (part=1)")
+          assert(getTableStats("t").sizeInBytes > 0)
+        }
+      }
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala
new file mode 100644
index 0000000000000..65492abf38cc0
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.client
+
+import java.io.File
+import java.net.URLClassLoader
+
+import org.apache.hadoop.conf.Configuration
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
+import org.apache.spark.util.Utils
+
+/**
+ * This test suite requires a clean JVM because it's testing the initialization of static codes in
+ * `org.apache.hadoop.util.VersionInfo`.
+ */
+class HadoopVersionInfoSuite extends SparkFunSuite {
+  override protected val enableAutoThreadAudit = false
+
+  test("SPARK-32256: Hadoop VersionInfo should be preloaded") {
+    val ivyPath =
+      Utils.createTempDir(namePrefix = s"${classOf[HadoopVersionInfoSuite].getSimpleName}-ivy")
+    try {
+      val hadoopConf = new Configuration()
+      hadoopConf.set("test", "success")
+      hadoopConf.set("datanucleus.schema.autoCreateAll", "true")
+      hadoopConf.set("hive.metastore.schema.verification", "false")
+
+      // Download jars for Hive 2.0
+      val client = IsolatedClientLoader.forVersion(
+        hiveMetastoreVersion = "2.0",
+        hadoopVersion = "2.7.4",
+        sparkConf = new SparkConf(),
+        hadoopConf = hadoopConf,
+        config = HiveClientBuilder.buildConf(Map.empty),
+        ivyPath = Some(ivyPath.getCanonicalPath),
+        sharesHadoopClasses = true)
+      val jars = client.classLoader.getParent.asInstanceOf[URLClassLoader].getURLs
+        .map(u => new File(u.toURI))
+        // Drop all Hadoop jars to use the existing Hadoop jars on the classpath
+        .filter(!_.getName.startsWith("org.apache.hadoop_hadoop-"))
+
+      val sparkConf = new SparkConf()
+      sparkConf.set(HiveUtils.HIVE_METASTORE_VERSION, "2.0")
+      sparkConf.set(
+        HiveUtils.HIVE_METASTORE_JARS,
+        jars.map(_.getCanonicalPath).mkString(File.pathSeparator))
+      HiveClientBuilder.buildConf(Map.empty).foreach { case (k, v) =>
+        hadoopConf.set(k, v)
+      }
+      new HiveExternalCatalog(sparkConf, hadoopConf).client.getState
+    } finally {
+      Utils.deleteRecursively(ivyPath)
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala
index ab73f668c6ca6..2ad3afcb214b3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala
@@ -33,7 +33,7 @@ private[client] object HiveClientBuilder {
       Some(new File(sys.props("java.io.tmpdir"), "hive-ivy-cache").getAbsolutePath))
   }
 
-  private def buildConf(extraConf: Map[String, String]) = {
+  private[client] def buildConf(extraConf: Map[String, String]): Map[String, String] = {
     lazy val warehousePath = Utils.createTempDir()
     lazy val metastorePath = Utils.createTempDir()
     metastorePath.delete()
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
index 2d615f6fdc261..edb1a27df806f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.types.{BooleanType, IntegerType, LongType, StructType}
+import org.apache.spark.sql.types.{BooleanType, IntegerType, LongType, StringType, StructType}
 import org.apache.spark.util.Utils
 
 class HivePartitionFilteringSuite(version: String)
@@ -276,6 +276,13 @@ class HivePartitionFilteringSuite(version: String)
     buildClient(new Configuration(), sharesHadoopClasses = false)
   }
 
+  test("getPartitionsByFilter: chunk in ('ab', 'ba') and ((cast(ds as string)>'20170102')") {
+    val day = (20170101 to 20170103, 0 to 4, Seq("ab", "ba"))
+    testMetastorePartitionFiltering(
+      attr("chunk").in("ab", "ba") && (attr("ds").cast(StringType) > "20170102"),
+      day :: Nil)
+  }
+
   private def testMetastorePartitionFiltering(
       filterExpr: Expression,
       expectedDs: Seq[Int],
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index 4760af7aa46ff..41b90db3b2e7e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -33,7 +33,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, Row}
 import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
-import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPermanentFunctionException}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPermanentFunctionException, PartitionsAlreadyExistException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Literal}
 import org.apache.spark.sql.catalyst.util.quietly
@@ -41,7 +41,7 @@ import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
 import org.apache.spark.sql.hive.test.TestHiveVersion
 import org.apache.spark.sql.types.IntegerType
 import org.apache.spark.sql.types.StructType
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, GitHubActionsUnstableTest}
 import org.apache.spark.util.{MutableURLClassLoader, Utils}
 
 /**
@@ -52,6 +52,7 @@ import org.apache.spark.util.{MutableURLClassLoader, Utils}
  */
 // TODO: Refactor this to `HiveClientSuite` and make it a subclass of `HiveVersionSuite`
 @ExtendedHiveTest
+@GitHubActionsUnstableTest
 class VersionsSuite extends SparkFunSuite with Logging {
 
   override protected val enableAutoThreadAudit = false
@@ -82,6 +83,18 @@ class VersionsSuite extends SparkFunSuite with Logging {
     assert("success" === client.getConf("test", null))
   }
 
+  test("override useless and side-effect hive configurations ") {
+    val hadoopConf = new Configuration()
+    // These hive flags should be reset by spark
+    hadoopConf.setBoolean("hive.cbo.enable", true)
+    hadoopConf.setBoolean("hive.session.history.enabled", true)
+    hadoopConf.set("hive.execution.engine", "tez")
+    val client = buildClient(HiveUtils.builtinHiveVersion, hadoopConf)
+    assert(!client.getConf("hive.cbo.enable", "true").toBoolean)
+    assert(!client.getConf("hive.session.history.enabled", "true").toBoolean)
+    assert(client.getConf("hive.execution.engine", "tez") === "mr")
+  }
+
   private def getNestedMessages(e: Throwable): String = {
     var causes = ""
     var lastException = e
@@ -142,10 +155,11 @@ class VersionsSuite extends SparkFunSuite with Logging {
         .client.version.fullVersion.startsWith(version))
     }
 
-    def table(database: String, tableName: String): CatalogTable = {
+    def table(database: String, tableName: String,
+        tableType: CatalogTableType = CatalogTableType.MANAGED): CatalogTable = {
       CatalogTable(
         identifier = TableIdentifier(tableName, Some(database)),
-        tableType = CatalogTableType.MANAGED,
+        tableType = tableType,
         schema = new StructType().add("key", "int"),
         storage = CatalogStorageFormat(
           locationUri = None,
@@ -261,7 +275,9 @@ class VersionsSuite extends SparkFunSuite with Logging {
 
     test(s"$version: createTable") {
       client.createTable(table("default", tableName = "src"), ignoreIfExists = false)
-      client.createTable(table("default", "temporary"), ignoreIfExists = false)
+      client.createTable(table("default", tableName = "temporary"), ignoreIfExists = false)
+      client.createTable(table("default", tableName = "view1", tableType = CatalogTableType.VIEW),
+        ignoreIfExists = false)
     }
 
     test(s"$version: loadTable") {
@@ -377,7 +393,7 @@ class VersionsSuite extends SparkFunSuite with Logging {
     }
 
     test(s"$version: listTables(database)") {
-      assert(client.listTables("default") === Seq("src", "temporary"))
+      assert(client.listTables("default") === Seq("src", "temporary", "view1"))
     }
 
     test(s"$version: listTables(database, pattern)") {
@@ -385,6 +401,13 @@ class VersionsSuite extends SparkFunSuite with Logging {
       assert(client.listTables("default", pattern = "nonexist").isEmpty)
     }
 
+    test(s"$version: listTablesByType(database, pattern, tableType)") {
+      assert(client.listTablesByType("default", pattern = "view1",
+        CatalogTableType.VIEW) === Seq("view1"))
+      assert(client.listTablesByType("default", pattern = "nonexist",
+        CatalogTableType.VIEW).isEmpty)
+    }
+
     test(s"$version: dropTable") {
       val versionsWithoutPurge =
         if (versions.contains("0.14")) versions.takeWhile(_ != "0.14") else Nil
@@ -400,6 +423,16 @@ class VersionsSuite extends SparkFunSuite with Logging {
           client.dropTable("default", tableName = "temporary", ignoreIfNotExists = false,
             purge = false)
       }
+      // Drop table with type CatalogTableType.VIEW.
+      try {
+        client.dropTable("default", tableName = "view1", ignoreIfNotExists = false,
+          purge = true)
+        assert(!versionsWithoutPurge.contains(version))
+      } catch {
+        case _: UnsupportedOperationException =>
+          client.dropTable("default", tableName = "view1", ignoreIfNotExists = false,
+            purge = false)
+      }
       assert(client.listTables("default") === Seq("src"))
     }
 
@@ -560,6 +593,27 @@ class VersionsSuite extends SparkFunSuite with Logging {
       assert(client.getPartitionOption("default", "src_part", spec).isEmpty)
     }
 
+    test(s"$version: createPartitions if already exists") {
+      val partitions = Seq(CatalogTablePartition(
+        Map("key1" -> "101", "key2" -> "102"),
+        storageFormat))
+      try {
+        client.createPartitions("default", "src_part", partitions, ignoreIfExists = false)
+        val errMsg = intercept[PartitionsAlreadyExistException] {
+          client.createPartitions("default", "src_part", partitions, ignoreIfExists = false)
+        }.getMessage
+        assert(errMsg.contains("partitions already exists"))
+      } finally {
+        client.dropPartitions(
+          "default",
+          "src_part",
+          partitions.map(_.spec),
+          ignoreIfNotExists = true,
+          purge = false,
+          retainData = false)
+      }
+    }
+
     ///////////////////////////////////////////////////////////////////////////
     // Function related API
     ///////////////////////////////////////////////////////////////////////////
@@ -950,7 +1004,7 @@ class VersionsSuite extends SparkFunSuite with Logging {
            """.stripMargin
           )
 
-          val errorMsg = "Cannot safely cast 'f0': DecimalType(2,1) to BinaryType"
+          val errorMsg = "Cannot safely cast 'f0': decimal(2,1) to binary"
 
           if (isPartitioned) {
             val insertStmt = s"INSERT OVERWRITE TABLE $tableName partition (ds='a') SELECT 1.3"
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
index f84b854048e8a..1dd2ad3837cc3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
@@ -31,6 +31,8 @@ import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.SlowHiveTest
+import org.apache.spark.unsafe.UnsafeAlignedOffset
 
 
 class ScalaAggregateFunction(schema: StructType) extends UserDefinedAggregateFunction {
@@ -203,11 +205,13 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
   }
 
   test("group by function") {
-    Seq((1, 2)).toDF("a", "b").createOrReplaceTempView("data")
+    withTempView("data") {
+      Seq((1, 2)).toDF("a", "b").createOrReplaceTempView("data")
 
-    checkAnswer(
-      sql("SELECT floor(a) AS a, collect_set(b) FROM data GROUP BY floor(a) ORDER BY a"),
-      Row(1, Array(2)) :: Nil)
+      checkAnswer(
+        sql("SELECT floor(a) AS a, collect_set(b) FROM data GROUP BY floor(a) ORDER BY a"),
+        Row(1, Array(2)) :: Nil)
+    }
   }
 
   test("empty table") {
@@ -799,43 +803,45 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
       (5, 8, 17),
       (6, 2, 11)).toDF("a", "b", "c")
 
-    covar_tab.createOrReplaceTempView("covar_tab")
+    withTempView("covar_tab") {
+      covar_tab.createOrReplaceTempView("covar_tab")
 
-    checkAnswer(
-      spark.sql(
-        """
-          |SELECT corr(b, c) FROM covar_tab WHERE a < 1
-        """.stripMargin),
-      Row(null) :: Nil)
+      checkAnswer(
+        spark.sql(
+          """
+            |SELECT corr(b, c) FROM covar_tab WHERE a < 1
+          """.stripMargin),
+        Row(null) :: Nil)
 
-    checkAnswer(
-      spark.sql(
-        """
-          |SELECT corr(b, c) FROM covar_tab WHERE a < 3
-        """.stripMargin),
-      Row(null) :: Nil)
+      checkAnswer(
+        spark.sql(
+          """
+            |SELECT corr(b, c) FROM covar_tab WHERE a < 3
+          """.stripMargin),
+        Row(null) :: Nil)
 
-    checkAnswer(
-      spark.sql(
-        """
-          |SELECT corr(b, c) FROM covar_tab WHERE a = 3
-        """.stripMargin),
-      Row(Double.NaN) :: Nil)
+      checkAnswer(
+        spark.sql(
+          """
+            |SELECT corr(b, c) FROM covar_tab WHERE a = 3
+          """.stripMargin),
+        Row(Double.NaN) :: Nil)
 
-    checkAnswer(
-      spark.sql(
-        """
-          |SELECT a, corr(b, c) FROM covar_tab GROUP BY a ORDER BY a
-        """.stripMargin),
-      Row(1, null) ::
-      Row(2, null) ::
-      Row(3, Double.NaN) ::
-      Row(4, Double.NaN) ::
-      Row(5, Double.NaN) ::
-      Row(6, Double.NaN) :: Nil)
-
-    val corr7 = spark.sql("SELECT corr(b, c) FROM covar_tab").collect()(0).getDouble(0)
-    assert(math.abs(corr7 - 0.6633880657639323) < 1e-12)
+      checkAnswer(
+        spark.sql(
+          """
+            |SELECT a, corr(b, c) FROM covar_tab GROUP BY a ORDER BY a
+          """.stripMargin),
+        Row(1, null) ::
+        Row(2, null) ::
+        Row(3, Double.NaN) ::
+        Row(4, Double.NaN) ::
+        Row(5, Double.NaN) ::
+        Row(6, Double.NaN) :: Nil)
+
+      val corr7 = spark.sql("SELECT corr(b, c) FROM covar_tab").collect()(0).getDouble(0)
+      assert(math.abs(corr7 - 0.6633880657639323) < 1e-12)
+    }
   }
 
   test("covariance: covar_pop and covar_samp") {
@@ -1049,36 +1055,42 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
 class HashAggregationQuerySuite extends AggregationQuerySuite
 
 
+@SlowHiveTest
 class HashAggregationQueryWithControlledFallbackSuite extends AggregationQuerySuite {
 
   override protected def checkAnswer(actual: => DataFrame, expectedAnswer: Seq[Row]): Unit = {
     Seq("true", "false").foreach { enableTwoLevelMaps =>
       withSQLConf(SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key ->
         enableTwoLevelMaps) {
-        (1 to 3).foreach { fallbackStartsAt =>
-          withSQLConf("spark.sql.TungstenAggregate.testFallbackStartsAt" ->
-            s"${(fallbackStartsAt - 1).toString}, ${fallbackStartsAt.toString}") {
-            // Create a new df to make sure its physical operator picks up
-            // spark.sql.TungstenAggregate.testFallbackStartsAt.
-            // todo: remove it?
-            val newActual = Dataset.ofRows(spark, actual.logicalPlan)
-
-            QueryTest.getErrorMessageInCheckAnswer(newActual, expectedAnswer) match {
-              case Some(errorMessage) =>
-                val newErrorMessage =
-                  s"""
-                     |The following aggregation query failed when using HashAggregate with
-                     |controlled fallback (it falls back to bytes to bytes map once it has processed
-                     |${fallbackStartsAt - 1} input rows and to sort-based aggregation once it has
-                     |processed $fallbackStartsAt input rows). The query is ${actual.queryExecution}
-                     |
-                    |$errorMessage
-                  """.stripMargin
-
-                fail(newErrorMessage)
-              case None => // Success
+        Seq(4, 8).foreach { uaoSize =>
+          UnsafeAlignedOffset.setUaoSize(uaoSize)
+          (1 to 3).foreach { fallbackStartsAt =>
+            withSQLConf("spark.sql.TungstenAggregate.testFallbackStartsAt" ->
+              s"${(fallbackStartsAt - 1).toString}, ${fallbackStartsAt.toString}") {
+              // Create a new df to make sure its physical operator picks up
+              // spark.sql.TungstenAggregate.testFallbackStartsAt.
+              // todo: remove it?
+              val newActual = Dataset.ofRows(spark, actual.logicalPlan)
+
+              QueryTest.getErrorMessageInCheckAnswer(newActual, expectedAnswer) match {
+                case Some(errorMessage) =>
+                  val newErrorMessage =
+                    s"""
+                       |The following aggregation query failed when using HashAggregate with
+                       |controlled fallback (it falls back to bytes to bytes map once it has
+                       |processed ${fallbackStartsAt - 1} input rows and to sort-based aggregation
+                       |once it has processed $fallbackStartsAt input rows).
+                       |The query is ${actual.queryExecution}
+                       |$errorMessage
+                    """.stripMargin
+
+                  fail(newErrorMessage)
+                case None => // Success
+              }
             }
           }
+          // reset static uaoSize to avoid affect other tests
+          UnsafeAlignedOffset.setUaoSize(0)
         }
       }
     }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
index dbbf2b29fe8b7..2b7cfe57fdae3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
@@ -117,6 +117,52 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     }
   }
 
+  test("show views") {
+    withView("show1a", "show2b", "global_temp.temp1", "temp2") {
+      sql("CREATE VIEW show1a AS SELECT 1 AS id")
+      sql("CREATE VIEW show2b AS SELECT 1 AS id")
+      sql("CREATE GLOBAL TEMP VIEW temp1 AS SELECT 1 AS id")
+      sql("CREATE TEMP VIEW temp2 AS SELECT 1 AS id")
+      checkAnswer(
+        sql("SHOW VIEWS"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) ::
+          Row("default", "parquet_view1", false) ::
+          Row("", "temp2", true) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS IN default"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) ::
+          Row("default", "parquet_view1", false) ::
+          Row("", "temp2", true) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS FROM default"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) ::
+          Row("default", "parquet_view1", false) ::
+          Row("", "temp2", true) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS FROM global_temp"),
+        Row("global_temp", "temp1", true) ::
+          Row("", "temp2", true) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS 'show1*|show2*'"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS LIKE 'show1*|show2*'"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS IN default 'show1*'"),
+        Row("default", "show1a", false) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS IN default LIKE 'show1*|show2*'"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) :: Nil)
+    }
+  }
+
   test("show tblproperties of data source tables - basic") {
     checkAnswer(
       sql("SHOW TBLPROPERTIES parquet_tab1").filter(s"key = 'my_key1'"),
@@ -133,7 +179,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     val message = intercept[AnalysisException] {
       sql("SHOW TBLPROPERTIES badtable")
     }.getMessage
-    assert(message.contains("Table not found: badtable"))
+    assert(message.contains("Table or view not found: badtable"))
 
     // When key is not found, a row containing the error is returned.
     checkAnswer(
@@ -147,21 +193,6 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     checkAnswer(sql("SHOW TBLPROPERTIES parquet_tab2('`prop2Key`')"), Row("prop2Val"))
   }
 
-  test("show tblproperties for spark temporary table - AnalysisException is thrown") {
-    withTempView("parquet_temp") {
-      sql(
-        """
-         |CREATE TEMPORARY VIEW parquet_temp (c1 INT, c2 STRING)
-         |USING org.apache.spark.sql.parquet.DefaultSource
-        """.stripMargin)
-
-      val message = intercept[AnalysisException] {
-        sql("SHOW TBLPROPERTIES parquet_temp")
-      }.getMessage
-      assert(message.contains("parquet_temp is a temp view not table"))
-    }
-  }
-
   Seq(true, false).foreach { local =>
     val loadQuery = if (local) "LOAD DATA LOCAL" else "LOAD DATA"
     test(loadQuery) {
@@ -444,7 +475,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       val message2 = intercept[AnalysisException] {
         sql("SHOW PARTITIONS parquet_tab4 PARTITION(abcd=2015, xyz=1)")
       }.getMessage
-      assert(message2.contains("Non-partitioning column(s) [abcd, xyz] are specified"))
+      assert(message2.contains("abcd is not a valid partition column"))
 
       val message3 = intercept[AnalysisException] {
         sql("SHOW PARTITIONS parquet_view1")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index 28e1db961f611..8b1f4c92755b9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -346,8 +346,7 @@ abstract class HiveComparisonTest
         val catalystResults = queryList.zip(hiveResults).map { case (queryString, hive) =>
           val query = new TestHiveQueryExecution(queryString.replace("../../data", testDataPath))
           def getResult(): Seq[String] = {
-            SQLExecution.withNewExecutionId(
-              query.sparkSession, query)(hiveResultString(query.executedPlan))
+            SQLExecution.withNewExecutionId(query)(hiveResultString(query.executedPlan))
           }
           try { (query, prepareAnswer(query, getResult())) } catch {
             case e: Throwable =>
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index ba48cfd4142f6..c8726c7ae4bc9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -28,7 +28,7 @@ import org.scalatest.BeforeAndAfterEach
 import org.apache.spark.SparkException
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, NoSuchPartitionsException, PartitionsAlreadyExistException, TableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.connector.catalog.CatalogManager
@@ -44,9 +44,11 @@ import org.apache.spark.sql.internal.SQLConf.ORC_IMPLEMENTATION
 import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.util.Utils
 
 // TODO(gatorsmile): combine HiveCatalogedDDLSuite and HiveDDLSuite
+@SlowHiveTest
 class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeAndAfterEach {
   override def afterEach(): Unit = {
     try {
@@ -404,6 +406,7 @@ class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeA
   }
 }
 
+@SlowHiveTest
 class HiveDDLSuite
   extends QueryTest with SQLTestUtils with TestHiveSingleton with BeforeAndAfterEach {
   import testImplicits._
@@ -602,8 +605,8 @@ class HiveDDLSuite
     val tab = "tab_with_partitions"
     withTempDir { tmpDir =>
       val basePath = new File(tmpDir.getCanonicalPath)
-      val part1Path = new File(basePath + "/part1")
-      val part2Path = new File(basePath + "/part2")
+      val part1Path = new File(new File(basePath, "part10"), "part11")
+      val part2Path = new File(new File(basePath, "part20"), "part21")
       val dirSet = part1Path :: part2Path :: Nil
 
       // Before data insertion, all the directory are empty
@@ -981,8 +984,8 @@ class HiveDDLSuite
     val expectedSerdePropsString =
       expectedSerdeProps.map { case (k, v) => s"'$k'='$v'" }.mkString(", ")
     val oldPart = catalog.getPartition(TableIdentifier("boxes"), Map("width" -> "4"))
-    assume(oldPart.storage.serde != Some(expectedSerde), "bad test: serde was already set")
-    assume(oldPart.storage.properties.filterKeys(expectedSerdeProps.contains) !=
+    assert(oldPart.storage.serde != Some(expectedSerde), "bad test: serde was already set")
+    assert(oldPart.storage.properties.filterKeys(expectedSerdeProps.contains) !=
       expectedSerdeProps, "bad test: serde properties were already set")
     sql(s"""ALTER TABLE boxes PARTITION (width=4)
       |    SET SERDE '$expectedSerde'
@@ -1188,7 +1191,7 @@ class HiveDDLSuite
       expectedDBUri,
       Map.empty))
     // the database directory was created
-    assert(fs.exists(dbPath) && fs.isDirectory(dbPath))
+    assert(fs.exists(dbPath) && fs.getFileStatus(dbPath).isDirectory)
     sql(s"USE $dbName")
 
     val tabName = "tab1"
@@ -1580,6 +1583,12 @@ class HiveDDLSuite
         "source table/view path should be different from target table path")
     }
 
+    if (DDLUtils.isHiveTable(targetTable)) {
+      assert(targetTable.tracksPartitionsInCatalog)
+    } else {
+      assert(targetTable.tracksPartitionsInCatalog == sourceTable.tracksPartitionsInCatalog)
+    }
+
     // The source table contents should not been seen in the target table.
     assert(spark.table(sourceTable.identifier).count() != 0, "the source table should be nonempty")
     assert(spark.table(targetTable.identifier).count() == 0, "the target table should be empty")
@@ -1729,7 +1738,7 @@ class HiveDDLSuite
     Seq("json", "parquet").foreach { format =>
       withTable("rectangles") {
         data.write.format(format).saveAsTable("rectangles")
-        assume(spark.table("rectangles").collect().nonEmpty,
+        assert(spark.table("rectangles").collect().nonEmpty,
           "bad test; table was empty to begin with")
 
         sql("TRUNCATE TABLE rectangles")
@@ -2309,37 +2318,39 @@ class HiveDDLSuite
 
     implicit val _sqlContext = spark.sqlContext
 
-    Seq((1, "one"), (2, "two"), (4, "four")).toDF("number", "word").createOrReplaceTempView("t1")
-    // Make a table and ensure it will be broadcast.
-    sql("""CREATE TABLE smallTable(word string, number int)
-          |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-          |STORED AS TEXTFILE
-        """.stripMargin)
+    withTempView("t1") {
+      Seq((1, "one"), (2, "two"), (4, "four")).toDF("number", "word").createOrReplaceTempView("t1")
+      // Make a table and ensure it will be broadcast.
+      sql("""CREATE TABLE smallTable(word string, number int)
+            |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+            |STORED AS TEXTFILE
+          """.stripMargin)
 
-    sql(
-      """INSERT INTO smallTable
-        |SELECT word, number from t1
-      """.stripMargin)
+      sql(
+        """INSERT INTO smallTable
+          |SELECT word, number from t1
+        """.stripMargin)
 
-    val inputData = MemoryStream[Int]
-    val joined = inputData.toDS().toDF()
-      .join(spark.table("smallTable"), $"value" === $"number")
+      val inputData = MemoryStream[Int]
+      val joined = inputData.toDS().toDF()
+        .join(spark.table("smallTable"), $"value" === $"number")
 
-    val sq = joined.writeStream
-      .format("memory")
-      .queryName("t2")
-      .start()
-    try {
-      inputData.addData(1, 2)
+      val sq = joined.writeStream
+        .format("memory")
+        .queryName("t2")
+        .start()
+      try {
+        inputData.addData(1, 2)
 
-      sq.processAllAvailable()
+        sq.processAllAvailable()
 
-      checkAnswer(
-        spark.table("t2"),
-        Seq(Row(1, "one", 1), Row(2, "two", 2))
-      )
-    } finally {
-      sq.stop()
+        checkAnswer(
+          spark.table("t2"),
+          Seq(Row(1, "one", 1), Row(2, "two", 2))
+        )
+      } finally {
+        sq.stop()
+      }
     }
   }
 
@@ -2698,29 +2709,74 @@ class HiveDDLSuite
     }
   }
 
-  test("SPARK-30098: create table without provider should " +
-    "use default data source under non-legacy mode") {
+  test("SPARK-30785: create table like a partitioned table") {
     val catalog = spark.sessionState.catalog
-    withSQLConf(
-      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "false") {
-      withTable("s") {
-        val defaultProvider = conf.defaultDataSourceName
-        sql("CREATE TABLE s(a INT, b INT)")
-        val table = catalog.getTableMetadata(TableIdentifier("s"))
-        assert(table.provider === Some(defaultProvider))
-      }
+    withTable("sc_part", "ta_part") {
+      sql("CREATE TABLE sc_part (key string, ts int) USING parquet PARTITIONED BY (ts)")
+      sql("CREATE TABLE ta_part like sc_part")
+      val sourceTable = catalog.getTableMetadata(TableIdentifier("sc_part", Some("default")))
+      val targetTable = catalog.getTableMetadata(TableIdentifier("ta_part", Some("default")))
+      assert(sourceTable.tracksPartitionsInCatalog)
+      assert(targetTable.tracksPartitionsInCatalog)
+      assert(targetTable.partitionColumnNames == Seq("ts"))
+      sql("ALTER TABLE ta_part ADD PARTITION (ts=10)") // no exception
+      checkAnswer(sql("SHOW PARTITIONS ta_part"), Row("ts=10") :: Nil)
     }
   }
 
-  test("SPARK-30098: create table without provider should " +
-    "use hive under legacy mode") {
-    val catalog = spark.sessionState.catalog
-    withSQLConf(
-      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") {
-      withTable("s") {
-        sql("CREATE TABLE s(a INT, b INT)")
-        val table = catalog.getTableMetadata(TableIdentifier("s"))
-        assert(table.provider === Some("hive"))
+  test("SPARK-31904: Fix case sensitive problem of char and varchar partition columns") {
+    withTable("t1", "t2") {
+      sql("CREATE TABLE t1(a STRING, B VARCHAR(10), C CHAR(10)) STORED AS parquet")
+      sql("CREATE TABLE t2 USING parquet PARTITIONED BY (b, c) AS SELECT * FROM t1")
+      // make sure there is no exception
+      assert(sql("SELECT * FROM t2 WHERE b = 'A'").collect().isEmpty)
+      assert(sql("SELECT * FROM t2 WHERE c = 'A'").collect().isEmpty)
+    }
+  }
+
+  test("SPARK-33742: partition already exists") {
+    withTable("t") {
+      sql(s"CREATE TABLE t (data string) PARTITIONED BY (id bigint)")
+      sql(s"ALTER TABLE t ADD PARTITION (id=2) LOCATION 'loc1'")
+
+      val errMsg = intercept[PartitionsAlreadyExistException] {
+        sql(s"ALTER TABLE t ADD PARTITION (id=1) LOCATION 'loc'" +
+          " PARTITION (id=2) LOCATION 'loc1'")
+      }.getMessage
+      assert(errMsg.contains("The following partitions already exists"))
+
+      sql(s"ALTER TABLE t ADD IF NOT EXISTS PARTITION (id=1) LOCATION 'loc'" +
+        " PARTITION (id=2) LOCATION 'loc1'")
+      checkAnswer(sql("SHOW PARTITIONS t"), Seq(Row("id=1"), Row("id=2")))
+    }
+  }
+
+  test("SPARK-33788: partition not exists") {
+    withTable("t") {
+      sql(s"CREATE TABLE t (data string) PARTITIONED BY (id bigint)")
+      sql(s"ALTER TABLE t ADD PARTITION (id=1)")
+
+      val errMsg = intercept[NoSuchPartitionsException] {
+        sql(s"ALTER TABLE t DROP PARTITION (id=1), PARTITION (id=2)")
+      }.getMessage
+      assert(errMsg.contains("partitions not found in table"))
+
+      checkAnswer(sql("SHOW PARTITIONS t"), Seq(Row("id=1")))
+      sql(s"ALTER TABLE t DROP IF EXISTS PARTITION (id=1), PARTITION (id=2)")
+      checkAnswer(sql("SHOW PARTITIONS t"), Seq.empty)
+    }
+  }
+
+  test("SPARK-33844: Insert overwrite directory should check schema too") {
+    withView("v") {
+      spark.range(1).createTempView("v")
+      withTempPath { path =>
+        val e = intercept[AnalysisException] {
+          spark.sql(s"INSERT OVERWRITE LOCAL DIRECTORY '${path.getCanonicalPath}' " +
+            s"STORED AS PARQUET SELECT ID, if(1=1, 1, 0), abs(id), '^-' FROM v")
+        }.getMessage
+        assert(e.contains("Attribute name \"(IF((1 = 1), 1, 0))\" contains" +
+          " invalid character(s) among \" ,;{}()\\n\\t=\". Please use alias to rename it."))
       }
     }
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
index f9a4e2cd210e3..7a913e99fdbe6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.metrics.source.HiveCatalogMetrics
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
 import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand
 import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHiveSingleton
@@ -133,22 +134,21 @@ class HiveExplainSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       "src")
   }
 
-  test("explain output of physical plan should contain proper codegen stage ID") {
-    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-      checkKeywordsExist(sql(
-        """
-          |EXPLAIN SELECT t1.id AS a, t2.id AS b FROM
-          |(SELECT * FROM range(3)) t1 JOIN
-          |(SELECT * FROM range(10)) t2 ON t1.id == t2.id % 3
-        """.stripMargin),
-        "== Physical Plan ==",
-        "*(2) Project ",
-        "+- *(2) BroadcastHashJoin ",
-        "   :- BroadcastExchange ",
-        "   :  +- *(1) Range ",
-        "   +- *(2) Range "
-      )
-    }
+  test("explain output of physical plan should contain proper codegen stage ID",
+    DisableAdaptiveExecution("Adaptive explain is different")) {
+    checkKeywordsExist(sql(
+      """
+        |EXPLAIN SELECT t1.id AS a, t2.id AS b FROM
+        |(SELECT * FROM range(3)) t1 JOIN
+        |(SELECT * FROM range(10)) t2 ON t1.id == t2.id % 3
+      """.stripMargin),
+      "== Physical Plan ==",
+      "*(2) Project ",
+      "+- *(2) BroadcastHashJoin ",
+      "   :- BroadcastExchange ",
+      "   :  +- *(1) Range ",
+      "   +- *(2) Range "
+    )
   }
 
   test("EXPLAIN CODEGEN command") {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 5a8365017a5ba..e798a352f72bf 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.hive.execution
 import java.io.File
 import java.net.URI
 import java.sql.Timestamp
-import java.util.{Locale, TimeZone}
+import java.util.Locale
 
 import scala.util.Try
 
@@ -39,6 +39,7 @@ import org.apache.spark.sql.hive.test.{HiveTestJars, TestHive}
 import org.apache.spark.sql.hive.test.TestHive._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
+import org.apache.spark.tags.SlowHiveTest
 
 case class TestData(a: Int, b: String)
 
@@ -46,10 +47,8 @@ case class TestData(a: Int, b: String)
  * A set of test cases expressed in Hive QL that are not covered by the tests
  * included in the hive distribution.
  */
+@SlowHiveTest
 class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAndAfter {
-  private val originalTimeZone = TimeZone.getDefault
-  private val originalLocale = Locale.getDefault
-
   import org.apache.spark.sql.hive.test.TestHive.implicits._
 
   private val originalCrossJoinEnabled = TestHive.conf.crossJoinEnabled
@@ -59,10 +58,6 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
   override def beforeAll(): Unit = {
     super.beforeAll()
     TestHive.setCacheTables(true)
-    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    // Add Locale setting
-    Locale.setDefault(Locale.US)
     // Ensures that cross joins are enabled so that we can test them
     TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, true)
   }
@@ -70,8 +65,6 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
   override def afterAll(): Unit = {
     try {
       TestHive.setCacheTables(false)
-      TimeZone.setDefault(originalTimeZone)
-      Locale.setDefault(originalLocale)
       sql("DROP TEMPORARY FUNCTION IF EXISTS udtf_count2")
       TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, originalCrossJoinEnabled)
     } finally {
@@ -565,33 +558,27 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
     assert(1 == res.getDouble(0))
   }
 
-  createQueryTest("timestamp cast #2",
-    "SELECT CAST(CAST(1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
-
-  test("timestamp cast #3") {
-    val res = sql("SELECT CAST(CAST(1200 AS TIMESTAMP) AS INT) FROM src LIMIT 1").collect().head
-    assert(1200 == res.getInt(0))
+  test("timestamp cast #2") {
+    val res = sql("SELECT CAST(CAST(-1 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1").collect().head
+    assert(-1 == res.get(0))
   }
 
-  createQueryTest("timestamp cast #4",
+  createQueryTest("timestamp cast #3",
     "SELECT CAST(CAST(1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
 
+  createQueryTest("timestamp cast #4",
+    "SELECT CAST(CAST(-1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
+
   test("timestamp cast #5") {
-    val res = sql("SELECT CAST(CAST(-1 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1").collect().head
-    assert(-1 == res.get(0))
+    val res = sql("SELECT CAST(CAST(1200 AS TIMESTAMP) AS INT) FROM src LIMIT 1").collect().head
+    assert(1200 == res.getInt(0))
   }
 
-  createQueryTest("timestamp cast #6",
-    "SELECT CAST(CAST(-1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
-
-  test("timestamp cast #7") {
+  test("timestamp cast #6") {
     val res = sql("SELECT CAST(CAST(-1200 AS TIMESTAMP) AS INT) FROM src LIMIT 1").collect().head
     assert(-1200 == res.getInt(0))
   }
 
-  createQueryTest("timestamp cast #8",
-    "SELECT CAST(CAST(-1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
-
   createQueryTest("select null from table",
     "SELECT null FROM src LIMIT 1")
 
@@ -698,15 +685,17 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
     "SELECT srcalias.KEY, SRCALIAS.value FROM sRc SrCAlias WHERE SrCAlias.kEy < 15")
 
   test("case sensitivity: created temporary view") {
-    val testData =
-      TestHive.sparkContext.parallelize(
-        TestData(1, "str1") ::
-        TestData(2, "str2") :: Nil)
-    testData.toDF().createOrReplaceTempView("REGisteredTABle")
-
-    assertResult(Array(Row(2, "str2"))) {
-      sql("SELECT tablealias.A, TABLEALIAS.b FROM reGisteredTABle TableAlias " +
-        "WHERE TableAliaS.a > 1").collect()
+    withTempView("REGisteredTABle") {
+      val testData =
+        TestHive.sparkContext.parallelize(
+          TestData(1, "str1") ::
+          TestData(2, "str2") :: Nil)
+      testData.toDF().createOrReplaceTempView("REGisteredTABle")
+
+      assertResult(Array(Row(2, "str2"))) {
+        sql("SELECT tablealias.A, TABLEALIAS.b FROM reGisteredTABle TableAlias " +
+          "WHERE TableAliaS.a > 1").collect()
+      }
     }
   }
 
@@ -725,16 +714,18 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
   }
 
   test("SPARK-2180: HAVING support in GROUP BY clauses (positive)") {
-    val fixture = List(("foo", 2), ("bar", 1), ("foo", 4), ("bar", 3))
-      .zipWithIndex.map {case ((value, attr), key) => HavingRow(key, value, attr)}
-    TestHive.sparkContext.parallelize(fixture).toDF().createOrReplaceTempView("having_test")
-    val results =
-      sql("SELECT value, max(attr) AS attr FROM having_test GROUP BY value HAVING attr > 3")
-      .collect()
-      .map(x => (x.getString(0), x.getInt(1)))
+    withTempView("having_test") {
+      val fixture = List(("foo", 2), ("bar", 1), ("foo", 4), ("bar", 3))
+        .zipWithIndex.map {case ((value, attr), key) => HavingRow(key, value, attr)}
+      TestHive.sparkContext.parallelize(fixture).toDF().createOrReplaceTempView("having_test")
+      val results =
+        sql("SELECT value, max(attr) AS attr FROM having_test GROUP BY value HAVING attr > 3")
+        .collect()
+        .map(x => (x.getString(0), x.getInt(1)))
 
-    assert(results === Array(("foo", 4)))
-    TestHive.reset()
+      assert(results === Array(("foo", 4)))
+      TestHive.reset()
+    }
   }
 
   test("SPARK-2180: HAVING with non-boolean clause raises no exceptions") {
@@ -966,22 +957,24 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
   }
 
   test("SPARK-3414 regression: should store analyzed logical plan when creating a temporary view") {
-    sparkContext.makeRDD(Seq.empty[LogEntry]).toDF().createOrReplaceTempView("rawLogs")
-    sparkContext.makeRDD(Seq.empty[LogFile]).toDF().createOrReplaceTempView("logFiles")
+    withTempView("rawLogs", "logFiles", "boom") {
+      sparkContext.makeRDD(Seq.empty[LogEntry]).toDF().createOrReplaceTempView("rawLogs")
+      sparkContext.makeRDD(Seq.empty[LogFile]).toDF().createOrReplaceTempView("logFiles")
 
-    sql(
-      """
-      SELECT name, message
-      FROM rawLogs
-      JOIN (
-        SELECT name
-        FROM logFiles
-      ) files
-      ON rawLogs.filename = files.name
-      """).createOrReplaceTempView("boom")
-
-    // This should be successfully analyzed
-    sql("SELECT * FROM boom").queryExecution.analyzed
+      sql(
+        """
+        SELECT name, message
+        FROM rawLogs
+        JOIN (
+          SELECT name
+          FROM logFiles
+        ) files
+        ON rawLogs.filename = files.name
+        """).createOrReplaceTempView("boom")
+
+      // This should be successfully analyzed
+      sql("SELECT * FROM boom").queryExecution.analyzed
+    }
   }
 
   test("SPARK-3810: PreprocessTableInsertion static partitioning support") {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala
index 5e6e114fc3fdc..fa43ff14fd796 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala
@@ -80,8 +80,8 @@ class HiveSQLViewSuite extends SQLViewSuite with TestHiveSingleton {
             val e = intercept[AnalysisException] {
               sql(s"CREATE VIEW view1 AS SELECT $tempFunctionName(id) from tab1")
             }.getMessage
-            assert(e.contains("Not allowed to create a permanent view `view1` by referencing " +
-              s"a temporary function `$tempFunctionName`"))
+            assert(e.contains("Not allowed to create a permanent view `default`.`view1` by " +
+              s"referencing a temporary function `$tempFunctionName`"))
           }
         }
       }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
index f8ba7bf2c1a62..ac9ae8c9229db 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
@@ -154,7 +154,7 @@ class HiveSerDeReadWriteSuite extends QueryTest with SQLTestUtils with TestHiveS
     }
   }
 
-  Seq("PARQUET", "ORC").foreach { fileFormat =>
+  Seq("PARQUET", "ORC", "TEXTFILE").foreach { fileFormat =>
     test(s"Read/Write Hive $fileFormat serde table") {
       // Numeric Types
       checkNumericTypes(fileFormat, "TINYINT", 2)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
index 9a1190af02fac..24b1e3405379c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
@@ -82,9 +82,12 @@ class HiveSerDeSuite extends HiveComparisonTest with PlanTest with BeforeAndAfte
     }.head
   }
 
+  // Make sure we set the config values to TestHive.conf.
+  override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit =
+    SQLConf.withExistingConf(TestHive.conf)(super.withSQLConf(pairs: _*)(f))
+
   test("Test the default fileformat for Hive-serde tables") {
-    withSQLConf("hive.default.fileformat" -> "orc",
-      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") {
+    withSQLConf("hive.default.fileformat" -> "orc") {
       val (desc, exists) = extractTableDesc(
         "CREATE TABLE IF NOT EXISTS fileformat_test (id int)")
       assert(exists)
@@ -93,8 +96,7 @@ class HiveSerDeSuite extends HiveComparisonTest with PlanTest with BeforeAndAfte
       assert(desc.storage.serde == Some("org.apache.hadoop.hive.ql.io.orc.OrcSerde"))
     }
 
-    withSQLConf("hive.default.fileformat" -> "parquet",
-      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") {
+    withSQLConf("hive.default.fileformat" -> "parquet") {
       val (desc, exists) = extractTableDesc("CREATE TABLE IF NOT EXISTS fileformat_test (id int)")
       assert(exists)
       val input = desc.storage.inputFormat
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 7bca2af379934..d3c8428b71806 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.execution.command.FunctionsCommand
 import org.apache.spark.sql.functions.max
+import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
@@ -658,6 +659,25 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
     }
   }
 
+  test("SPARK-32877: add test for Hive UDF complex decimal type") {
+    assume(HiveUtils.isHive23)
+    withUserDefinedFunction("testArraySum" -> false) {
+      sql(s"CREATE FUNCTION testArraySum AS '${classOf[ArraySumUDF].getName}'")
+      checkAnswer(
+        sql("SELECT testArraySum(array(1, 1.1, 1.2))"),
+        Seq(Row(3.3)))
+
+      val msg = intercept[AnalysisException] {
+        sql("SELECT testArraySum(1)")
+      }.getMessage
+      assert(msg.contains(s"No handler for UDF/UDAF/UDTF '${classOf[ArraySumUDF].getName}'"))
+
+      val msg2 = intercept[AnalysisException] {
+        sql("SELECT testArraySum(1, 2)")
+      }.getMessage
+      assert(msg2.contains(s"No handler for UDF/UDAF/UDTF '${classOf[ArraySumUDF].getName}'"))
+    }
+  }
 }
 
 class TestPair(x: Int, y: Int) extends Writable with Serializable {
@@ -741,3 +761,14 @@ class StatelessUDF extends UDF {
     result
   }
 }
+
+class ArraySumUDF extends UDF {
+  import scala.collection.JavaConverters._
+  def evaluate(values: java.util.List[java.lang.Double]): java.lang.Double = {
+    var r = 0d
+    for (v <- values.asScala) {
+      r += v
+    }
+    r
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
index b20ef035594da..6f37e39a532d6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
@@ -27,13 +27,14 @@ import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.sql.types._
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
 import org.apache.spark.util.Utils
 
 /**
  * A separate set of DDL tests that uses Hive 2.1 libraries, which behave a little differently
  * from the built-in ones.
  */
+@SlowHiveTest
 @ExtendedHiveTest
 class Hive_2_1_DDLSuite extends SparkFunSuite with TestHiveSingleton with BeforeAndAfterEach
   with BeforeAndAfterAll {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala
index 16668f93bd4e7..4d6dafd598a2e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala
@@ -17,24 +17,13 @@
 
 package org.apache.spark.sql.hive.execution
 
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.metric.SQLMetricsTestUtils
 import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.internal.SQLConf
 
-class SQLMetricsSuite extends SQLMetricsTestUtils with TestHiveSingleton {
-
-  var originalValue: String = _
-  // With AQE on/off, the metric info is different.
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
+// Disable AQE because metric info is different with AQE on/off
+class SQLMetricsSuite extends SQLMetricsTestUtils with TestHiveSingleton
+  with DisableAdaptiveExecutionSuite {
 
   test("writing data out metrics: hive") {
     testMetricsNonDynamicPartition("hive", "t1")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 539b464743461..a46db32b9adee 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, Functio
 import org.apache.spark.sql.catalyst.catalog.{CatalogTableType, CatalogUtils, HiveTableRelation}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
+import org.apache.spark.sql.execution.adaptive.{DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
 import org.apache.spark.sql.execution.command.{FunctionsCommand, LoadDataCommand}
 import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
 import org.apache.spark.sql.functions._
@@ -42,6 +43,7 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.util.Utils
 
 case class Nested1(f1: Nested2)
@@ -67,7 +69,7 @@ case class Order(
  * Hive to generate them (in contrast to HiveQuerySuite).  Often this is because the query is
  * valid, but Hive currently cannot execute it.
  */
-class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
+abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHiveSingleton {
   import hiveContext._
   import spark.implicits._
 
@@ -88,24 +90,28 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   }
 
   test("script") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    assume(TestUtils.testCommandAvailable("echo | sed"))
-    val scriptFilePath = getTestResourcePath("test_script.sh")
-    val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
-    df.createOrReplaceTempView("script_table")
-    val query1 = sql(
-      s"""
-        |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
-        |REDUCE c1, c2, c3 USING 'bash $scriptFilePath' AS
-        |(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
-    checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
+    withTempView("script_table") {
+      assume(TestUtils.testCommandAvailable("/bin/bash"))
+      assume(TestUtils.testCommandAvailable("echo | sed"))
+      val scriptFilePath = getTestResourcePath("test_script.sh")
+      val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
+      df.createOrReplaceTempView("script_table")
+      val query1 = sql(
+        s"""
+          |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
+          |REDUCE c1, c2, c3 USING 'bash $scriptFilePath' AS
+          |(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
+      checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
+    }
   }
 
   test("SPARK-6835: udtf in lateral view") {
-    val df = Seq((1, 1)).toDF("c1", "c2")
-    df.createOrReplaceTempView("table1")
-    val query = sql("SELECT c1, v FROM table1 LATERAL VIEW stack(3, 1, c1 + 1, c1 + 2) d AS v")
-    checkAnswer(query, Row(1, 1) :: Row(1, 2) :: Row(1, 3) :: Nil)
+    withTempView("table1") {
+      val df = Seq((1, 1)).toDF("c1", "c2")
+      df.createOrReplaceTempView("table1")
+      val query = sql("SELECT c1, v FROM table1 LATERAL VIEW stack(3, 1, c1 + 1, c1 + 2) d AS v")
+      checkAnswer(query, Row(1, 1) :: Row(1, 2) :: Row(1, 3) :: Nil)
+    }
   }
 
   test("SPARK-13651: generator outputs shouldn't be resolved from its child's output") {
@@ -119,70 +125,72 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   }
 
   test("SPARK-6851: Self-joined converted parquet tables") {
-    val orders = Seq(
-      Order(1, "Atlas", "MTB", 234, "2015-01-07", "John D", "Pacifica", "CA", 20151),
-      Order(3, "Swift", "MTB", 285, "2015-01-17", "John S", "Redwood City", "CA", 20151),
-      Order(4, "Atlas", "Hybrid", 303, "2015-01-23", "Jones S", "San Mateo", "CA", 20151),
-      Order(7, "Next", "MTB", 356, "2015-01-04", "Jane D", "Daly City", "CA", 20151),
-      Order(10, "Next", "YFlikr", 187, "2015-01-09", "John D", "Fremont", "CA", 20151),
-      Order(11, "Swift", "YFlikr", 187, "2015-01-23", "John D", "Hayward", "CA", 20151),
-      Order(2, "Next", "Hybrid", 324, "2015-02-03", "Jane D", "Daly City", "CA", 20152),
-      Order(5, "Next", "Street", 187, "2015-02-08", "John D", "Fremont", "CA", 20152),
-      Order(6, "Atlas", "Street", 154, "2015-02-09", "John D", "Pacifica", "CA", 20152),
-      Order(8, "Swift", "Hybrid", 485, "2015-02-19", "John S", "Redwood City", "CA", 20152),
-      Order(9, "Atlas", "Split", 303, "2015-02-28", "Jones S", "San Mateo", "CA", 20152))
-
-    val orderUpdates = Seq(
-      Order(1, "Atlas", "MTB", 434, "2015-01-07", "John D", "Pacifica", "CA", 20151),
-      Order(11, "Swift", "YFlikr", 137, "2015-01-23", "John D", "Hayward", "CA", 20151))
-
-    orders.toDF.createOrReplaceTempView("orders1")
-    orderUpdates.toDF.createOrReplaceTempView("orderupdates1")
-
-    withTable("orders", "orderupdates") {
-      sql(
-        """CREATE TABLE orders(
-          |  id INT,
-          |  make String,
-          |  type String,
-          |  price INT,
-          |  pdate String,
-          |  customer String,
-          |  city String)
-          |PARTITIONED BY (state STRING, month INT)
-          |STORED AS PARQUET
-        """.stripMargin)
+    withTempView("orders1", "orderupdates1") {
+      val orders = Seq(
+        Order(1, "Atlas", "MTB", 234, "2015-01-07", "John D", "Pacifica", "CA", 20151),
+        Order(3, "Swift", "MTB", 285, "2015-01-17", "John S", "Redwood City", "CA", 20151),
+        Order(4, "Atlas", "Hybrid", 303, "2015-01-23", "Jones S", "San Mateo", "CA", 20151),
+        Order(7, "Next", "MTB", 356, "2015-01-04", "Jane D", "Daly City", "CA", 20151),
+        Order(10, "Next", "YFlikr", 187, "2015-01-09", "John D", "Fremont", "CA", 20151),
+        Order(11, "Swift", "YFlikr", 187, "2015-01-23", "John D", "Hayward", "CA", 20151),
+        Order(2, "Next", "Hybrid", 324, "2015-02-03", "Jane D", "Daly City", "CA", 20152),
+        Order(5, "Next", "Street", 187, "2015-02-08", "John D", "Fremont", "CA", 20152),
+        Order(6, "Atlas", "Street", 154, "2015-02-09", "John D", "Pacifica", "CA", 20152),
+        Order(8, "Swift", "Hybrid", 485, "2015-02-19", "John S", "Redwood City", "CA", 20152),
+        Order(9, "Atlas", "Split", 303, "2015-02-28", "Jones S", "San Mateo", "CA", 20152))
+
+      val orderUpdates = Seq(
+        Order(1, "Atlas", "MTB", 434, "2015-01-07", "John D", "Pacifica", "CA", 20151),
+        Order(11, "Swift", "YFlikr", 137, "2015-01-23", "John D", "Hayward", "CA", 20151))
+
+      orders.toDF.createOrReplaceTempView("orders1")
+      orderUpdates.toDF.createOrReplaceTempView("orderupdates1")
+
+      withTable("orders", "orderupdates") {
+        sql(
+          """CREATE TABLE orders(
+            |  id INT,
+            |  make String,
+            |  type String,
+            |  price INT,
+            |  pdate String,
+            |  customer String,
+            |  city String)
+            |PARTITIONED BY (state STRING, month INT)
+            |STORED AS PARQUET
+          """.stripMargin)
 
-      sql(
-        """CREATE TABLE orderupdates(
-          |  id INT,
-          |  make String,
-          |  type String,
-          |  price INT,
-          |  pdate String,
-          |  customer String,
-          |  city String)
-          |PARTITIONED BY (state STRING, month INT)
-          |STORED AS PARQUET
-        """.stripMargin)
+        sql(
+          """CREATE TABLE orderupdates(
+            |  id INT,
+            |  make String,
+            |  type String,
+            |  price INT,
+            |  pdate String,
+            |  customer String,
+            |  city String)
+            |PARTITIONED BY (state STRING, month INT)
+            |STORED AS PARQUET
+          """.stripMargin)
 
-      sql("set hive.exec.dynamic.partition.mode=nonstrict")
-      sql("INSERT INTO TABLE orders PARTITION(state, month) SELECT * FROM orders1")
-      sql("INSERT INTO TABLE orderupdates PARTITION(state, month) SELECT * FROM orderupdates1")
+        sql("set hive.exec.dynamic.partition.mode=nonstrict")
+        sql("INSERT INTO TABLE orders PARTITION(state, month) SELECT * FROM orders1")
+        sql("INSERT INTO TABLE orderupdates PARTITION(state, month) SELECT * FROM orderupdates1")
 
-      checkAnswer(
-        sql(
-          """
-            |select orders.state, orders.month
-            |from orders
-            |join (
-            |  select distinct orders.state,orders.month
-            |  from orders
-            |  join orderupdates
-            |    on orderupdates.id = orders.id) ao
-            |  on ao.state = orders.state and ao.month = orders.month
-          """.stripMargin),
-        (1 to 6).map(_ => Row("CA", 20151)))
+        checkAnswer(
+          sql(
+            """
+              |select orders.state, orders.month
+              |from orders
+              |join (
+              |  select distinct orders.state,orders.month
+              |  from orders
+              |  join orderupdates
+              |    on orderupdates.id = orders.id) ao
+              |  on ao.state = orders.state and ao.month = orders.month
+            """.stripMargin),
+          (1 to 6).map(_ => Row("CA", 20151)))
+      }
     }
   }
 
@@ -335,71 +343,76 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   }
 
   test("SPARK-5371: union with null and sum") {
-    val df = Seq((1, 1)).toDF("c1", "c2")
-    df.createOrReplaceTempView("table1")
-
-    val query = sql(
-      """
-        |SELECT
-        |  MIN(c1),
-        |  MIN(c2)
-        |FROM (
-        |  SELECT
-        |    SUM(c1) c1,
-        |    NULL c2
-        |  FROM table1
-        |  UNION ALL
-        |  SELECT
-        |    NULL c1,
-        |    SUM(c2) c2
-        |  FROM table1
-        |) a
-      """.stripMargin)
-    checkAnswer(query, Row(1, 1) :: Nil)
-  }
-
-  test("CTAS with WITH clause") {
+    withTempView("table1") {
+      val df = Seq((1, 1)).toDF("c1", "c2")
+      df.createOrReplaceTempView("table1")
 
-    val df = Seq((1, 1)).toDF("c1", "c2")
-    df.createOrReplaceTempView("table1")
-    withTable("with_table1") {
-      sql(
+      val query = sql(
         """
-          |CREATE TABLE with_table1 AS
-          |WITH T AS (
-          |  SELECT *
+          |SELECT
+          |  MIN(c1),
+          |  MIN(c2)
+          |FROM (
+          |  SELECT
+          |    SUM(c1) c1,
+          |    NULL c2
+          |  FROM table1
+          |  UNION ALL
+          |  SELECT
+          |    NULL c1,
+          |    SUM(c2) c2
           |  FROM table1
-          |)
-          |SELECT *
-          |FROM T
+          |) a
         """.stripMargin)
-      val query = sql("SELECT * FROM with_table1")
       checkAnswer(query, Row(1, 1) :: Nil)
     }
   }
 
+  test("CTAS with WITH clause") {
+    withTempView("table1") {
+      val df = Seq((1, 1)).toDF("c1", "c2")
+      df.createOrReplaceTempView("table1")
+      withTable("with_table1") {
+        sql(
+          """
+            |CREATE TABLE with_table1 AS
+            |WITH T AS (
+            |  SELECT *
+            |  FROM table1
+            |)
+            |SELECT *
+            |FROM T
+          """.stripMargin)
+        val query = sql("SELECT * FROM with_table1")
+        checkAnswer(query, Row(1, 1) :: Nil)
+      }
+    }
+  }
+
   test("explode nested Field") {
-    Seq(NestedArray1(NestedArray2(Seq(1, 2, 3)))).toDF.createOrReplaceTempView("nestedArray")
-    checkAnswer(
-      sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"),
-      Row(1) :: Row(2) :: Row(3) :: Nil)
+    withTempView("nestedArray") {
+      Seq(NestedArray1(NestedArray2(Seq(1, 2, 3)))).toDF.createOrReplaceTempView("nestedArray")
+      checkAnswer(
+        sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"),
+        Row(1) :: Row(2) :: Row(3) :: Nil)
 
-    checkAnswer(
-      sql("SELECT `ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"),
-      Row(1) :: Row(2) :: Row(3) :: Nil)
+      checkAnswer(
+        sql("SELECT `ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"),
+        Row(1) :: Row(2) :: Row(3) :: Nil)
 
-    checkAnswer(
-      sql("SELECT `a`.`ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"),
-      Row(1) :: Row(2) :: Row(3) :: Nil)
+      checkAnswer(
+        sql("SELECT `a`.`ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"),
+        Row(1) :: Row(2) :: Row(3) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT `weird``tab`.`weird``col`
-          |FROM nestedArray
-          |LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col`
-        """.stripMargin),
-      Row(1) :: Row(2) :: Row(3) :: Nil)
+      checkAnswer(
+        sql(
+          """
+            |SELECT `weird``tab`.`weird``col`
+            |FROM nestedArray
+            |LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col`
+          """.stripMargin),
+        Row(1) :: Row(2) :: Row(3) :: Nil)
+    }
   }
 
   test("SPARK-4512 Fix attribute reference resolution error when using SORT BY") {
@@ -741,20 +754,22 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   }
 
   test("double nested data") {
-    withTable("test_ctas_1234") {
-      sparkContext.parallelize(Nested1(Nested2(Nested3(1))) :: Nil)
-        .toDF().createOrReplaceTempView("nested")
-      checkAnswer(
-        sql("SELECT f1.f2.f3 FROM nested"),
-        Row(1))
+    withTempView("nested") {
+      withTable("test_ctas_1234") {
+        sparkContext.parallelize(Nested1(Nested2(Nested3(1))) :: Nil)
+          .toDF().createOrReplaceTempView("nested")
+        checkAnswer(
+          sql("SELECT f1.f2.f3 FROM nested"),
+          Row(1))
 
-      sql("CREATE TABLE test_ctas_1234 AS SELECT * from nested")
-      checkAnswer(
-        sql("SELECT * FROM test_ctas_1234"),
-        sql("SELECT * FROM nested").collect().toSeq)
+        sql("CREATE TABLE test_ctas_1234 AS SELECT * from nested")
+        checkAnswer(
+          sql("SELECT * FROM test_ctas_1234"),
+          sql("SELECT * FROM nested").collect().toSeq)
 
-      intercept[AnalysisException] {
-        sql("CREATE TABLE test_ctas_1234 AS SELECT * from notexists").collect()
+        intercept[AnalysisException] {
+          sql("CREATE TABLE test_ctas_1234 AS SELECT * from notexists").collect()
+        }
       }
     }
   }
@@ -831,13 +846,15 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   }
 
   test("SPARK-4963 DataFrame sample on mutable row return wrong result") {
-    sql("SELECT * FROM src WHERE key % 2 = 0")
-      .sample(withReplacement = false, fraction = 0.3)
-      .createOrReplaceTempView("sampled")
-    (1 to 10).foreach { i =>
-      checkAnswer(
-        sql("SELECT * FROM sampled WHERE key % 2 = 1"),
-        Seq.empty[Row])
+    withTempView("sampled") {
+      sql("SELECT * FROM src WHERE key % 2 = 0")
+        .sample(withReplacement = false, fraction = 0.3)
+        .createOrReplaceTempView("sampled")
+      (1 to 10).foreach { i =>
+        checkAnswer(
+          sql("SELECT * FROM sampled WHERE key % 2 = 1"),
+          Seq.empty[Row])
+      }
     }
   }
 
@@ -898,33 +915,38 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   }
 
   test("resolve udtf in projection #1") {
-    val ds = (1 to 5).map(i => s"""{"a":[$i, ${i + 1}]}""").toDS()
-    read.json(ds).createOrReplaceTempView("data")
-    val df = sql("SELECT explode(a) AS val FROM data")
-    val col = df("val")
+    withTempView("data") {
+      val ds = (1 to 5).map(i => s"""{"a":[$i, ${i + 1}]}""").toDS()
+      read.json(ds).createOrReplaceTempView("data")
+      sql("SELECT explode(a) AS val FROM data")
+    }
   }
 
   test("resolve udtf in projection #2") {
-    val ds = (1 to 2).map(i => s"""{"a":[$i, ${i + 1}]}""").toDS()
-    read.json(ds).createOrReplaceTempView("data")
-    checkAnswer(sql("SELECT explode(map(1, 1)) FROM data LIMIT 1"), Row(1, 1) :: Nil)
-    checkAnswer(sql("SELECT explode(map(1, 1)) as (k1, k2) FROM data LIMIT 1"), Row(1, 1) :: Nil)
-    intercept[AnalysisException] {
-      sql("SELECT explode(map(1, 1)) as k1 FROM data LIMIT 1")
-    }
+    withTempView("data") {
+      val ds = (1 to 2).map(i => s"""{"a":[$i, ${i + 1}]}""").toDS()
+      read.json(ds).createOrReplaceTempView("data")
+      checkAnswer(sql("SELECT explode(map(1, 1)) FROM data LIMIT 1"), Row(1, 1) :: Nil)
+      checkAnswer(sql("SELECT explode(map(1, 1)) as (k1, k2) FROM data LIMIT 1"), Row(1, 1) :: Nil)
+      intercept[AnalysisException] {
+        sql("SELECT explode(map(1, 1)) as k1 FROM data LIMIT 1")
+      }
 
-    intercept[AnalysisException] {
-      sql("SELECT explode(map(1, 1)) as (k1, k2, k3) FROM data LIMIT 1")
+      intercept[AnalysisException] {
+        sql("SELECT explode(map(1, 1)) as (k1, k2, k3) FROM data LIMIT 1")
+      }
     }
   }
 
   // TGF with non-TGF in project is allowed in Spark SQL, but not in Hive
   test("TGF with non-TGF in projection") {
-    val ds = Seq("""{"a": "1", "b":"1"}""").toDS()
-    read.json(ds).createOrReplaceTempView("data")
-    checkAnswer(
-      sql("SELECT explode(map(a, b)) as (k1, k2), a, b FROM data"),
-      Row("1", "1", "1", "1") :: Nil)
+    withTempView("data") {
+      val ds = Seq("""{"a": "1", "b":"1"}""").toDS()
+      read.json(ds).createOrReplaceTempView("data")
+      checkAnswer(
+        sql("SELECT explode(map(a, b)) as (k1, k2), a, b FROM data"),
+        Row("1", "1", "1", "1") :: Nil)
+    }
   }
 
   test("logical.Project should not be resolved if it contains aggregates or generators") {
@@ -974,47 +996,58 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   }
 
   test("SPARK-5203 union with different decimal precision") {
-    Seq.empty[(java.math.BigDecimal, java.math.BigDecimal)]
-      .toDF("d1", "d2")
-      .select($"d1".cast(DecimalType(10, 5)).as("d"))
-      .createOrReplaceTempView("dn")
+    withTempView("dn") {
+      Seq.empty[(java.math.BigDecimal, java.math.BigDecimal)]
+        .toDF("d1", "d2")
+        .select($"d1".cast(DecimalType(10, 5)).as("d"))
+        .createOrReplaceTempView("dn")
 
-    sql("select d from dn union all select d * 2 from dn")
-      .queryExecution.analyzed
+      sql("select d from dn union all select d * 2 from dn")
+        .queryExecution.analyzed
+    }
   }
 
   test("Star Expansion - script transform") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    val data = (1 to 100000).map { i => (i, i, i) }
-    data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
-    assert(100000 === sql("SELECT TRANSFORM (*) USING 'cat' FROM script_trans").count())
+    withTempView("script_trans") {
+      assume(TestUtils.testCommandAvailable("/bin/bash"))
+      val data = (1 to 100000).map { i => (i, i, i) }
+      data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
+      assert(100000 === sql("SELECT TRANSFORM (*) USING 'cat' FROM script_trans").count())
+    }
   }
 
   test("test script transform for stdout") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    val data = (1 to 100000).map { i => (i, i, i) }
-    data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
-    assert(100000 ===
-      sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat' AS (a,b,c) FROM script_trans").count())
+    withTempView("script_trans") {
+      assume(TestUtils.testCommandAvailable("/bin/bash"))
+      val data = (1 to 100000).map { i => (i, i, i) }
+      data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
+      assert(100000 ===
+        sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat' AS (a,b,c) FROM script_trans").count())
+    }
   }
 
   test("test script transform for stderr") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    val data = (1 to 100000).map { i => (i, i, i) }
-    data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
-    assert(0 ===
-      sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat 1>&2' AS (a,b,c) FROM script_trans").count())
+    withTempView("script_trans") {
+      assume(TestUtils.testCommandAvailable("/bin/bash"))
+      val data = (1 to 100000).map { i => (i, i, i) }
+      data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
+      assert(0 ===
+        sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat 1>&2' AS (a,b,c) FROM script_trans").count())
+    }
   }
 
   test("test script transform data type") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    val data = (1 to 5).map { i => (i, i) }
-    data.toDF("key", "value").createOrReplaceTempView("test")
-    checkAnswer(
-      sql("""FROM
-          |(FROM test SELECT TRANSFORM(key, value) USING 'cat' AS (`thing1` int, thing2 string)) t
-          |SELECT thing1 + 1
-        """.stripMargin), (2 to 6).map(i => Row(i)))
+    withTempView("test") {
+      assume(TestUtils.testCommandAvailable("/bin/bash"))
+      val data = (1 to 5).map { i => (i, i) }
+      data.toDF("key", "value").createOrReplaceTempView("test")
+      checkAnswer(
+        sql(
+          """FROM
+            |(FROM test SELECT TRANSFORM(key, value) USING 'cat' AS (`thing1` int, thing2 string)) t
+            |SELECT thing1 + 1
+          """.stripMargin), (2 to 6).map(i => Row(i)))
+    }
   }
 
   test("Sorting columns are not in Generate") {
@@ -1045,27 +1078,31 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   }
 
   test("test case key when") {
-    (1 to 5).map(i => (i, i.toString)).toDF("k", "v").createOrReplaceTempView("t")
-    checkAnswer(
-      sql("SELECT CASE k WHEN 2 THEN 22 WHEN 4 THEN 44 ELSE 0 END, v FROM t"),
-      Row(0, "1") :: Row(22, "2") :: Row(0, "3") :: Row(44, "4") :: Row(0, "5") :: Nil)
+    withTempView("t") {
+      (1 to 5).map(i => (i, i.toString)).toDF("k", "v").createOrReplaceTempView("t")
+      checkAnswer(
+        sql("SELECT CASE k WHEN 2 THEN 22 WHEN 4 THEN 44 ELSE 0 END, v FROM t"),
+        Row(0, "1") :: Row(22, "2") :: Row(0, "3") :: Row(44, "4") :: Row(0, "5") :: Nil)
+    }
   }
 
   test("SPARK-7269 Check analysis failed in case in-sensitive") {
-    Seq(1, 2, 3).map { i =>
-      (i.toString, i.toString)
-    }.toDF("key", "value").createOrReplaceTempView("df_analysis")
-    sql("SELECT kEy from df_analysis group by key").collect()
-    sql("SELECT kEy+3 from df_analysis group by key+3").collect()
-    sql("SELECT kEy+3, a.kEy, A.kEy from df_analysis A group by key").collect()
-    sql("SELECT cast(kEy+1 as Int) from df_analysis A group by cast(key+1 as int)").collect()
-    sql("SELECT cast(kEy+1 as Int) from df_analysis A group by key+1").collect()
-    sql("SELECT 2 from df_analysis A group by key+1").collect()
-    intercept[AnalysisException] {
-      sql("SELECT kEy+1 from df_analysis group by key+3")
-    }
-    intercept[AnalysisException] {
-      sql("SELECT cast(key+2 as Int) from df_analysis A group by cast(key+1 as int)")
+    withTempView("df_analysis") {
+      Seq(1, 2, 3).map { i =>
+        (i.toString, i.toString)
+      }.toDF("key", "value").createOrReplaceTempView("df_analysis")
+      sql("SELECT kEy from df_analysis group by key").collect()
+      sql("SELECT kEy+3 from df_analysis group by key+3").collect()
+      sql("SELECT kEy+3, a.kEy, A.kEy from df_analysis A group by key").collect()
+      sql("SELECT cast(kEy+1 as Int) from df_analysis A group by cast(key+1 as int)").collect()
+      sql("SELECT cast(kEy+1 as Int) from df_analysis A group by key+1").collect()
+      sql("SELECT 2 from df_analysis A group by key+1").collect()
+      intercept[AnalysisException] {
+        sql("SELECT kEy+1 from df_analysis group by key+3")
+      }
+      intercept[AnalysisException] {
+        sql("SELECT cast(key+2 as Int) from df_analysis A group by cast(key+1 as int)")
+      }
     }
   }
 
@@ -1178,10 +1215,12 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   }
 
   test("SPARK-9371: fix the support for special chars in column names for hive context") {
-    val ds = Seq("""{"a": {"c.b": 1}, "b.$q": [{"a@!.q": 1}], "q.w": {"w.i&": [1]}}""").toDS()
-    read.json(ds).createOrReplaceTempView("t")
+    withTempView("t") {
+      val ds = Seq("""{"a": {"c.b": 1}, "b.$q": [{"a@!.q": 1}], "q.w": {"w.i&": [1]}}""").toDS()
+      read.json(ds).createOrReplaceTempView("t")
 
-    checkAnswer(sql("SELECT a.`c.b`, `b.$q`[0].`a@!.q`, `q.w`.`w.i&`[0] FROM t"), Row(1, 1, 1))
+      checkAnswer(sql("SELECT a.`c.b`, `b.$q`[0].`a@!.q`, `q.w`.`w.i&`[0] FROM t"), Row(1, 1, 1))
+    }
   }
 
   test("specifying database name for a temporary view is not allowed") {
@@ -1236,43 +1275,47 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
 
   ignore("SPARK-10310: " +
     "script transformation using default input/output SerDe and record reader/writer") {
-    spark
-      .range(5)
-      .selectExpr("id AS a", "id AS b")
-      .createOrReplaceTempView("test")
+    withTempView("test") {
+      spark
+        .range(5)
+        .selectExpr("id AS a", "id AS b")
+        .createOrReplaceTempView("test")
 
-    val scriptFilePath = getTestResourcePath("data")
-    checkAnswer(
-      sql(
-        s"""FROM(
-          |  FROM test SELECT TRANSFORM(a, b)
-          |  USING 'python $scriptFilePath/scripts/test_transform.py "\t"'
-          |  AS (c STRING, d STRING)
-          |) t
-          |SELECT c
-        """.stripMargin),
-      (0 until 5).map(i => Row(i + "#")))
+      val scriptFilePath = getTestResourcePath("data")
+      checkAnswer(
+        sql(
+          s"""FROM(
+            |  FROM test SELECT TRANSFORM(a, b)
+            |  USING 'python $scriptFilePath/scripts/test_transform.py "\t"'
+            |  AS (c STRING, d STRING)
+            |) t
+            |SELECT c
+          """.stripMargin),
+        (0 until 5).map(i => Row(i + "#")))
+    }
   }
 
   ignore("SPARK-10310: script transformation using LazySimpleSerDe") {
-    spark
-      .range(5)
-      .selectExpr("id AS a", "id AS b")
-      .createOrReplaceTempView("test")
-
-    val scriptFilePath = getTestResourcePath("data")
-    val df = sql(
-      s"""FROM test
-        |SELECT TRANSFORM(a, b)
-        |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-        |WITH SERDEPROPERTIES('field.delim' = '|')
-        |USING 'python $scriptFilePath/scripts/test_transform.py "|"'
-        |AS (c STRING, d STRING)
-        |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-        |WITH SERDEPROPERTIES('field.delim' = '|')
-      """.stripMargin)
+    withTempView("test") {
+      spark
+        .range(5)
+        .selectExpr("id AS a", "id AS b")
+        .createOrReplaceTempView("test")
+
+      val scriptFilePath = getTestResourcePath("data")
+      val df = sql(
+        s"""FROM test
+          |SELECT TRANSFORM(a, b)
+          |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+          |WITH SERDEPROPERTIES('field.delim' = '|')
+          |USING 'python $scriptFilePath/scripts/test_transform.py "|"'
+          |AS (c STRING, d STRING)
+          |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+          |WITH SERDEPROPERTIES('field.delim' = '|')
+        """.stripMargin)
 
-    checkAnswer(df, (0 until 5).map(i => Row(i + "#", i + "#")))
+      checkAnswer(df, (0 until 5).map(i => Row(i + "#", i + "#")))
+    }
   }
 
   test("SPARK-10741: Sort on Aggregate using parquet") {
@@ -2493,50 +2536,32 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
     }
   }
 
-  test("SPARK-26560 Spark should be able to run Hive UDF using jar regardless of " +
-    "current thread context classloader") {
-    // force to use Spark classloader as other test (even in other test suites) may change the
-    // current thread's context classloader to jar classloader
-    Utils.withContextClassLoader(Utils.getSparkClassLoader) {
-      withUserDefinedFunction("udtf_count3" -> false) {
-        val sparkClassLoader = Thread.currentThread().getContextClassLoader
-
-        // This jar file should not be placed to the classpath; GenericUDTFCount3 is slightly
-        // modified version of GenericUDTFCount2 in hive/contrib, which emits the count for
-        // three times.
-        val jarPath = "src/test/noclasspath/TestUDTF-spark-26560.jar"
-        val jarURL = s"file://${System.getProperty("user.dir")}/$jarPath"
-
-        sql(
-          s"""
-             |CREATE FUNCTION udtf_count3
-             |AS 'org.apache.hadoop.hive.contrib.udtf.example.GenericUDTFCount3'
-             |USING JAR '$jarURL'
-          """.stripMargin)
-
-        assert(Thread.currentThread().getContextClassLoader eq sparkClassLoader)
+  test("SPARK-31522: hive metastore related configurations should be static") {
+    Seq("spark.sql.hive.metastore.version",
+      "spark.sql.hive.metastore.jars",
+      "spark.sql.hive.metastore.sharedPrefixes",
+      "spark.sql.hive.metastore.barrierPrefixes").foreach { key =>
+      val e = intercept[AnalysisException](sql(s"set $key=abc"))
+      assert(e.getMessage.contains("Cannot modify the value of a static config"))
+    }
+  }
 
-        // JAR will be loaded at first usage, and it will change the current thread's
-        // context classloader to jar classloader in sharedState.
-        // See SessionState.addJar for details.
-        checkAnswer(
-          sql("SELECT udtf_count3(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"),
-          Row(3) :: Row(3) :: Row(3) :: Nil)
-
-        assert(Thread.currentThread().getContextClassLoader ne sparkClassLoader)
-        assert(Thread.currentThread().getContextClassLoader eq
-          spark.sqlContext.sharedState.jarClassLoader)
-
-        // Roll back to the original classloader and run query again. Without this line, the test
-        // would pass, as thread's context classloader is changed to jar classloader. But thread
-        // context classloader can be changed from others as well which would fail the query; one
-        // example is spark-shell, which thread context classloader rolls back automatically. This
-        // mimics the behavior of spark-shell.
-        Thread.currentThread().setContextClassLoader(sparkClassLoader)
-        checkAnswer(
-          sql("SELECT udtf_count3(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"),
-          Row(3) :: Row(3) :: Row(3) :: Nil)
+  test("SPARK-29295: dynamic partition map parsed from partition path should be case insensitive") {
+    withTable("t") {
+      withSQLConf("hive.exec.dynamic.partition" -> "true",
+        "hive.exec.dynamic.partition.mode" -> "nonstrict") {
+        withTempDir { loc =>
+          sql(s"CREATE TABLE t(c1 INT) PARTITIONED BY(P1 STRING) LOCATION '${loc.getAbsolutePath}'")
+          sql("INSERT OVERWRITE TABLE t PARTITION(P1) VALUES(1, 'caseSensitive')")
+          checkAnswer(sql("select * from t"), Row(1, "caseSensitive"))
+        }
       }
     }
   }
 }
+
+@SlowHiveTest
+class SQLQuerySuite extends SQLQuerySuiteBase with DisableAdaptiveExecutionSuite
+@SlowHiveTest
+class SQLQuerySuiteAE extends SQLQuerySuiteBase with EnableAdaptiveExecutionSuite
+
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
index 80a50c18bcb93..0d1fe20130c0e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.hive.execution
 
+import java.sql.Timestamp
+
 import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
 import org.scalatest.Assertions._
 import org.scalatest.BeforeAndAfterEach
@@ -24,15 +26,19 @@ import org.scalatest.exceptions.TestFailedException
 
 import org.apache.spark.{SparkException, TaskContext, TestUtils}
 import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.Column
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution.{SparkPlan, SparkPlanTest, UnaryExecNode}
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types.StringType
 
-class ScriptTransformationSuite extends SparkPlanTest with TestHiveSingleton with
-  BeforeAndAfterEach {
+class ScriptTransformationSuite extends SparkPlanTest with SQLTestUtils with TestHiveSingleton
+  with BeforeAndAfterEach {
   import spark.implicits._
 
   private val noSerdeIOSchema = HiveScriptIOSchema(
@@ -56,6 +62,14 @@ class ScriptTransformationSuite extends SparkPlanTest with TestHiveSingleton wit
 
   private val uncaughtExceptionHandler = new TestUncaughtExceptionHandler
 
+  // In Hive 1.2, the string representation of a decimal omits trailing zeroes.
+  // But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
+  val decimalToString: Column => Column = if (HiveUtils.isHive23) {
+    c => c.cast("string")
+  } else {
+    c => c.cast("decimal(1, 0)").cast("string")
+  }
+
   protected override def beforeAll(): Unit = {
     super.beforeAll()
     defaultUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler
@@ -186,6 +200,132 @@ class ScriptTransformationSuite extends SparkPlanTest with TestHiveSingleton wit
       rowsDf.select("name").collect())
     assert(uncaughtExceptionHandler.exception.isEmpty)
   }
+
+  test("SPARK-25990: TRANSFORM should handle different data types correctly") {
+    assume(TestUtils.testCommandAvailable("python"))
+    val scriptFilePath = getTestResourcePath("test_script.py")
+
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
+        (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
+        (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
+      ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
+      df.createTempView("v")
+
+      val query = sql(
+        s"""
+          |SELECT
+          |TRANSFORM(a, b, c, d, e)
+          |USING 'python $scriptFilePath' AS (a, b, c, d, e)
+          |FROM v
+        """.stripMargin)
+
+      checkAnswer(query, identity, df.select(
+        'a.cast("string"),
+        'b.cast("string"),
+        'c.cast("string"),
+        decimalToString('d),
+        'e.cast("string")).collect())
+    }
+  }
+
+  test("SPARK-30973: TRANSFORM should wait for the termination of the script (no serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+    val e = intercept[SparkException] {
+      val plan =
+        new ScriptTransformationExec(
+          input = Seq(rowsDf.col("a").expr),
+          script = "some_non_existent_command",
+          output = Seq(AttributeReference("a", StringType)()),
+          child = rowsDf.queryExecution.sparkPlan,
+          ioschema = noSerdeIOSchema)
+      SparkPlanTest.executePlan(plan, hiveContext)
+    }
+    assert(e.getMessage.contains("Subprocess exited with status"))
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  test("SPARK-30973: TRANSFORM should wait for the termination of the script (with serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+    val e = intercept[SparkException] {
+      val plan =
+        new ScriptTransformationExec(
+          input = Seq(rowsDf.col("a").expr),
+          script = "some_non_existent_command",
+          output = Seq(AttributeReference("a", StringType)()),
+          child = rowsDf.queryExecution.sparkPlan,
+          ioschema = serdeIOSchema)
+      SparkPlanTest.executePlan(plan, hiveContext)
+    }
+    assert(e.getMessage.contains("Subprocess exited with status"))
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+
+  test("SPARK-32608: Script Transform ROW FORMAT DELIMIT value should format value") {
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
+        (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
+        (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
+      ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
+      df.createTempView("v")
+
+      // input/output with same delimit
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, cast(e as string))
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY ','
+             |  COLLECTION ITEMS TERMINATED BY '#'
+             |  MAP KEYS TERMINATED BY '@'
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'null'
+             |  USING 'cat' AS (a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY ','
+             |  COLLECTION ITEMS TERMINATED BY '#'
+             |  MAP KEYS TERMINATED BY '@'
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'NULL'
+             |FROM v
+        """.stripMargin), identity, df.select(
+          'a.cast("string"),
+          'b.cast("string"),
+          'c.cast("string"),
+          'd.cast("string"),
+          'e.cast("string")).collect())
+
+      // input/output with different delimit and show result
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, cast(e as string))
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY ','
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'null'
+             |  USING 'cat' AS (value)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '&'
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'NULL'
+             |FROM v
+        """.stripMargin), identity, df.select(
+          concat_ws(",",
+            'a.cast("string"),
+            'b.cast("string"),
+            'c.cast("string"),
+            'd.cast("string"),
+            'e.cast("string"))).collect())
+    }
+  }
 }
 
 private case class ExceptionInjectingOperator(child: SparkPlan) extends UnaryExecNode {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala
index e6856a58b0ea9..1f1a5568b0201 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala
@@ -119,6 +119,27 @@ object CountSerDeAgg extends Aggregator[Int, CountSerDeSQL, CountSerDeSQL] {
   def outputEncoder: Encoder[CountSerDeSQL] = ExpressionEncoder[CountSerDeSQL]()
 }
 
+object ArrayDataAgg extends Aggregator[Array[Double], Array[Double], Array[Double]] {
+  def zero: Array[Double] = Array(0.0, 0.0, 0.0)
+  def reduce(s: Array[Double], array: Array[Double]): Array[Double] = {
+    require(s.length == array.length)
+    for ( j <- 0 until s.length ) {
+      s(j) += array(j)
+    }
+    s
+  }
+  def merge(s1: Array[Double], s2: Array[Double]): Array[Double] = {
+    require(s1.length == s2.length)
+    for ( j <- 0 until s1.length ) {
+      s1(j) += s2(j)
+    }
+    s1
+  }
+  def finish(s: Array[Double]): Array[Double] = s
+  def bufferEncoder: Encoder[Array[Double]] = ExpressionEncoder[Array[Double]]
+  def outputEncoder: Encoder[Array[Double]] = ExpressionEncoder[Array[Double]]
+}
+
 abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   import testImplicits._
 
@@ -156,20 +177,11 @@ abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSi
       (3, null, null)).toDF("key", "value1", "value2")
     data2.write.saveAsTable("agg2")
 
-    val data3 = Seq[(Seq[Integer], Integer, Integer)](
-      (Seq[Integer](1, 1), 10, -10),
-      (Seq[Integer](null), -60, 60),
-      (Seq[Integer](1, 1), 30, -30),
-      (Seq[Integer](1), 30, 30),
-      (Seq[Integer](2), 1, 1),
-      (null, -10, 10),
-      (Seq[Integer](2, 3), -1, null),
-      (Seq[Integer](2, 3), 1, 1),
-      (Seq[Integer](2, 3, 4), null, 1),
-      (Seq[Integer](null), 100, -10),
-      (Seq[Integer](3), null, 3),
-      (null, null, null),
-      (Seq[Integer](3), null, null)).toDF("key", "value1", "value2")
+    val data3 = Seq[(Seq[Double], Int)](
+      (Seq(1.0, 2.0, 3.0), 0),
+      (Seq(4.0, 5.0, 6.0), 0),
+      (Seq(7.0, 8.0, 9.0), 0)
+    ).toDF("data", "dummy")
     data3.write.saveAsTable("agg3")
 
     val data4 = Seq[Boolean](true, false, true).toDF("boolvalues")
@@ -184,6 +196,7 @@ abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSi
     spark.udf.register("mydoublesum", udaf(MyDoubleSumAgg))
     spark.udf.register("mydoubleavg", udaf(MyDoubleAvgAgg))
     spark.udf.register("longProductSum", udaf(LongProductSumAgg))
+    spark.udf.register("arraysum", udaf(ArrayDataAgg))
   }
 
   override def afterAll(): Unit = {
@@ -354,6 +367,12 @@ abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSi
         Row(3, 0, null, 1, 3, 0, 0, 0, null, 1, 3, 0, 2, 2) :: Nil)
   }
 
+  test("SPARK-32159: array encoders should be resolved in analyzer") {
+    checkAnswer(
+      spark.sql("SELECT arraysum(data) FROM agg3"),
+      Row(Seq(12.0, 15.0, 18.0)) :: Nil)
+  }
+
   test("verify aggregator ser/de behavior") {
     val data = sparkContext.parallelize((1 to 100).toSeq, 3).toDF("value1")
     val agg = udaf(CountSerDeAgg)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala
index 990d9425fb7fc..12ee5bea7c2f9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala
@@ -288,4 +288,32 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton {
       }
     }
   }
+
+  test("SPARK-32234 read ORC table with column names all starting with '_col'") {
+    Seq("native", "hive").foreach { orcImpl =>
+      Seq("false", "true").foreach { vectorized =>
+        withSQLConf(
+          SQLConf.ORC_IMPLEMENTATION.key -> orcImpl,
+          SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized) {
+          withTable("test_hive_orc_impl") {
+            spark.sql(
+              s"""
+                 | CREATE TABLE test_hive_orc_impl
+                 | (_col1 INT, _col2 STRING, _col3 INT)
+                 | STORED AS ORC
+               """.stripMargin)
+            spark.sql(
+              s"""
+                 | INSERT INTO
+                 | test_hive_orc_impl
+                 | VALUES(9, '12', 2020)
+               """.stripMargin)
+
+            val df = spark.sql("SELECT _col2 FROM test_hive_orc_impl")
+            checkAnswer(df, Row("12"))
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
index f3e712d6c0a4a..91fd8a47339fc 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
@@ -320,4 +320,11 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
       }
     }
   }
+
+  test("SPARK-31580: Read a file written before ORC-569") {
+    assume(HiveUtils.isHive23) // Hive 1.2 doesn't use Apache ORC
+    // Test ORC file came from ORC-621
+    val df = readResourceOrcFile("test-data/TestStringDictionary.testRowIndex.orc")
+    assert(df.where("str < 'row 001000'").count() === 1000)
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
index cc4592a5caf68..a3e2444cae887 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -40,6 +40,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OneRowRelation}
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 import org.apache.spark.sql.execution.{QueryExecution, SQLExecution}
 import org.apache.spark.sql.execution.command.CacheTableCommand
 import org.apache.spark.sql.hive._
@@ -212,7 +213,7 @@ private[hive] class TestHiveSparkSession(
     }
   }
 
-  assume(sc.conf.get(CATALOG_IMPLEMENTATION) == "hive")
+  assert(sc.conf.get(CATALOG_IMPLEMENTATION) == "hive")
 
   @transient
   override lazy val sharedState: TestHiveSharedState = {
@@ -233,16 +234,16 @@ private[hive] class TestHiveSparkSession(
    * Dataset.ofRows that creates a TestHiveQueryExecution (rather than a normal QueryExecution
    * which wouldn't load all the test tables).
    */
-  override def sql(sqlText: String): DataFrame = {
+  override def sql(sqlText: String): DataFrame = withActive {
     val plan = sessionState.sqlParser.parsePlan(sqlText)
     Dataset.ofRows(self, plan)
   }
 
-  override def newSession(): TestHiveSparkSession = {
+  override def newSession(): TestHiveSparkSession = withActive {
     new TestHiveSparkSession(sc, Some(sharedState), None, loadTestTables)
   }
 
-  override def cloneSession(): SparkSession = {
+  override def cloneSession(): SparkSession = withActive {
     val result = new TestHiveSparkSession(
       sparkContext,
       Some(sharedState),
@@ -263,7 +264,10 @@ private[hive] class TestHiveSparkSession(
   System.clearProperty("spark.hostPort")
 
   // For some hive test case which contain ${system:test.tmp.dir}
-  System.setProperty("test.tmp.dir", Utils.createTempDir().toURI.getPath)
+  // Make sure it is not called again when cloning sessions.
+  if (parentSessionState.isEmpty) {
+    System.setProperty("test.tmp.dir", Utils.createTempDir().toURI.getPath)
+  }
 
   /** The location of the compiled hive distribution */
   lazy val hiveHome = envVarToFile("HIVE_HOME")
@@ -501,7 +505,7 @@ private[hive] class TestHiveSparkSession(
       // has already set the execution id.
       if (sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) == null) {
         // We don't actually have a `QueryExecution` here, use a fake one instead.
-        SQLExecution.withNewExecutionId(this, new QueryExecution(this, OneRowRelation())) {
+        SQLExecution.withNewExecutionId(new QueryExecution(this, OneRowRelation())) {
           createCmds.foreach(_())
         }
       } else {
@@ -586,22 +590,31 @@ private[hive] class TestHiveQueryExecution(
     this(TestHive.sparkSession, sql)
   }
 
-  override lazy val analyzed: LogicalPlan = {
+  override lazy val analyzed: LogicalPlan = sparkSession.withActive {
     val describedTables = logical match {
-      case CacheTableCommand(tbl, _, _, _) => tbl.table :: Nil
+      case CacheTableCommand(tbl, _, _, _) => tbl :: Nil
       case _ => Nil
     }
 
     // Make sure any test tables referenced are loaded.
     val referencedTables =
       describedTables ++
-        logical.collect { case UnresolvedRelation(ident) => ident.last }
+        logical.collect { case UnresolvedRelation(ident) => ident.asTableIdentifier }
     val resolver = sparkSession.sessionState.conf.resolver
-    val referencedTestTables = sparkSession.testTables.keys.filter { testTable =>
-      referencedTables.exists(resolver(_, testTable))
+    val referencedTestTables = referencedTables.flatMap { tbl =>
+      val testTableOpt = sparkSession.testTables.keys.find(resolver(_, tbl.table))
+      testTableOpt.map(testTable => tbl.copy(table = testTable))
+    }
+    logDebug(s"Query references test tables: ${referencedTestTables.map(_.table).mkString(", ")}")
+    referencedTestTables.foreach { tbl =>
+      val curDB = sparkSession.catalog.currentDatabase
+      try {
+        tbl.database.foreach(db => sparkSession.catalog.setCurrentDatabase(db))
+        sparkSession.loadTestTable(tbl.table)
+      } finally {
+        tbl.database.foreach(_ => sparkSession.catalog.setCurrentDatabase(curDB))
+      }
     }
-    logDebug(s"Query references test tables: ${referencedTestTables.mkString(", ")}")
-    referencedTestTables.foreach(sparkSession.loadTestTable)
     // Proceed with analysis.
     sparkSession.sessionState.analyzer.executeAndCheck(logical, tracker)
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala
index f277f99805a4a..35dab79ff6dff 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala
@@ -23,6 +23,6 @@ import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 class BucketedReadWithHiveSupportSuite extends BucketedReadSuite with TestHiveSingleton {
   protected override def beforeAll(): Unit = {
     super.beforeAll()
-    assume(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive")
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive")
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteWithHiveSupportSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteWithHiveSupportSuite.scala
index 454e2f65d5d88..bdbdcc2951072 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteWithHiveSupportSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteWithHiveSupportSuite.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 class BucketedWriteWithHiveSupportSuite extends BucketedWriteSuite with TestHiveSingleton {
   protected override def beforeAll(): Unit = {
     super.beforeAll()
-    assume(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive")
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive")
   }
 
   override protected def fileFormatsToTest: Seq[String] = Seq("parquet", "orc")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
index 4ada5077aec7f..cbea74103343e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.execution.DataSourceScanExec
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
 
@@ -145,40 +146,52 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
 
           val seed = System.nanoTime()
           withClue(s"Random data generated with the seed: ${seed}") {
-            val dataGenerator = RandomDataGenerator.forType(
-              dataType = dataType,
-              nullable = true,
-              new Random(seed)
-            ).getOrElse {
-              fail(s"Failed to create data generator for schema $dataType")
+            val java8ApiConfValues = if (dataType == DateType || dataType == TimestampType) {
+              Seq(false, true)
+            } else {
+              Seq(false)
+            }
+            java8ApiConfValues.foreach { java8Api =>
+              withSQLConf(
+                SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString,
+                SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString,
+                SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+                val dataGenerator = RandomDataGenerator.forType(
+                  dataType = dataType,
+                  nullable = true,
+                  new Random(seed)
+                ).getOrElse {
+                  fail(s"Failed to create data generator for schema $dataType")
+                }
+
+                // Create a DF for the schema with random data. The index field is used to sort the
+                // DataFrame.  This is a workaround for SPARK-10591.
+                val schema = new StructType()
+                  .add("index", IntegerType, nullable = false)
+                  .add("col", dataType, nullable = true)
+                val rdd =
+                  spark.sparkContext.parallelize((1 to 20).map(i => Row(i, dataGenerator())))
+                val df = spark.createDataFrame(rdd, schema).orderBy("index").coalesce(1)
+
+                df.write
+                  .mode("overwrite")
+                  .format(dataSourceName)
+                  .option("dataSchema", df.schema.json)
+                  .options(extraOptions)
+                  .save(path)
+
+                val loadedDF = spark
+                  .read
+                  .format(dataSourceName)
+                  .option("dataSchema", df.schema.json)
+                  .schema(df.schema)
+                  .options(extraOptions)
+                  .load(path)
+                  .orderBy("index")
+
+                checkAnswer(loadedDF, df)
+              }
             }
-
-            // Create a DF for the schema with random data. The index field is used to sort the
-            // DataFrame.  This is a workaround for SPARK-10591.
-            val schema = new StructType()
-              .add("index", IntegerType, nullable = false)
-              .add("col", dataType, nullable = true)
-            val rdd =
-              spark.sparkContext.parallelize((1 to 10).map(i => Row(i, dataGenerator())))
-            val df = spark.createDataFrame(rdd, schema).orderBy("index").coalesce(1)
-
-            df.write
-              .mode("overwrite")
-              .format(dataSourceName)
-              .option("dataSchema", df.schema.json)
-              .options(extraOptions)
-              .save(path)
-
-            val loadedDF = spark
-              .read
-              .format(dataSourceName)
-              .option("dataSchema", df.schema.json)
-              .schema(df.schema)
-              .options(extraOptions)
-              .load(path)
-              .orderBy("index")
-
-            checkAnswer(loadedDF, df)
           }
         }
       }
diff --git a/sql/mkdocs.yml b/sql/mkdocs.yml
index c34c891bb9e42..4463e11f17d1f 100644
--- a/sql/mkdocs.yml
+++ b/sql/mkdocs.yml
@@ -15,5 +15,8 @@
 
 site_name: Spark SQL, Built-in Functions
 theme: readthedocs
-pages:
+nav:
   - 'Functions': 'index.md'
+markdown_extensions:
+  - toc:
+      anchorlink: True
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 87af6388e1118..251570496d805 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala b/streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala
new file mode 100644
index 0000000000000..bb80bd7072e8e
--- /dev/null
+++ b/streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.streaming
+
+import java.util.concurrent.TimeUnit
+
+import org.apache.spark.internal.config.ConfigBuilder
+import org.apache.spark.streaming.util.OpenHashMapBasedStateMap.DELTA_CHAIN_LENGTH_THRESHOLD
+
+object StreamingConf {
+
+  private[streaming] val BACKPRESSURE_ENABLED =
+    ConfigBuilder("spark.streaming.backpressure.enabled")
+      .version("1.5.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[streaming] val RECEIVER_MAX_RATE =
+    ConfigBuilder("spark.streaming.receiver.maxRate")
+      .version("1.0.2")
+      .longConf
+      .createWithDefault(Long.MaxValue)
+
+  private[streaming] val BACKPRESSURE_INITIAL_RATE =
+    ConfigBuilder("spark.streaming.backpressure.initialRate")
+      .version("2.0.0")
+      .fallbackConf(RECEIVER_MAX_RATE)
+
+  private[streaming] val BLOCK_INTERVAL =
+    ConfigBuilder("spark.streaming.blockInterval")
+      .version("0.8.0")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .createWithDefaultString("200ms")
+
+  private[streaming] val RECEIVER_WAL_ENABLE_CONF_KEY =
+    ConfigBuilder("spark.streaming.receiver.writeAheadLog.enable")
+      .version("1.2.1")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[streaming] val RECEIVER_WAL_CLASS_CONF_KEY =
+    ConfigBuilder("spark.streaming.receiver.writeAheadLog.class")
+      .version("1.4.0")
+      .stringConf
+      .createOptional
+
+  private[streaming] val RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY =
+    ConfigBuilder("spark.streaming.receiver.writeAheadLog.rollingIntervalSecs")
+      .version("1.4.0")
+      .intConf
+      .createWithDefault(60)
+
+  private[streaming] val RECEIVER_WAL_MAX_FAILURES_CONF_KEY =
+    ConfigBuilder("spark.streaming.receiver.writeAheadLog.maxFailures")
+      .version("1.2.0")
+      .intConf
+      .createWithDefault(3)
+
+  private[streaming] val RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY =
+    ConfigBuilder("spark.streaming.receiver.writeAheadLog.closeFileAfterWrite")
+      .version("1.6.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[streaming] val DRIVER_WAL_CLASS_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.class")
+      .version("1.4.0")
+      .stringConf
+      .createOptional
+
+  private[streaming] val DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.rollingIntervalSecs")
+      .version("1.4.0")
+      .intConf
+      .createWithDefault(60)
+
+  private[streaming] val DRIVER_WAL_MAX_FAILURES_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.maxFailures")
+      .version("1.4.0")
+      .intConf
+      .createWithDefault(3)
+
+  private[streaming] val DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.closeFileAfterWrite")
+      .version("1.6.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[streaming] val DRIVER_WAL_BATCHING_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.allowBatching")
+      .version("1.6.0")
+      .booleanConf
+      .createWithDefault(true)
+
+  private[streaming] val DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.batchingTimeout")
+      .version("1.6.0")
+      .longConf
+      .createWithDefault(5000)
+
+  private[streaming] val STREAMING_UNPERSIST =
+    ConfigBuilder("spark.streaming.unpersist")
+      .version("0.9.0")
+      .booleanConf
+      .createWithDefault(true)
+
+  private[streaming] val STOP_GRACEFULLY_ON_SHUTDOWN =
+    ConfigBuilder("spark.streaming.stopGracefullyOnShutdown")
+      .version("1.4.0")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[streaming] val UI_RETAINED_BATCHES =
+    ConfigBuilder("spark.streaming.ui.retainedBatches")
+      .version("1.0.0")
+      .intConf
+      .createWithDefault(1000)
+
+  private[streaming] val SESSION_BY_KEY_DELTA_CHAIN_THRESHOLD =
+    ConfigBuilder("spark.streaming.sessionByKey.deltaChainThreshold")
+      .version("1.6.0")
+      .intConf
+      .createWithDefault(DELTA_CHAIN_LENGTH_THRESHOLD)
+
+  private[streaming] val BACKPRESSURE_RATE_ESTIMATOR =
+    ConfigBuilder("spark.streaming.backpressure.rateEstimator")
+      .version("1.5.0")
+      .stringConf
+      .createWithDefault("pid")
+
+  private[streaming] val BACKPRESSURE_PID_PROPORTIONAL =
+    ConfigBuilder("spark.streaming.backpressure.pid.proportional")
+      .version("1.5.0")
+      .doubleConf
+      .createWithDefault(1.0)
+
+  private[streaming] val BACKPRESSURE_PID_INTEGRAL =
+    ConfigBuilder("spark.streaming.backpressure.pid.integral")
+      .version("1.5.0")
+      .doubleConf
+      .createWithDefault(0.2)
+
+  private[streaming] val BACKPRESSURE_PID_DERIVED =
+    ConfigBuilder("spark.streaming.backpressure.pid.derived")
+      .version("1.5.0")
+      .doubleConf
+      .createWithDefault(0.0)
+
+  private[streaming] val BACKPRESSURE_PID_MIN_RATE =
+    ConfigBuilder("spark.streaming.backpressure.pid.minRate")
+      .version("1.5.0")
+      .doubleConf
+      .createWithDefault(100)
+
+  private[streaming] val CONCURRENT_JOBS =
+    ConfigBuilder("spark.streaming.concurrentJobs")
+      .version("0.7.0")
+      .intConf
+      .createWithDefault(1)
+
+  private[streaming] val GRACEFUL_STOP_TIMEOUT =
+    ConfigBuilder("spark.streaming.gracefulStopTimeout")
+      .version("1.0.0")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .createOptional
+
+  private[streaming] val MANUAL_CLOCK_JUMP =
+    ConfigBuilder("spark.streaming.manualClock.jump")
+      .version("0.7.0")
+      .longConf
+      .createWithDefault(0)
+
+}
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
index 440b653e45de1..e3459c96335b3 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
@@ -41,6 +41,7 @@ import org.apache.spark.rdd.{RDD, RDDOperationScope}
 import org.apache.spark.scheduler.LiveListenerBus
 import org.apache.spark.serializer.SerializationDebugger
 import org.apache.spark.storage.StorageLevel
+import org.apache.spark.streaming.StreamingConf.STOP_GRACEFULLY_ON_SHUTDOWN
 import org.apache.spark.streaming.StreamingContextState._
 import org.apache.spark.streaming.dstream._
 import org.apache.spark.streaming.receiver.Receiver
@@ -717,7 +718,7 @@ class StreamingContext private[streaming] (
   }
 
   private def stopOnShutdown(): Unit = {
-    val stopGracefully = conf.getBoolean("spark.streaming.stopGracefullyOnShutdown", false)
+    val stopGracefully = conf.get(STOP_GRACEFULLY_ON_SHUTDOWN)
     logInfo(s"Invoking stop(stopGracefully=$stopGracefully) from shutdown hook")
     // Do not stop SparkContext, let its own shutdown hook stop it
     stop(stopSparkContext = false, stopGracefully = stopGracefully)
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala
index 6c981b293ac76..e037f26088347 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala
@@ -31,6 +31,7 @@ import org.apache.spark.internal.io.SparkHadoopWriterUtils
 import org.apache.spark.rdd.{BlockRDD, RDD, RDDOperationScope}
 import org.apache.spark.storage.StorageLevel
 import org.apache.spark.streaming._
+import org.apache.spark.streaming.StreamingConf.STREAMING_UNPERSIST
 import org.apache.spark.streaming.StreamingContext.rddToFileName
 import org.apache.spark.streaming.scheduler.Job
 import org.apache.spark.ui.{UIUtils => SparkUIUtils}
@@ -447,7 +448,7 @@ abstract class DStream[T: ClassTag] (
    * this to clear their own metadata along with the generated RDDs.
    */
   private[streaming] def clearMetadata(time: Time): Unit = {
-    val unpersistData = ssc.conf.getBoolean("spark.streaming.unpersist", true)
+    val unpersistData = ssc.conf.get(STREAMING_UNPERSIST)
     val oldRDDs = generatedRDDs.filter(_._1 <= (time - rememberDuration))
     logDebug("Clearing references to old RDDs: [" +
       oldRDDs.map(x => s"${x._1} -> ${x._2.id}").mkString(", ") + "]")
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/receiver/BlockGenerator.scala b/streaming/src/main/scala/org/apache/spark/streaming/receiver/BlockGenerator.scala
index 2533c53883cac..d641f55fa7f6f 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/receiver/BlockGenerator.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/receiver/BlockGenerator.scala
@@ -24,6 +24,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.internal.Logging
 import org.apache.spark.storage.StreamBlockId
+import org.apache.spark.streaming.StreamingConf.BLOCK_INTERVAL
 import org.apache.spark.streaming.util.RecurringTimer
 import org.apache.spark.util.{Clock, SystemClock}
 
@@ -100,8 +101,8 @@ private[streaming] class BlockGenerator(
   }
   import GeneratorState._
 
-  private val blockIntervalMs = conf.getTimeAsMs("spark.streaming.blockInterval", "200ms")
-  require(blockIntervalMs > 0, s"'spark.streaming.blockInterval' should be a positive value")
+  private val blockIntervalMs = conf.get(BLOCK_INTERVAL)
+  require(blockIntervalMs > 0, s"'${BLOCK_INTERVAL.key}' should be a positive value")
 
   private val blockIntervalTimer =
     new RecurringTimer(clock, blockIntervalMs, updateCurrentBuffer, "BlockGenerator")
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/receiver/RateLimiter.scala b/streaming/src/main/scala/org/apache/spark/streaming/receiver/RateLimiter.scala
index c620074b4e44d..f77ca3e8fdb45 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/receiver/RateLimiter.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/receiver/RateLimiter.scala
@@ -21,6 +21,7 @@ import com.google.common.util.concurrent.{RateLimiter => GuavaRateLimiter}
 
 import org.apache.spark.SparkConf
 import org.apache.spark.internal.Logging
+import org.apache.spark.streaming.StreamingConf.{BACKPRESSURE_INITIAL_RATE, RECEIVER_MAX_RATE}
 
 /**
  * Provides waitToPush() method to limit the rate at which receivers consume data.
@@ -37,7 +38,7 @@ import org.apache.spark.internal.Logging
 private[receiver] abstract class RateLimiter(conf: SparkConf) extends Logging {
 
   // treated as an upper limit
-  private val maxRateLimit = conf.getLong("spark.streaming.receiver.maxRate", Long.MaxValue)
+  private val maxRateLimit = conf.get(RECEIVER_MAX_RATE)
   private lazy val rateLimiter = GuavaRateLimiter.create(getInitialRateLimit().toDouble)
 
   def waitToPush(): Unit = {
@@ -68,6 +69,6 @@ private[receiver] abstract class RateLimiter(conf: SparkConf) extends Logging {
    * Get the initial rateLimit to initial rateLimiter
    */
   private def getInitialRateLimit(): Long = {
-    math.min(conf.getLong("spark.streaming.backpressure.initialRate", maxRateLimit), maxRateLimit)
+    math.min(conf.get(BACKPRESSURE_INITIAL_RATE), maxRateLimit)
   }
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobGenerator.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobGenerator.scala
index 7e8449ee5aa7e..8008a5c495e9d 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobGenerator.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobGenerator.scala
@@ -23,7 +23,7 @@ import scala.util.{Failure, Success, Try}
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
-import org.apache.spark.streaming.{Checkpoint, CheckpointWriter, Time}
+import org.apache.spark.streaming.{Checkpoint, CheckpointWriter, StreamingConf, Time}
 import org.apache.spark.streaming.api.python.PythonDStream
 import org.apache.spark.streaming.util.RecurringTimer
 import org.apache.spark.util.{Clock, EventLoop, ManualClock, Utils}
@@ -115,7 +115,7 @@ class JobGenerator(jobScheduler: JobScheduler) extends Logging {
       logInfo("Stopping JobGenerator gracefully")
       val timeWhenStopStarted = System.nanoTime()
       val stopTimeoutMs = conf.getTimeAsMs(
-        "spark.streaming.gracefulStopTimeout", s"${10 * ssc.graph.batchDuration.milliseconds}ms")
+        StreamingConf.GRACEFUL_STOP_TIMEOUT.key, s"${10 * ssc.graph.batchDuration.milliseconds}ms")
       val pollTime = 100
 
       // To prevent graceful stop to get stuck permanently
@@ -206,7 +206,7 @@ class JobGenerator(jobScheduler: JobScheduler) extends Logging {
     // or if the property is defined set it to that time
     if (clock.isInstanceOf[ManualClock]) {
       val lastTime = ssc.initialCheckpoint.checkpointTime.milliseconds
-      val jumpTime = ssc.sc.conf.getLong("spark.streaming.manualClock.jump", 0)
+      val jumpTime = ssc.sc.conf.get(StreamingConf.MANUAL_CLOCK_JUMP)
       clock.asInstanceOf[ManualClock].setTime(lastTime + jumpTime)
     }
 
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala
index 7eea57cc083ed..a6d8dccd7e722 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala
@@ -47,7 +47,7 @@ class JobScheduler(val ssc: StreamingContext) extends Logging {
   // Use of ConcurrentHashMap.keySet later causes an odd runtime problem due to Java 7/8 diff
   // https://gist.github.com/AlainODea/1375759b8720a3f9f094
   private val jobSets: java.util.Map[Time, JobSet] = new ConcurrentHashMap[Time, JobSet]
-  private val numConcurrentJobs = ssc.conf.getInt("spark.streaming.concurrentJobs", 1)
+  private val numConcurrentJobs = ssc.conf.get(StreamingConf.CONCURRENT_JOBS)
   private val jobExecutor =
     ThreadUtils.newDaemonFixedThreadPool(numConcurrentJobs, "streaming-job-executor")
   private[streaming] val jobGenerator = new JobGenerator(this)
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/RateController.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/RateController.scala
index 7774e85f778a6..88f191fa022c6 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/RateController.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/RateController.scala
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicLong
 import scala.concurrent.{ExecutionContext, Future}
 
 import org.apache.spark.SparkConf
+import org.apache.spark.streaming.StreamingConf.BACKPRESSURE_ENABLED
 import org.apache.spark.streaming.scheduler.rate.RateEstimator
 import org.apache.spark.util.{ThreadUtils, Utils}
 
@@ -86,5 +87,5 @@ private[streaming] abstract class RateController(val streamUID: Int, rateEstimat
 
 object RateController {
   def isBackPressureEnabled(conf: SparkConf): Boolean =
-    conf.getBoolean("spark.streaming.backpressure.enabled", false)
+    conf.get(BACKPRESSURE_ENABLED)
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/rate/RateEstimator.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/rate/RateEstimator.scala
index e4b9dffee04f4..7f4d0f298e8f2 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/rate/RateEstimator.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/rate/RateEstimator.scala
@@ -19,6 +19,7 @@ package org.apache.spark.streaming.scheduler.rate
 
 import org.apache.spark.SparkConf
 import org.apache.spark.streaming.Duration
+import org.apache.spark.streaming.StreamingConf._
 
 /**
  * A component that estimates the rate at which an `InputDStream` should ingest
@@ -57,12 +58,12 @@ object RateEstimator {
    * @throws IllegalArgumentException if the configured RateEstimator is not `pid`.
    */
   def create(conf: SparkConf, batchInterval: Duration): RateEstimator =
-    conf.get("spark.streaming.backpressure.rateEstimator", "pid") match {
+    conf.get(BACKPRESSURE_RATE_ESTIMATOR) match {
       case "pid" =>
-        val proportional = conf.getDouble("spark.streaming.backpressure.pid.proportional", 1.0)
-        val integral = conf.getDouble("spark.streaming.backpressure.pid.integral", 0.2)
-        val derived = conf.getDouble("spark.streaming.backpressure.pid.derived", 0.0)
-        val minRate = conf.getDouble("spark.streaming.backpressure.pid.minRate", 100)
+        val proportional = conf.get(BACKPRESSURE_PID_PROPORTIONAL)
+        val integral = conf.get(BACKPRESSURE_PID_INTEGRAL)
+        val derived = conf.get(BACKPRESSURE_PID_DERIVED)
+        val minRate = conf.get(BACKPRESSURE_PID_MIN_RATE)
         new PIDRateEstimator(batchInterval.milliseconds, proportional, integral, derived, minRate)
 
       case estimator =>
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala
index de73762beb860..da351ecf1889c 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala
@@ -24,7 +24,7 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable.{HashMap, Queue}
 
 import org.apache.spark.scheduler._
-import org.apache.spark.streaming.{StreamingContext, Time}
+import org.apache.spark.streaming.{StreamingConf, StreamingContext, Time}
 import org.apache.spark.streaming.scheduler._
 
 private[spark] class StreamingJobProgressListener(ssc: StreamingContext)
@@ -33,7 +33,7 @@ private[spark] class StreamingJobProgressListener(ssc: StreamingContext)
   private val waitingBatchUIData = new HashMap[Time, BatchUIData]
   private val runningBatchUIData = new HashMap[Time, BatchUIData]
   private val completedBatchUIData = new Queue[BatchUIData]
-  private val batchUIDataLimit = ssc.conf.getInt("spark.streaming.ui.retainedBatches", 1000)
+  private val batchUIDataLimit = ssc.conf.get(StreamingConf.UI_RETAINED_BATCHES)
   private var totalCompletedBatches = 0L
   private var totalReceivedRecords = 0L
   private var totalProcessedRecords = 0L
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
index d47287b6077f8..bbfb013c8dfdd 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
@@ -80,9 +80,10 @@ private[ui] class StreamingPage(parent: StreamingTab)
   /** Render the page */
   def render(request: HttpServletRequest): Seq[Node] = {
     val resources = generateLoadResources(request)
+    val onClickTimelineFunc = generateOnClickTimelineFunction()
     val basicInfo = generateBasicInfo()
     val content = resources ++
-      basicInfo ++
+      onClickTimelineFunc ++ basicInfo ++
       listener.synchronized {
         generateStatTable() ++
           generateBatchListTables()
@@ -101,6 +102,12 @@ private[ui] class StreamingPage(parent: StreamingTab)
     // scalastyle:on
   }
 
+  /** Generate html that will set onClickTimeline declared in streaming-page.js */
+  private def generateOnClickTimelineFunction(): Seq[Node] = {
+    val js = "onClickTimeline = getOnClickTimelineFunction();"
+    <script>{Unparsed(js)}</script>
+  }
+
   /** Generate basic information of the streaming program */
   private def generateBasicInfo(): Seq[Node] = {
     val timeSinceStart = System.currentTimeMillis() - startTime
@@ -140,6 +147,16 @@ private[ui] class StreamingPage(parent: StreamingTab)
     <script>{Unparsed(js)}</script>
   }
 
+  private def generateTimeTipStrings(times: Seq[Long]): Seq[Node] = {
+    // We leverage timeFormat as the value would be same as timeFormat. This means it is
+    // sensitive to the order - generateTimeMap should be called earlier than this.
+    val js = "var timeTipStrings = {};\n" + times.map { time =>
+      s"timeTipStrings[$time] = timeFormat[$time];"
+    }.mkString("\n")
+
+    <script>{Unparsed(js)}</script>
+  }
+
   private def generateStatTable(): Seq[Node] = {
     val batches = listener.retainedBatches
 
@@ -313,7 +330,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
     </table>
     // scalastyle:on
 
-    generateTimeMap(batchTimes) ++ table ++ jsCollector.toHtml
+    generateTimeMap(batchTimes) ++ generateTimeTipStrings(batchTimes) ++ table ++
+      jsCollector.toHtml
   }
 
   private def generateInputDStreamsTable(
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
index 146577214de17..006bcad5d68c2 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
@@ -58,7 +58,7 @@ private[streaming] object HdfsUtils {
         // If we are really unlucky, the file may be deleted as we're opening the stream.
         // This can happen as clean up is performed by daemon threads that may be left over from
         // previous runs.
-        if (!dfs.isFile(dfsPath)) null else throw e
+        if (!dfs.getFileStatus(dfsPath).isFile) null else throw e
     }
   }
 
@@ -92,6 +92,10 @@ private[streaming] object HdfsUtils {
   def checkFileExists(path: String, conf: Configuration): Boolean = {
     val hdpPath = new Path(path)
     val fs = getFileSystemForPath(hdpPath, conf)
-    fs.isFile(hdpPath)
+    try {
+      fs.getFileStatus(hdpPath).isFile
+    } catch {
+      case _: FileNotFoundException => false
+    }
   }
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/StateMap.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/StateMap.scala
index 618c036377aee..4224cef1cbae1 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/StateMap.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/StateMap.scala
@@ -26,6 +26,7 @@ import com.esotericsoftware.kryo.io.{Input, Output}
 
 import org.apache.spark.SparkConf
 import org.apache.spark.serializer.{KryoInputObjectInputBridge, KryoOutputObjectOutputBridge}
+import org.apache.spark.streaming.StreamingConf.SESSION_BY_KEY_DELTA_CHAIN_THRESHOLD
 import org.apache.spark.streaming.util.OpenHashMapBasedStateMap._
 import org.apache.spark.util.collection.OpenHashMap
 
@@ -61,8 +62,7 @@ private[streaming] object StateMap {
   def empty[K, S]: StateMap[K, S] = new EmptyStateMap[K, S]
 
   def create[K: ClassTag, S: ClassTag](conf: SparkConf): StateMap[K, S] = {
-    val deltaChainThreshold = conf.getInt("spark.streaming.sessionByKey.deltaChainThreshold",
-      DELTA_CHAIN_LENGTH_THRESHOLD)
+    val deltaChainThreshold = conf.get(SESSION_BY_KEY_DELTA_CHAIN_THRESHOLD)
     new OpenHashMapBasedStateMap[K, S](deltaChainThreshold)
   }
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/WriteAheadLogUtils.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/WriteAheadLogUtils.scala
index b0a4c98fc9e57..224e782066f60 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/WriteAheadLogUtils.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/WriteAheadLogUtils.scala
@@ -23,52 +23,34 @@ import org.apache.hadoop.conf.Configuration
 
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.internal.Logging
+import org.apache.spark.streaming.StreamingConf._
 import org.apache.spark.util.Utils
 
 /** A helper class with utility functions related to the WriteAheadLog interface */
 private[streaming] object WriteAheadLogUtils extends Logging {
-  val RECEIVER_WAL_ENABLE_CONF_KEY = "spark.streaming.receiver.writeAheadLog.enable"
-  val RECEIVER_WAL_CLASS_CONF_KEY = "spark.streaming.receiver.writeAheadLog.class"
-  val RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY =
-    "spark.streaming.receiver.writeAheadLog.rollingIntervalSecs"
-  val RECEIVER_WAL_MAX_FAILURES_CONF_KEY = "spark.streaming.receiver.writeAheadLog.maxFailures"
-  val RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY =
-    "spark.streaming.receiver.writeAheadLog.closeFileAfterWrite"
-
-  val DRIVER_WAL_CLASS_CONF_KEY = "spark.streaming.driver.writeAheadLog.class"
-  val DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY =
-    "spark.streaming.driver.writeAheadLog.rollingIntervalSecs"
-  val DRIVER_WAL_MAX_FAILURES_CONF_KEY = "spark.streaming.driver.writeAheadLog.maxFailures"
-  val DRIVER_WAL_BATCHING_CONF_KEY = "spark.streaming.driver.writeAheadLog.allowBatching"
-  val DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY = "spark.streaming.driver.writeAheadLog.batchingTimeout"
-  val DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY =
-    "spark.streaming.driver.writeAheadLog.closeFileAfterWrite"
-
-  val DEFAULT_ROLLING_INTERVAL_SECS = 60
-  val DEFAULT_MAX_FAILURES = 3
 
   def enableReceiverLog(conf: SparkConf): Boolean = {
-    conf.getBoolean(RECEIVER_WAL_ENABLE_CONF_KEY, false)
+    conf.get(RECEIVER_WAL_ENABLE_CONF_KEY)
   }
 
   def getRollingIntervalSecs(conf: SparkConf, isDriver: Boolean): Int = {
     if (isDriver) {
-      conf.getInt(DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY, DEFAULT_ROLLING_INTERVAL_SECS)
+      conf.get(DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY)
     } else {
-      conf.getInt(RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY, DEFAULT_ROLLING_INTERVAL_SECS)
+      conf.get(RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY)
     }
   }
 
   def getMaxFailures(conf: SparkConf, isDriver: Boolean): Int = {
     if (isDriver) {
-      conf.getInt(DRIVER_WAL_MAX_FAILURES_CONF_KEY, DEFAULT_MAX_FAILURES)
+      conf.get(DRIVER_WAL_MAX_FAILURES_CONF_KEY)
     } else {
-      conf.getInt(RECEIVER_WAL_MAX_FAILURES_CONF_KEY, DEFAULT_MAX_FAILURES)
+      conf.get(RECEIVER_WAL_MAX_FAILURES_CONF_KEY)
     }
   }
 
   def isBatchingEnabled(conf: SparkConf, isDriver: Boolean): Boolean = {
-    isDriver && conf.getBoolean(DRIVER_WAL_BATCHING_CONF_KEY, defaultValue = true)
+    isDriver && conf.get(DRIVER_WAL_BATCHING_CONF_KEY)
   }
 
   /**
@@ -76,14 +58,14 @@ private[streaming] object WriteAheadLogUtils extends Logging {
    * before we fail the write attempt to unblock receivers.
    */
   def getBatchingTimeout(conf: SparkConf): Long = {
-    conf.getLong(DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY, defaultValue = 5000)
+    conf.get(DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY)
   }
 
   def shouldCloseFileAfterWrite(conf: SparkConf, isDriver: Boolean): Boolean = {
     if (isDriver) {
-      conf.getBoolean(DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY, defaultValue = false)
+      conf.get(DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY)
     } else {
-      conf.getBoolean(RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY, defaultValue = false)
+      conf.get(RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY)
     }
   }
 
@@ -126,9 +108,9 @@ private[streaming] object WriteAheadLogUtils extends Logging {
     ): WriteAheadLog = {
 
     val classNameOption = if (isDriver) {
-      sparkConf.getOption(DRIVER_WAL_CLASS_CONF_KEY)
+      sparkConf.get(DRIVER_WAL_CLASS_CONF_KEY)
     } else {
-      sparkConf.getOption(RECEIVER_WAL_CLASS_CONF_KEY)
+      sparkConf.get(RECEIVER_WAL_CLASS_CONF_KEY)
     }
     val wal = classNameOption.map { className =>
       try {
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala
index 5e2ce25c7c441..6b332206e8f6d 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala
@@ -22,6 +22,7 @@ import scala.util.Random
 import org.apache.spark.{SparkConf, SparkEnv}
 import org.apache.spark.rdd.BlockRDD
 import org.apache.spark.storage.{StorageLevel, StreamBlockId}
+import org.apache.spark.streaming.StreamingConf.RECEIVER_WAL_ENABLE_CONF_KEY
 import org.apache.spark.streaming.dstream.ReceiverInputDStream
 import org.apache.spark.streaming.rdd.WriteAheadLogBackedBlockRDD
 import org.apache.spark.streaming.receiver.{BlockManagerBasedStoreResult, Receiver, WriteAheadLogBasedStoreResult}
@@ -117,7 +118,7 @@ class ReceiverInputDStreamSuite
   private def runTest(enableWAL: Boolean, body: ReceiverInputDStream[_] => Unit): Unit = {
     val conf = new SparkConf()
     conf.setMaster("local[4]").setAppName("ReceiverInputDStreamSuite")
-    conf.set(WriteAheadLogUtils.RECEIVER_WAL_ENABLE_CONF_KEY, enableWAL.toString)
+    conf.set(StreamingConf.RECEIVER_WAL_ENABLE_CONF_KEY.key, enableWAL.toString)
     require(WriteAheadLogUtils.enableReceiverLog(conf) === enableWAL)
     ssc = new StreamingContext(conf, Seconds(1))
     val receiverStream = new ReceiverInputDStream[Int](ssc) {
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala
index 1d6637861511f..4eff464dcdafb 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala
@@ -293,7 +293,8 @@ class StreamingContextSuite
     }
   }
 
-  test("stop gracefully") {
+  // TODO (SPARK-31728): re-enable it
+  ignore("stop gracefully") {
     val conf = new SparkConf().setMaster(master).setAppName(appName)
     conf.set("spark.dummyTimeConfig", "3600s")
     val sc = new SparkContext(conf)
diff --git a/tools/pom.xml b/tools/pom.xml
index e380e869f55c7..41662b1957703 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>